mirror of
https://github.com/wabbajack-tools/wabbajack.git
synced 2024-08-30 18:42:17 +00:00
List validation, can heal HTTP downloads (and probably Nexus archives)
This commit is contained in:
Timothy Baldridge
parent
74a332d6cb
commit
65cac27403
@ -26,6 +26,7 @@ namespace Wabbajack.BuildServer.Test
|
||||
{
|
||||
public ModListValidationTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@ -143,9 +144,10 @@ namespace Wabbajack.BuildServer.Test
|
||||
data = await ModlistMetadata.LoadFromGithub();
|
||||
Assert.Single(data);
|
||||
Assert.Equal(0, data.First().ValidationSummary.Failed);
|
||||
Assert.Equal(1, data.First().ValidationSummary.Passed);
|
||||
Assert.Equal(0, data.First().ValidationSummary.Passed);
|
||||
Assert.Equal(1, data.First().ValidationSummary.Updating);
|
||||
|
||||
await CheckListFeeds(0, 1);
|
||||
await CheckListFeeds(1, 0);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -26,7 +26,7 @@ namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
await SQL.IngestMetric(new Metric
|
||||
{
|
||||
MetricsKey = Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault(),
|
||||
MetricsKey = Request?.Headers[Consts.MetricsKeyHeader].FirstOrDefault() ?? "",
|
||||
Subject = subject,
|
||||
Action = verb,
|
||||
Timestamp = DateTime.UtcNow
|
||||
|
||||
@ -33,6 +33,7 @@ namespace Wabbajack.BuildServer.Controllers
|
||||
|
||||
public ListValidation(ILogger<ListValidation> logger, SqlService sql, AppSettings settings) : base(logger, sql)
|
||||
{
|
||||
_updater = new ModlistUpdater(null, sql, settings);
|
||||
_settings = settings;
|
||||
}
|
||||
|
||||
@ -42,21 +43,32 @@ namespace Wabbajack.BuildServer.Controllers
|
||||
|
||||
using var queue = new WorkQueue();
|
||||
|
||||
var results = data.ModLists.PMap(queue, list =>
|
||||
var results = await data.ModLists.PMap(queue, async list =>
|
||||
{
|
||||
var (metadata, modList) = list;
|
||||
var archives = modList.Archives.Select(archive => ValidateArchive(data, archive)).ToList();
|
||||
var archives = await modList.Archives.PMap(queue, async archive =>
|
||||
{
|
||||
var (_, result) = ValidateArchive(data, archive);
|
||||
if (result == ArchiveStatus.InValid)
|
||||
{
|
||||
return await TryToFix(data, archive);
|
||||
}
|
||||
|
||||
return (archive, result);
|
||||
});
|
||||
|
||||
var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid);
|
||||
var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated);
|
||||
var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating);
|
||||
|
||||
var summary = new ModListSummary
|
||||
{
|
||||
Checked = DateTime.UtcNow,
|
||||
Failed = failedCount,
|
||||
Passed = passCount,
|
||||
Updating = updatingCount,
|
||||
MachineURL = metadata.Links.MachineURL,
|
||||
Name = metadata.Title,
|
||||
Passed = passCount
|
||||
};
|
||||
|
||||
var detailed = new DetailedStatus
|
||||
@ -68,14 +80,14 @@ namespace Wabbajack.BuildServer.Controllers
|
||||
MachineName = metadata.Links.MachineURL,
|
||||
Archives = archives.Select(a => new DetailedStatusItem
|
||||
{
|
||||
Archive = a.archive, IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating
|
||||
Archive = a.Item1, IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating
|
||||
}).ToList()
|
||||
};
|
||||
|
||||
return (summary, detailed);
|
||||
});
|
||||
|
||||
return await results;
|
||||
return results;
|
||||
}
|
||||
|
||||
private static (Archive archive, ArchiveStatus) ValidateArchive(SqlService.ValidationData data, Archive archive)
|
||||
@ -106,190 +118,16 @@ namespace Wabbajack.BuildServer.Controllers
|
||||
private async Task<(Archive, ArchiveStatus)> TryToFix(SqlService.ValidationData data, Archive archive)
|
||||
{
|
||||
using var _ = await _findPatchLock.Wait();
|
||||
try
|
||||
|
||||
var result = await _updater.GetAlternative(archive.Hash.ToHex());
|
||||
return result switch
|
||||
{
|
||||
// Find all possible patches
|
||||
var patches = data.ArchivePatches
|
||||
.Where(patch =>
|
||||
patch.SrcHash == archive.Hash &&
|
||||
patch.SrcState.PrimaryKeyString == archive.State.PrimaryKeyString)
|
||||
.ToList();
|
||||
|
||||
// Any that are finished
|
||||
if (patches.Where(patch => patch.DestHash != default)
|
||||
.Where(patch =>
|
||||
ValidateArchive(data, new Archive {State = patch.DestState, Hash = patch.DestHash}).Item2 ==
|
||||
ArchiveStatus.Valid)
|
||||
.Any(patch => patch.CDNPath != null))
|
||||
return (archive, ArchiveStatus.Updated);
|
||||
|
||||
// Any that are in progress
|
||||
if (patches.Any(patch => patch.CDNPath == null))
|
||||
return (archive, ArchiveStatus.Updating);
|
||||
|
||||
// Can't upgrade, don't have the original archive
|
||||
if (_settings.PathForArchive(archive.Hash) == default)
|
||||
return (archive, ArchiveStatus.InValid);
|
||||
|
||||
|
||||
switch (archive.State)
|
||||
{
|
||||
case NexusDownloader.State nexusState:
|
||||
{
|
||||
var otherFiles = await SQL.GetModFiles(nexusState.Game, nexusState.ModID);
|
||||
var modInfo = await SQL.GetNexusModInfoString(nexusState.Game, nexusState.ModID);
|
||||
if (modInfo == null || !modInfo.available || otherFiles == null || !otherFiles.files.Any())
|
||||
return (archive, ArchiveStatus.InValid);
|
||||
|
||||
|
||||
|
||||
var file = otherFiles.files
|
||||
.Where(f => f.category_name != null)
|
||||
.OrderByDescending(f => f.uploaded_time)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (file == null) return (archive, ArchiveStatus.InValid);
|
||||
|
||||
var destState = new NexusDownloader.State
|
||||
{
|
||||
Game = nexusState.Game,
|
||||
ModID = nexusState.ModID,
|
||||
FileID = file.file_id,
|
||||
Name = file.category_name,
|
||||
};
|
||||
var existingState = await SQL.DownloadStateByPrimaryKey(destState.PrimaryKeyString);
|
||||
|
||||
Hash destHash = default;
|
||||
if (existingState != null)
|
||||
{
|
||||
destHash = existingState.Hash;
|
||||
}
|
||||
|
||||
var patch = new SqlService.ArchivePatch
|
||||
{
|
||||
SrcHash = archive.Hash, SrcState = archive.State, DestHash = destHash, DestState = destState,
|
||||
};
|
||||
|
||||
await SQL.UpsertArchivePatch(patch);
|
||||
BeginPatching(patch);
|
||||
break;
|
||||
}
|
||||
case HTTPDownloader.State httpState:
|
||||
{
|
||||
var indexJob = new IndexJob {Archive = new Archive {State = httpState}};
|
||||
await indexJob.Execute(SQL, _settings);
|
||||
|
||||
var patch = new SqlService.ArchivePatch
|
||||
{
|
||||
SrcHash = archive.Hash,
|
||||
DestHash = indexJob.DownloadedHash,
|
||||
SrcState = archive.State,
|
||||
DestState = archive.State,
|
||||
};
|
||||
await SQL.UpsertArchivePatch(patch);
|
||||
BeginPatching(patch);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return (archive, ArchiveStatus.InValid);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return (archive, ArchiveStatus.InValid);
|
||||
}
|
||||
OkResult ok => (archive, ArchiveStatus.Updated),
|
||||
AcceptedResult accept => (archive, ArchiveStatus.Updating),
|
||||
_ => (archive, ArchiveStatus.InValid)
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
private void BeginPatching(SqlService.ArchivePatch patch)
|
||||
{
|
||||
Task.Run(async () =>
|
||||
{
|
||||
if (patch.DestHash == default)
|
||||
{
|
||||
patch.DestHash = await DownloadAndHash(patch.DestState);
|
||||
}
|
||||
|
||||
patch.SrcDownload = _settings.PathForArchive(patch.SrcHash).RelativeTo(_settings.ArchivePath);
|
||||
patch.DestDownload = _settings.PathForArchive(patch.DestHash).RelativeTo(_settings.ArchivePath);
|
||||
|
||||
if (patch.SrcDownload == default || patch.DestDownload == default)
|
||||
{
|
||||
throw new InvalidDataException("Src or Destination files do not exist");
|
||||
}
|
||||
|
||||
var result = await PatchArchive(patch);
|
||||
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
public static AbsolutePath CdnPath(SqlService.ArchivePatch patch)
|
||||
{
|
||||
return $"updates/{patch.SrcHash.ToHex()}_{patch.DestHash.ToHex()}".RelativeTo(AbsolutePath.EntryPoint);
|
||||
}
|
||||
private async Task<bool> PatchArchive(SqlService.ArchivePatch patch)
|
||||
{
|
||||
if (patch.SrcHash == patch.DestHash)
|
||||
return true;
|
||||
|
||||
Utils.Log($"Creating Patch ({patch.SrcHash} -> {patch.DestHash})");
|
||||
var cdnPath = CdnPath(patch);
|
||||
cdnPath.Parent.CreateDirectory();
|
||||
|
||||
if (cdnPath.Exists)
|
||||
return true;
|
||||
|
||||
Utils.Log($"Calculating Patch ({patch.SrcHash} -> {patch.DestHash})");
|
||||
await using var fs = cdnPath.Create();
|
||||
await using (var srcStream = patch.SrcDownload.RelativeTo(_settings.ArchivePath).OpenRead())
|
||||
await using (var destStream = patch.DestDownload.RelativeTo(_settings.ArchivePath).OpenRead())
|
||||
await using (var sigStream = cdnPath.WithExtension(Consts.OctoSig).Create())
|
||||
{
|
||||
OctoDiff.Create(destStream, srcStream, sigStream, fs);
|
||||
}
|
||||
fs.Position = 0;
|
||||
|
||||
Utils.Log($"Uploading Patch ({patch.SrcHash} -> {patch.DestHash})");
|
||||
|
||||
int retries = 0;
|
||||
|
||||
if (_settings.BunnyCDN_User == "TEST" && _settings.BunnyCDN_Password == "TEST")
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
TOP:
|
||||
using (var client = new FtpClient("storage.bunnycdn.com"))
|
||||
{
|
||||
client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password);
|
||||
await client.ConnectAsync();
|
||||
try
|
||||
{
|
||||
await client.UploadAsync(fs, cdnPath.RelativeTo(AbsolutePath.EntryPoint).ToString(), progress: new UploadToCDN.Progress(cdnPath.FileName));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (retries > 10) throw;
|
||||
Utils.Log(ex.ToString());
|
||||
Utils.Log("Retrying FTP Upload");
|
||||
retries++;
|
||||
goto TOP;
|
||||
}
|
||||
}
|
||||
|
||||
patch.CDNPath = new Uri($"https://wabbajackpush.b-cdn.net/{cdnPath}");
|
||||
await SQL.UpsertArchivePatch(patch);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task<Hash> DownloadAndHash(AbstractDownloadState state)
|
||||
{
|
||||
var indexJob = new IndexJob();
|
||||
await indexJob.Execute(SQL, _settings);
|
||||
return indexJob.DownloadedHash;
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("status.json")]
|
||||
@ -353,6 +191,7 @@ namespace Wabbajack.BuildServer.Controllers
|
||||
");
|
||||
|
||||
private AppSettings _settings;
|
||||
private ModlistUpdater _updater;
|
||||
|
||||
[HttpGet]
|
||||
[Route("status/{Name}.html")]
|
||||
|
||||
@ -91,43 +91,34 @@ namespace Wabbajack.BuildServer.Controllers
|
||||
Utils.Log($"Alternative requested for {startingHash}");
|
||||
await Metric("requested_upgrade", startingHash.ToString());
|
||||
|
||||
var state = await SQL.GetNexusStateByHash(startingHash);
|
||||
|
||||
/*.DownloadStates.AsQueryable()
|
||||
.Where(s => s.Hash == startingHash)
|
||||
.Where(s => s.State is NexusDownloader.State)
|
||||
.OrderByDescending(s => s.LastValidationTime).FirstOrDefaultAsync();*/
|
||||
var archive = await SQL.GetStateByHash(startingHash);
|
||||
|
||||
if (state == null)
|
||||
if (archive == null)
|
||||
{
|
||||
Utils.Log($"No original state for {startingHash}");
|
||||
return NotFound("Original state not found");
|
||||
}
|
||||
|
||||
var nexusState = state.State as NexusDownloader.State;
|
||||
var nexusGame = nexusState.Game;
|
||||
var nexusModFiles = await SQL.GetModFiles(nexusGame, nexusState.ModID);
|
||||
if (nexusModFiles == null)
|
||||
Archive newArchive;
|
||||
IActionResult result;
|
||||
switch (archive.State)
|
||||
{
|
||||
Utils.Log($"No nexus mod files for {startingHash}");
|
||||
return NotFound("No nexus info");
|
||||
}
|
||||
var mod_files = nexusModFiles.files;
|
||||
|
||||
if (mod_files.Any(f => f.category_name != null && f.file_id == nexusState.FileID))
|
||||
{
|
||||
Utils.Log($"No available upgrade required for {nexusState.PrimaryKey}");
|
||||
await Metric("not_required_upgrade", startingHash.ToString());
|
||||
return BadRequest("Upgrade Not Required");
|
||||
case NexusDownloader.State _:
|
||||
{
|
||||
(result, newArchive) = await FindNexusAlternative(archive);
|
||||
if (newArchive == null)
|
||||
return result;
|
||||
break;
|
||||
}
|
||||
case HTTPDownloader.State _:
|
||||
(result, newArchive) = await FindHttpAlternative(archive);
|
||||
if (newArchive == null)
|
||||
return result;
|
||||
break;
|
||||
default:
|
||||
return NotFound("No alternative");
|
||||
}
|
||||
|
||||
Utils.Log($"Found original, looking for alternatives to {startingHash}");
|
||||
var newArchive = await FindAlternatives(nexusState, startingHash);
|
||||
if (newArchive == null)
|
||||
{
|
||||
Utils.Log($"No available upgrade for {nexusState.PrimaryKey}");
|
||||
return NotFound("No alternative available");
|
||||
}
|
||||
|
||||
Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}");
|
||||
if (newArchive.Hash == Hash.Empty)
|
||||
@ -168,7 +159,61 @@ namespace Wabbajack.BuildServer.Controllers
|
||||
return Ok(newArchive.ToJson());
|
||||
}
|
||||
|
||||
private async Task<Archive> FindAlternatives(NexusDownloader.State state, Hash srcHash)
|
||||
|
||||
private async Task<(IActionResult, Archive)> FindHttpAlternative(Archive archive)
|
||||
{
|
||||
try
|
||||
{
|
||||
var valid = await archive.State.Verify(archive);
|
||||
|
||||
if (valid)
|
||||
{
|
||||
Utils.Log($"Http file {archive.Hash} is still valid");
|
||||
return (NotFound("Http file still valid"), null);
|
||||
}
|
||||
|
||||
archive.Hash = default;
|
||||
archive.Size = 0;
|
||||
return (Ok("Index"), archive);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Utils.Log($"Http file {archive.Hash} no longer exists");
|
||||
return (NotFound("Http file no longer exists"), null);
|
||||
}
|
||||
}
|
||||
private async Task<(IActionResult, Archive)> FindNexusAlternative(Archive archive)
|
||||
{
|
||||
var nexusState = (NexusDownloader.State)archive.State;
|
||||
var nexusGame = nexusState.Game;
|
||||
var nexusModFiles = await SQL.GetModFiles(nexusGame, nexusState.ModID);
|
||||
if (nexusModFiles == null)
|
||||
{
|
||||
Utils.Log($"No nexus mod files for {archive.Hash}");
|
||||
return (NotFound("No nexus info"), null);
|
||||
}
|
||||
var mod_files = nexusModFiles.files;
|
||||
|
||||
if (mod_files.Any(f => f.category_name != null && f.file_id == nexusState.FileID))
|
||||
{
|
||||
Utils.Log($"No available upgrade required for {nexusState.PrimaryKey}");
|
||||
await Metric("not_required_upgrade", archive.Hash.ToString());
|
||||
return (BadRequest("Upgrade Not Required"), null);
|
||||
}
|
||||
|
||||
Utils.Log($"Found original, looking for alternatives to {archive.Hash}");
|
||||
var newArchive = await FindNexusAlternative(nexusState, archive.Hash);
|
||||
if (newArchive != null)
|
||||
{
|
||||
return (Ok(archive), archive);
|
||||
}
|
||||
|
||||
Utils.Log($"No available upgrade for {nexusState.PrimaryKey}");
|
||||
return (NotFound("No alternative available"), null);
|
||||
|
||||
}
|
||||
|
||||
private async Task<Archive> FindNexusAlternative(NexusDownloader.State state, Hash srcHash)
|
||||
{
|
||||
var origSize = _settings.PathForArchive(srcHash).Size;
|
||||
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
|
||||
|
||||
@ -1,8 +1,10 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.Common.Serialization.Json;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.JobQueue
|
||||
{
|
||||
[JsonName("Job")]
|
||||
public class Job
|
||||
{
|
||||
public enum JobPriority : int
|
||||
|
||||
@ -18,6 +18,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
|
||||
public class IndexJob : AJobPayload, IBackEndJob
|
||||
{
|
||||
public Archive Archive { get; set; }
|
||||
|
||||
public bool ForceIndex { get; set; }
|
||||
public override string Description => $"Index ${Archive.State.PrimaryKeyString} and save the download/file state";
|
||||
public override bool UsesNexus { get => Archive.State is NexusDownloader.State; }
|
||||
public Hash DownloadedHash { get; set; }
|
||||
@ -33,10 +35,10 @@ namespace Wabbajack.BuildServer.Models.Jobs
|
||||
var pkStr = string.Join("|",pk.Select(p => p.ToString()));
|
||||
|
||||
var found = await sql.DownloadStateByPrimaryKey(pkStr);
|
||||
if (found != null)
|
||||
if (found != null && !ForceIndex)
|
||||
return JobResult.Success();
|
||||
|
||||
string fileName = Archive.Name;
|
||||
string fileName = Archive.Name ?? Guid.NewGuid().ToString();
|
||||
string folder = Guid.NewGuid().ToString();
|
||||
Utils.Log($"Indexer is downloading {fileName}");
|
||||
var downloadDest = settings.DownloadPath.Combine(folder, fileName);
|
||||
|
||||
@ -538,7 +538,7 @@ namespace Wabbajack.BuildServer.Model.Models
|
||||
public async Task<Archive> GetNexusStateByHash(Hash startingHash)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var result = await conn.QueryFirstOrDefaultAsync<string>(@"SELECT JsonState FROM dbo.DownloadStates
|
||||
var result = await conn.QueryFirstOrDefaultAsync<string>(@"SELECT JsonState, Size FROM dbo.DownloadStates
|
||||
WHERE Hash = @hash AND PrimaryKey like 'NexusDownloader+State|%'",
|
||||
new {Hash = (long)startingHash});
|
||||
return result == null ? null : new Archive
|
||||
@ -548,6 +548,21 @@ namespace Wabbajack.BuildServer.Model.Models
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<Archive> GetStateByHash(Hash startingHash)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
var result = await conn.QueryFirstOrDefaultAsync<(string, long)>(@"SELECT JsonState, indexed.Size FROM dbo.DownloadStates state
|
||||
LEFT JOIN dbo.IndexedFile indexed ON indexed.Hash = state.Hash
|
||||
WHERE state.Hash = @hash",
|
||||
new {Hash = (long)startingHash});
|
||||
return result == default ? null : new Archive
|
||||
{
|
||||
State = result.Item1.FromJsonString<AbstractDownloadState>(),
|
||||
Hash = startingHash,
|
||||
Size = result.Item2
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<Archive> DownloadStateByPrimaryKey(string primaryKey)
|
||||
{
|
||||
await using var conn = await Open();
|
||||
|
||||
@ -121,6 +121,9 @@ namespace Wabbajack.Lib.ModListRegistry
|
||||
public int Failed { get; set; }
|
||||
[JsonProperty("passed")]
|
||||
public int Passed { get; set; }
|
||||
[JsonProperty("updating")]
|
||||
public int Updating { get; set; }
|
||||
|
||||
[JsonProperty("link")]
|
||||
public string Link => $"/lists/status/{MachineURL}.json";
|
||||
[JsonProperty("report")]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user