diff --git a/Wabbajack.BuildServer.Test/ModListValidationTests.cs b/Wabbajack.BuildServer.Test/ModListValidationTests.cs index fa150e7d..6afbd7db 100644 --- a/Wabbajack.BuildServer.Test/ModListValidationTests.cs +++ b/Wabbajack.BuildServer.Test/ModListValidationTests.cs @@ -26,6 +26,7 @@ namespace Wabbajack.BuildServer.Test { public ModListValidationTests(ITestOutputHelper output, SingletonAdaptor fixture) : base(output, fixture) { + } [Fact] @@ -143,9 +144,10 @@ namespace Wabbajack.BuildServer.Test data = await ModlistMetadata.LoadFromGithub(); Assert.Single(data); Assert.Equal(0, data.First().ValidationSummary.Failed); - Assert.Equal(1, data.First().ValidationSummary.Passed); + Assert.Equal(0, data.First().ValidationSummary.Passed); + Assert.Equal(1, data.First().ValidationSummary.Updating); - await CheckListFeeds(0, 1); + await CheckListFeeds(1, 0); } diff --git a/Wabbajack.BuildServer/Controllers/AControllerBase.cs b/Wabbajack.BuildServer/Controllers/AControllerBase.cs index 3820fdca..c0bafbe7 100644 --- a/Wabbajack.BuildServer/Controllers/AControllerBase.cs +++ b/Wabbajack.BuildServer/Controllers/AControllerBase.cs @@ -26,7 +26,7 @@ namespace Wabbajack.BuildServer.Controllers { await SQL.IngestMetric(new Metric { - MetricsKey = Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault(), + MetricsKey = Request?.Headers[Consts.MetricsKeyHeader].FirstOrDefault() ?? "", Subject = subject, Action = verb, Timestamp = DateTime.UtcNow diff --git a/Wabbajack.BuildServer/Controllers/ListValidation.cs b/Wabbajack.BuildServer/Controllers/ListValidation.cs index 822cf156..2b3ab30e 100644 --- a/Wabbajack.BuildServer/Controllers/ListValidation.cs +++ b/Wabbajack.BuildServer/Controllers/ListValidation.cs @@ -33,6 +33,7 @@ namespace Wabbajack.BuildServer.Controllers public ListValidation(ILogger logger, SqlService sql, AppSettings settings) : base(logger, sql) { + _updater = new ModlistUpdater(null, sql, settings); _settings = settings; } @@ -42,21 +43,32 @@ namespace Wabbajack.BuildServer.Controllers using var queue = new WorkQueue(); - var results = data.ModLists.PMap(queue, list => + var results = await data.ModLists.PMap(queue, async list => { var (metadata, modList) = list; - var archives = modList.Archives.Select(archive => ValidateArchive(data, archive)).ToList(); + var archives = await modList.Archives.PMap(queue, async archive => + { + var (_, result) = ValidateArchive(data, archive); + if (result == ArchiveStatus.InValid) + { + return await TryToFix(data, archive); + } + + return (archive, result); + }); var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid); var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated); + var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating); var summary = new ModListSummary { Checked = DateTime.UtcNow, Failed = failedCount, + Passed = passCount, + Updating = updatingCount, MachineURL = metadata.Links.MachineURL, Name = metadata.Title, - Passed = passCount }; var detailed = new DetailedStatus @@ -68,14 +80,14 @@ namespace Wabbajack.BuildServer.Controllers MachineName = metadata.Links.MachineURL, Archives = archives.Select(a => new DetailedStatusItem { - Archive = a.archive, IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating + Archive = a.Item1, IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating }).ToList() }; return (summary, detailed); }); - return await results; + return results; } private static (Archive archive, ArchiveStatus) ValidateArchive(SqlService.ValidationData data, Archive archive) @@ -106,190 +118,16 @@ namespace Wabbajack.BuildServer.Controllers private async Task<(Archive, ArchiveStatus)> TryToFix(SqlService.ValidationData data, Archive archive) { using var _ = await _findPatchLock.Wait(); - try + + var result = await _updater.GetAlternative(archive.Hash.ToHex()); + return result switch { - // Find all possible patches - var patches = data.ArchivePatches - .Where(patch => - patch.SrcHash == archive.Hash && - patch.SrcState.PrimaryKeyString == archive.State.PrimaryKeyString) - .ToList(); - - // Any that are finished - if (patches.Where(patch => patch.DestHash != default) - .Where(patch => - ValidateArchive(data, new Archive {State = patch.DestState, Hash = patch.DestHash}).Item2 == - ArchiveStatus.Valid) - .Any(patch => patch.CDNPath != null)) - return (archive, ArchiveStatus.Updated); - - // Any that are in progress - if (patches.Any(patch => patch.CDNPath == null)) - return (archive, ArchiveStatus.Updating); - - // Can't upgrade, don't have the original archive - if (_settings.PathForArchive(archive.Hash) == default) - return (archive, ArchiveStatus.InValid); - - - switch (archive.State) - { - case NexusDownloader.State nexusState: - { - var otherFiles = await SQL.GetModFiles(nexusState.Game, nexusState.ModID); - var modInfo = await SQL.GetNexusModInfoString(nexusState.Game, nexusState.ModID); - if (modInfo == null || !modInfo.available || otherFiles == null || !otherFiles.files.Any()) - return (archive, ArchiveStatus.InValid); - - - - var file = otherFiles.files - .Where(f => f.category_name != null) - .OrderByDescending(f => f.uploaded_time) - .FirstOrDefault(); - - if (file == null) return (archive, ArchiveStatus.InValid); - - var destState = new NexusDownloader.State - { - Game = nexusState.Game, - ModID = nexusState.ModID, - FileID = file.file_id, - Name = file.category_name, - }; - var existingState = await SQL.DownloadStateByPrimaryKey(destState.PrimaryKeyString); - - Hash destHash = default; - if (existingState != null) - { - destHash = existingState.Hash; - } - - var patch = new SqlService.ArchivePatch - { - SrcHash = archive.Hash, SrcState = archive.State, DestHash = destHash, DestState = destState, - }; - - await SQL.UpsertArchivePatch(patch); - BeginPatching(patch); - break; - } - case HTTPDownloader.State httpState: - { - var indexJob = new IndexJob {Archive = new Archive {State = httpState}}; - await indexJob.Execute(SQL, _settings); - - var patch = new SqlService.ArchivePatch - { - SrcHash = archive.Hash, - DestHash = indexJob.DownloadedHash, - SrcState = archive.State, - DestState = archive.State, - }; - await SQL.UpsertArchivePatch(patch); - BeginPatching(patch); - break; - } - } - - return (archive, ArchiveStatus.InValid); - } - catch (Exception) - { - return (archive, ArchiveStatus.InValid); - } + OkResult ok => (archive, ArchiveStatus.Updated), + AcceptedResult accept => (archive, ArchiveStatus.Updating), + _ => (archive, ArchiveStatus.InValid) + }; } - - private void BeginPatching(SqlService.ArchivePatch patch) - { - Task.Run(async () => - { - if (patch.DestHash == default) - { - patch.DestHash = await DownloadAndHash(patch.DestState); - } - - patch.SrcDownload = _settings.PathForArchive(patch.SrcHash).RelativeTo(_settings.ArchivePath); - patch.DestDownload = _settings.PathForArchive(patch.DestHash).RelativeTo(_settings.ArchivePath); - - if (patch.SrcDownload == default || patch.DestDownload == default) - { - throw new InvalidDataException("Src or Destination files do not exist"); - } - - var result = await PatchArchive(patch); - - - }); - } - - public static AbsolutePath CdnPath(SqlService.ArchivePatch patch) - { - return $"updates/{patch.SrcHash.ToHex()}_{patch.DestHash.ToHex()}".RelativeTo(AbsolutePath.EntryPoint); - } - private async Task PatchArchive(SqlService.ArchivePatch patch) - { - if (patch.SrcHash == patch.DestHash) - return true; - - Utils.Log($"Creating Patch ({patch.SrcHash} -> {patch.DestHash})"); - var cdnPath = CdnPath(patch); - cdnPath.Parent.CreateDirectory(); - - if (cdnPath.Exists) - return true; - - Utils.Log($"Calculating Patch ({patch.SrcHash} -> {patch.DestHash})"); - await using var fs = cdnPath.Create(); - await using (var srcStream = patch.SrcDownload.RelativeTo(_settings.ArchivePath).OpenRead()) - await using (var destStream = patch.DestDownload.RelativeTo(_settings.ArchivePath).OpenRead()) - await using (var sigStream = cdnPath.WithExtension(Consts.OctoSig).Create()) - { - OctoDiff.Create(destStream, srcStream, sigStream, fs); - } - fs.Position = 0; - - Utils.Log($"Uploading Patch ({patch.SrcHash} -> {patch.DestHash})"); - - int retries = 0; - - if (_settings.BunnyCDN_User == "TEST" && _settings.BunnyCDN_Password == "TEST") - { - return true; - } - - TOP: - using (var client = new FtpClient("storage.bunnycdn.com")) - { - client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password); - await client.ConnectAsync(); - try - { - await client.UploadAsync(fs, cdnPath.RelativeTo(AbsolutePath.EntryPoint).ToString(), progress: new UploadToCDN.Progress(cdnPath.FileName)); - } - catch (Exception ex) - { - if (retries > 10) throw; - Utils.Log(ex.ToString()); - Utils.Log("Retrying FTP Upload"); - retries++; - goto TOP; - } - } - - patch.CDNPath = new Uri($"https://wabbajackpush.b-cdn.net/{cdnPath}"); - await SQL.UpsertArchivePatch(patch); - - return true; - } - - private async Task DownloadAndHash(AbstractDownloadState state) - { - var indexJob = new IndexJob(); - await indexJob.Execute(SQL, _settings); - return indexJob.DownloadedHash; - } [HttpGet] [Route("status.json")] @@ -353,6 +191,7 @@ namespace Wabbajack.BuildServer.Controllers "); private AppSettings _settings; + private ModlistUpdater _updater; [HttpGet] [Route("status/{Name}.html")] diff --git a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs index 1ccb2e11..9a0fe1a7 100644 --- a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs +++ b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs @@ -91,43 +91,34 @@ namespace Wabbajack.BuildServer.Controllers Utils.Log($"Alternative requested for {startingHash}"); await Metric("requested_upgrade", startingHash.ToString()); - var state = await SQL.GetNexusStateByHash(startingHash); - - /*.DownloadStates.AsQueryable() - .Where(s => s.Hash == startingHash) - .Where(s => s.State is NexusDownloader.State) - .OrderByDescending(s => s.LastValidationTime).FirstOrDefaultAsync();*/ + var archive = await SQL.GetStateByHash(startingHash); - if (state == null) + if (archive == null) { Utils.Log($"No original state for {startingHash}"); return NotFound("Original state not found"); } - var nexusState = state.State as NexusDownloader.State; - var nexusGame = nexusState.Game; - var nexusModFiles = await SQL.GetModFiles(nexusGame, nexusState.ModID); - if (nexusModFiles == null) + Archive newArchive; + IActionResult result; + switch (archive.State) { - Utils.Log($"No nexus mod files for {startingHash}"); - return NotFound("No nexus info"); - } - var mod_files = nexusModFiles.files; - - if (mod_files.Any(f => f.category_name != null && f.file_id == nexusState.FileID)) - { - Utils.Log($"No available upgrade required for {nexusState.PrimaryKey}"); - await Metric("not_required_upgrade", startingHash.ToString()); - return BadRequest("Upgrade Not Required"); + case NexusDownloader.State _: + { + (result, newArchive) = await FindNexusAlternative(archive); + if (newArchive == null) + return result; + break; + } + case HTTPDownloader.State _: + (result, newArchive) = await FindHttpAlternative(archive); + if (newArchive == null) + return result; + break; + default: + return NotFound("No alternative"); } - Utils.Log($"Found original, looking for alternatives to {startingHash}"); - var newArchive = await FindAlternatives(nexusState, startingHash); - if (newArchive == null) - { - Utils.Log($"No available upgrade for {nexusState.PrimaryKey}"); - return NotFound("No alternative available"); - } Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}"); if (newArchive.Hash == Hash.Empty) @@ -168,7 +159,61 @@ namespace Wabbajack.BuildServer.Controllers return Ok(newArchive.ToJson()); } - private async Task FindAlternatives(NexusDownloader.State state, Hash srcHash) + + private async Task<(IActionResult, Archive)> FindHttpAlternative(Archive archive) + { + try + { + var valid = await archive.State.Verify(archive); + + if (valid) + { + Utils.Log($"Http file {archive.Hash} is still valid"); + return (NotFound("Http file still valid"), null); + } + + archive.Hash = default; + archive.Size = 0; + return (Ok("Index"), archive); + } + catch + { + Utils.Log($"Http file {archive.Hash} no longer exists"); + return (NotFound("Http file no longer exists"), null); + } + } + private async Task<(IActionResult, Archive)> FindNexusAlternative(Archive archive) + { + var nexusState = (NexusDownloader.State)archive.State; + var nexusGame = nexusState.Game; + var nexusModFiles = await SQL.GetModFiles(nexusGame, nexusState.ModID); + if (nexusModFiles == null) + { + Utils.Log($"No nexus mod files for {archive.Hash}"); + return (NotFound("No nexus info"), null); + } + var mod_files = nexusModFiles.files; + + if (mod_files.Any(f => f.category_name != null && f.file_id == nexusState.FileID)) + { + Utils.Log($"No available upgrade required for {nexusState.PrimaryKey}"); + await Metric("not_required_upgrade", archive.Hash.ToString()); + return (BadRequest("Upgrade Not Required"), null); + } + + Utils.Log($"Found original, looking for alternatives to {archive.Hash}"); + var newArchive = await FindNexusAlternative(nexusState, archive.Hash); + if (newArchive != null) + { + return (Ok(archive), archive); + } + + Utils.Log($"No available upgrade for {nexusState.PrimaryKey}"); + return (NotFound("No alternative available"), null); + + } + + private async Task FindNexusAlternative(NexusDownloader.State state, Hash srcHash) { var origSize = _settings.PathForArchive(srcHash).Size; var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); diff --git a/Wabbajack.BuildServer/Models/JobQueue/Job.cs b/Wabbajack.BuildServer/Models/JobQueue/Job.cs index f0353222..b00a84aa 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/Job.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/Job.cs @@ -1,8 +1,10 @@ using System; using System.Threading.Tasks; +using Wabbajack.Common.Serialization.Json; namespace Wabbajack.BuildServer.Models.JobQueue { + [JsonName("Job")] public class Job { public enum JobPriority : int diff --git a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs index 5f5f9bf2..583323e1 100644 --- a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs @@ -18,6 +18,8 @@ namespace Wabbajack.BuildServer.Models.Jobs public class IndexJob : AJobPayload, IBackEndJob { public Archive Archive { get; set; } + + public bool ForceIndex { get; set; } public override string Description => $"Index ${Archive.State.PrimaryKeyString} and save the download/file state"; public override bool UsesNexus { get => Archive.State is NexusDownloader.State; } public Hash DownloadedHash { get; set; } @@ -33,10 +35,10 @@ namespace Wabbajack.BuildServer.Models.Jobs var pkStr = string.Join("|",pk.Select(p => p.ToString())); var found = await sql.DownloadStateByPrimaryKey(pkStr); - if (found != null) + if (found != null && !ForceIndex) return JobResult.Success(); - string fileName = Archive.Name; + string fileName = Archive.Name ?? Guid.NewGuid().ToString(); string folder = Guid.NewGuid().ToString(); Utils.Log($"Indexer is downloading {fileName}"); var downloadDest = settings.DownloadPath.Combine(folder, fileName); diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index 8a1305df..4a541cb8 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -538,7 +538,7 @@ namespace Wabbajack.BuildServer.Model.Models public async Task GetNexusStateByHash(Hash startingHash) { await using var conn = await Open(); - var result = await conn.QueryFirstOrDefaultAsync(@"SELECT JsonState FROM dbo.DownloadStates + var result = await conn.QueryFirstOrDefaultAsync(@"SELECT JsonState, Size FROM dbo.DownloadStates WHERE Hash = @hash AND PrimaryKey like 'NexusDownloader+State|%'", new {Hash = (long)startingHash}); return result == null ? null : new Archive @@ -548,6 +548,21 @@ namespace Wabbajack.BuildServer.Model.Models }; } + public async Task GetStateByHash(Hash startingHash) + { + await using var conn = await Open(); + var result = await conn.QueryFirstOrDefaultAsync<(string, long)>(@"SELECT JsonState, indexed.Size FROM dbo.DownloadStates state + LEFT JOIN dbo.IndexedFile indexed ON indexed.Hash = state.Hash + WHERE state.Hash = @hash", + new {Hash = (long)startingHash}); + return result == default ? null : new Archive + { + State = result.Item1.FromJsonString(), + Hash = startingHash, + Size = result.Item2 + }; + } + public async Task DownloadStateByPrimaryKey(string primaryKey) { await using var conn = await Open(); diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index e68df7e2..a74b52d3 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -121,6 +121,9 @@ namespace Wabbajack.Lib.ModListRegistry public int Failed { get; set; } [JsonProperty("passed")] public int Passed { get; set; } + [JsonProperty("updating")] + public int Updating { get; set; } + [JsonProperty("link")] public string Link => $"/lists/status/{MachineURL}.json"; [JsonProperty("report")]