From a481f546c1c2ac862380c2b1b7e59ec2e013a656 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 9 Apr 2020 21:54:02 -0600 Subject: [PATCH] Fix a bunch of tests with auto-healing --- .../ABuildServerSystemTest.cs | 1 + Wabbajack.BuildServer.Test/ModlistUpdater.cs | 50 ++++++++++++++++++- .../Controllers/ModlistUpdater.cs | 13 ++++- Wabbajack.BuildServer/Models/PatchArchive.cs | 9 ++++ .../Models/Sql/SqlService.cs | 5 +- Wabbajack.Lib/ClientAPI.cs | 11 +++- 6 files changed, 82 insertions(+), 7 deletions(-) diff --git a/Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs b/Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs index a624db80..5dcb5083 100644 --- a/Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs +++ b/Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs @@ -28,6 +28,7 @@ namespace Wabbajack.BuildServer.Test public AbsolutePath ServerPublicFolder => "public".RelativeTo(AbsolutePath.EntryPoint); public AbsolutePath ServerArchivesFolder => "archives".RelativeTo(AbsolutePath.EntryPoint); + public AbsolutePath ServerUpdatesFolder => "updates".RelativeTo(AbsolutePath.EntryPoint); public BuildServerFixture() diff --git a/Wabbajack.BuildServer.Test/ModlistUpdater.cs b/Wabbajack.BuildServer.Test/ModlistUpdater.cs index be3a5605..3295ea96 100644 --- a/Wabbajack.BuildServer.Test/ModlistUpdater.cs +++ b/Wabbajack.BuildServer.Test/ModlistUpdater.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Dapper; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.Jobs; @@ -22,7 +23,7 @@ namespace Wabbajack.BuildServer.Test } [Fact, Priority(0)] - public async Task CanIndexFiles() + public async Task CanIndexAndUpdateFiles() { var sql = Fixture.GetService(); var modId = long.MaxValue >> 1; @@ -76,6 +77,53 @@ namespace Wabbajack.BuildServer.Test Assert.Equal($"Oldfile_{oldDataHash.ToHex()}_".RelativeTo(Fixture.ServerArchivesFolder), settings.PathForArchive(oldDataHash)); Assert.Equal($"Newfile_{newDataHash.ToHex()}_".RelativeTo(Fixture.ServerArchivesFolder), settings.PathForArchive(newDataHash)); + Utils.Log($"Download Updating {oldDataHash} -> {newDataHash}"); + await using var conn = await sql.Open(); + + await conn.ExecuteAsync("DELETE FROM dbo.DownloadStates WHERE Hash in (@OldHash, @NewHash);", + new {OldHash = (long)oldDataHash, NewHash = (long)newDataHash}); + + await sql.AddDownloadState(oldDataHash, new NexusDownloader.State + { + Game = Game.Oblivion, + ModID = modId, + FileID = oldFileId + }); + + await sql.AddDownloadState(newDataHash, new NexusDownloader.State + { + Game = Game.Oblivion, + ModID = modId, + FileID = newFileId + }); + + Assert.NotNull(await sql.GetNexusStateByHash(oldDataHash)); + Assert.NotNull(await sql.GetNexusStateByHash(newDataHash)); + + // No nexus info, so no upgrade + var noUpgrade = await ClientAPI.GetModUpgrade(oldDataHash); + Assert.Null(noUpgrade); + + // Add Nexus info + await sql.AddNexusModFiles(Game.Oblivion, modId, DateTime.Now, + new NexusApiClient.GetModFilesResponse + { + files = new List + { + new NexusFileInfo {category_name = "MAIN", file_id = newFileId, file_name = "New File"}, + new NexusFileInfo {category_name = null, file_id = oldFileId, file_name = "Old File"} + } + }); + + + var enqueuedUpgrade = await ClientAPI.GetModUpgrade(oldDataHash); + + // Not Null because upgrade was enqueued + Assert.NotNull(enqueuedUpgrade); + + await RunAllJobs(); + + Assert.True($"{oldDataHash.ToHex()}_{newDataHash.ToHex()}".RelativeTo(Fixture.ServerUpdatesFolder).IsFile); } diff --git a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs index 56257a57..1ccb2e11 100644 --- a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs +++ b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs @@ -99,14 +99,24 @@ namespace Wabbajack.BuildServer.Controllers .OrderByDescending(s => s.LastValidationTime).FirstOrDefaultAsync();*/ if (state == null) + { + Utils.Log($"No original state for {startingHash}"); return NotFound("Original state not found"); + } var nexusState = state.State as NexusDownloader.State; var nexusGame = nexusState.Game; - var mod_files = (await SQL.GetModFiles(nexusGame, nexusState.ModID)).files; + var nexusModFiles = await SQL.GetModFiles(nexusGame, nexusState.ModID); + if (nexusModFiles == null) + { + Utils.Log($"No nexus mod files for {startingHash}"); + return NotFound("No nexus info"); + } + var mod_files = nexusModFiles.files; if (mod_files.Any(f => f.category_name != null && f.file_id == nexusState.FileID)) { + Utils.Log($"No available upgrade required for {nexusState.PrimaryKey}"); await Metric("not_required_upgrade", startingHash.ToString()); return BadRequest("Upgrade Not Required"); } @@ -115,6 +125,7 @@ namespace Wabbajack.BuildServer.Controllers var newArchive = await FindAlternatives(nexusState, startingHash); if (newArchive == null) { + Utils.Log($"No available upgrade for {nexusState.PrimaryKey}"); return NotFound("No alternative available"); } diff --git a/Wabbajack.BuildServer/Models/PatchArchive.cs b/Wabbajack.BuildServer/Models/PatchArchive.cs index 5a8d4b7c..d3855623 100644 --- a/Wabbajack.BuildServer/Models/PatchArchive.cs +++ b/Wabbajack.BuildServer/Models/PatchArchive.cs @@ -6,9 +6,11 @@ using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.Common; +using Wabbajack.Common.Serialization.Json; namespace Wabbajack.BuildServer.Models { + [JsonName("PatchArchive")] public class PatchArchive : AJobPayload { public override string Description => "Create a archive update patch"; @@ -25,6 +27,7 @@ namespace Wabbajack.BuildServer.Models Utils.Log($"Creating Patch ({Src} -> {DestPK})"); var cdnPath = CdnPath(Src, destHash); + cdnPath.Parent.CreateDirectory(); if (cdnPath.Exists) return JobResult.Success(); @@ -42,6 +45,12 @@ namespace Wabbajack.BuildServer.Models Utils.Log($"Uploading Patch ({Src} -> {DestPK})"); int retries = 0; + + if (settings.BunnyCDN_User == "TEST" && settings.BunnyCDN_Password == "TEST") + { + return JobResult.Success(); + } + TOP: using (var client = new FtpClient("storage.bunnycdn.com")) { diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index 5b6fc763..b1756323 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -517,9 +517,8 @@ namespace Wabbajack.BuildServer.Model.Models { await using var conn = await Open(); var result = await conn.QueryFirstOrDefaultAsync(@"SELECT JsonState FROM dbo.DownloadStates - WHERE Hash = @hash AND PrimaryKey like 'NexusDownloader+State|%' - ORDER BY LastValidated DESC", - new {Hash = startingHash}); + WHERE Hash = @hash AND PrimaryKey like 'NexusDownloader+State|%'", + new {Hash = (long)startingHash}); return result == null ? null : new Archive { State = result.FromJsonString(), diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index 7c562d97..c750782b 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -17,8 +17,15 @@ namespace Wabbajack.Lib public static async Task GetModUpgrade(Hash hash) { using var response = await GetClient() - .GetAsync($"https://{Consts.WabbajackCacheHostname}/alternative/{hash.ToHex()}"); - return !response.IsSuccessStatusCode ? null : (await response.Content.ReadAsStringAsync()).FromJsonString(); + .GetAsync($"{Consts.WabbajackBuildServerUri}alternative/{hash.ToHex()}"); + if (response.IsSuccessStatusCode) + { + return (await response.Content.ReadAsStringAsync()).FromJsonString(); + } + + Utils.Log($"No Upgrade for {hash}"); + Utils.Log(await response.Content.ReadAsStringAsync()); + return null; } ///