From 74a332d6cb517bab479058cea76662b755343fbe Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 14 Apr 2020 07:31:03 -0600 Subject: [PATCH] WIP Archive Patching --- .../ModListValidationTests.cs | 52 +++- .../sql/wabbajack_db.sql | 23 ++ .../Controllers/ListValidation.cs | 272 +++++++++++++++--- Wabbajack.BuildServer/Models/Jobs/IndexJob.cs | 5 + .../Models/Sql/SqlService.cs | 109 ++++++- Wabbajack.BuildServer/Startup.cs | 1 + 6 files changed, 422 insertions(+), 40 deletions(-) diff --git a/Wabbajack.BuildServer.Test/ModListValidationTests.cs b/Wabbajack.BuildServer.Test/ModListValidationTests.cs index 208a2f88..fa150e7d 100644 --- a/Wabbajack.BuildServer.Test/ModListValidationTests.cs +++ b/Wabbajack.BuildServer.Test/ModListValidationTests.cs @@ -18,6 +18,7 @@ using Wabbajack.Lib.FileUploader; using Wabbajack.Lib.ModListRegistry; using Xunit; using Xunit.Abstractions; +using IndexedFile = Wabbajack.BuildServer.Models.IndexedFile; namespace Wabbajack.BuildServer.Test { @@ -109,6 +110,51 @@ namespace Wabbajack.BuildServer.Test } + [Fact] + public async Task CanUpgradeHttpDownloads() + { + await ClearJobQueue(); + var modlists = await MakeModList(); + + await IndexFile(ModListData.Archives.First()); + + Consts.ModlistMetadataURL = modlists.ToString(); + Utils.Log("Updating modlists"); + await RevalidateLists(); + + Utils.Log("Checking validated results"); + var data = await ModlistMetadata.LoadFromGithub(); + Assert.Single(data); + Assert.Equal(0, data.First().ValidationSummary.Failed); + Assert.Equal(1, data.First().ValidationSummary.Passed); + + await CheckListFeeds(0, 1); + + var archive = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder); + archive.Delete(); + await archive.WriteAllBytesAsync(Encoding.UTF8.GetBytes("More Cheese for Everyone!")); + + var evalService = new ValidateNonNexusArchives(Fixture.GetService(), Fixture.GetService()); + await evalService.Execute(); + await RevalidateLists(); + + + Utils.Log("Checking updated results"); + data = await ModlistMetadata.LoadFromGithub(); + Assert.Single(data); + Assert.Equal(0, data.First().ValidationSummary.Failed); + Assert.Equal(1, data.First().ValidationSummary.Passed); + + await CheckListFeeds(0, 1); + + } + + private async Task IndexFile(Archive archive) + { + var job = new IndexJob {Archive = archive}; + await job.Execute(Fixture.GetService(), Fixture.GetService()); + } + private async Task RevalidateLists() { var sql = Fixture.GetService(); @@ -142,7 +188,7 @@ namespace Wabbajack.BuildServer.Test - var modListData = new ModList + ModListData = new ModList { Archives = new List { @@ -163,7 +209,7 @@ namespace Wabbajack.BuildServer.Test using var za = new ZipArchive(fs, ZipArchiveMode.Create); var entry = za.CreateEntry("modlist.json"); await using var es = entry.Open(); - modListData.ToJson(es); + ModListData.ToJson(es); } ModListMetaData = new List @@ -193,6 +239,8 @@ namespace Wabbajack.BuildServer.Test return new Uri(MakeURL("test_mod_list_metadata.json")); } + public ModList ModListData { get; set; } + public List ModListMetaData { get; set; } } } diff --git a/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql b/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql index 4bacb98f..fbfcd194 100644 --- a/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql +++ b/Wabbajack.BuildServer.Test/sql/wabbajack_db.sql @@ -361,6 +361,29 @@ CREATE TABLE [dbo].[ModListArchiveStatus]( ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] GO +/****** Object: Table [dbo].[ArchivePatches] Script Date: 4/13/2020 9:39:25 PM ******/ +CREATE TABLE [dbo].[ArchivePatches]( +[SrcPrimaryKeyStringHash] [binary](32) NOT NULL, +[SrcPrimaryKeyString] [nvarchar](max) NOT NULL, +[SrcHash] [bigint] NOT NULL, +[DestPrimaryKeyStringHash] [binary](32) NOT NULL, +[DestPrimaryKeyString] [nvarchar](max) NOT NULL, +[DestHash] [bigint] NOT NULL, +[SrcState] [nvarchar](max) NOT NULL, +[DestState] [nvarchar](max) NOT NULL, +[SrcDownload] [nvarchar](max) NULL, +[DestDownload] [nvarchar](max) NULL, +[CDNPath] [nvarchar](max) NULL, +CONSTRAINT [PK_ArchivePatches] PRIMARY KEY CLUSTERED + ( + [SrcPrimaryKeyStringHash] ASC, + [SrcHash] ASC, + [DestPrimaryKeyStringHash] ASC, + [DestHash] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO + /****** Object: Table [dbo].[Metrics] Script Date: 3/28/2020 4:58:59 PM ******/ SET ANSI_NULLS ON diff --git a/Wabbajack.BuildServer/Controllers/ListValidation.cs b/Wabbajack.BuildServer/Controllers/ListValidation.cs index 0706e066..822cf156 100644 --- a/Wabbajack.BuildServer/Controllers/ListValidation.cs +++ b/Wabbajack.BuildServer/Controllers/ListValidation.cs @@ -1,14 +1,19 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net; using System.Threading.Tasks; +using FluentFTP; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Nettle; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.Common; +using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.ModListRegistry; @@ -18,8 +23,17 @@ namespace Wabbajack.BuildServer.Controllers [Route("/lists")] public class ListValidation : AControllerBase { - public ListValidation(ILogger logger, SqlService sql) : base(logger, sql) + enum ArchiveStatus { + Valid, + InValid, + Updating, + Updated, + } + + public ListValidation(ILogger logger, SqlService sql, AppSettings settings) : base(logger, sql) + { + _settings = settings; } public async Task> GetSummaries() @@ -30,52 +44,31 @@ namespace Wabbajack.BuildServer.Controllers var results = data.ModLists.PMap(queue, list => { - var archives = list.ModList.Archives.Select(archive => - { - switch (archive.State) - { - case NexusDownloader.State nexusState when data.NexusFiles.Contains(( - nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)): - return (archive, true); - case NexusDownloader.State nexusState: - return (archive, false); - case ManualDownloader.State _: - return (archive, true); - default: - { - if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash), - out var isValid)) - { - return (archive, isValid); - } + var (metadata, modList) = list; + var archives = modList.Archives.Select(archive => ValidateArchive(data, archive)).ToList(); - return (archive, false); - } - } - }).ToList(); - - var failedCount = archives.Count(f => !f.Item2); - var passCount = archives.Count(f => f.Item2); + var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid); + var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated); var summary = new ModListSummary { Checked = DateTime.UtcNow, Failed = failedCount, - MachineURL = list.Metadata.Links.MachineURL, - Name = list.Metadata.Title, + MachineURL = metadata.Links.MachineURL, + Name = metadata.Title, Passed = passCount }; var detailed = new DetailedStatus { - Name = list.Metadata.Title, + Name = metadata.Title, Checked = DateTime.UtcNow, - DownloadMetaData = list.Metadata.DownloadMetadata, + DownloadMetaData = metadata.DownloadMetadata, HasFailures = failedCount > 0, - MachineName = list.Metadata.Links.MachineURL, + MachineName = metadata.Links.MachineURL, Archives = archives.Select(a => new DetailedStatusItem { - Archive = a.archive, IsFailing = !a.Item2 + Archive = a.archive, IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating }).ToList() }; @@ -84,7 +77,220 @@ namespace Wabbajack.BuildServer.Controllers return await results; } + + private static (Archive archive, ArchiveStatus) ValidateArchive(SqlService.ValidationData data, Archive archive) + { + switch (archive.State) + { + case NexusDownloader.State nexusState when data.NexusFiles.Contains(( + nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)): + return (archive, ArchiveStatus.Valid); + case NexusDownloader.State nexusState: + return (archive, ArchiveStatus.InValid); + case ManualDownloader.State _: + return (archive, ArchiveStatus.Valid); + default: + { + if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash), + out bool isValid)) + { + return isValid ? (archive, ArchiveStatus.Valid) : (archive, ArchiveStatus.InValid); + } + + return (archive, ArchiveStatus.InValid); + } + } + } + + private static AsyncLock _findPatchLock = new AsyncLock(); + private async Task<(Archive, ArchiveStatus)> TryToFix(SqlService.ValidationData data, Archive archive) + { + using var _ = await _findPatchLock.Wait(); + try + { + // Find all possible patches + var patches = data.ArchivePatches + .Where(patch => + patch.SrcHash == archive.Hash && + patch.SrcState.PrimaryKeyString == archive.State.PrimaryKeyString) + .ToList(); + + // Any that are finished + if (patches.Where(patch => patch.DestHash != default) + .Where(patch => + ValidateArchive(data, new Archive {State = patch.DestState, Hash = patch.DestHash}).Item2 == + ArchiveStatus.Valid) + .Any(patch => patch.CDNPath != null)) + return (archive, ArchiveStatus.Updated); + + // Any that are in progress + if (patches.Any(patch => patch.CDNPath == null)) + return (archive, ArchiveStatus.Updating); + + // Can't upgrade, don't have the original archive + if (_settings.PathForArchive(archive.Hash) == default) + return (archive, ArchiveStatus.InValid); + + + switch (archive.State) + { + case NexusDownloader.State nexusState: + { + var otherFiles = await SQL.GetModFiles(nexusState.Game, nexusState.ModID); + var modInfo = await SQL.GetNexusModInfoString(nexusState.Game, nexusState.ModID); + if (modInfo == null || !modInfo.available || otherFiles == null || !otherFiles.files.Any()) + return (archive, ArchiveStatus.InValid); + + + + var file = otherFiles.files + .Where(f => f.category_name != null) + .OrderByDescending(f => f.uploaded_time) + .FirstOrDefault(); + + if (file == null) return (archive, ArchiveStatus.InValid); + + var destState = new NexusDownloader.State + { + Game = nexusState.Game, + ModID = nexusState.ModID, + FileID = file.file_id, + Name = file.category_name, + }; + var existingState = await SQL.DownloadStateByPrimaryKey(destState.PrimaryKeyString); + + Hash destHash = default; + if (existingState != null) + { + destHash = existingState.Hash; + } + + var patch = new SqlService.ArchivePatch + { + SrcHash = archive.Hash, SrcState = archive.State, DestHash = destHash, DestState = destState, + }; + + await SQL.UpsertArchivePatch(patch); + BeginPatching(patch); + break; + } + case HTTPDownloader.State httpState: + { + var indexJob = new IndexJob {Archive = new Archive {State = httpState}}; + await indexJob.Execute(SQL, _settings); + + var patch = new SqlService.ArchivePatch + { + SrcHash = archive.Hash, + DestHash = indexJob.DownloadedHash, + SrcState = archive.State, + DestState = archive.State, + }; + await SQL.UpsertArchivePatch(patch); + BeginPatching(patch); + break; + } + } + + return (archive, ArchiveStatus.InValid); + } + catch (Exception) + { + return (archive, ArchiveStatus.InValid); + } + } + + private void BeginPatching(SqlService.ArchivePatch patch) + { + Task.Run(async () => + { + if (patch.DestHash == default) + { + patch.DestHash = await DownloadAndHash(patch.DestState); + } + + patch.SrcDownload = _settings.PathForArchive(patch.SrcHash).RelativeTo(_settings.ArchivePath); + patch.DestDownload = _settings.PathForArchive(patch.DestHash).RelativeTo(_settings.ArchivePath); + + if (patch.SrcDownload == default || patch.DestDownload == default) + { + throw new InvalidDataException("Src or Destination files do not exist"); + } + + var result = await PatchArchive(patch); + + + }); + } + + public static AbsolutePath CdnPath(SqlService.ArchivePatch patch) + { + return $"updates/{patch.SrcHash.ToHex()}_{patch.DestHash.ToHex()}".RelativeTo(AbsolutePath.EntryPoint); + } + private async Task PatchArchive(SqlService.ArchivePatch patch) + { + if (patch.SrcHash == patch.DestHash) + return true; + + Utils.Log($"Creating Patch ({patch.SrcHash} -> {patch.DestHash})"); + var cdnPath = CdnPath(patch); + cdnPath.Parent.CreateDirectory(); + + if (cdnPath.Exists) + return true; + + Utils.Log($"Calculating Patch ({patch.SrcHash} -> {patch.DestHash})"); + await using var fs = cdnPath.Create(); + await using (var srcStream = patch.SrcDownload.RelativeTo(_settings.ArchivePath).OpenRead()) + await using (var destStream = patch.DestDownload.RelativeTo(_settings.ArchivePath).OpenRead()) + await using (var sigStream = cdnPath.WithExtension(Consts.OctoSig).Create()) + { + OctoDiff.Create(destStream, srcStream, sigStream, fs); + } + fs.Position = 0; + + Utils.Log($"Uploading Patch ({patch.SrcHash} -> {patch.DestHash})"); + + int retries = 0; + + if (_settings.BunnyCDN_User == "TEST" && _settings.BunnyCDN_Password == "TEST") + { + return true; + } + + TOP: + using (var client = new FtpClient("storage.bunnycdn.com")) + { + client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password); + await client.ConnectAsync(); + try + { + await client.UploadAsync(fs, cdnPath.RelativeTo(AbsolutePath.EntryPoint).ToString(), progress: new UploadToCDN.Progress(cdnPath.FileName)); + } + catch (Exception ex) + { + if (retries > 10) throw; + Utils.Log(ex.ToString()); + Utils.Log("Retrying FTP Upload"); + retries++; + goto TOP; + } + } + + patch.CDNPath = new Uri($"https://wabbajackpush.b-cdn.net/{cdnPath}"); + await SQL.UpsertArchivePatch(patch); + + return true; + } + + private async Task DownloadAndHash(AbstractDownloadState state) + { + var indexJob = new IndexJob(); + await indexJob.Execute(SQL, _settings); + return indexJob.DownloadedHash; + } + [HttpGet] [Route("status.json")] public async Task> HandleGetLists() @@ -146,6 +352,8 @@ namespace Wabbajack.BuildServer.Controllers "); + private AppSettings _settings; + [HttpGet] [Route("status/{Name}.html")] public async Task HandleGetListHtml(string Name) diff --git a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs index 9aca15f7..5f5f9bf2 100644 --- a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs @@ -20,6 +20,8 @@ namespace Wabbajack.BuildServer.Models.Jobs public Archive Archive { get; set; } public override string Description => $"Index ${Archive.State.PrimaryKeyString} and save the download/file state"; public override bool UsesNexus { get => Archive.State is NexusDownloader.State; } + public Hash DownloadedHash { get; set; } + public override async Task Execute(SqlService sql, AppSettings settings) { if (Archive.State is ManualDownloader.State) @@ -46,6 +48,8 @@ namespace Wabbajack.BuildServer.Models.Jobs await vfs.AddRoot(settings.DownloadPath.Combine(folder)); var archive = vfs.Index.ByRootPath.First().Value; + DownloadedHash = archive.Hash; + await sql.MergeVirtualFile(archive); await sql.AddDownloadState(archive.Hash, Archive.State); @@ -63,6 +67,7 @@ namespace Wabbajack.BuildServer.Models.Jobs return JobResult.Success(); } + protected override IEnumerable PrimaryKey => Archive.State.PrimaryKey; } diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index f6638181..8a1305df 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -256,20 +256,35 @@ namespace Wabbajack.BuildServer.Model.Models static SqlService() { - SqlMapper.AddTypeHandler(new PayloadMapper()); SqlMapper.AddTypeHandler(new HashMapper()); + SqlMapper.AddTypeHandler(new RelativePathMapper()); + SqlMapper.AddTypeHandler(new JsonMapper()); + SqlMapper.AddTypeHandler(new JsonMapper()); } - public class PayloadMapper : SqlMapper.TypeHandler + public class JsonMapper : SqlMapper.TypeHandler { - public override void SetValue(IDbDataParameter parameter, AJobPayload value) + public override void SetValue(IDbDataParameter parameter, T value) { parameter.Value = value.ToJson(); } - public override AJobPayload Parse(object value) + public override T Parse(object value) { - return ((string)value).FromJsonString(); + return ((string)value).FromJsonString(); + } + } + + public class RelativePathMapper : SqlMapper.TypeHandler + { + public override void SetValue(IDbDataParameter parameter, RelativePath value) + { + parameter.Value = value.ToJson(); + } + + public override RelativePath Parse(object value) + { + return (RelativePath)(string)value; } } @@ -690,12 +705,14 @@ namespace Wabbajack.BuildServer.Model.Models var nexusFiles = AllNexusFiles(); var archiveStatus = AllModListArchivesStatus(); var modLists = AllModLists(); + var archivePatches = AllArchivePatches(); return new ValidationData { NexusFiles = await nexusFiles, ArchiveStatus = await archiveStatus, - ModLists = await modLists + ModLists = await modLists, + ArchivePatches = await archivePatches }; } @@ -731,6 +748,86 @@ namespace Wabbajack.BuildServer.Model.Models public HashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; } public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; } public List<(ModlistMetadata Metadata, ModList ModList)> ModLists { get; set; } + public List ArchivePatches { get; set; } } + + + #region ArchivePatches + + public class ArchivePatch + { + public Hash SrcHash { get; set; } + public AbstractDownloadState SrcState { get; set; } + public Hash DestHash { get; set; } + public AbstractDownloadState DestState { get; set; } + + public RelativePath DestDownload { get; set; } + public RelativePath SrcDownload { get; set; } + public Uri CDNPath { get; set; } + } + + public async Task UpsertArchivePatch(ArchivePatch patch) + { + await using var conn = await Open(); + + await using var trans = conn.BeginTransaction(); + await conn.ExecuteAsync(@"DELETE FROM dbo.ArchivePatches + WHERE SrcHash = @SrcHash + AND DestHash = @DestHash + AND SrcPrimaryKeyStringHash = HASHBYTES('SHA2-256', @SrcPrimaryKeyString) + AND DestPrimaryKeyStringHash = HASHBYTES('SHA2-256', @DestPrimaryKeyString)", + new + { + SrcHash = patch.SrcHash, + DestHash = patch.DestHash, + SrcPrimaryKeyString = patch.SrcState.PrimaryKeyString, + DestPrimaryKeyString = patch.DestState.PrimaryKeyString + }, trans); + + await conn.ExecuteAsync(@"INSERT INTO dbo.ArchivePatches + (SrcHash, SrcPrimaryKeyString, SrcPrimaryKeyStringHash, SrcState, + DestHash, DestPrimaryKeyString, DestPrimaryKeyStringHash, DestState, + + SrcDownload, DestDownload, CDNPath) + VALUES (@SrcHash, @SrcPrimaryKeyString, HASHBYTES('SHA2-256', @SrcPrimaryKeyString), @SrcState, + @DestHash, @DestPrimaryKeyString, HASHBYTES('SHA2-256', @DestPrimaryKeyString), @DestState, + @SrcDownload, @DestDownload, @CDNPAth)", + new + { + SrcHash = patch.SrcHash, + DestHash = patch.DestHash, + SrcPrimaryKeyString = patch.SrcState.PrimaryKeyString, + DestPrimaryKeyString = patch.DestState.PrimaryKeyString, + SrcState = patch.SrcState.ToJson(), + DestState = patch.DestState.ToString(), + DestDownload = patch.DestDownload, + SrcDownload = patch.SrcDownload, + CDNPath = patch.CDNPath + }, trans); + + await trans.CommitAsync(); + } + + public async Task> AllArchivePatches() + { + await using var conn = await Open(); + + var results = + await conn.QueryAsync<(Hash, AbstractDownloadState, Hash, AbstractDownloadState, RelativePath, RelativePath, Uri)>( + @"SELECT SrcHash, SrcState, DestHash, DestState, SrcDownload, DestDownload, CDNPath FROM dbo.ArchivePatches"); + return results.Select(a => new ArchivePatch + { + SrcHash = a.Item1, + SrcState = a.Item2, + DestHash = a.Item3, + DestState = a.Item4, + SrcDownload = a.Item5, + DestDownload = a.Item6, + CDNPath = a.Item7 + }).ToList(); + } + + + #endregion } } diff --git a/Wabbajack.BuildServer/Startup.cs b/Wabbajack.BuildServer/Startup.cs index 2e4b5962..7b5581ff 100644 --- a/Wabbajack.BuildServer/Startup.cs +++ b/Wabbajack.BuildServer/Startup.cs @@ -143,6 +143,7 @@ namespace Wabbajack.BuildServer FileProvider = new PhysicalFileProvider( Path.Combine(Directory.GetCurrentDirectory(), "public")), StaticFileOptions = {ServeUnknownFileTypes = true}, + }); app.UseEndpoints(endpoints =>