diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 13827c09..c4b227ec 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -144,6 +144,8 @@ namespace Wabbajack.Common public static Uri WabbajackOrg = new Uri("https://www.wabbajack.org/"); public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2; + + public static string ArchiveUpdatesCDNFolder = "archive_updates"; } } diff --git a/Wabbajack.Common/Hash.cs b/Wabbajack.Common/Hash.cs index cd23e1d3..eab70e4f 100644 --- a/Wabbajack.Common/Hash.cs +++ b/Wabbajack.Common/Hash.cs @@ -225,7 +225,8 @@ namespace Wabbajack.Common { await using var fs = file.OpenRead(); var config = new xxHashConfig {HashSizeInBits = 64}; - var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(fs); + await using var hs = new StatusFileStream(fs, $"Hashing {file}"); + var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(hs); return new Hash(BitConverter.ToUInt64(value.Hash)); } catch (IOException) diff --git a/Wabbajack.Common/Http/Client.cs b/Wabbajack.Common/Http/Client.cs index a58aed49..da58a7cb 100644 --- a/Wabbajack.Common/Http/Client.cs +++ b/Wabbajack.Common/Http/Client.cs @@ -19,11 +19,17 @@ namespace Wabbajack.Common.Http return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions); } + public async Task GetAsync(Uri url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true) + { + var request = new HttpRequestMessage(HttpMethod.Get, url); + return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions); + } - public async Task PostAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead) + + public async Task PostAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true) { var request = new HttpRequestMessage(HttpMethod.Post, url) {Content = content}; - return await SendAsync(request, responseHeadersRead); + return await SendAsync(request, responseHeadersRead, errorsAsExceptions); } public async Task PutAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead) @@ -79,7 +85,6 @@ namespace Wabbajack.Common.Http if (errorsAsExceptions) throw new HttpRequestException( $"Http Exception {response.StatusCode} - {response.ReasonPhrase} - {msg.RequestUri}"); - ; return response; } catch (Exception ex) diff --git a/Wabbajack.Common/OctoDiff.cs b/Wabbajack.Common/OctoDiff.cs index bb17c9e8..271274bb 100644 --- a/Wabbajack.Common/OctoDiff.cs +++ b/Wabbajack.Common/OctoDiff.cs @@ -28,7 +28,7 @@ namespace Wabbajack.Common return sigStream; } - private static void CreateSignature(Stream oldData, FileStream sigStream) + private static void CreateSignature(Stream oldData, Stream sigStream) { Utils.Status("Creating Patch Signature"); var signatureBuilder = new SignatureBuilder(); @@ -36,7 +36,7 @@ namespace Wabbajack.Common sigStream.Position = 0; } - public static void Create(Stream oldData, FileStream newData, FileStream signature, FileStream output) + public static void Create(Stream oldData, Stream newData, Stream signature, Stream output) { CreateSignature(oldData, signature); var db = new DeltaBuilder {ProgressReporter = reporter}; diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index adbe3dea..eae25994 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -337,6 +337,13 @@ namespace Wabbajack.Common await fs.WriteAsync(data); } + public async Task WriteAllAsync(Stream data, bool disposeAfter = true) + { + await using var fs = Create(); + await data.CopyToAsync(fs); + if (disposeAfter) await data.DisposeAsync(); + } + public void AppendAllText(string text) { File.AppendAllText(_path, text); diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 50bf4976..6784c8a1 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -298,7 +298,7 @@ namespace Wabbajack.Lib .Where(e => e.Extension != Consts.HashFileExtension) .ToList(); - Utils.Log($"Found {toHash} files to hash"); + Utils.Log($"Found {toHash.Count} files to hash"); var hashResults = await toHash diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index 4eadf302..802e6bcf 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -1,11 +1,46 @@ using System; using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; using System.Threading.Tasks; using Wabbajack.Common; +using Wabbajack.Common.Serialization.Json; +using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.Exceptions; namespace Wabbajack.Lib { + [JsonName("ModUpgradeRequest")] + public class ModUpgradeRequest + { + public Archive OldArchive { get; set; } + public Archive NewArchive { get; set; } + + public ModUpgradeRequest(Archive oldArchive, Archive newArchive) + { + OldArchive = oldArchive; + NewArchive = newArchive; + } + + public bool IsValid + { + get + { + if (OldArchive.Hash == NewArchive.Hash && OldArchive.State.PrimaryKeyString == NewArchive.State.PrimaryKeyString) return false; + if (OldArchive.State.GetType() != NewArchive.State.GetType()) + return false; + if (OldArchive.State is IUpgradingState u) + { + return u.ValidateUpgrade(NewArchive.State); + } + + return false; + } + } + } + public class ClientAPI { public static Common.Http.Client GetClient() @@ -16,18 +51,39 @@ namespace Wabbajack.Lib return client; } - public static async Task GetModUpgrade(Hash hash) + + + public static async Task GetModUpgrade(Archive oldArchive, Archive newArchive, TimeSpan? maxWait = null, TimeSpan? waitBetweenTries = null) { - using var response = await GetClient() - .GetAsync($"{Consts.WabbajackBuildServerUri}alternative/{hash.ToHex()}"); + maxWait ??= TimeSpan.FromMinutes(10); + waitBetweenTries ??= TimeSpan.FromSeconds(15); + + var request = new ModUpgradeRequest( oldArchive, newArchive); + var start = DateTime.UtcNow; + + RETRY: + + var response = await GetClient() + .PostAsync($"{Consts.WabbajackBuildServerUri}mod_upgrade", new StringContent(request.ToJson(), Encoding.UTF8, "application/json")); + if (response.IsSuccessStatusCode) { - return (await response.Content.ReadAsStringAsync()).FromJsonString(); + switch (response.StatusCode) + { + case HttpStatusCode.OK: + return new Uri(await response.Content.ReadAsStringAsync()); + case HttpStatusCode.Accepted: + Utils.Log($"Waiting for patch processing on the server for {oldArchive.Name}, sleeping for another 15 seconds"); + await Task.Delay(TimeSpan.FromSeconds(15)); + response.Dispose(); + if (DateTime.UtcNow - start > maxWait) + throw new HttpException(response); + goto RETRY; + } } - - Utils.Log($"No Upgrade for {hash}"); - Utils.Log(await response.Content.ReadAsStringAsync()); - return null; + var ex = new HttpException(response); + response.Dispose(); + throw ex; } /// diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index bdd3eb9a..7a19665f 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Net.Http; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; @@ -95,40 +97,49 @@ namespace Wabbajack.Lib.Downloaders return true; } - Utils.Log($"Download failed, looking for upgrade"); - var upgrade = await ClientAPI.GetModUpgrade(archive.Hash); - if (upgrade == null) + if (!(archive.State is IUpgradingState)) { - Utils.Log($"No upgrade found for {archive.Hash}"); + Utils.Log($"Download failed for {archive.Name} and no upgrade from this download source is possible"); return false; } - Utils.Log($"Upgrading via {upgrade.State.PrimaryKeyString}"); + + var upgrade = (IUpgradingState)archive.State; + + Utils.Log($"Trying to find solution to broken download for {archive.Name}"); - Utils.Log($"Upgrading {archive.Hash}"); - var upgradePath = destination.Parent.Combine("_Upgrade_" + archive.Name); - var upgradeResult = await Download(upgrade, upgradePath); - if (!upgradeResult) return false; - - var patchName = $"{archive.Hash.ToHex()}_{upgrade.Hash.ToHex()}"; - var patchPath = destination.Parent.Combine("_Patch_" + patchName); - - var patchState = new Archive(new HTTPDownloader.State($"https://wabbajackcdn.b-cdn.net/updates/{patchName}")) + var result = await upgrade.FindUpgrade(archive); + if (result == default) { - Name = patchName, - }; + Utils.Log( + $"No solution for broken download {archive.Name} {archive.State.PrimaryKeyString} could be found"); + return false; - var patchResult = await Download(patchState, patchPath); - if (!patchResult) return false; - - Utils.Status($"Applying Upgrade to {archive.Hash}"); - await using (var patchStream = patchPath.OpenRead()) - await using (var srcStream = upgradePath.OpenRead()) - await using (var destStream = destination.Create()) - { - OctoDiff.Apply(srcStream, patchStream, destStream); } - await destination.FileHashCachedAsync(); + Utils.Log($"Looking for patch for {archive.Name}"); + var patchResult = await ClientAPI.GetModUpgrade(archive, result.Archive!); + + Utils.Log($"Downloading patch for {archive.Name}"); + + var tempFile = new TempFile(); + + using var response = await (new Common.Http.Client()).GetAsync(patchResult); + await tempFile.Path.WriteAllAsync(await response.Content.ReadAsStreamAsync()); + response.Dispose(); + + Utils.Log($"Applying patch to {archive.Name}"); + await using(var src = result.NewFile.Path.OpenShared()) + await using (var final = destination.Create()) + { + Utils.ApplyPatch(src, () => tempFile.Path.OpenShared(), final); + } + + var hash = await destination.FileHashCachedAsync(); + if (hash != archive.Hash && archive.Hash != default) + { + Utils.Log("Archive hash didn't match after patching"); + return false; + } return true; } diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index 912306a2..2836693d 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Net.Http; using System.Net.Http.Headers; using System.Threading.Tasks; +using AngleSharp.Css; using Newtonsoft.Json; using Wabbajack.Common; using Wabbajack.Common.Serialization.Json; @@ -46,7 +47,7 @@ namespace Wabbajack.Lib.Downloaders } [JsonName("HttpDownloader")] - public class State : AbstractDownloadState + public class State : AbstractDownloadState, IUpgradingState { public string Url { get; } @@ -210,6 +211,35 @@ TOP: return new [] {"[General]", $"directURL={Url}"}; } + + public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a) + { + var tmpFile = new TempFile(); + + var newArchive = new Archive(this) {Name = a.Name}; + + try + { + if (!await Download(newArchive, tmpFile.Path)) + return default; + } + catch (HttpRequestException) + { + return default; + } + + newArchive.Hash = await tmpFile.Path.FileHashAsync(); + newArchive.Size = tmpFile.Path.Size; + + return (newArchive, tmpFile); + + } + + public bool ValidateUpgrade(AbstractDownloadState newArchiveState) + { + var httpState = (State)newArchiveState; + return httpState.Url == Url; + } } } } diff --git a/Wabbajack.Lib/Downloaders/IUpgradingState.cs b/Wabbajack.Lib/Downloaders/IUpgradingState.cs new file mode 100644 index 00000000..139b0a11 --- /dev/null +++ b/Wabbajack.Lib/Downloaders/IUpgradingState.cs @@ -0,0 +1,18 @@ +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.Lib.Downloaders +{ + public interface IUpgradingState + { + /// + /// Find a possible archive that can be combined with a server generated patch to get the input archive + /// state; + /// + /// + /// + public Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a); + + bool ValidateUpgrade(AbstractDownloadState newArchiveState); + } +} diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index d4749087..d2955862 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -140,7 +140,7 @@ namespace Wabbajack.Lib.Downloaders } [JsonName("NexusDownloader")] - public class State : AbstractDownloadState, IMetaState + public class State : AbstractDownloadState, IMetaState, IUpgradingState { [JsonIgnore] public Uri URL => new Uri($"http://nexusmods.com/{Game.MetaData().NexusName}/mods/{ModID}"); @@ -240,6 +240,41 @@ namespace Wabbajack.Lib.Downloaders { return new[] {"[General]", $"gameName={Game.MetaData().MO2ArchiveName}", $"modID={ModID}", $"fileID={FileID}"}; } + + public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a) + { + var client = await NexusApiClient.Get(); + + var mod = await client.GetModInfo(Game, ModID); + var files = await client.GetModFiles(Game, ModID); + var oldFile = files.files.FirstOrDefault(f => f.file_id == FileID); + var newFile = files.files.OrderByDescending(f => f.uploaded_timestamp).FirstOrDefault(); + + if (!mod.available || oldFile == default || newFile == default) + { + return default; + } + + var tempFile = new TempFile(); + + var newArchive = new Archive(new State {Game = Game, ModID = ModID, FileID = FileID}) + { + Name = newFile.file_name, + }; + + await newArchive.State.Download(newArchive, tempFile.Path); + + newArchive.Size = tempFile.Path.Size; + newArchive.Hash = await tempFile.Path.FileHashAsync(); + + return (newArchive, tempFile); + } + + public bool ValidateUpgrade(AbstractDownloadState newArchiveState) + { + var state = (State)newArchiveState; + return Game == state.Game && ModID == state.ModID; + } } } } diff --git a/Wabbajack.Lib/Exceptions/HttpException.cs b/Wabbajack.Lib/Exceptions/HttpException.cs index af0a3bb0..1902ca04 100644 --- a/Wabbajack.Lib/Exceptions/HttpException.cs +++ b/Wabbajack.Lib/Exceptions/HttpException.cs @@ -1,4 +1,5 @@ using System; +using System.Net.Http; namespace Wabbajack.Lib.Exceptions { @@ -12,5 +13,12 @@ namespace Wabbajack.Lib.Exceptions Code = code; Reason = reason; } + + public HttpException(HttpResponseMessage response) : base( + $"Http Error {response.StatusCode} - {response.ReasonPhrase}") + { + Code = (int)response.StatusCode; + Reason = response.ReasonPhrase; + } } } diff --git a/Wabbajack.Server.Test/ABuildServerSystemTest.cs b/Wabbajack.Server.Test/ABuildServerSystemTest.cs index d5a8fa61..eaaf9838 100644 --- a/Wabbajack.Server.Test/ABuildServerSystemTest.cs +++ b/Wabbajack.Server.Test/ABuildServerSystemTest.cs @@ -1,12 +1,18 @@ using System; +using System.Collections.Generic; +using System.IO.Compression; using System.Reactive.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Hosting; using Wabbajack.Common; using Wabbajack.Common.Http; using Wabbajack.Common.StatusFeed; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.FileUploader; +using Wabbajack.Lib.ModListRegistry; using Wabbajack.Server; using Wabbajack.Server.DataLayer; using Xunit; @@ -182,5 +188,79 @@ namespace Wabbajack.BuildServer.Test _unsubErr.Dispose(); } + + protected async Task MakeModList(string modFileName) + { + var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!"); + var test_archive_path = modFileName.RelativeTo(Fixture.ServerPublicFolder); + await test_archive_path.WriteAllBytesAsync(archive_data); + + + + ModListData = new ModList(); + ModListData.Archives.Add( + new Archive(new HTTPDownloader.State(MakeURL(modFileName.ToString()))) + { + Hash = await test_archive_path.FileHashAsync(), + Name = "test_archive", + Size = test_archive_path.Size, + }); + + var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder); + + await using (var fs = modListPath.Create()) + { + using var za = new ZipArchive(fs, ZipArchiveMode.Create); + var entry = za.CreateEntry("modlist"); + await using var es = entry.Open(); + ModListData.ToJson(es); + } + + ModListMetaData = new List + { + new ModlistMetadata + { + Official = false, + Author = "Test Suite", + Description = "A test", + DownloadMetadata = new DownloadMetadata + { + Hash = await modListPath.FileHashAsync(), + Size = modListPath.Size + }, + Links = new ModlistMetadata.LinksObject + { + MachineURL = "test_list", + Download = MakeURL("test_modlist.wabbajack") + } + }, + new ModlistMetadata + { + Official = true, + Author = "Test Suite", + Description = "A list with a broken hash", + DownloadMetadata = new DownloadMetadata() + { + Hash = Hash.FromLong(42), + Size = 42 + }, + Links = new ModlistMetadata.LinksObject + { + MachineURL = "broken_list", + Download = MakeURL("test_modlist.wabbajack") + } + } + }; + + var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder); + + ModListMetaData.ToJson(metadataPath); + + return new Uri(MakeURL("test_mod_list_metadata.json")); + } + + public ModList ModListData { get; set; } + + public List ModListMetaData { get; set; } } } diff --git a/Wabbajack.Server.Test/ModListValidationTests.cs b/Wabbajack.Server.Test/ModListValidationTests.cs index d20c39a7..0396a632 100644 --- a/Wabbajack.Server.Test/ModListValidationTests.cs +++ b/Wabbajack.Server.Test/ModListValidationTests.cs @@ -25,7 +25,7 @@ namespace Wabbajack.BuildServer.Test [Fact] public async Task CanLoadMetadataFromTestServer() { - var modlist = await MakeModList(); + var modlist = await MakeModList("CanLoadMetadataFromTestServer.txt"); Consts.ModlistMetadataURL = modlist.ToString(); var data = await ModlistMetadata.LoadFromGithub(); Assert.Equal(2, data.Count); @@ -35,11 +35,11 @@ namespace Wabbajack.BuildServer.Test [Fact] public async Task CanIngestModLists() { - var modlist = await MakeModList(); + var modlist = await MakeModList("CanIngestModLists.txt"); Consts.ModlistMetadataURL = modlist.ToString(); var sql = Fixture.GetService(); var downloader = Fixture.GetService(); - Assert.Equal(2, await downloader.CheckForNewLists()); + await downloader.CheckForNewLists(); foreach (var list in ModListMetaData) { @@ -54,7 +54,7 @@ namespace Wabbajack.BuildServer.Test [Fact] public async Task CanValidateModLists() { - var modlists = await MakeModList(); + var modlists = await MakeModList("can_validate_file.txt"); Consts.ModlistMetadataURL = modlists.ToString(); Utils.Log("Updating modlists"); await RevalidateLists(true); @@ -68,7 +68,7 @@ namespace Wabbajack.BuildServer.Test await CheckListFeeds(0, 1); Utils.Log("Break List"); - var archive = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder); + var archive = "can_validate_file.txt".RelativeTo(Fixture.ServerPublicFolder); await archive.MoveToAsync(archive.WithExtension(new Extension(".moved")), true); // We can revalidate but the non-nexus archives won't be checked yet since the list didn't change @@ -101,6 +101,58 @@ namespace Wabbajack.BuildServer.Test await CheckListFeeds(0, 1); + } + + [Fact] + public async Task CanHealLists() + { + var modlists = await MakeModList("CanHealLists.txt"); + Consts.ModlistMetadataURL = modlists.ToString(); + Utils.Log("Updating modlists"); + await RevalidateLists(true); + + Utils.Log("Checking validated results"); + var data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list"); + Assert.NotNull(data); + Assert.Equal(0, data.ValidationSummary.Failed); + Assert.Equal(1, data.ValidationSummary.Passed); + + await CheckListFeeds(0, 1); + + Utils.Log("Break List by changing the file"); + var archive = "CanHealLists.txt".RelativeTo(Fixture.ServerPublicFolder); + await archive.WriteAllTextAsync("broken"); + + // We can revalidate but the non-nexus archives won't be checked yet since the list didn't change + await RevalidateLists(false); + + data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list"); + Assert.NotNull(data); + Assert.Equal(0, data.ValidationSummary.Failed); + Assert.Equal(1, data.ValidationSummary.Passed); + + // Run the non-nexus validator + await RevalidateLists(true); + + data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list"); + Assert.NotNull(data); + Assert.Equal(0, data.ValidationSummary.Failed); + Assert.Equal(0, data.ValidationSummary.Passed); + Assert.Equal(1, data.ValidationSummary.Updating); + + var patcher = Fixture.GetService(); + Assert.Equal(1, await patcher.Execute()); + + await RevalidateLists(false); + + data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list"); + Assert.NotNull(data); + Assert.Equal(0, data.ValidationSummary.Failed); + Assert.Equal(1, data.ValidationSummary.Passed); + Assert.Equal(0, data.ValidationSummary.Updating); + + + } private async Task RevalidateLists(bool runNonNexus) @@ -117,6 +169,9 @@ namespace Wabbajack.BuildServer.Test var validator = Fixture.GetService(); await validator.Execute(); + + var archiver = Fixture.GetService(); + await archiver.Execute(); } private async Task CheckListFeeds(int failed, int passed) @@ -135,79 +190,7 @@ namespace Wabbajack.BuildServer.Test } - private async Task MakeModList() - { - var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!"); - var test_archive_path = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder); - await test_archive_path.WriteAllBytesAsync(archive_data); - - ModListData = new ModList(); - ModListData.Archives.Add( - new Archive(new HTTPDownloader.State(MakeURL("test_archive.txt"))) - { - Hash = await test_archive_path.FileHashAsync(), - Name = "test_archive", - Size = test_archive_path.Size, - }); - - var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder); - - await using (var fs = modListPath.Create()) - { - using var za = new ZipArchive(fs, ZipArchiveMode.Create); - var entry = za.CreateEntry("modlist"); - await using var es = entry.Open(); - ModListData.ToJson(es); - } - - ModListMetaData = new List - { - new ModlistMetadata - { - Official = false, - Author = "Test Suite", - Description = "A test", - DownloadMetadata = new DownloadMetadata - { - Hash = await modListPath.FileHashAsync(), - Size = modListPath.Size - }, - Links = new ModlistMetadata.LinksObject - { - MachineURL = "test_list", - Download = MakeURL("test_modlist.wabbajack") - } - }, - new ModlistMetadata - { - Official = true, - Author = "Test Suite", - Description = "A list with a broken hash", - DownloadMetadata = new DownloadMetadata() - { - Hash = Hash.FromLong(42), - Size = 42 - }, - Links = new ModlistMetadata.LinksObject - { - MachineURL = "broken_list", - Download = MakeURL("test_modlist.wabbajack") - } - } - }; - - var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder); - - ModListMetaData.ToJson(metadataPath); - - return new Uri(MakeURL("test_mod_list_metadata.json")); - } - - public ModList ModListData { get; set; } - - public List ModListMetaData { get; set; } - } } diff --git a/Wabbajack.Server.Test/ModlistUpdater.cs b/Wabbajack.Server.Test/ModlistUpdater.cs new file mode 100644 index 00000000..c81e8c0d --- /dev/null +++ b/Wabbajack.Server.Test/ModlistUpdater.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Dapper; +using Wabbajack.BuildServer; +using Wabbajack.BuildServer.Test; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.Exceptions; +using Wabbajack.Lib.ModListRegistry; +using Wabbajack.Lib.NexusApi; +using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DTOs; +using Wabbajack.Server.Services; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.Server.Test +{ + public class ModlistUpdater : ABuildServerSystemTest + { + public ModlistUpdater(ITestOutputHelper output, SingletonAdaptor fixture) : base(output, + fixture) + { + } + + [Fact] + public async Task CanIndexAndUpdateFiles() + { + var validator = Fixture.GetService(); + var nonNexus = Fixture.GetService(); + var modLists = await MakeModList("CanIndexAndUpdateFiles.txt"); + Consts.ModlistMetadataURL = modLists.ToString(); + + + var listDownloader = Fixture.GetService(); + var downloader = Fixture.GetService(); + var archiver = Fixture.GetService(); + var patcher = Fixture.GetService(); + + var sql = Fixture.GetService(); + var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!"); + var newFileData = Encoding.UTF8.GetBytes("Forks for Everyone!"); + var oldDataHash = oldFileData.xxHash(); + var newDataHash = newFileData.xxHash(); + + var oldArchive = new Archive(new NexusDownloader.State {Game = Game.Enderal, ModID = 42, FileID = 10}) + { + Size = oldFileData.Length, + Hash = oldDataHash + }; + var newArchive = new Archive(new NexusDownloader.State {Game = Game.Enderal, ModID = 42, FileID = 11}) + { + Size = newFileData.Length, + Hash = newDataHash + }; + + await IngestData(archiver, oldFileData); + await IngestData(archiver, newFileData); + + await sql.EnqueueDownload(oldArchive); + var oldDownload = await sql.GetNextPendingDownload(); + await oldDownload.Finish(sql); + + await sql.EnqueueDownload(newArchive); + var newDownload = await sql.GetNextPendingDownload(); + await newDownload.Finish(sql); + + + await Assert.ThrowsAsync(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); + Assert.Equal(1, await patcher.Execute()); + + Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_updates/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); + } + + [Fact] + public async Task TestEndToEndArchiveUpdating() + { + var modLists = await MakeModList("TestEndToEndArchiveUpdating.txt"); + Consts.ModlistMetadataURL = modLists.ToString(); + + + var downloader = Fixture.GetService(); + var archiver = Fixture.GetService(); + var patcher = Fixture.GetService(); + + var sql = Fixture.GetService(); + var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!"); + var newFileData = Encoding.UTF8.GetBytes("Forks for Everyone!"); + var oldDataHash = oldFileData.xxHash(); + var newDataHash = newFileData.xxHash(); + + await "TestEndToEndArchiveUpdating.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(oldFileData); + + var oldArchive = new Archive(new HTTPDownloader.State(MakeURL("TestEndToEndArchiveUpdating.txt"))) + { + Size = oldFileData.Length, + Hash = oldDataHash + }; + + await IngestData(archiver, oldFileData); + await sql.EnqueueDownload(oldArchive); + var oldDownload = await sql.GetNextPendingDownload(); + await oldDownload.Finish(sql); + + + // Now update the file + await"TestEndToEndArchiveUpdating.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(newFileData); + + + using var tempFile = new TempFile(); + var pendingRequest = DownloadDispatcher.DownloadWithPossibleUpgrade(oldArchive, tempFile.Path); + + for (var times = 0; await downloader.Execute() == 0 && times < 40; times ++) + { + await Task.Delay(TimeSpan.FromMilliseconds(200)); + } + + + for (var times = 0; await patcher.Execute() == 0 && times < 40; times ++) + { + await Task.Delay(TimeSpan.FromMilliseconds(200)); + } + + Assert.True(await pendingRequest); + Assert.Equal(oldDataHash, await tempFile.Path.FileHashAsync()); + } + + private async Task IngestData(ArchiveMaintainer am, byte[] data) + { + using var f = new TempFile(); + await f.Path.WriteAllBytesAsync(data); + await am.Ingest(f.Path); + } + } +} diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index 09a9754a..69502283 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -551,6 +551,39 @@ CONSTRAINT [PK_NexusModFilesSlow] PRIMARY KEY CLUSTERED ) ON [PRIMARY] GO +/****** Object: Table [dbo].[Patches] Script Date: 5/18/2020 6:26:07 AM ******/ + +CREATE TABLE [dbo].[Patches]( +[SrcId] [uniqueidentifier] NOT NULL, +[DestId] [uniqueidentifier] NOT NULL, +[PatchSize] [bigint] NULL, +[Finished] [datetime] NULL, +[IsFailed] [tinyint] NULL, +[FailMessage] [varchar](MAX) NULL, +CONSTRAINT [PK_Patches] PRIMARY KEY CLUSTERED + ( + [SrcId] ASC, + [DestId] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO + +ALTER TABLE [dbo].[Patches] WITH CHECK ADD CONSTRAINT [FK_DestId] FOREIGN KEY([DestId]) + REFERENCES [dbo].[ArchiveDownloads] ([Id]) +GO + +ALTER TABLE [dbo].[Patches] CHECK CONSTRAINT [FK_DestId] +GO + +ALTER TABLE [dbo].[Patches] WITH CHECK ADD CONSTRAINT [FK_SrcId] FOREIGN KEY([SrcId]) + REFERENCES [dbo].[ArchiveDownloads] ([Id]) +GO + +ALTER TABLE [dbo].[Patches] CHECK CONSTRAINT [FK_SrcId] +GO + + + /****** Object: Table [dbo].[NexusKeys] Script Date: 5/15/2020 5:20:02 PM ******/ SET ANSI_NULLS ON GO diff --git a/Wabbajack.Server/Controllers/ModUpgrade.cs b/Wabbajack.Server/Controllers/ModUpgrade.cs new file mode 100644 index 00000000..4f54359d --- /dev/null +++ b/Wabbajack.Server/Controllers/ModUpgrade.cs @@ -0,0 +1,59 @@ +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Server.DataLayer; +using Wabbajack.Server.Services; + +namespace Wabbajack.BuildServer.Controllers +{ + [ApiController] + public class ModUpgrade : ControllerBase + { + private ILogger _logger; + private SqlService _sql; + private DiscordWebHook _discord; + private AppSettings _settings; + + public ModUpgrade(ILogger logger, SqlService sql, DiscordWebHook discord, AppSettings settings) + { + _logger = logger; + _sql = sql; + _discord = discord; + _settings = settings; + } + + [HttpPost] + [Route("/mod_upgrade")] + public async Task PostModUpgrade() + { + var request = (await Request.Body.ReadAllTextAsync()).FromJsonString(); + if (!request.IsValid) + { + return BadRequest("Invalid mod upgrade"); + } + + var oldDownload = await _sql.GetOrEnqueueArchive(request.OldArchive); + var newDownload = await _sql.GetOrEnqueueArchive(request.NewArchive); + + _logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash}"); + var patch = await _sql.FindOrEnqueuePatch(oldDownload.Id, newDownload.Id); + if (patch.Finished.HasValue) + { + if (patch.PatchSize != 0) + { + return + Ok( + $"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{Consts.ArchiveUpdatesCDNFolder}/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); + } + + return NotFound("Patch creation failed"); + } + + // Still processing + return Accepted(); + } + + } +} diff --git a/Wabbajack.Server/DTOs/ArchiveDownload.cs b/Wabbajack.Server/DTOs/ArchiveDownload.cs index bf43b82e..6ffe0871 100644 --- a/Wabbajack.Server/DTOs/ArchiveDownload.cs +++ b/Wabbajack.Server/DTOs/ArchiveDownload.cs @@ -23,7 +23,7 @@ namespace Wabbajack.Server.DTOs public async Task Finish(SqlService service) { - IsFailed = true; + IsFailed = false; DownloadFinished = DateTime.UtcNow; await service.UpdatePendingDownload(this); } diff --git a/Wabbajack.Server/DTOs/Patch.cs b/Wabbajack.Server/DTOs/Patch.cs new file mode 100644 index 00000000..9d980e0c --- /dev/null +++ b/Wabbajack.Server/DTOs/Patch.cs @@ -0,0 +1,38 @@ +using System; +using System.Threading.Tasks; +using Microsoft.VisualBasic; +using Wabbajack.Common; +using Wabbajack.Server.DataLayer; + +namespace Wabbajack.Server.DTOs +{ + public class Patch + { + public ArchiveDownload Src { get; set; } + public ArchiveDownload Dest { get; set; } + public long PatchSize { get; set; } + public DateTime? Finished { get; set; } + public bool? IsFailed { get; set; } + public string FailMessage { get; set; } + + public async Task Finish(SqlService sql, long size) + { + IsFailed = false; + Finished = DateTime.UtcNow; + PatchSize = size; + await sql.FinializePatch(this); + } + + + public async Task Fail(SqlService sql, string msg) + { + IsFailed = true; + Finished = DateTime.UtcNow; + FailMessage = msg; + await sql.FinializePatch(this); + } + + + + } +} diff --git a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs index 359a9fd1..8a2b84d2 100644 --- a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs +++ b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs @@ -13,6 +13,26 @@ namespace Wabbajack.Server.DataLayer { public partial class SqlService { + public async Task AddKnownDownload(Archive a, DateTime downloadFinished) + { + await using var conn = await Open(); + var Id = Guid.NewGuid(); + await conn.ExecuteAsync( + "INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader, DownloadFinished, IsFailed) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader, @DownloadFinished, @IsFailed)", + new + { + Id = Id, + PrimaryKeyString = a.State.PrimaryKeyString, + Size = a.Size == 0 ? null : (long?)a.Size, + Hash = a.Hash == default ? null : (Hash?)a.Hash, + DownloadState = a.State, + Downloader = AbstractDownloadState.TypeToName[a.State.GetType()], + DownloadFinished = downloadFinished, + IsFailed = false + }); + return Id; + } + public async Task EnqueueDownload(Archive a) { await using var conn = await Open(); @@ -26,7 +46,7 @@ namespace Wabbajack.Server.DataLayer Size = a.Size == 0 ? null : (long?)a.Size, Hash = a.Hash == default ? null : (Hash?)a.Hash, DownloadState = a.State, - Downloader = AbstractDownloadState.TypeToName[a.State.GetType()] + Downloader = AbstractDownloadState.TypeToName[a.State.GetType()], }); return Id; } @@ -37,6 +57,93 @@ namespace Wabbajack.Server.DataLayer return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, PrimaryKeyString FROM ArchiveDownloads")).ToHashSet(); } + + public async Task GetArchiveDownload(Guid id) + { + await using var conn = await Open(); + var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>( + "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE Id = @id", + new {Id = id}); + if (result == default) + return null; + + return new ArchiveDownload + { + Id = result.Item1, + IsFailed = result.Item4, + DownloadFinished = result.Item6, + Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default} + }; + + } + + public async Task GetArchiveDownload(string primaryKeyString, Hash hash, long size) + { + await using var conn = await Open(); + var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>( + "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size", + new + { + PrimaryKeyString = primaryKeyString, + Hash = hash, + Size = size + }); + if (result == default) + return null; + + return new ArchiveDownload + { + Id = result.Item1, + IsFailed = result.Item4, + DownloadFinished = result.Item6, + Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default} + }; + + } + + + public async Task GetOrEnqueueArchive(Archive a) + { + await using var conn = await Open(); + using var trans = await conn.BeginTransactionAsync(); + var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>( + "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size", + new + { + PrimaryKeyString = a.State.PrimaryKeyString, + Hash = a.Hash, + Size = a.Size + }, trans); + if (result != default) + { + return new ArchiveDownload + { + Id = result.Item1, + IsFailed = result.Item4, + DownloadFinished = result.Item6, + Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default} + }; + } + + var id = Guid.NewGuid(); + await conn.ExecuteAsync( + "INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)", + new + { + Id = id, + PrimaryKeyString = a.State.PrimaryKeyString, + Size = a.Size == 0 ? null : (long?)a.Size, + Hash = a.Hash == default ? null : (Hash?)a.Hash, + DownloadState = a.State, + Downloader = AbstractDownloadState.TypeToName[a.State.GetType()] + }, trans); + + await trans.CommitAsync(); + + return new ArchiveDownload {Id = id, Archive = a,}; + + } + public async Task GetNextPendingDownload(bool ignoreNexus = false) { await using var conn = await Open(); diff --git a/Wabbajack.Server/DataLayer/Patches.cs b/Wabbajack.Server/DataLayer/Patches.cs new file mode 100644 index 00000000..60c2d969 --- /dev/null +++ b/Wabbajack.Server/DataLayer/Patches.cs @@ -0,0 +1,148 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Dapper; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Server.DTOs; + +namespace Wabbajack.Server.DataLayer +{ + public partial class SqlService + { + /// + /// Adds a patch record + /// + /// + /// + public async Task AddPatch(Patch patch) + { + await using var conn = await Open(); + await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)", + new {SrcId = patch.Src.Id, DestId = patch.Dest.Id}); + } + + /// + /// Adds a patch record + /// + /// + /// + public async Task FinializePatch(Patch patch) + { + await using var conn = await Open(); + await conn.ExecuteAsync("UPDATE dbo.Patches SET PatchSize = @PatchSize, Finished = @Finished, IsFailed = @IsFailed, FailMessage = @FailMessage WHERE SrcId = @SrcId AND DestID = @DestId", + new + { + SrcId = patch.Src.Id, + DestId = patch.Dest.Id, + PatchSize = patch.PatchSize, + Finished = patch.Finished, + IsFailed = patch.IsFailed, + FailMessage = patch.FailMessage + }); + } + + public async Task FindPatch(Guid src, Guid dest) + { + await using var conn = await Open(); + var patch = await conn.QueryFirstOrDefaultAsync<(long, DateTime?, bool?, string)>( + @"SELECT p.PatchHash, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage + FROM dbo.Patches p + LEFT JOIN dbo.ArchiveDownloads src ON p.SrcId = src.Id + LEFT JOIN dbo.ArchiveDownloads dest ON p.SrcId = dest.Id + WHERE SrcId = @SrcId + AND DestId = @DestId + AND src.DownloadFinished IS NOT NULL + AND dest.DownloadFinished IS NOT NULL", + new + { + SrcId = src, + DestId = dest + }); + if (patch == default) + return default; + + return new Patch { + Src = await GetArchiveDownload(src), + Dest = await GetArchiveDownload(dest), + PatchSize = patch.Item1, + Finished = patch.Item2, + IsFailed = patch.Item3, + FailMessage = patch.Item4 + }; + } + + public async Task FindOrEnqueuePatch(Guid src, Guid dest) + { + await using var conn = await Open(); + var trans = await conn.BeginTransactionAsync(); + var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>( + "SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestId", + new + { + SrcId = src, + DestId = dest + }, trans); + if (patch == default) + { + await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)", + new {SrcId = src, DestId = dest}, trans); + await trans.CommitAsync(); + return new Patch {Src = await GetArchiveDownload(src), Dest = await GetArchiveDownload(dest),}; + } + else + { + return new Patch { + Src = await GetArchiveDownload(src), + Dest = await GetArchiveDownload(dest), + PatchSize = patch.Item3, + Finished = patch.Item4, + IsFailed = patch.Item5, + FailMessage = patch.Item6 + }; + + } + + } + + public async Task GetPendingPatch() + { + await using var conn = await Open(); + var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>( + "SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE Finished is NULL"); + if (patch == default) + return default(Patch); + + return new Patch { + Src = await GetArchiveDownload(patch.Item1), + Dest = await GetArchiveDownload(patch.Item2), + PatchSize = patch.Item3, + Finished = patch.Item4, + IsFailed = patch.Item5, + FailMessage = patch.Item6 + }; + } + + public async Task> PatchesForSource(Guid sourceDownload) + { + await using var conn = await Open(); + var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>( + "SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId", new {SrcId = sourceDownload}); + + List results = new List(); + foreach (var (srcId, destId, patchSize, finished, isFinished, failMessage) in patches) + { + results.Add( new Patch { + Src = await GetArchiveDownload(srcId), + Dest = await GetArchiveDownload(destId), + PatchSize = patchSize, + Finished = finished, + IsFailed = isFinished, + FailMessage = failMessage + }); + } + return results; + } + } +} diff --git a/Wabbajack.Server/GlobalInformation.cs b/Wabbajack.Server/GlobalInformation.cs index 2b51f673..9695be8d 100644 --- a/Wabbajack.Server/GlobalInformation.cs +++ b/Wabbajack.Server/GlobalInformation.cs @@ -5,7 +5,7 @@ namespace Wabbajack.Server public class GlobalInformation { public TimeSpan NexusRSSPollRate = TimeSpan.FromMinutes(1); - public TimeSpan NexusAPIPollRate = TimeSpan.FromHours(24); + public TimeSpan NexusAPIPollRate = TimeSpan.FromMinutes(15); public DateTime LastNexusSyncUTC { get; set; } public TimeSpan TimeSinceLastNexusSync => DateTime.UtcNow - LastNexusSyncUTC; } diff --git a/Wabbajack.Server/Services/ArchiveDownloader.cs b/Wabbajack.Server/Services/ArchiveDownloader.cs index fbf733ad..510b9e16 100644 --- a/Wabbajack.Server/Services/ArchiveDownloader.cs +++ b/Wabbajack.Server/Services/ArchiveDownloader.cs @@ -67,6 +67,7 @@ namespace Wabbajack.Server.Services if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash) { + _logger.Log(LogLevel.Warning, $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}"); await nextDownload.Fail(_sql, "Invalid Hash"); continue; } diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index ce2955c5..b2453a7a 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -24,17 +24,19 @@ namespace Wabbajack.Server.Services private SqlService _sql; private DiscordWebHook _discord; private NexusKeyMaintainance _nexus; + private ArchiveMaintainer _archives; public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries { get; private set; } = new (ModListSummary Summary, DetailedStatus Detailed)[0]; - public ListValidator(ILogger logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus) - : base(logger, settings, TimeSpan.FromMinutes(10)) + public ListValidator(ILogger logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives) + : base(logger, settings, TimeSpan.FromMinutes(5)) { _sql = sql; _discord = discord; _nexus = nexus; + _archives = archives; } public override async Task Execute() @@ -42,14 +44,19 @@ namespace Wabbajack.Server.Services var data = await _sql.GetValidationData(); using var queue = new WorkQueue(); + var oldSummaries = Summaries; var results = await data.ModLists.PMap(queue, async list => { + var oldSummary = + oldSummaries.FirstOrDefault(s => s.Summary.MachineURL == list.Metadata.Links.MachineURL); + var (metadata, modList) = list; var archives = await modList.Archives.PMap(queue, async archive => { var (_, result) = await ValidateArchive(data, archive); - // TODO : auto-healing goes here + if (result == ArchiveStatus.InValid) + return await TryToHeal(data, archive); return (archive, result); }); @@ -80,12 +87,103 @@ namespace Wabbajack.Server.Services }).ToList() }; + if (oldSummary != default && oldSummary.Summary.Failed != summary.Failed) + { + _logger.Log(LogLevel.Information, $"Number of failures {oldSummary.Summary.Failed} -> {summary.Failed}"); + + if (summary.HasFailures) + { + await _discord.Send(Channel.Ham, + new DiscordMessage + { + Embeds = new[] + { + new DiscordEmbed + { + Description = + $"Number of failures in {summary.Name} (`{summary.MachineURL}`) was {oldSummary.Summary.Failed} is now {summary.Failed}", + Url = new Uri( + $"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html") + } + } + }); + } + + if (!summary.HasFailures) + { + await _discord.Send(Channel.Ham, + new DiscordMessage + { + Embeds = new[] + { + new DiscordEmbed + { + Description = $"{summary.Name} (`{summary.MachineURL}`) is now passing.", + } + } + }); + } + + } + return (summary, detailed); }); Summaries = results; return Summaries.Count(s => s.Summary.HasFailures); } - + + private AsyncLock _healLock = new AsyncLock(); + private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive) + { + using var _ = await _healLock.WaitAsync(); + + if (!(archive.State is IUpgradingState)) + return (archive, ArchiveStatus.InValid); + + var srcDownload = await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size); + if (srcDownload == null || srcDownload.IsFailed == true) + { + return (archive, ArchiveStatus.InValid); + } + + + var patches = await _sql.PatchesForSource(srcDownload.Id); + foreach (var patch in patches) + { + if (patch.Finished is null) + return (archive, ArchiveStatus.Updating); + + if (patch.IsFailed == true) + continue; + + var (_, status) = await ValidateArchive(data, patch.Dest.Archive); + if (status == ArchiveStatus.Valid) + return (archive, ArchiveStatus.Updated); + } + + + var upgradeTime = DateTime.UtcNow; + var upgrade = await (archive.State as IUpgradingState)?.FindUpgrade(archive); + if (upgrade == default) + { + return (archive, ArchiveStatus.InValid); + } + + await _archives.Ingest(upgrade.NewFile.Path); + + var id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime); + var destDownload = await _sql.GetArchiveDownload(id); + + await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload}); + + _logger.Log(LogLevel.Information, $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}"); + await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}" }); + + upgrade.NewFile.Dispose(); + + return (archive, ArchiveStatus.Updating); + } + private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(ValidationData data, Archive archive) { switch (archive.State) @@ -108,7 +206,7 @@ namespace Wabbajack.Server.Services return isValid ? (archive, ArchiveStatus.Valid) : (archive, ArchiveStatus.InValid); } - return (archive, ArchiveStatus.InValid); + return (archive, ArchiveStatus.Valid); } } } diff --git a/Wabbajack.Server/Services/NexusPoll.cs b/Wabbajack.Server/Services/NexusPoll.cs index ef68b47e..b1e77953 100644 --- a/Wabbajack.Server/Services/NexusPoll.cs +++ b/Wabbajack.Server/Services/NexusPoll.cs @@ -66,7 +66,7 @@ namespace Wabbajack.Server.Services public async Task UpdateNexusCacheAPI() { using var _ = _logger.BeginScope("Nexus Update via API"); - _logger.Log(LogLevel.Information, "Starting"); + _logger.Log(LogLevel.Information, "Starting Nexus Update via API"); var api = await NexusApiClient.Get(); var gameTasks = GameRegistry.Games.Values @@ -117,7 +117,7 @@ namespace Wabbajack.Server.Services public void Start() { if (!_settings.RunBackEndJobs) return; - + /* Task.Run(async () => { while (true) @@ -133,7 +133,7 @@ namespace Wabbajack.Server.Services await Task.Delay(_globalInformation.NexusRSSPollRate); } }); - +*/ Task.Run(async () => { while (true) diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs new file mode 100644 index 00000000..e9c56470 --- /dev/null +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -0,0 +1,108 @@ +using System; +using System.IO; +using System.Net; +using System.Threading.Tasks; +using FluentFTP; +using Microsoft.Extensions.Logging; +using Splat; +using Wabbajack.BuildServer; +using Wabbajack.Common; +using Wabbajack.Lib.CompilationSteps; +using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DTOs; + +namespace Wabbajack.Server.Services +{ + public class PatchBuilder : AbstractService + { + private DiscordWebHook _discordWebHook; + private SqlService _sql; + private ArchiveMaintainer _maintainer; + + public PatchBuilder(ILogger logger, SqlService sql, AppSettings settings, ArchiveMaintainer maintainer, + DiscordWebHook discordWebHook) : base(logger, settings, TimeSpan.FromMinutes(1)) + { + _discordWebHook = discordWebHook; + _sql = sql; + _maintainer = maintainer; + } + + public override async Task Execute() + { + int count = 0; + while (true) + { + var patch = await _sql.GetPendingPatch(); + if (patch == default) break; + + try + { + + _logger.LogInformation( + $"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"); + await _discordWebHook.Send(Channel.Spam, + new DiscordMessage + { + Content = + $"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}" + }); + + _maintainer.TryGetPath(patch.Src.Archive.Hash, out var srcPath); + _maintainer.TryGetPath(patch.Dest.Archive.Hash, out var destPath); + + var patchName = $"{Consts.ArchiveUpdatesCDNFolder}\\{patch.Src.Archive.Hash.ToHex()}_{patch.Dest.Archive.Hash.ToHex()}"; + + using var sigFile = new TempFile(); + await using var srcStream = srcPath.OpenShared(); + await using var destStream = destPath.OpenShared(); + await using var sigStream = sigFile.Path.Create(); + using var ftpClient = await GetBunnyCdnFtpClient(); + + if (!await ftpClient.DirectoryExistsAsync(Consts.ArchiveUpdatesCDNFolder)) + await ftpClient.CreateDirectoryAsync(Consts.ArchiveUpdatesCDNFolder); + + + await using var patchOutput = await ftpClient.OpenWriteAsync(patchName); + OctoDiff.Create(destStream, srcStream, sigStream, patchOutput); + + await patchOutput.DisposeAsync(); + + var size = await ftpClient.GetFileSizeAsync(patchName); + + await patch.Finish(_sql, size); + await _discordWebHook.Send(Channel.Spam, + new DiscordMessage + { + Content = + $"Built {size.ToFileSizeString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}" + }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error while building patch"); + await patch.Fail(_sql, ex.ToString()); + await _discordWebHook.Send(Channel.Spam, + new DiscordMessage + { + Content = + $"Failure building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}" + }); + + } + + count++; + } + + return count; + } + + private async Task GetBunnyCdnFtpClient() + { + var info = Utils.FromEncryptedJson("bunny-cdn-ftp-info"); + var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)}; + await client.ConnectAsync(); + return client; + } + + } +} diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index e97dcd6b..b6000912 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -65,6 +65,7 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddMvc(); services.AddControllers() @@ -116,6 +117,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); + app.UseService(); app.Use(next => {