From 03f5ff6c928c19de4f52e979096d7a21d41814c3 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 19 May 2020 21:25:41 -0600 Subject: [PATCH] Can heal a simple pre-indexed download --- Wabbajack.Common/Http/Client.cs | 11 +- Wabbajack.Common/OctoDiff.cs | 4 +- Wabbajack.Common/Paths.cs | 7 ++ Wabbajack.Lib/ClientAPI.cs | 72 ++++++++++-- .../Downloaders/DownloadDispatcher.cs | 63 ++++++----- Wabbajack.Lib/Downloaders/IUpgradingState.cs | 18 +++ Wabbajack.Lib/Downloaders/NexusDownloader.cs | 37 +++++- Wabbajack.Lib/Exceptions/HttpException.cs | 8 ++ Wabbajack.Server.Test/ModlistUpdater.cs | 107 +++++------------- Wabbajack.Server.Test/sql/wabbajack_db.sql | 33 ++++++ Wabbajack.Server/Controllers/AuthoredFiles.cs | 2 +- Wabbajack.Server/Controllers/ModUpgrade.cs | 58 ++++++++++ Wabbajack.Server/DTOs/Patch.cs | 26 ++++- .../DataLayer/ArchiveDownloads.cs | 67 +++++++++++ Wabbajack.Server/DataLayer/Patches.cs | 78 +++++++++++-- Wabbajack.Server/GlobalInformation.cs | 2 +- Wabbajack.Server/Services/NexusPoll.cs | 6 +- Wabbajack.Server/Services/PatchBuilder.cs | 95 ++++++++++++++++ Wabbajack.Server/Startup.cs | 2 + 19 files changed, 564 insertions(+), 132 deletions(-) create mode 100644 Wabbajack.Lib/Downloaders/IUpgradingState.cs create mode 100644 Wabbajack.Server/Controllers/ModUpgrade.cs create mode 100644 Wabbajack.Server/Services/PatchBuilder.cs diff --git a/Wabbajack.Common/Http/Client.cs b/Wabbajack.Common/Http/Client.cs index a58aed49..da58a7cb 100644 --- a/Wabbajack.Common/Http/Client.cs +++ b/Wabbajack.Common/Http/Client.cs @@ -19,11 +19,17 @@ namespace Wabbajack.Common.Http return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions); } + public async Task GetAsync(Uri url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true) + { + var request = new HttpRequestMessage(HttpMethod.Get, url); + return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions); + } - public async Task PostAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead) + + public async Task PostAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true) { var request = new HttpRequestMessage(HttpMethod.Post, url) {Content = content}; - return await SendAsync(request, responseHeadersRead); + return await SendAsync(request, responseHeadersRead, errorsAsExceptions); } public async Task PutAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead) @@ -79,7 +85,6 @@ namespace Wabbajack.Common.Http if (errorsAsExceptions) throw new HttpRequestException( $"Http Exception {response.StatusCode} - {response.ReasonPhrase} - {msg.RequestUri}"); - ; return response; } catch (Exception ex) diff --git a/Wabbajack.Common/OctoDiff.cs b/Wabbajack.Common/OctoDiff.cs index bb17c9e8..271274bb 100644 --- a/Wabbajack.Common/OctoDiff.cs +++ b/Wabbajack.Common/OctoDiff.cs @@ -28,7 +28,7 @@ namespace Wabbajack.Common return sigStream; } - private static void CreateSignature(Stream oldData, FileStream sigStream) + private static void CreateSignature(Stream oldData, Stream sigStream) { Utils.Status("Creating Patch Signature"); var signatureBuilder = new SignatureBuilder(); @@ -36,7 +36,7 @@ namespace Wabbajack.Common sigStream.Position = 0; } - public static void Create(Stream oldData, FileStream newData, FileStream signature, FileStream output) + public static void Create(Stream oldData, Stream newData, Stream signature, Stream output) { CreateSignature(oldData, signature); var db = new DeltaBuilder {ProgressReporter = reporter}; diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index adbe3dea..612e0ed1 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -337,6 +337,13 @@ namespace Wabbajack.Common await fs.WriteAsync(data); } + public async Task WriteAllAsync(Stream data, bool disposeAfter = true) + { + await using var fs = Create(); + await fs.CopyToAsync(data); + if (disposeAfter) await data.DisposeAsync(); + } + public void AppendAllText(string text) { File.AppendAllText(_path, text); diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index 4eadf302..802e6bcf 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -1,11 +1,46 @@ using System; using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; using System.Threading.Tasks; using Wabbajack.Common; +using Wabbajack.Common.Serialization.Json; +using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.Exceptions; namespace Wabbajack.Lib { + [JsonName("ModUpgradeRequest")] + public class ModUpgradeRequest + { + public Archive OldArchive { get; set; } + public Archive NewArchive { get; set; } + + public ModUpgradeRequest(Archive oldArchive, Archive newArchive) + { + OldArchive = oldArchive; + NewArchive = newArchive; + } + + public bool IsValid + { + get + { + if (OldArchive.Hash == NewArchive.Hash && OldArchive.State.PrimaryKeyString == NewArchive.State.PrimaryKeyString) return false; + if (OldArchive.State.GetType() != NewArchive.State.GetType()) + return false; + if (OldArchive.State is IUpgradingState u) + { + return u.ValidateUpgrade(NewArchive.State); + } + + return false; + } + } + } + public class ClientAPI { public static Common.Http.Client GetClient() @@ -16,18 +51,39 @@ namespace Wabbajack.Lib return client; } - public static async Task GetModUpgrade(Hash hash) + + + public static async Task GetModUpgrade(Archive oldArchive, Archive newArchive, TimeSpan? maxWait = null, TimeSpan? waitBetweenTries = null) { - using var response = await GetClient() - .GetAsync($"{Consts.WabbajackBuildServerUri}alternative/{hash.ToHex()}"); + maxWait ??= TimeSpan.FromMinutes(10); + waitBetweenTries ??= TimeSpan.FromSeconds(15); + + var request = new ModUpgradeRequest( oldArchive, newArchive); + var start = DateTime.UtcNow; + + RETRY: + + var response = await GetClient() + .PostAsync($"{Consts.WabbajackBuildServerUri}mod_upgrade", new StringContent(request.ToJson(), Encoding.UTF8, "application/json")); + if (response.IsSuccessStatusCode) { - return (await response.Content.ReadAsStringAsync()).FromJsonString(); + switch (response.StatusCode) + { + case HttpStatusCode.OK: + return new Uri(await response.Content.ReadAsStringAsync()); + case HttpStatusCode.Accepted: + Utils.Log($"Waiting for patch processing on the server for {oldArchive.Name}, sleeping for another 15 seconds"); + await Task.Delay(TimeSpan.FromSeconds(15)); + response.Dispose(); + if (DateTime.UtcNow - start > maxWait) + throw new HttpException(response); + goto RETRY; + } } - - Utils.Log($"No Upgrade for {hash}"); - Utils.Log(await response.Content.ReadAsStringAsync()); - return null; + var ex = new HttpException(response); + response.Dispose(); + throw ex; } /// diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index bdd3eb9a..7a19665f 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Net.Http; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; @@ -95,40 +97,49 @@ namespace Wabbajack.Lib.Downloaders return true; } - Utils.Log($"Download failed, looking for upgrade"); - var upgrade = await ClientAPI.GetModUpgrade(archive.Hash); - if (upgrade == null) + if (!(archive.State is IUpgradingState)) { - Utils.Log($"No upgrade found for {archive.Hash}"); + Utils.Log($"Download failed for {archive.Name} and no upgrade from this download source is possible"); return false; } - Utils.Log($"Upgrading via {upgrade.State.PrimaryKeyString}"); + + var upgrade = (IUpgradingState)archive.State; + + Utils.Log($"Trying to find solution to broken download for {archive.Name}"); - Utils.Log($"Upgrading {archive.Hash}"); - var upgradePath = destination.Parent.Combine("_Upgrade_" + archive.Name); - var upgradeResult = await Download(upgrade, upgradePath); - if (!upgradeResult) return false; - - var patchName = $"{archive.Hash.ToHex()}_{upgrade.Hash.ToHex()}"; - var patchPath = destination.Parent.Combine("_Patch_" + patchName); - - var patchState = new Archive(new HTTPDownloader.State($"https://wabbajackcdn.b-cdn.net/updates/{patchName}")) + var result = await upgrade.FindUpgrade(archive); + if (result == default) { - Name = patchName, - }; + Utils.Log( + $"No solution for broken download {archive.Name} {archive.State.PrimaryKeyString} could be found"); + return false; - var patchResult = await Download(patchState, patchPath); - if (!patchResult) return false; - - Utils.Status($"Applying Upgrade to {archive.Hash}"); - await using (var patchStream = patchPath.OpenRead()) - await using (var srcStream = upgradePath.OpenRead()) - await using (var destStream = destination.Create()) - { - OctoDiff.Apply(srcStream, patchStream, destStream); } - await destination.FileHashCachedAsync(); + Utils.Log($"Looking for patch for {archive.Name}"); + var patchResult = await ClientAPI.GetModUpgrade(archive, result.Archive!); + + Utils.Log($"Downloading patch for {archive.Name}"); + + var tempFile = new TempFile(); + + using var response = await (new Common.Http.Client()).GetAsync(patchResult); + await tempFile.Path.WriteAllAsync(await response.Content.ReadAsStreamAsync()); + response.Dispose(); + + Utils.Log($"Applying patch to {archive.Name}"); + await using(var src = result.NewFile.Path.OpenShared()) + await using (var final = destination.Create()) + { + Utils.ApplyPatch(src, () => tempFile.Path.OpenShared(), final); + } + + var hash = await destination.FileHashCachedAsync(); + if (hash != archive.Hash && archive.Hash != default) + { + Utils.Log("Archive hash didn't match after patching"); + return false; + } return true; } diff --git a/Wabbajack.Lib/Downloaders/IUpgradingState.cs b/Wabbajack.Lib/Downloaders/IUpgradingState.cs new file mode 100644 index 00000000..139b0a11 --- /dev/null +++ b/Wabbajack.Lib/Downloaders/IUpgradingState.cs @@ -0,0 +1,18 @@ +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.Lib.Downloaders +{ + public interface IUpgradingState + { + /// + /// Find a possible archive that can be combined with a server generated patch to get the input archive + /// state; + /// + /// + /// + public Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a); + + bool ValidateUpgrade(AbstractDownloadState newArchiveState); + } +} diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index d4749087..d2955862 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -140,7 +140,7 @@ namespace Wabbajack.Lib.Downloaders } [JsonName("NexusDownloader")] - public class State : AbstractDownloadState, IMetaState + public class State : AbstractDownloadState, IMetaState, IUpgradingState { [JsonIgnore] public Uri URL => new Uri($"http://nexusmods.com/{Game.MetaData().NexusName}/mods/{ModID}"); @@ -240,6 +240,41 @@ namespace Wabbajack.Lib.Downloaders { return new[] {"[General]", $"gameName={Game.MetaData().MO2ArchiveName}", $"modID={ModID}", $"fileID={FileID}"}; } + + public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a) + { + var client = await NexusApiClient.Get(); + + var mod = await client.GetModInfo(Game, ModID); + var files = await client.GetModFiles(Game, ModID); + var oldFile = files.files.FirstOrDefault(f => f.file_id == FileID); + var newFile = files.files.OrderByDescending(f => f.uploaded_timestamp).FirstOrDefault(); + + if (!mod.available || oldFile == default || newFile == default) + { + return default; + } + + var tempFile = new TempFile(); + + var newArchive = new Archive(new State {Game = Game, ModID = ModID, FileID = FileID}) + { + Name = newFile.file_name, + }; + + await newArchive.State.Download(newArchive, tempFile.Path); + + newArchive.Size = tempFile.Path.Size; + newArchive.Hash = await tempFile.Path.FileHashAsync(); + + return (newArchive, tempFile); + } + + public bool ValidateUpgrade(AbstractDownloadState newArchiveState) + { + var state = (State)newArchiveState; + return Game == state.Game && ModID == state.ModID; + } } } } diff --git a/Wabbajack.Lib/Exceptions/HttpException.cs b/Wabbajack.Lib/Exceptions/HttpException.cs index af0a3bb0..1902ca04 100644 --- a/Wabbajack.Lib/Exceptions/HttpException.cs +++ b/Wabbajack.Lib/Exceptions/HttpException.cs @@ -1,4 +1,5 @@ using System; +using System.Net.Http; namespace Wabbajack.Lib.Exceptions { @@ -12,5 +13,12 @@ namespace Wabbajack.Lib.Exceptions Code = code; Reason = reason; } + + public HttpException(HttpResponseMessage response) : base( + $"Http Error {response.StatusCode} - {response.ReasonPhrase}") + { + Code = (int)response.StatusCode; + Reason = response.ReasonPhrase; + } } } diff --git a/Wabbajack.Server.Test/ModlistUpdater.cs b/Wabbajack.Server.Test/ModlistUpdater.cs index acf413b5..418af94e 100644 --- a/Wabbajack.Server.Test/ModlistUpdater.cs +++ b/Wabbajack.Server.Test/ModlistUpdater.cs @@ -9,6 +9,7 @@ using Wabbajack.BuildServer.Test; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.Exceptions; using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.NexusApi; using Wabbajack.Server.DataLayer; @@ -38,97 +39,49 @@ namespace Wabbajack.Server.Test var listDownloader = Fixture.GetService(); var downloader = Fixture.GetService(); var archiver = Fixture.GetService(); + var patcher = Fixture.GetService(); var sql = Fixture.GetService(); - var modId = long.MaxValue >> 1; - var oldFileId = long.MaxValue >> 2; - var newFileId = (long.MaxValue >> 2) + 1; - var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!"); var newFileData = Encoding.UTF8.GetBytes("Forks for Everyone!"); var oldDataHash = oldFileData.xxHash(); var newDataHash = newFileData.xxHash(); - - Assert.Equal(2, await listDownloader.CheckForNewLists()); - Assert.Equal(1, await downloader.Execute()); - Assert.Equal(0, await nonNexus.Execute()); - Assert.Equal(0, await validator.Execute()); - - Assert.True(archiver.HaveArchive(oldDataHash)); - Assert.False(archiver.HaveArchive(newDataHash)); - - var status = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list"); - Assert.Equal(0, status.ValidationSummary.Failed); - - - // Update the archive - await "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(newFileData); - - // Nothing new to do - Assert.Equal(0, await listDownloader.CheckForNewLists()); - Assert.Equal(0, await downloader.Execute()); - - // List now fails after we check the manual link - Assert.Equal(1, await nonNexus.Execute()); - Assert.Equal(1, await validator.Execute()); - - /* - Assert.True(await sql.HaveIndexdFile(oldDataHash)); - Assert.True(await sql.HaveIndexdFile(newDataHash)); - - var settings = Fixture.GetService(); - Assert.Equal($"Oldfile_{oldDataHash.ToHex()}_".RelativeTo(Fixture.ServerArchivesFolder), settings.PathForArchive(oldDataHash)); - Assert.Equal($"Newfile_{newDataHash.ToHex()}_".RelativeTo(Fixture.ServerArchivesFolder), settings.PathForArchive(newDataHash)); - - Utils.Log($"Download Updating {oldDataHash} -> {newDataHash}"); - await using var conn = await sql.Open(); - - await conn.ExecuteAsync("DELETE FROM dbo.DownloadStates WHERE Hash in (@OldHash, @NewHash);", - new {OldHash = (long)oldDataHash, NewHash = (long)newDataHash}); - - await sql.AddDownloadState(oldDataHash, new NexusDownloader.State + var oldArchive = new Archive(new NexusDownloader.State {Game = Game.Enderal, ModID = 42, FileID = 10}) { - Game = Game.Oblivion, - ModID = modId, - FileID = oldFileId - }); - - await sql.AddDownloadState(newDataHash, new NexusDownloader.State + Size = oldFileData.Length, + Hash = oldDataHash + }; + var newArchive = new Archive(new NexusDownloader.State {Game = Game.Enderal, ModID = 42, FileID = 11}) { - Game = Game.Oblivion, - ModID = modId, - FileID = newFileId - }); + Size = newFileData.Length, + Hash = newDataHash + }; + + await IngestData(archiver, oldFileData); + await IngestData(archiver, newFileData); + + await sql.EnqueueDownload(oldArchive); + var oldDownload = await sql.GetNextPendingDownload(); + await oldDownload.Finish(sql); + + await sql.EnqueueDownload(newArchive); + var newDownload = await sql.GetNextPendingDownload(); + await newDownload.Finish(sql); - Assert.NotNull(await sql.GetNexusStateByHash(oldDataHash)); - Assert.NotNull(await sql.GetNexusStateByHash(newDataHash)); - // No nexus info, so no upgrade - var noUpgrade = await ClientAPI.GetModUpgrade(oldDataHash); - Assert.Null(noUpgrade); + await Assert.ThrowsAsync(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); + Assert.Equal(1, await patcher.Execute()); - // Add Nexus info - await sql.AddNexusModFiles(Game.Oblivion, modId, DateTime.Now, - new NexusApiClient.GetModFilesResponse - { - files = new List - { - new NexusFileInfo {category_name = "MAIN", file_id = newFileId, file_name = "New File"}, - new NexusFileInfo {category_name = null, file_id = oldFileId, file_name = "Old File"} - } - }); + Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_upgrades/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); - - var enqueuedUpgrade = await ClientAPI.GetModUpgrade(oldDataHash); - - // Not Null because upgrade was enqueued - Assert.NotNull(enqueuedUpgrade); + } - await RunAllJobs(); - - Assert.True($"{oldDataHash.ToHex()}_{newDataHash.ToHex()}".RelativeTo(Fixture.ServerUpdatesFolder).IsFile); -*/ + private async Task IngestData(ArchiveMaintainer am, byte[] data) + { + using var f = new TempFile(); + await f.Path.WriteAllBytesAsync(data); + await am.Ingest(f.Path); } } } diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index 09a9754a..69502283 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -551,6 +551,39 @@ CONSTRAINT [PK_NexusModFilesSlow] PRIMARY KEY CLUSTERED ) ON [PRIMARY] GO +/****** Object: Table [dbo].[Patches] Script Date: 5/18/2020 6:26:07 AM ******/ + +CREATE TABLE [dbo].[Patches]( +[SrcId] [uniqueidentifier] NOT NULL, +[DestId] [uniqueidentifier] NOT NULL, +[PatchSize] [bigint] NULL, +[Finished] [datetime] NULL, +[IsFailed] [tinyint] NULL, +[FailMessage] [varchar](MAX) NULL, +CONSTRAINT [PK_Patches] PRIMARY KEY CLUSTERED + ( + [SrcId] ASC, + [DestId] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO + +ALTER TABLE [dbo].[Patches] WITH CHECK ADD CONSTRAINT [FK_DestId] FOREIGN KEY([DestId]) + REFERENCES [dbo].[ArchiveDownloads] ([Id]) +GO + +ALTER TABLE [dbo].[Patches] CHECK CONSTRAINT [FK_DestId] +GO + +ALTER TABLE [dbo].[Patches] WITH CHECK ADD CONSTRAINT [FK_SrcId] FOREIGN KEY([SrcId]) + REFERENCES [dbo].[ArchiveDownloads] ([Id]) +GO + +ALTER TABLE [dbo].[Patches] CHECK CONSTRAINT [FK_SrcId] +GO + + + /****** Object: Table [dbo].[NexusKeys] Script Date: 5/15/2020 5:20:02 PM ******/ SET ANSI_NULLS ON GO diff --git a/Wabbajack.Server/Controllers/AuthoredFiles.cs b/Wabbajack.Server/Controllers/AuthoredFiles.cs index b4197485..5ca8eaf2 100644 --- a/Wabbajack.Server/Controllers/AuthoredFiles.cs +++ b/Wabbajack.Server/Controllers/AuthoredFiles.cs @@ -99,7 +99,7 @@ namespace Wabbajack.BuildServer.Controllers private async Task GetBunnyCdnFtpClient() { - var info = Utils.FromEncryptedJson("bunny-cdn-ftp-info"); + var info = Utils.FromEncryptedJson("bunny-cdn-patch-ftp-info"); var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)}; await client.ConnectAsync(); return client; diff --git a/Wabbajack.Server/Controllers/ModUpgrade.cs b/Wabbajack.Server/Controllers/ModUpgrade.cs new file mode 100644 index 00000000..ea6c919e --- /dev/null +++ b/Wabbajack.Server/Controllers/ModUpgrade.cs @@ -0,0 +1,58 @@ +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Server.DataLayer; +using Wabbajack.Server.Services; + +namespace Wabbajack.BuildServer.Controllers +{ + [ApiController] + public class ModUpgrade : ControllerBase + { + private ILogger _logger; + private SqlService _sql; + private DiscordWebHook _discord; + private AppSettings _settings; + + public ModUpgrade(ILogger logger, SqlService sql, DiscordWebHook discord, AppSettings settings) + { + _logger = logger; + _sql = sql; + _discord = discord; + _settings = settings; + } + + [HttpPost] + [Route("/mod_upgrade")] + public async Task PostModUpgrade() + { + var request = (await Request.Body.ReadAllTextAsync()).FromJsonString(); + if (!request.IsValid) + { + return BadRequest("Invalid mod upgrade"); + } + + var oldDownload = await _sql.GetOrEnqueueArchive(request.OldArchive); + var newDownload = await _sql.GetOrEnqueueArchive(request.NewArchive); + + var patch = await _sql.FindOrEnqueuePatch(oldDownload.Id, newDownload.Id); + if (patch.Finished.HasValue) + { + if (patch.PatchSize != 0) + { + return + Ok( + $"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/archive_upgrades/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); + } + + return NotFound("Patch creation failed"); + } + + // Still processing + return Accepted(); + } + + } +} diff --git a/Wabbajack.Server/DTOs/Patch.cs b/Wabbajack.Server/DTOs/Patch.cs index 11a54472..9d980e0c 100644 --- a/Wabbajack.Server/DTOs/Patch.cs +++ b/Wabbajack.Server/DTOs/Patch.cs @@ -1,5 +1,8 @@ using System; +using System.Threading.Tasks; +using Microsoft.VisualBasic; using Wabbajack.Common; +using Wabbajack.Server.DataLayer; namespace Wabbajack.Server.DTOs { @@ -7,8 +10,29 @@ namespace Wabbajack.Server.DTOs { public ArchiveDownload Src { get; set; } public ArchiveDownload Dest { get; set; } - public Hash PatchHash { get; set; } public long PatchSize { get; set; } public DateTime? Finished { get; set; } + public bool? IsFailed { get; set; } + public string FailMessage { get; set; } + + public async Task Finish(SqlService sql, long size) + { + IsFailed = false; + Finished = DateTime.UtcNow; + PatchSize = size; + await sql.FinializePatch(this); + } + + + public async Task Fail(SqlService sql, string msg) + { + IsFailed = true; + Finished = DateTime.UtcNow; + FailMessage = msg; + await sql.FinializePatch(this); + } + + + } } diff --git a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs index f4dbe2c7..d45724d6 100644 --- a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs +++ b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs @@ -56,6 +56,73 @@ namespace Wabbajack.Server.DataLayer }; } + + public async Task GetArchiveDownload(string primaryKeyString, Hash hash, long size) + { + await using var conn = await Open(); + var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>( + "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size", + new + { + PrimaryKeyString = primaryKeyString, + Hash = hash, + Size = size + }); + if (result == default) + return null; + + return new ArchiveDownload + { + Id = result.Item1, + IsFailed = result.Item4, + DownloadFinished = result.Item6, + Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default} + }; + + } + + + public async Task GetOrEnqueueArchive(Archive a) + { + await using var conn = await Open(); + using var trans = await conn.BeginTransactionAsync(); + var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>( + "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size", + new + { + PrimaryKeyString = a.State.PrimaryKeyString, + Hash = a.Hash, + Size = a.Size + }, trans); + if (result != default) + { + return new ArchiveDownload + { + Id = result.Item1, + IsFailed = result.Item4, + DownloadFinished = result.Item6, + Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default} + }; + } + + var id = Guid.NewGuid(); + await conn.ExecuteAsync( + "INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)", + new + { + Id = id, + PrimaryKeyString = a.State.PrimaryKeyString, + Size = a.Size == 0 ? null : (long?)a.Size, + Hash = a.Hash == default ? null : (Hash?)a.Hash, + DownloadState = a.State, + Downloader = AbstractDownloadState.TypeToName[a.State.GetType()] + }, trans); + + await trans.CommitAsync(); + + return new ArchiveDownload {Id = id, Archive = a,}; + + } public async Task GetNextPendingDownload(bool ignoreNexus = false) { diff --git a/Wabbajack.Server/DataLayer/Patches.cs b/Wabbajack.Server/DataLayer/Patches.cs index fe1125fb..f79ae4c4 100644 --- a/Wabbajack.Server/DataLayer/Patches.cs +++ b/Wabbajack.Server/DataLayer/Patches.cs @@ -28,38 +28,98 @@ namespace Wabbajack.Server.DataLayer public async Task FinializePatch(Patch patch) { await using var conn = await Open(); - await conn.ExecuteAsync("UPDATE dbo.Patches SET PatchSize = @Size, PatchHash = @PatchHash, Finished = @Finished WHERE SrcId = @SrcId AND DestID = @DestId", + await conn.ExecuteAsync("UPDATE dbo.Patches SET PatchSize = @PatchSize, Finished = @Finished, IsFailed = @IsFailed, FailMessage = @FailMessage WHERE SrcId = @SrcId AND DestID = @DestId", new { SrcId = patch.Src.Id, DestId = patch.Dest.Id, - PatchHash = patch.PatchHash, PatchSize = patch.PatchSize, - Finshed = patch.Finished + Finished = patch.Finished, + IsFailed = patch.IsFailed, + FailMessage = patch.FailMessage }); } public async Task FindPatch(Guid src, Guid dest) { await using var conn = await Open(); - var patch = await conn.QueryFirstOrDefaultAsync<(Hash, long, DateTime?)>( - "SELECT PatchHash, PatchSize, Finished FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestId", + var patch = await conn.QueryFirstOrDefaultAsync<(long, DateTime?, bool?, string)>( + @"SELECT p.PatchHash, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage + FROM dbo.Patches p + LEFT JOIN dbo.ArchiveDownloads src ON p.SrcId = src.Id + LEFT JOIN dbo.ArchiveDownloads dest ON p.SrcId = dest.Id + WHERE SrcId = @SrcId + AND DestId = @DestId + AND src.DownloadFinished IS NOT NULL + AND dest.DownloadFinished IS NOT NULL", new { SrcId = src, DestId = dest }); if (patch == default) - return default(Patch); + return default; return new Patch { Src = await GetArchiveDownload(src), Dest = await GetArchiveDownload(dest), - PatchHash = patch.Item1, - PatchSize = patch.Item2, - Finished = patch.Item3 + PatchSize = patch.Item1, + Finished = patch.Item2, + IsFailed = patch.Item3, + FailMessage = patch.Item4 }; + } + + public async Task FindOrEnqueuePatch(Guid src, Guid dest) + { + await using var conn = await Open(); + var trans = await conn.BeginTransactionAsync(); + var patch = await conn.QueryFirstOrDefaultAsync<(long, DateTime?, bool, string)>( + "SELECT PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestId", + new + { + SrcId = src, + DestId = dest + }, trans); + if (patch == default) + { + await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)", + new {SrcId = src, DestId = dest}, trans); + await trans.CommitAsync(); + return new Patch {Src = await GetArchiveDownload(src), Dest = await GetArchiveDownload(dest),}; + } + else + { + await trans.CommitAsync(); + return new Patch { + Src = await GetArchiveDownload(src), + Dest = await GetArchiveDownload(dest), + PatchSize = patch.Item1, + Finished = patch.Item2, + IsFailed = patch.Item3, + FailMessage = patch.Item4 + }; + + } } + + public async Task GetPendingPatch() + { + await using var conn = await Open(); + var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>( + "SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE Finished is NULL"); + if (patch == default) + return default(Patch); + + return new Patch { + Src = await GetArchiveDownload(patch.Item1), + Dest = await GetArchiveDownload(patch.Item2), + PatchSize = patch.Item3, + Finished = patch.Item4, + IsFailed = patch.Item5, + FailMessage = patch.Item6 + }; + } } } diff --git a/Wabbajack.Server/GlobalInformation.cs b/Wabbajack.Server/GlobalInformation.cs index 2b51f673..9695be8d 100644 --- a/Wabbajack.Server/GlobalInformation.cs +++ b/Wabbajack.Server/GlobalInformation.cs @@ -5,7 +5,7 @@ namespace Wabbajack.Server public class GlobalInformation { public TimeSpan NexusRSSPollRate = TimeSpan.FromMinutes(1); - public TimeSpan NexusAPIPollRate = TimeSpan.FromHours(24); + public TimeSpan NexusAPIPollRate = TimeSpan.FromMinutes(15); public DateTime LastNexusSyncUTC { get; set; } public TimeSpan TimeSinceLastNexusSync => DateTime.UtcNow - LastNexusSyncUTC; } diff --git a/Wabbajack.Server/Services/NexusPoll.cs b/Wabbajack.Server/Services/NexusPoll.cs index ef68b47e..b1e77953 100644 --- a/Wabbajack.Server/Services/NexusPoll.cs +++ b/Wabbajack.Server/Services/NexusPoll.cs @@ -66,7 +66,7 @@ namespace Wabbajack.Server.Services public async Task UpdateNexusCacheAPI() { using var _ = _logger.BeginScope("Nexus Update via API"); - _logger.Log(LogLevel.Information, "Starting"); + _logger.Log(LogLevel.Information, "Starting Nexus Update via API"); var api = await NexusApiClient.Get(); var gameTasks = GameRegistry.Games.Values @@ -117,7 +117,7 @@ namespace Wabbajack.Server.Services public void Start() { if (!_settings.RunBackEndJobs) return; - + /* Task.Run(async () => { while (true) @@ -133,7 +133,7 @@ namespace Wabbajack.Server.Services await Task.Delay(_globalInformation.NexusRSSPollRate); } }); - +*/ Task.Run(async () => { while (true) diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs new file mode 100644 index 00000000..b150d73c --- /dev/null +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -0,0 +1,95 @@ +using System; +using System.IO; +using System.Net; +using System.Threading.Tasks; +using FluentFTP; +using Microsoft.Extensions.Logging; +using Splat; +using Wabbajack.BuildServer; +using Wabbajack.Common; +using Wabbajack.Lib.CompilationSteps; +using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DTOs; + +namespace Wabbajack.Server.Services +{ + public class PatchBuilder : AbstractService + { + private DiscordWebHook _discordWebHook; + private SqlService _sql; + private ArchiveMaintainer _maintainer; + + public PatchBuilder(ILogger logger, SqlService sql, AppSettings settings, ArchiveMaintainer maintainer, + DiscordWebHook discordWebHook) : base(logger, settings, TimeSpan.FromMinutes(1)) + { + _discordWebHook = discordWebHook; + _sql = sql; + _maintainer = maintainer; + } + + public override async Task Execute() + { + int count = 0; + while (true) + { + var patch = await _sql.GetPendingPatch(); + if (patch == default) break; + + try + { + + _logger.LogInformation( + $"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"); + await _discordWebHook.Send(Channel.Spam, + new DiscordMessage + { + Content = + $"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}" + }); + + _maintainer.TryGetPath(patch.Src.Archive.Hash, out var srcPath); + _maintainer.TryGetPath(patch.Dest.Archive.Hash, out var destPath); + + var patchName = $"archive_updates\\{patch.Src.Archive.Hash}_{patch.Dest.Archive.Hash}"; + + using var sigFile = new TempFile(); + await using var srcStream = srcPath.OpenShared(); + await using var destStream = destPath.OpenShared(); + await using var sigStream = sigFile.Path.Create(); + using var ftpClient = await GetBunnyCdnFtpClient(); + + if (!await ftpClient.DirectoryExistsAsync("archive_updates")) + await ftpClient.CreateDirectoryAsync("archive_updates"); + + + await using var patchOutput = await ftpClient.OpenWriteAsync(patchName); + OctoDiff.Create(destStream, srcStream, sigStream, patchOutput); + + await patchOutput.DisposeAsync(); + + var size = await ftpClient.GetFileSizeAsync(patchName); + + await patch.Finish(_sql, size); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error while building patch"); + await patch.Fail(_sql, ex.ToString()); + } + + count++; + } + + return count; + } + + private async Task GetBunnyCdnFtpClient() + { + var info = Utils.FromEncryptedJson("bunny-cdn-ftp-info"); + var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)}; + await client.ConnectAsync(); + return client; + } + + } +} diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index e97dcd6b..b6000912 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -65,6 +65,7 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddMvc(); services.AddControllers() @@ -116,6 +117,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); + app.UseService(); app.Use(next => {