Can heal a simple pre-indexed download

Wabbajack.Common/Http/Client.cs

@@ -19,11 +19,17 @@ namespace Wabbajack.Common.Http
             return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions);
         }
         
+        public async Task<HttpResponseMessage> GetAsync(Uri url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true)
+        {
+            var request = new HttpRequestMessage(HttpMethod.Get, url);
+            return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions);
+        }
         
-        public async Task<HttpResponseMessage> PostAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead)
+        
+        public async Task<HttpResponseMessage> PostAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true)
         {
             var request = new HttpRequestMessage(HttpMethod.Post, url) {Content = content};
-            return await SendAsync(request, responseHeadersRead);
+            return await SendAsync(request, responseHeadersRead, errorsAsExceptions);
         }
         
         public async Task<HttpResponseMessage> PutAsync(string url, HttpContent content, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead)
@@ -79,7 +85,6 @@ namespace Wabbajack.Common.Http
                 if (errorsAsExceptions)
                     throw new HttpRequestException(
                         $"Http Exception {response.StatusCode} - {response.ReasonPhrase} - {msg.RequestUri}");
-                ;
                 return response;
             }
             catch (Exception ex)

Wabbajack.Common/OctoDiff.cs

@@ -28,7 +28,7 @@ namespace Wabbajack.Common
             return sigStream;
         }
         
-        private static void CreateSignature(Stream oldData, FileStream sigStream)
+        private static void CreateSignature(Stream oldData, Stream sigStream)
         {
             Utils.Status("Creating Patch Signature");
             var signatureBuilder = new SignatureBuilder();
@@ -36,7 +36,7 @@ namespace Wabbajack.Common
             sigStream.Position = 0;
         }
         
-        public static void Create(Stream oldData, FileStream newData, FileStream signature, FileStream output)
+        public static void Create(Stream oldData, Stream newData, Stream signature, Stream output)
         {
             CreateSignature(oldData, signature);
             var db = new DeltaBuilder {ProgressReporter = reporter};

Wabbajack.Common/Paths.cs

@@ -337,6 +337,13 @@ namespace Wabbajack.Common
             await fs.WriteAsync(data);
         }
 
+        public async Task WriteAllAsync(Stream data, bool disposeAfter = true)
+        {
+            await using var fs = Create();
+            await fs.CopyToAsync(data);
+            if (disposeAfter) await data.DisposeAsync();
+        }
+
         public void AppendAllText(string text)
         {
             File.AppendAllText(_path, text);

Wabbajack.Lib/ClientAPI.cs

@@ -1,11 +1,46 @@
 using System;
 using System.Collections.Generic;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Text;
 using System.Threading.Tasks;
 using Wabbajack.Common;
+using Wabbajack.Common.Serialization.Json;
+using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.Exceptions;
 
 namespace Wabbajack.Lib
 {
+    [JsonName("ModUpgradeRequest")]
+    public class ModUpgradeRequest
+    {
+        public Archive OldArchive { get; set; }
+        public Archive NewArchive { get; set; }
+
+        public ModUpgradeRequest(Archive oldArchive, Archive newArchive)
+        {
+            OldArchive = oldArchive;
+            NewArchive = newArchive;
+        }
+
+        public bool IsValid
+        {
+            get
+            {
+                if (OldArchive.Hash == NewArchive.Hash && OldArchive.State.PrimaryKeyString == NewArchive.State.PrimaryKeyString) return false;
+                if (OldArchive.State.GetType() != NewArchive.State.GetType())
+                    return false;
+                if (OldArchive.State is IUpgradingState u)
+                {
+                    return u.ValidateUpgrade(NewArchive.State);
+                }
+
+                return false;
+            }
+        }
+    }
+    
     public class ClientAPI
     {
         public static Common.Http.Client GetClient()
@@ -16,18 +51,39 @@ namespace Wabbajack.Lib
             return client;
         }
 
-        public static async Task<Archive?> GetModUpgrade(Hash hash)
+        
+
+        public static async Task<Uri> GetModUpgrade(Archive oldArchive, Archive newArchive, TimeSpan? maxWait = null, TimeSpan? waitBetweenTries = null)
         {
-            using var response = await GetClient()
+            maxWait ??= TimeSpan.FromMinutes(10);
-                .GetAsync($"{Consts.WabbajackBuildServerUri}alternative/{hash.ToHex()}");
+            waitBetweenTries ??= TimeSpan.FromSeconds(15);
+            
+            var request = new ModUpgradeRequest( oldArchive, newArchive);
+            var start = DateTime.UtcNow;
+            
+            RETRY:
+            
+            var response = await GetClient()
+                .PostAsync($"{Consts.WabbajackBuildServerUri}mod_upgrade", new StringContent(request.ToJson(), Encoding.UTF8, "application/json"));
+
             if (response.IsSuccessStatusCode)
             {
-                return (await response.Content.ReadAsStringAsync()).FromJsonString<Archive>();
+                switch (response.StatusCode)
+                {
+                    case HttpStatusCode.OK:
+                        return new Uri(await response.Content.ReadAsStringAsync());
+                    case HttpStatusCode.Accepted:
+                        Utils.Log($"Waiting for patch processing on the server for {oldArchive.Name}, sleeping for another 15 seconds");
+                        await Task.Delay(TimeSpan.FromSeconds(15));
+                        response.Dispose();
+                        if (DateTime.UtcNow - start > maxWait)
+                            throw new HttpException(response);
+                        goto RETRY;
                 }
-
+            }
-            Utils.Log($"No Upgrade for {hash}");
+            var ex = new HttpException(response);
-            Utils.Log(await response.Content.ReadAsStringAsync());
+            response.Dispose();
-            return null;
+            throw ex;
         }
 
         /// <summary>

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
+using System.Net.Http;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
@@ -95,40 +97,49 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
-            Utils.Log($"Download failed, looking for upgrade");
+            if (!(archive.State is IUpgradingState))
-            var upgrade = await ClientAPI.GetModUpgrade(archive.Hash);
-            if (upgrade == null)
             {
-                Utils.Log($"No upgrade found for {archive.Hash}");
+                Utils.Log($"Download failed for {archive.Name} and no upgrade from this download source is possible");
                 return false;
             }
-            Utils.Log($"Upgrading via {upgrade.State.PrimaryKeyString}");
 
-            Utils.Log($"Upgrading {archive.Hash}");
+            var upgrade = (IUpgradingState)archive.State;
-            var upgradePath = destination.Parent.Combine("_Upgrade_" + archive.Name);
-            var upgradeResult = await Download(upgrade, upgradePath);
-            if (!upgradeResult) return false;
 
-            var patchName = $"{archive.Hash.ToHex()}_{upgrade.Hash.ToHex()}";
+            Utils.Log($"Trying to find solution to broken download for {archive.Name}");
-            var patchPath = destination.Parent.Combine("_Patch_" + patchName);
             
-            var patchState = new Archive(new HTTPDownloader.State($"https://wabbajackcdn.b-cdn.net/updates/{patchName}"))
+            var result = await upgrade.FindUpgrade(archive);
+            if (result == default)
             {
-                Name = patchName,
+                Utils.Log(
-            };
+                    $"No solution for broken download {archive.Name} {archive.State.PrimaryKeyString} could be found");
+                return false;
 
-            var patchResult = await Download(patchState, patchPath);
-            if (!patchResult) return false;
-
-            Utils.Status($"Applying Upgrade to {archive.Hash}");
-            await using (var patchStream = patchPath.OpenRead())
-            await using (var srcStream = upgradePath.OpenRead())
-            await using (var destStream = destination.Create())
-            {
-                OctoDiff.Apply(srcStream, patchStream, destStream);
             }
 
-            await destination.FileHashCachedAsync();
+            Utils.Log($"Looking for patch for {archive.Name}");
+            var patchResult = await ClientAPI.GetModUpgrade(archive, result.Archive!);
+
+            Utils.Log($"Downloading patch for {archive.Name}");
+            
+            var tempFile = new TempFile();
+
+            using var response = await (new Common.Http.Client()).GetAsync(patchResult);
+            await tempFile.Path.WriteAllAsync(await response.Content.ReadAsStreamAsync());
+            response.Dispose();
+
+            Utils.Log($"Applying patch to {archive.Name}");
+            await using(var src = result.NewFile.Path.OpenShared())
+            await using (var final = destination.Create())
+            {
+                Utils.ApplyPatch(src, () => tempFile.Path.OpenShared(), final);
+            }
+
+            var hash = await destination.FileHashCachedAsync();
+            if (hash != archive.Hash && archive.Hash != default)
+            {
+                Utils.Log("Archive hash didn't match after patching");
+                return false;
+            }
 
             return true;
         }

Wabbajack.Lib/Downloaders/IUpgradingState.cs

@@ -0,0 +1,18 @@
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.Downloaders
+{
+    public interface IUpgradingState
+    {
+        /// <summary>
+        /// Find a possible archive that can be combined with a server generated patch to get the input archive
+        /// state;
+        /// </summary>
+        /// <param name="a"></param>
+        /// <returns></returns>
+        public Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a);
+
+        bool ValidateUpgrade(AbstractDownloadState newArchiveState);
+    }
+}

Wabbajack.Lib/Downloaders/NexusDownloader.cs

@@ -140,7 +140,7 @@ namespace Wabbajack.Lib.Downloaders
         }
 
         [JsonName("NexusDownloader")]
-        public class State : AbstractDownloadState, IMetaState
+        public class State : AbstractDownloadState, IMetaState, IUpgradingState
         {
             [JsonIgnore]
             public Uri URL => new Uri($"http://nexusmods.com/{Game.MetaData().NexusName}/mods/{ModID}");
@@ -240,6 +240,41 @@ namespace Wabbajack.Lib.Downloaders
             {
                 return new[] {"[General]", $"gameName={Game.MetaData().MO2ArchiveName}", $"modID={ModID}", $"fileID={FileID}"};
             }
+
+            public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
+            {
+                var client = await NexusApiClient.Get();
+
+                var mod = await client.GetModInfo(Game, ModID);
+                var files = await client.GetModFiles(Game, ModID);
+                var oldFile = files.files.FirstOrDefault(f => f.file_id == FileID);
+                var newFile = files.files.OrderByDescending(f => f.uploaded_timestamp).FirstOrDefault();
+
+                if (!mod.available || oldFile == default || newFile == default)
+                {
+                    return default;
+                }
+                
+                var tempFile = new TempFile();
+
+                var newArchive = new Archive(new State {Game = Game, ModID = ModID, FileID = FileID})
+                {
+                    Name = newFile.file_name,
+                };
+
+                await newArchive.State.Download(newArchive, tempFile.Path);
+
+                newArchive.Size = tempFile.Path.Size;
+                newArchive.Hash = await tempFile.Path.FileHashAsync();
+
+                return (newArchive, tempFile);
+            }
+
+            public bool ValidateUpgrade(AbstractDownloadState newArchiveState)
+            {
+                var state = (State)newArchiveState;
+                return Game == state.Game && ModID == state.ModID;
+            }
         }
     }
 }

Wabbajack.Lib/Exceptions/HttpException.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Net.Http;
 
 namespace Wabbajack.Lib.Exceptions
 {
@@ -12,5 +13,12 @@ namespace Wabbajack.Lib.Exceptions
             Code = code;
             Reason = reason;
         }
+
+        public HttpException(HttpResponseMessage response) : base(
+            $"Http Error {response.StatusCode} - {response.ReasonPhrase}")
+        {
+            Code = (int)response.StatusCode;
+            Reason = response.ReasonPhrase;
+        }
     }
 }

Wabbajack.Server.Test/ModlistUpdater.cs

@@ -9,6 +9,7 @@ using Wabbajack.BuildServer.Test;
 using Wabbajack.Common;
 using Wabbajack.Lib;
 using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.Exceptions;
 using Wabbajack.Lib.ModListRegistry;
 using Wabbajack.Lib.NexusApi;
 using Wabbajack.Server.DataLayer;
@@ -38,97 +39,49 @@ namespace Wabbajack.Server.Test
             var listDownloader = Fixture.GetService<ModListDownloader>();
             var downloader = Fixture.GetService<ArchiveDownloader>();
             var archiver = Fixture.GetService<ArchiveMaintainer>();
+            var patcher = Fixture.GetService<PatchBuilder>();
             
             var sql = Fixture.GetService<SqlService>();
-            var modId = long.MaxValue >> 1;
-            var oldFileId = long.MaxValue >> 2;
-            var newFileId = (long.MaxValue >> 2) + 1;
-
             var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!");
             var newFileData = Encoding.UTF8.GetBytes("Forks for Everyone!");
             var oldDataHash = oldFileData.xxHash();
             var newDataHash = newFileData.xxHash();
             
-            Assert.Equal(2, await listDownloader.CheckForNewLists());
+            var oldArchive = new Archive(new NexusDownloader.State {Game = Game.Enderal, ModID = 42, FileID = 10})
-            Assert.Equal(1, await downloader.Execute());
-            Assert.Equal(0, await nonNexus.Execute());
-            Assert.Equal(0, await validator.Execute());
-            
-            
-            Assert.True(archiver.HaveArchive(oldDataHash));
-            Assert.False(archiver.HaveArchive(newDataHash));
-
-            var status = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
-            Assert.Equal(0, status.ValidationSummary.Failed);
-            
-
-            // Update the archive
-            await "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(newFileData);
-            
-            // Nothing new to do
-            Assert.Equal(0, await listDownloader.CheckForNewLists());
-            Assert.Equal(0, await downloader.Execute());
-            
-            // List now fails after we check the manual link
-            Assert.Equal(1, await nonNexus.Execute());
-            Assert.Equal(1, await validator.Execute());
-
-            /*
-            Assert.True(await sql.HaveIndexdFile(oldDataHash));
-            Assert.True(await sql.HaveIndexdFile(newDataHash));
-
-            var settings = Fixture.GetService<AppSettings>();
-            Assert.Equal($"Oldfile_{oldDataHash.ToHex()}_".RelativeTo(Fixture.ServerArchivesFolder), settings.PathForArchive(oldDataHash));
-            Assert.Equal($"Newfile_{newDataHash.ToHex()}_".RelativeTo(Fixture.ServerArchivesFolder), settings.PathForArchive(newDataHash));
-
-            Utils.Log($"Download Updating {oldDataHash} -> {newDataHash}");
-            await using var conn = await sql.Open();
-
-            await conn.ExecuteAsync("DELETE FROM dbo.DownloadStates WHERE Hash in (@OldHash, @NewHash);",
-                new {OldHash = (long)oldDataHash, NewHash = (long)newDataHash});
-
-            await sql.AddDownloadState(oldDataHash, new NexusDownloader.State
             {
-                Game = Game.Oblivion,
+                Size = oldFileData.Length,
-                ModID = modId,
+                Hash = oldDataHash
-                FileID = oldFileId
+            };
-            });
+            var newArchive =  new Archive(new NexusDownloader.State {Game = Game.Enderal, ModID = 42, FileID = 11})
-
-            await sql.AddDownloadState(newDataHash, new NexusDownloader.State
             {
-                Game = Game.Oblivion,
+                Size = newFileData.Length,
-                ModID = modId,
+                Hash = newDataHash
-                FileID = newFileId
+            };
-            });
 
-            Assert.NotNull(await sql.GetNexusStateByHash(oldDataHash));
+            await IngestData(archiver, oldFileData);
-            Assert.NotNull(await sql.GetNexusStateByHash(newDataHash));
+            await IngestData(archiver, newFileData);
 
-            // No nexus info, so no upgrade
+            await sql.EnqueueDownload(oldArchive);
-            var noUpgrade = await ClientAPI.GetModUpgrade(oldDataHash);
+            var oldDownload = await sql.GetNextPendingDownload();
-            Assert.Null(noUpgrade);
+            await oldDownload.Finish(sql);
+
+            await sql.EnqueueDownload(newArchive);
+            var newDownload = await sql.GetNextPendingDownload();
+            await newDownload.Finish(sql);
+            
+
+            await Assert.ThrowsAsync<HttpException>(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
+            Assert.Equal(1, await patcher.Execute());
+
+            Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_upgrades/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
 
-            // Add Nexus info
-            await sql.AddNexusModFiles(Game.Oblivion, modId, DateTime.Now,
-                new NexusApiClient.GetModFilesResponse
-                {
-                    files = new List<NexusFileInfo>
-                    {
-                        new NexusFileInfo {category_name = "MAIN", file_id = newFileId, file_name = "New File"},
-                        new NexusFileInfo {category_name = null, file_id = oldFileId, file_name = "Old File"}
         }
-                });
 
-            
+        private async Task IngestData(ArchiveMaintainer am, byte[] data)
-            var enqueuedUpgrade = await ClientAPI.GetModUpgrade(oldDataHash);
+        {
-            
+            using var f = new TempFile();
-            // Not Null because upgrade was enqueued
+            await f.Path.WriteAllBytesAsync(data);
-            Assert.NotNull(enqueuedUpgrade);
+            await am.Ingest(f.Path);
-
-            await RunAllJobs();
-
-            Assert.True($"{oldDataHash.ToHex()}_{newDataHash.ToHex()}".RelativeTo(Fixture.ServerUpdatesFolder).IsFile);
-*/
         }
     }
 }

Wabbajack.Server.Test/sql/wabbajack_db.sql

@@ -551,6 +551,39 @@ CONSTRAINT [PK_NexusModFilesSlow] PRIMARY KEY CLUSTERED
 ) ON [PRIMARY]
 GO
 
+/****** Object:  Table [dbo].[Patches]    Script Date: 5/18/2020 6:26:07 AM ******/
+
+CREATE TABLE [dbo].[Patches](
+[SrcId] [uniqueidentifier] NOT NULL,
+[DestId] [uniqueidentifier] NOT NULL,
+[PatchSize] [bigint] NULL,
+[Finished] [datetime] NULL,
+[IsFailed] [tinyint] NULL,
+[FailMessage] [varchar](MAX) NULL,
+CONSTRAINT [PK_Patches] PRIMARY KEY CLUSTERED
+    (
+     [SrcId] ASC,
+     [DestId] ASC
+        )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+) ON [PRIMARY]
+GO
+
+ALTER TABLE [dbo].[Patches]  WITH CHECK ADD  CONSTRAINT [FK_DestId] FOREIGN KEY([DestId])
+    REFERENCES [dbo].[ArchiveDownloads] ([Id])
+GO
+
+ALTER TABLE [dbo].[Patches] CHECK CONSTRAINT [FK_DestId]
+GO
+
+ALTER TABLE [dbo].[Patches]  WITH CHECK ADD  CONSTRAINT [FK_SrcId] FOREIGN KEY([SrcId])
+    REFERENCES [dbo].[ArchiveDownloads] ([Id])
+GO
+
+ALTER TABLE [dbo].[Patches] CHECK CONSTRAINT [FK_SrcId]
+GO
+
+
+
 /****** Object:  Table [dbo].[NexusKeys]    Script Date: 5/15/2020 5:20:02 PM ******/
 SET ANSI_NULLS ON
 GO

Wabbajack.Server/Controllers/AuthoredFiles.cs

@@ -99,7 +99,7 @@ namespace Wabbajack.BuildServer.Controllers
 
         private async Task<FtpClient> GetBunnyCdnFtpClient()
         {
-            var info = Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");
+            var info = Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-patch-ftp-info");
             var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
             await client.ConnectAsync();
             return client;

Wabbajack.Server/Controllers/ModUpgrade.cs

@@ -0,0 +1,58 @@
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.Services;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [ApiController]
+    public class ModUpgrade : ControllerBase
+    {
+        private ILogger<ModUpgrade> _logger;
+        private SqlService _sql;
+        private DiscordWebHook _discord;
+        private AppSettings _settings;
+
+        public ModUpgrade(ILogger<ModUpgrade> logger, SqlService sql, DiscordWebHook discord, AppSettings settings)
+        {
+            _logger = logger;
+            _sql = sql;
+            _discord = discord;
+            _settings = settings;
+        }
+        
+        [HttpPost]
+        [Route("/mod_upgrade")]
+        public async Task<IActionResult> PostModUpgrade()
+        {
+            var request = (await Request.Body.ReadAllTextAsync()).FromJsonString<ModUpgradeRequest>();
+            if (!request.IsValid)
+            {
+                return BadRequest("Invalid mod upgrade");
+            }
+
+            var oldDownload = await _sql.GetOrEnqueueArchive(request.OldArchive);
+            var newDownload = await _sql.GetOrEnqueueArchive(request.NewArchive);
+
+            var patch = await _sql.FindOrEnqueuePatch(oldDownload.Id, newDownload.Id);
+            if (patch.Finished.HasValue)
+            {
+                if (patch.PatchSize != 0)
+                {
+                    return
+                        Ok(
+                            $"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/archive_upgrades/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
+                }
+
+                return NotFound("Patch creation failed");
+            }
+            
+            // Still processing
+            return Accepted();
+        }
+
+    }
+}

Wabbajack.Server/DTOs/Patch.cs

@@ -1,5 +1,8 @@
 using System;
+using System.Threading.Tasks;
+using Microsoft.VisualBasic;
 using Wabbajack.Common;
+using Wabbajack.Server.DataLayer;
 
 namespace Wabbajack.Server.DTOs
 {
@@ -7,8 +10,29 @@ namespace Wabbajack.Server.DTOs
     {
         public ArchiveDownload Src { get; set; }
         public ArchiveDownload Dest { get; set; }
-        public Hash PatchHash { get; set; }
         public long PatchSize { get; set; }
         public DateTime? Finished { get; set; }
+        public bool? IsFailed { get; set; }
+        public string FailMessage { get; set; }
+
+        public async Task Finish(SqlService sql, long size)
+        {
+            IsFailed = false;
+            Finished = DateTime.UtcNow;
+            PatchSize = size;
+            await sql.FinializePatch(this);
+        }
+
+
+        public async Task Fail(SqlService sql, string msg)
+        {
+            IsFailed = true;
+            Finished = DateTime.UtcNow;
+            FailMessage = msg;
+            await sql.FinializePatch(this);
+        }
+
+        
+        
     }
 }

Wabbajack.Server/DataLayer/ArchiveDownloads.cs

@@ -57,6 +57,73 @@ namespace Wabbajack.Server.DataLayer
 
         }
         
+        public async Task<ArchiveDownload> GetArchiveDownload(string primaryKeyString, Hash hash, long size)
+        {
+            await using var conn = await Open();
+            var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
+                "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size",
+                new
+                {
+                    PrimaryKeyString = primaryKeyString,
+                    Hash = hash,
+                    Size = size
+                });
+            if (result == default)
+                return null;
+
+            return new ArchiveDownload
+            {
+                Id = result.Item1,
+                IsFailed = result.Item4,
+                DownloadFinished = result.Item6,
+                Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
+            };
+
+        }
+
+        
+        public async Task<ArchiveDownload> GetOrEnqueueArchive(Archive a)
+        {
+            await using var conn = await Open();
+            using var trans = await conn.BeginTransactionAsync();
+            var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>(
+                "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size",
+                new
+                {
+                    PrimaryKeyString = a.State.PrimaryKeyString,
+                    Hash = a.Hash,
+                    Size = a.Size
+                }, trans);
+            if (result != default)
+            {
+                return new ArchiveDownload
+                {
+                    Id = result.Item1,
+                    IsFailed = result.Item4,
+                    DownloadFinished = result.Item6,
+                    Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
+                };
+            }
+
+            var id = Guid.NewGuid();
+            await conn.ExecuteAsync(
+                "INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)",
+                new
+                {
+                    Id = id,
+                    PrimaryKeyString = a.State.PrimaryKeyString,
+                    Size = a.Size == 0 ? null : (long?)a.Size,
+                    Hash = a.Hash == default ? null : (Hash?)a.Hash,
+                    DownloadState = a.State,
+                    Downloader = AbstractDownloadState.TypeToName[a.State.GetType()]
+                }, trans);
+
+            await trans.CommitAsync();
+
+            return new ArchiveDownload {Id = id, Archive = a,};
+
+        }
+
         public async Task<ArchiveDownload> GetNextPendingDownload(bool ignoreNexus = false)
         {
             await using var conn = await Open();

Wabbajack.Server/DataLayer/Patches.cs

@@ -28,38 +28,98 @@ namespace Wabbajack.Server.DataLayer
         public async Task FinializePatch(Patch patch)
         {
             await using var conn = await Open();
-            await conn.ExecuteAsync("UPDATE dbo.Patches SET PatchSize = @Size, PatchHash = @PatchHash, Finished = @Finished WHERE SrcId = @SrcId AND DestID = @DestId",
+            await conn.ExecuteAsync("UPDATE dbo.Patches SET PatchSize = @PatchSize, Finished = @Finished, IsFailed = @IsFailed, FailMessage = @FailMessage WHERE SrcId = @SrcId AND DestID = @DestId",
                 new
                 {
                     SrcId = patch.Src.Id, 
                     DestId = patch.Dest.Id,
-                    PatchHash = patch.PatchHash,
                     PatchSize = patch.PatchSize,
-                    Finshed = patch.Finished
+                    Finished = patch.Finished,
+                    IsFailed = patch.IsFailed,
+                    FailMessage = patch.FailMessage
                 });
         }
 
         public async Task<Patch> FindPatch(Guid src, Guid dest)
         {
             await using var conn = await Open();
-            var patch = await conn.QueryFirstOrDefaultAsync<(Hash, long, DateTime?)>(
+            var patch = await conn.QueryFirstOrDefaultAsync<(long, DateTime?, bool?, string)>(
-                "SELECT PatchHash, PatchSize, Finished FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestId",
+                @"SELECT p.PatchHash, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage 
+                      FROM dbo.Patches p
+                      LEFT JOIN dbo.ArchiveDownloads src ON p.SrcId = src.Id
+                      LEFT JOIN dbo.ArchiveDownloads dest ON p.SrcId = dest.Id
+                      WHERE SrcId = @SrcId 
+                        AND DestId = @DestId
+                        AND src.DownloadFinished IS NOT NULL
+                        AND dest.DownloadFinished IS NOT NULL",
                 new
                 {
                     SrcId = src,
                     DestId = dest
                 });
             if (patch == default)
-                return default(Patch);
+                return default;
 
             return new Patch {
                 Src = await GetArchiveDownload(src), 
                 Dest = await GetArchiveDownload(dest),
-                PatchHash = patch.Item1,
+                PatchSize = patch.Item1,
-                PatchSize = patch.Item2,
+                Finished = patch.Item2,
-                Finished = patch.Item3
+                IsFailed = patch.Item3,
+                FailMessage = patch.Item4
+            };
+        }
+        
+        public async Task<Patch> FindOrEnqueuePatch(Guid src, Guid dest)
+        {
+            await using var conn = await Open();
+            var trans = await conn.BeginTransactionAsync();
+            var patch = await conn.QueryFirstOrDefaultAsync<(long, DateTime?, bool, string)>(
+                "SELECT PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestId",
+                new
+                {
+                    SrcId = src,
+                    DestId = dest
+                }, trans);
+            if (patch == default)
+            {
+                await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)",
+                    new {SrcId = src, DestId = dest}, trans);
+                await trans.CommitAsync();
+                return new Patch {Src = await GetArchiveDownload(src), Dest = await GetArchiveDownload(dest),};
+            }
+            else
+            {
+                await trans.CommitAsync();
+                return new Patch {
+                    Src = await GetArchiveDownload(src), 
+                    Dest = await GetArchiveDownload(dest),
+                    PatchSize = patch.Item1,
+                    Finished = patch.Item2,
+                    IsFailed = patch.Item3,
+                    FailMessage = patch.Item4
                 };
                 
             }
+
+        }
+
+        public async Task<Patch> GetPendingPatch()
+        {
+            await using var conn = await Open();
+            var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
+                "SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE Finished is NULL");
+            if (patch == default)
+                return default(Patch);
+
+            return new Patch {
+                Src = await GetArchiveDownload(patch.Item1), 
+                Dest = await GetArchiveDownload(patch.Item2),
+                PatchSize = patch.Item3,
+                Finished = patch.Item4,
+                IsFailed = patch.Item5,
+                FailMessage = patch.Item6
+            };
+        }
     }
 }

Wabbajack.Server/GlobalInformation.cs

@@ -5,7 +5,7 @@ namespace Wabbajack.Server
     public class GlobalInformation
     {
         public TimeSpan NexusRSSPollRate = TimeSpan.FromMinutes(1);
-        public TimeSpan NexusAPIPollRate = TimeSpan.FromHours(24);
+        public TimeSpan NexusAPIPollRate = TimeSpan.FromMinutes(15);
         public DateTime LastNexusSyncUTC { get; set; }
         public TimeSpan TimeSinceLastNexusSync => DateTime.UtcNow - LastNexusSyncUTC;
     }

Wabbajack.Server/Services/NexusPoll.cs

@@ -66,7 +66,7 @@ namespace Wabbajack.Server.Services
         public async Task UpdateNexusCacheAPI()
         {
             using var _ = _logger.BeginScope("Nexus Update via API");
-            _logger.Log(LogLevel.Information, "Starting");
+            _logger.Log(LogLevel.Information, "Starting Nexus Update via API");
             var api = await NexusApiClient.Get();
             
             var gameTasks = GameRegistry.Games.Values
@@ -117,7 +117,7 @@ namespace Wabbajack.Server.Services
         public void Start()
         {
             if (!_settings.RunBackEndJobs) return;
-            
+            /*
             Task.Run(async () =>
             {
                 while (true)
@@ -133,7 +133,7 @@ namespace Wabbajack.Server.Services
                     await Task.Delay(_globalInformation.NexusRSSPollRate);
                 }
             });
-
+*/
             Task.Run(async () =>
             {
                 while (true)

Wabbajack.Server/Services/PatchBuilder.cs

@@ -0,0 +1,95 @@
+using System;
+using System.IO;
+using System.Net;
+using System.Threading.Tasks;
+using FluentFTP;
+using Microsoft.Extensions.Logging;
+using Splat;
+using Wabbajack.BuildServer;
+using Wabbajack.Common;
+using Wabbajack.Lib.CompilationSteps;
+using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DTOs;
+
+namespace Wabbajack.Server.Services
+{
+    public class PatchBuilder : AbstractService<PatchBuilder, int>
+    {
+        private DiscordWebHook _discordWebHook;
+        private SqlService _sql;
+        private ArchiveMaintainer _maintainer;
+
+        public PatchBuilder(ILogger<PatchBuilder> logger, SqlService sql, AppSettings settings, ArchiveMaintainer maintainer,
+            DiscordWebHook discordWebHook) : base(logger, settings, TimeSpan.FromMinutes(1))
+        {
+            _discordWebHook = discordWebHook;
+            _sql = sql;
+            _maintainer = maintainer;
+        }
+
+        public override async Task<int> Execute()
+        {
+            int count = 0;
+            while (true)
+            {
+                var patch = await _sql.GetPendingPatch();
+                if (patch == default) break;
+
+                try
+                {
+
+                    _logger.LogInformation(
+                        $"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}");
+                    await _discordWebHook.Send(Channel.Spam,
+                        new DiscordMessage
+                        {
+                            Content =
+                                $"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
+                        });
+
+                    _maintainer.TryGetPath(patch.Src.Archive.Hash, out var srcPath);
+                    _maintainer.TryGetPath(patch.Dest.Archive.Hash, out var destPath);
+
+                    var patchName = $"archive_updates\\{patch.Src.Archive.Hash}_{patch.Dest.Archive.Hash}";
+
+                    using var sigFile = new TempFile();
+                    await using var srcStream = srcPath.OpenShared();
+                    await using var destStream = destPath.OpenShared();
+                    await using var sigStream = sigFile.Path.Create();
+                    using var ftpClient = await GetBunnyCdnFtpClient();
+
+                    if (!await ftpClient.DirectoryExistsAsync("archive_updates")) 
+                        await ftpClient.CreateDirectoryAsync("archive_updates");
+
+                    
+                    await using var patchOutput = await ftpClient.OpenWriteAsync(patchName);
+                    OctoDiff.Create(destStream, srcStream, sigStream, patchOutput);
+                    
+                    await patchOutput.DisposeAsync();
+                    
+                    var size = await ftpClient.GetFileSizeAsync(patchName);
+                    
+                    await patch.Finish(_sql, size);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError(ex, "Error while building patch");
+                    await patch.Fail(_sql, ex.ToString());
+                }
+
+                count++;
+            }
+
+            return count;
+        }
+        
+        private async Task<FtpClient> GetBunnyCdnFtpClient()
+        {
+            var info = Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");
+            var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
+            await client.ConnectAsync();
+            return client;
+        }
+
+    }
+}

Wabbajack.Server/Startup.cs

@@ -65,6 +65,7 @@ namespace Wabbajack.Server
             services.AddSingleton<ArchiveDownloader>();
             services.AddSingleton<DiscordWebHook>();
             services.AddSingleton<NexusKeyMaintainance>();
+            services.AddSingleton<PatchBuilder>();
             
             services.AddMvc();
             services.AddControllers()
@@ -116,6 +117,7 @@ namespace Wabbajack.Server
             app.UseService<ArchiveDownloader>();
             app.UseService<DiscordWebHook>();
             app.UseService<NexusKeyMaintainance>();
+            app.UseService<PatchBuilder>();
 
             app.Use(next =>
             {