Merge pull request #1008 from wabbajack-tools/mirrored-files

Mirrored files

Wabbajack.CLI/OptionsDefinition.cs

@@ -26,7 +26,9 @@ namespace Wabbajack.CLI
             typeof(InlinedFileReport),
             typeof(ExtractBSA),
             typeof(PurgeNexusCache),
-            typeof(ForceHealing)
+            typeof(ForceHealing),
+            typeof(HashVariants),
+            typeof(ParseMeta)
         };
     }
 }

Wabbajack.CLI/Verbs/HashVariants.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Threading.Tasks;
+using CommandLine;
+using Wabbajack.Common;
+
+namespace Wabbajack.CLI.Verbs
+{
+    [Verb("hash-variants", HelpText = "Print all the known variants (formats) of a hash")]
+    public class HashVariants : AVerb
+    {
+        [Option('i', "input", Required = true, HelpText = "Input Hash")]
+        public string Input { get; set; } = "";
+        
+        protected override async Task<ExitCode> Run()
+        {
+            var hash = Hash.Interpret(Input);
+            Console.WriteLine($"Base64: {hash.ToBase64()}");
+            Console.WriteLine($"Hex: {hash.ToHex()}");
+            Console.WriteLine($"Long: {(long)hash}");
+            Console.WriteLine($"ULong (uncommon): {(ulong)hash}");
+            return ExitCode.Ok;
+        }
+    }
+}

Wabbajack.CLI/Verbs/ParseMeta.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Threading.Tasks;
+using CommandLine;
+using Wabbajack.Common;
+using Wabbajack.Lib.Downloaders;
+
+namespace Wabbajack.CLI.Verbs
+{
+    [Verb("parse-meta", HelpText = "Parse a .meta file, figure out the download state and print it")]
+    public class ParseMeta : AVerb
+    {
+        
+        [Option('i', "input", Required = true, HelpText = "Input meta file to parse")]
+        public string Input { get; set; } = "";
+        protected override async Task<ExitCode> Run()
+        {
+            var meta = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(((AbsolutePath)Input).LoadIniFile());
+            if (meta == null)
+            {
+                Console.WriteLine("Cannot resolve meta!");
+                return ExitCode.Error;
+            }
+
+                Console.WriteLine($"PrimaryKeyString : {meta.PrimaryKeyString}");
+            return ExitCode.Ok;
+        }
+    }
+}

Wabbajack.Common/Consts.cs

@@ -148,7 +148,7 @@ namespace Wabbajack.Common
         
         public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2;
         
-        public static string ArchiveUpdatesCDNFolder = "archive_updates";
+        public static Uri WabbajackMirror = new Uri("https://wabbajack-mirror.b-cdn.net");
 
     }
 }

Wabbajack.Common/Hash.cs

@@ -103,6 +103,16 @@ namespace Wabbajack.Common
         {
             return BitConverter.GetBytes(_code);
         }
+        
+        public static Hash Interpret(string input)
+        {
+            return input.Length switch
+            {
+                16 => FromHex(input),
+                12 when input.EndsWith('=') => FromBase64(input),
+                _ => FromLong(long.Parse(input))
+            };
+        }
     }
     
     public static partial class Utils

Wabbajack.Lib/ACompiler.cs

@@ -150,6 +150,9 @@ namespace Wabbajack.Lib
             await using (var of = await ModListOutputFolder.Combine("modlist").Create()) 
                 ModList.ToJson(of);
 
+            await ModListOutputFolder.Combine("sig")
+                .WriteAllBytesAsync((await ModListOutputFolder.Combine("modlist").FileHashAsync()).ToArray());
+
             await ClientAPI.SendModListDefinition(ModList);
 
             await ModListOutputFile.DeleteAsync();
@@ -157,6 +160,7 @@ namespace Wabbajack.Lib
             await using (var fs = await ModListOutputFile.Create())
             {
                 using var za = new ZipArchive(fs, ZipArchiveMode.Create);
+                
                 await ModListOutputFolder.EnumerateFiles()
                     .DoProgress("Compressing ModList",
                 async f =>

Wabbajack.Lib/AuthorApi/Client.cs

@@ -39,7 +39,7 @@ namespace Wabbajack.Lib.AuthorApi
         }
 
 
-        public async Task<CDNFileDefinition> GenerateFileDefinition(WorkQueue queue, AbsolutePath path, Action<string, Percent> progressFn)
+        public static async Task<CDNFileDefinition> GenerateFileDefinition(WorkQueue queue, AbsolutePath path, Action<string, Percent> progressFn)
         {
             IEnumerable<CDNFilePartDefinition> Blocks(AbsolutePath path)
             {

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -93,8 +93,14 @@ namespace Wabbajack.Lib.Downloaders
 
         public static async Task<bool> DownloadWithPossibleUpgrade(Archive archive, AbsolutePath destination)
         {
-            var success = await Download(archive, destination);
+            if (await Download(archive, destination))
-            if (success)
+            {
+                await destination.FileHashCachedAsync();
+                return true;
+            }
+
+            
+            if (await DownloadFromMirror(archive, destination))
             {
                 await destination.FileHashCachedAsync();
                 return true;
@@ -147,6 +153,24 @@ namespace Wabbajack.Lib.Downloaders
             return true;
         }
 
+        private static async Task<bool> DownloadFromMirror(Archive archive, AbsolutePath destination)
+        {
+            try
+            {
+                var newArchive =
+                    new Archive(
+                        new WabbajackCDNDownloader.State(new Uri($"{Consts.WabbajackMirror}{archive.Hash.ToHex()}")))
+                    {
+                        Hash = archive.Hash, Size = archive.Size, Name = archive.Name
+                    };
+                return await Download(newArchive, destination);
+            }
+            catch (Exception ex)
+            {
+                return false;
+            }
+        }
+
         private static async Task<bool> Download(Archive archive, AbsolutePath destination)
         {
             try

Wabbajack.Server.Test/ABuildServerSystemTest.cs

@@ -61,6 +61,7 @@ namespace Wabbajack.BuildServer.Test
             _token = new CancellationTokenSource();
             _task = _host.RunAsync(_token.Token);
             Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080");
+            Consts.WabbajackMirror = new Uri("https://wabbajack-test.b-cdn.net");
 
             await "ServerWhitelist.yaml".RelativeTo(ServerPublicFolder).WriteAllTextAsync(
                 "GoogleIDs:\nAllowedPrefixes:\n    - http://localhost");

Wabbajack.Server.Test/MirroredFilesTests.cs

@@ -0,0 +1,55 @@
+using System;
+using System.Threading.Tasks;
+using Wabbajack.BuildServer.Test;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DTOs;
+using Wabbajack.Server.Services;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Wabbajack.Server.Test
+{
+    public class MirroredFilesTests : ABuildServerSystemTest
+    {
+        public MirroredFilesTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
+        {
+        }
+
+        [Fact]
+        public async Task CanUploadAndDownloadMirroredFiles()
+        {
+            var file = new TempFile();
+            await file.Path.WriteAllBytesAsync(RandomData(1024 * 1024 * 6));
+            var dataHash = await file.Path.FileHashAsync();
+
+            await Fixture.GetService<ArchiveMaintainer>().Ingest(file.Path);
+            Assert.True(Fixture.GetService<ArchiveMaintainer>().HaveArchive(dataHash));
+
+            var sql = Fixture.GetService<SqlService>();
+            
+            await sql.UpsertMirroredFile(new MirroredFile
+            {
+                Created = DateTime.UtcNow,
+                Rationale = "Test File", 
+                Hash = dataHash
+            });
+
+            var uploader = Fixture.GetService<MirrorUploader>();
+            Assert.Equal(1, await uploader.Execute());
+            
+            
+            var archive = new Archive(new HTTPDownloader.State(MakeURL(dataHash.ToString())))
+            {
+                Hash = dataHash,
+                Size = file.Path.Size
+            };
+            
+            var file2 = new TempFile();
+            await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, file2.Path);
+        }
+        
+    }
+}

Wabbajack.Server.Test/ModlistUpdater.cs

@@ -70,7 +70,7 @@ namespace Wabbajack.Server.Test
             await Assert.ThrowsAsync<HttpException>(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
             Assert.True(await patcher.Execute() > 1);
 
-            Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_updates/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
+            Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
         }
 
         [Fact]

Wabbajack.Server.Test/Wabbajack.Server.Test.csproj

@@ -2,7 +2,7 @@
 
     <PropertyGroup>
         <TargetFramework>netcoreapp3.1</TargetFramework>
-
+        <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
         <IsPackable>false</IsPackable>
     </PropertyGroup>
 

Wabbajack.Server.Test/sql/wabbajack_db.sql

@@ -669,6 +669,50 @@ CONSTRAINT [PK_NexusModPermissions] PRIMARY KEY CLUSTERED
 ) ON [PRIMARY]
 GO
 
+/****** Object:  Table [dbo].[MirroredArchives]    Script Date: 8/3/2020 8:39:33 PM ******/
+SET ANSI_NULLS ON
+GO
+
+SET QUOTED_IDENTIFIER ON
+GO
+
+CREATE TABLE [dbo].[MirroredArchives](
+                                         [Hash] [bigint] NOT NULL,
+                                         [Created] [datetime] NOT NULL,
+                                         [Uploaded] [datetime] NULL,
+                                         [Rationale] [nvarchar](max) NOT NULL,
+                                         [FailMessage] [nvarchar](max) NULL,
+                                         CONSTRAINT [PK_MirroredArchives] PRIMARY KEY CLUSTERED
+                                             (
+                                              [Hash] ASC
+                                                 )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
+GO
+
+/****** Object:  Table [dbo].[GameMetadata]    Script Date: 8/3/2020 8:39:33 PM ******/
+SET ANSI_NULLS ON
+GO
+
+SET QUOTED_IDENTIFIER ON
+GO
+
+CREATE TABLE [dbo].[GameMetadata](
+ [NexusGameId] [bigint] NULL,
+ [WabbajackName] [nvarchar](50) NOT NULL,
+ CONSTRAINT [PK_GameMetadata] PRIMARY KEY CLUSTERED
+     (
+      [WabbajackName] ASC
+         )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+)
+GO
+
+CREATE NONCLUSTERED INDEX [IDX_GameAndName-20200804-164236] ON [dbo].[GameMetadata]
+    (
+     [NexusGameId] ASC,
+     [WabbajackName] ASC
+        )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+GO
+
 
 
 /****** Object:  StoredProcedure [dbo].[MergeAllFilesInArchive]    Script Date: 3/28/2020 4:58:59 PM ******/

Wabbajack.Server/Controllers/AuthoredFiles.cs

@@ -106,7 +106,7 @@ namespace Wabbajack.BuildServer.Controllers
 
         private async Task<FtpClient> GetBunnyCdnFtpClient()
         {
-            var info = await Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");
+            var info = await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles);
             var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
             await client.ConnectAsync();
             return client;

Wabbajack.Server/Controllers/ModUpgrade.cs

@@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging;
 using Wabbajack.Common;
 using Wabbajack.Lib;
 using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DTOs;
 using Wabbajack.Server.Services;
 
 namespace Wabbajack.BuildServer.Controllers
@@ -20,6 +21,7 @@ namespace Wabbajack.BuildServer.Controllers
         private DiscordWebHook _discord;
         private AppSettings _settings;
         private QuickSync _quickSync;
+        private Task<BunnyCdnFtpInfo> _creds;
 
         public ModUpgrade(ILogger<ModUpgrade> logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync, AppSettings settings)
         {
@@ -28,6 +30,7 @@ namespace Wabbajack.BuildServer.Controllers
             _discord = discord;
             _settings = settings;
             _quickSync = quickSync;
+            _creds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Patches);
         }
         
         [HttpPost]
@@ -93,7 +96,7 @@ namespace Wabbajack.BuildServer.Controllers
                     await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id);
                     return
                         Ok(
-                            $"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{Consts.ArchiveUpdatesCDNFolder}/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
+                            $"https://{(await _creds).Username}.b-cdn.net/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
                 }
                 _logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed");
 

Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs

@@ -1,9 +1,25 @@
-namespace Wabbajack.Server.DTOs
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.Server.DTOs
 {
+    public enum StorageSpace
+    {
+        AuthoredFiles,
+        Patches,
+        Mirrors
+    }
+    
     public class BunnyCdnFtpInfo
     {
         public string Username { get; set; }
         public string Password { get; set; }
         public string Hostname { get; set; }
+
+        public static async Task<BunnyCdnFtpInfo> GetCreds(StorageSpace space)
+        {
+            return (await Utils.FromEncryptedJson<Dictionary<string, BunnyCdnFtpInfo>>("bunnycdn"))[space.ToString()];
+        }
     }
 }

Wabbajack.Server/DTOs/MirroredFile.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.Server.DataLayer;
+
+namespace Wabbajack.Server.DTOs
+{
+    public class MirroredFile
+    {
+        public Hash Hash { get; set; }
+        public DateTime Created { get; set; }
+        public DateTime? Uploaded { get; set; }
+        public string Rationale { get; set; }
+        
+        public string FailMessage { get; set; }
+
+        public async Task Finish(SqlService sql)
+        {
+            Uploaded = DateTime.UtcNow;
+            await sql.UpsertMirroredFile(this);
+        }
+        
+        public async Task Fail(SqlService sql, string message)
+        {
+            Uploaded = DateTime.UtcNow;
+            FailMessage = message;
+            await sql.UpsertMirroredFile(this);
+        }
+    }
+}

Wabbajack.Server/DataLayer/ArchiveDownloads.cs

@@ -57,6 +57,12 @@ namespace Wabbajack.Server.DataLayer
             return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, PrimaryKeyString FROM ArchiveDownloads")).ToHashSet();
         }
         
+        public async Task<HashSet<(Hash Hash, AbstractDownloadState State)>> GetAllArchiveDownloadStates()
+        {
+            await using var conn = await Open();
+            return (await conn.QueryAsync<(Hash, AbstractDownloadState)>("SELECT Hash, DownloadState FROM ArchiveDownloads")).ToHashSet();
+        }
+
         
         public async Task<ArchiveDownload> GetArchiveDownload(Guid id)
         {

Wabbajack.Server/DataLayer/MirroredFiles.cs

@@ -0,0 +1,79 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using AngleSharp.Common;
+using Dapper;
+using Wabbajack.Common;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.NexusApi;
+using Wabbajack.Server.DTOs;
+
+namespace Wabbajack.Server.DataLayer
+{
+    public partial class SqlService
+    {
+        public async Task<MirroredFile> GetNextMirroredFile()
+        {
+            await using var conn = await Open();
+            var result = await conn.QueryFirstOrDefaultAsync<(Hash, DateTime, DateTime, string, string)>(
+                "SELECT Hash, Created, Uploaded, Rationale, FailMessage from dbo.MirroredArchives WHERE Uploaded IS NULL");
+            if (result == default) return null;
+            return new MirroredFile
+            {
+                Hash = result.Item1, Created = result.Item2, Uploaded = result.Item3, Rationale = result.Item4, FailMessage = result.Item5
+            };
+        }
+        
+        public async Task<HashSet<Hash>> GetAllMirroredHashes()
+        {
+            await using var conn = await Open();
+            return (await conn.QueryAsync<Hash>("SELECT Hash FROM dbo.MirroredArchives")).ToHashSet();
+        }
+        
+        public async Task UpsertMirroredFile(MirroredFile file)
+        {
+            await using var conn = await Open();
+            await using var trans = await conn.BeginTransactionAsync();
+
+            await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash", new {file.Hash}, trans);
+            await conn.ExecuteAsync(
+                "INSERT INTO dbo.MirroredArchives (Hash, Created, Uploaded, Rationale, FailMessage) VALUES (@Hash, @Created, @Uploaded, @Rationale, @FailMessage)",
+                new
+                {
+                    Hash = file.Hash,
+                    Created = file.Created,
+                    Uploaded = file.Uploaded,
+                    Rationale = file.Rationale,
+                    FailMessage = file.FailMessage
+                }, trans);
+            await trans.CommitAsync();
+        }
+
+        public async Task InsertAllNexusMirrors()
+        {
+            var permissions = (await GetNexusPermissions()).Where(p => p.Value == HTMLInterface.PermissionValue.Yes);
+            var downloads = (await GetAllArchiveDownloadStates()).Where(a => a.State is NexusDownloader.State).ToDictionary(a =>
+            {
+                var nd = (NexusDownloader.State)a.State;
+                return (nd.Game, nd.ModID);
+            }, a => a.Hash);
+            
+            var existing = await GetAllMirroredHashes();
+
+            foreach (var (key, _) in permissions)
+            {
+                if (!downloads.TryGetValue(key, out var hash)) continue;
+                if (existing.Contains(hash)) continue;
+
+                await UpsertMirroredFile(new MirroredFile
+                {
+                    Hash = hash,
+                    Created = DateTime.UtcNow,
+                    Rationale =
+                        $"Mod ({key.Item1} {key.Item2}) has allowed re-upload permissions on the Nexus"
+                });
+            }
+        }
+    }
+}

Wabbajack.Server/DataLayer/Nexus.cs

@@ -146,5 +146,38 @@ namespace Wabbajack.Server.DataLayer
             await tx.CommitAsync();
 
         }
+
+        public async Task UpdateGameMetadata()
+        {
+            await using var conn = await Open();
+            var existing = (await conn.QueryAsync<string>("SELECT WabbajackName FROM dbo.GameMetadata")).ToHashSet();
+
+            var missing = GameRegistry.Games.Values.Where(g => !existing.Contains(g.Game.ToString())).ToList();
+            foreach (var add in missing.Where(g => g.NexusGameId != 0))
+            {
+                await conn.ExecuteAsync(
+                    "INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, @WabbajackName)",
+                    new {NexusGameId = add.NexusGameId, WabbajackName = add.Game.ToString()});
+            }
+        }
+
+        public async Task SetNexusPermission(Game game, long modId, HTMLInterface.PermissionValue perm)
+        {
+            await using var conn = await Open();
+            var tx = await conn.BeginTransactionAsync();
+
+            await conn.ExecuteAsync("DELETE FROM NexusModPermissions WHERE GameID = @GameID AND ModID = @ModID", new
+                {
+                    GameID = game.MetaData().NexusGameId,
+                    ModID = modId
+                },
+                transaction:tx);
+            
+            await conn.ExecuteAsync(
+                "INSERT INTO NexusModPermissions (NexusGameID, ModID, Permissions) VALUES (@NexusGameID, @ModID, @Permissions)",
+                new {NexusGameID = game.MetaData().NexusGameId, ModID = modId, Permissions = (int)perm}, tx);
+
+            await tx.CommitAsync();
+        }
     }
 }

Wabbajack.Server/Services/AbstractService.cs

@@ -17,6 +17,7 @@ namespace Wabbajack.Server.Services
         private TimeSpan _delay;
         protected ILogger<TP> _logger;
         protected QuickSync _quickSync;
+        private bool _isSetup;
 
         public AbstractService(ILogger<TP> logger, AppSettings settings, QuickSync quickSync, TimeSpan delay)
         {
@@ -24,19 +25,32 @@ namespace Wabbajack.Server.Services
             _delay = delay;
             _logger = logger;
             _quickSync = quickSync;
+
+            _isSetup = false;
+        }
+
+        public virtual async Task Setup()
+        {
+            
         }
 
         public void Start()
         {
+
             if (_settings.RunBackEndJobs)
             {
                 Task.Run(async () =>
                 {
+                    await Setup();
+                    _isSetup = true;
+
+                    
                     while (true)
                     {
                         await _quickSync.ResetToken<TP>();
                         try
                         {
+                            _logger.LogInformation($"Running: {GetType().Name}");
                             await Execute();
                         }
                         catch (Exception ex)

Wabbajack.Server/Services/MirrorUploader.cs

@@ -0,0 +1,111 @@
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Net;
+using System.Threading.Tasks;
+using FluentFTP;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer;
+using Wabbajack.BuildServer.Controllers;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.AuthorApi;
+using Wabbajack.Lib.FileUploader;
+using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DTOs;
+
+namespace Wabbajack.Server.Services
+{
+    public class MirrorUploader : AbstractService<MirrorUploader, int>
+    {
+        private SqlService _sql;
+        private ArchiveMaintainer _archives;
+
+        public MirrorUploader(ILogger<MirrorUploader> logger, AppSettings settings, SqlService sql, QuickSync quickSync, ArchiveMaintainer archives) : base(logger, settings, quickSync, TimeSpan.FromHours(1))
+        {
+            _sql = sql;
+            _archives = archives;
+        }
+
+        public override async Task<int> Execute()
+        {
+        
+            int uploaded = 0;
+            TOP:
+            var toUpload = await _sql.GetNextMirroredFile();
+            if (toUpload == default) return uploaded;
+            uploaded += 1;
+
+            try
+            {
+                using var queue = new WorkQueue();
+                if (_archives.TryGetPath(toUpload.Hash, out var path))
+                {
+                    _logger.LogInformation($"Uploading mirror file {toUpload.Hash} {path.Size.FileSizeToString()}");
+
+                    var definition = await Client.GenerateFileDefinition(queue, path, (s, percent) => { });
+
+                    var creds = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
+                    using (var client = await GetClient(creds))
+                    {
+                        await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}");
+                        await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}/parts");
+                    }
+
+                    string MakePath(long idx)
+                    {
+                        return $"{definition.Hash.ToHex()}/parts/{idx}";
+                    }
+
+                    await definition.Parts.PMap(queue, async part =>
+                    {
+                        _logger.LogInformation($"Uploading mirror part ({part.Index}/{definition.Parts.Length})");
+                        var name = MakePath(part.Index);
+                        var buffer = new byte[part.Size];
+                        await using (var fs = await path.OpenShared())
+                        {
+                            fs.Position = part.Offset;
+                            await fs.ReadAsync(buffer);
+                        }
+
+                        using var client = await GetClient(creds);
+                        await client.UploadAsync(new MemoryStream(buffer), name);
+                    });
+
+                    using (var client = await GetClient(creds))
+                    {
+                        _logger.LogInformation($"Finishing mirror upload");
+                        await using var ms = new MemoryStream();
+                        await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
+                        {
+                            definition.ToJson(gz);
+                        }
+
+                        ms.Position = 0;
+                        await client.UploadAsync(ms, $"{definition.Hash.ToHex()}/definition.json.gz");
+                    }
+
+                    await toUpload.Finish(_sql);
+                }
+                else
+                {
+                    await toUpload.Fail(_sql, "Archive not found");
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogInformation($"{toUpload.Created} {toUpload.Uploaded}");
+                _logger.LogError(ex, "Error uploading");
+                await toUpload.Fail(_sql, ex.ToString());
+            }
+            goto TOP;
+        }
+
+        private static async Task<FtpClient> GetClient(BunnyCdnFtpInfo creds)
+        {
+            var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password));
+            await ftpClient.ConnectAsync();
+            return ftpClient;
+        }
+    }
+}

Wabbajack.Server/Services/NexusPermissionsUpdater.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Wabbajack.BuildServer;
@@ -16,7 +17,9 @@ namespace Wabbajack.Server.Services
         private DiscordWebHook _discord;
         private SqlService _sql;
         
-        public NexusPermissionsUpdater(ILogger<NexusKeyMaintainance> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromHours(4))
+        public static TimeSpan MaxSync = TimeSpan.FromHours(4);
+
+        public NexusPermissionsUpdater(ILogger<NexusKeyMaintainance> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromSeconds(1))
         {
             _discord = discord;
             _sql = sql;
@@ -24,7 +27,8 @@ namespace Wabbajack.Server.Services
 
         public override async Task<int> Execute()
         {
-            var permissions = await _sql.GetNexusPermissions();
+            await _sql.UpdateGameMetadata();
+            
             
             var data = await _sql.ModListArchives();
             var nexusArchives = data.Select(a => a.State).OfType<NexusDownloader.State>().Select(d => (d.Game, d.ModID))
@@ -33,38 +37,37 @@ namespace Wabbajack.Server.Services
             
             _logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods");
             
-            using var queue = new WorkQueue();
+            using var queue = new WorkQueue(1);
 
-            var results = await nexusArchives.PMap(queue, async archive =>
+            var prev = await _sql.GetNexusPermissions();
-            {
-                var permissions = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID);
-                return (archive.Game, archive.ModID, permissions);
-            });
 
-            var updated = 0;
+            var lag = MaxSync / nexusArchives.Count * 2; 
-            foreach (var result in results)
+
+
+            await nexusArchives.PMap(queue, async archive =>
             {
-                if (permissions.TryGetValue((result.Game, result.ModID), out var oldPermission))
+                _logger.LogInformation($"Checking permissions for {archive.Game} {archive.ModID}");
+                var result = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID);
+                await _sql.SetNexusPermission(archive.Game, archive.ModID, result);
+                
+                if (prev.TryGetValue((archive.Game, archive.ModID), out var oldPermission))
                 {
-                    if (oldPermission != result.permissions)
+                    if (oldPermission != result)
                     {
                         await _discord.Send(Channel.Spam,
                             new DiscordMessage {
-                                Content = $"Permissions status of {result.Game} {result.ModID} was {oldPermission} is now {result.permissions} "
+                                Content = $"Permissions status of {archive.Game} {archive.ModID} was {oldPermission} is now {result}"
                             });
-                        await _sql.PurgeNexusCache(result.ModID);
+                        await _sql.PurgeNexusCache(archive.ModID);
-                        updated += 1;
-                    }
-                }
-            }
-
-            await _sql.SetNexusPermissions(results);
-
-            if (updated > 0)
                         await _quickSync.Notify<ListValidator>();
+                    }
+                }
                 
+                await Task.Delay(lag);
+            });
+
+            return 1;
+        }
 
-            return updated;
-        }
     }
 }

Wabbajack.Server/Services/PatchBuilder.cs

@@ -125,7 +125,7 @@ namespace Wabbajack.Server.Services
 
         private static string PatchName(Hash oldHash, Hash newHash)
         {
-            return $"{Consts.ArchiveUpdatesCDNFolder}\\{oldHash.ToHex()}_{newHash.ToHex()}";
+            return $"\\{oldHash.ToHex()}_{newHash.ToHex()}";
         }
 
         private async Task CleanupOldPatches()
@@ -155,7 +155,7 @@ namespace Wabbajack.Server.Services
                 if (pendingPatch != default) break;
             }
 
-            var files = await client.GetListingAsync($"{Consts.ArchiveUpdatesCDNFolder}\\");
+            var files = await client.GetListingAsync($"\\");
             _logger.LogInformation($"Found {files.Length} on the CDN");
 
             var sqlFiles = await _sql.AllPatchHashes();
@@ -206,9 +206,6 @@ namespace Wabbajack.Server.Services
                         $"Uploading {patchFile.Size.ToFileSizeString()} patch file to CDN");
                     using var client = await GetBunnyCdnFtpClient();
                     
-                    if (!await client.DirectoryExistsAsync(Consts.ArchiveUpdatesCDNFolder)) 
-                        await client.CreateDirectoryAsync(Consts.ArchiveUpdatesCDNFolder);
-                    
                     await client.UploadFileAsync((string)patchFile, patchName, FtpRemoteExists.Overwrite);
                     return;
                 }
@@ -230,7 +227,7 @@ namespace Wabbajack.Server.Services
 
         private async Task<FtpClient> GetBunnyCdnFtpClient()
         {
-            var info = await Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");
+            var info = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Patches);
             var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
             await client.ConnectAsync();
             return client;

Wabbajack.Server/Startup.cs

@@ -69,6 +69,7 @@ namespace Wabbajack.Server
             services.AddSingleton<PatchBuilder>();
             services.AddSingleton<CDNMirrorList>();
             services.AddSingleton<NexusPermissionsUpdater>();
+            services.AddSingleton<MirrorUploader>();
             
             services.AddMvc();
             services.AddControllers()
@@ -125,6 +126,7 @@ namespace Wabbajack.Server
             app.UseService<PatchBuilder>();
             app.UseService<CDNMirrorList>();
             app.UseService<NexusPermissionsUpdater>();
+            app.UseService<MirrorUploader>();
 
             app.Use(next =>
             {