Cdn tweaks (#1362)

Mirrored file tweaks to reduce CDN disk usage

Wabbajack.Common/CircuitBreaker/WithAutoRetry.cs

@@ -58,6 +58,54 @@ namespace Wabbajack.Common
             }
         }
         
+        
+        public static async ValueTask WithAutoRetryAllAsync(Func<ValueTask> f, TimeSpan? delay = null, int? multipler = null, int? maxRetries = null)
+        {
+            int retries = 0;
+            delay ??= DEFAULT_DELAY;
+            multipler ??= DEFAULT_DELAY_MULTIPLIER;
+            maxRetries ??= DEFAULT_RETRIES;
+
+            TOP:
+            try
+            {
+                await f();
+            }
+            catch (Exception ex)
+            {
+                retries += 1;
+                if (retries > maxRetries)
+                    throw;
+                Utils.Log($"(Retry {retries} of {maxRetries}), got exception {ex.Message}, waiting {delay!.Value.TotalMilliseconds}ms");
+                await Task.Delay(delay.Value);
+                delay = delay * multipler;
+                goto TOP;
+            }
+        }
+        public static async ValueTask<T> WithAutoRetryAllAsync<T>(Func<ValueTask<T>> f, TimeSpan? delay = null, int? multipler = null, int? maxRetries = null)
+        {
+            int retries = 0;
+            delay ??= DEFAULT_DELAY;
+            multipler ??= DEFAULT_DELAY_MULTIPLIER;
+            maxRetries ??= DEFAULT_RETRIES;
+
+            TOP:
+            try
+            {
+                return await f();
+            }
+            catch (Exception ex)
+            {
+                retries += 1;
+                if (retries > maxRetries)
+                    throw;
+                Utils.Log($"(Retry {retries} of {maxRetries}), got exception {ex.Message}, waiting {delay!.Value.TotalMilliseconds}ms");
+                await Task.Delay(delay.Value);
+                delay = delay * multipler;
+                goto TOP;
+            }
+        }
+        
         public static void WithAutoRetry<TE>(Action f, TimeSpan? delay = null, int? multipler = null, int? maxRetries = null) where TE : Exception
         {
             int retries = 0;

Wabbajack.Common/Consts.cs

@@ -159,8 +159,6 @@ namespace Wabbajack.Common
         public static Uri WabbajackOrg = new Uri("https://www.wabbajack.org/");
         
         public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2;
-        
-        public static Uri WabbajackMirror = new Uri("https://wabbajack-mirror.b-cdn.net");
 
     }
 }

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -125,7 +125,7 @@ namespace Wabbajack.Lib.Downloaders
             Utils.Log($"Trying to find solution to broken download for {archive.Name}");
             
             var result = await FindUpgrade(archive);
-            if (result == default)
+            if (result == default )
             {
                 result = await AbstractDownloadState.ServerFindUpgrade(archive);
                 if (result == default)

Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs

@@ -26,7 +26,7 @@ namespace Wabbajack.Lib.Downloaders
             {"wabbajacktest.b-cdn.net", "test-files.wabbajack.org"}
         };
 
-        
+
         public string[]? Mirrors;
         public long TotalRetries;
         
@@ -70,6 +70,11 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
+            public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
+            {
+                return default;
+            }
+
             public override async Task<bool> Download(Archive a, AbsolutePath destination)
             {
                 destination.Parent.CreateDirectory();

Wabbajack.Server.Test/ABuildServerSystemTest.cs

@@ -62,7 +62,7 @@ namespace Wabbajack.BuildServer.Test
             _token = new CancellationTokenSource();
             _task = _host.RunAsync(_token.Token);
             Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080");
-            Consts.WabbajackMirror = new Uri("https://wabbajack-test.b-cdn.net");
+            Consts.TestMode = true;
 
             await "ServerWhitelist.yaml".RelativeTo(ServerPublicFolder).WriteAllTextAsync(
                 "GoogleIDs:\nAllowedPrefixes:\n    - http://localhost");

Wabbajack.Server.Test/AuthoredFilesTests.cs

@@ -83,26 +83,5 @@ namespace Wabbajack.BuildServer.Test
             Assert.Empty(toDelete.SQLDelete);
 
         }
-
-        [Fact]
-        public async Task ServerGetsEdgeServerInfo()
-        {
-            var service = Fixture.GetService<CDNMirrorList>();
-            Assert.True(await service.Execute() > 0);
-            Assert.NotEmpty(service.Mirrors);
-            Assert.True(DateTime.UtcNow - service.LastUpdate < TimeSpan.FromMinutes(1));
-
-            var servers = await ClientAPI.GetCDNMirrorList();
-            Assert.Equal(service.Mirrors, servers);
-            
-            var state = new WabbajackCDNDownloader.State(new Uri("https://wabbajack.b-cdn.net/this_file_doesn_t_exist"));
-            await DownloadDispatcher.PrepareAll(new[] {state});
-            await using var tmp = new TempFile();
-            
-            await Assert.ThrowsAsync<HttpException>(async () => await state.Download(new Archive(state) {Name = "test"}, tmp.Path));
-            var downloader = DownloadDispatcher.GetInstance<WabbajackCDNDownloader>();
-            Assert.Null(downloader.Mirrors); // Now works through a host remap
-        }
-
     }
 }

Wabbajack.Server.Test/MirroredFilesTests.cs

@@ -39,6 +39,7 @@ namespace Wabbajack.Server.Test
             });
 
             var uploader = Fixture.GetService<MirrorUploader>();
+            uploader.ActiveFileSyncEnabled = false;
             Assert.Equal(1, await uploader.Execute());
             
             
@@ -51,6 +52,23 @@ namespace Wabbajack.Server.Test
             await using var file2 = new TempFile();
             await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, file2.Path);
             Assert.Equal(dataHash!.Value, await file2.Path.FileHashAsync());
+
+            var onServer = await uploader.GetHashesOnCDN();
+            Assert.Contains(dataHash.Value, onServer);
+
+            await uploader.DeleteOldMirrorFiles();
+            
+            // Still in SQL so it will still exist
+            await using var file3 = new TempFile();
+            await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, file3.Path);
+            Assert.Equal(dataHash!.Value, await file3.Path.FileHashAsync());
+
+            // Enabling the sync should kill off the unattached file
+            uploader.ActiveFileSyncEnabled = true;
+            Assert.Equal(0, await uploader.Execute());
+            
+            var onServer2 = await uploader.GetHashesOnCDN();
+            Assert.DoesNotContain(dataHash.Value, onServer2);
         }
 
         [Fact]

Wabbajack.Server.Test/ModListValidationTests.cs

@@ -141,7 +141,7 @@ namespace Wabbajack.BuildServer.Test
 
             data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
             Assert.NotNull(data);
-            Assert.Equal(0, data.ValidationSummary.Failed);
+            Assert.Equal(1, data.ValidationSummary.Failed);
             Assert.Equal(0, data.ValidationSummary.Passed);
             Assert.Equal(1, data.ValidationSummary.Updating);
 

Wabbajack.Server.Test/ModlistUpdater.cs

@@ -76,7 +76,7 @@ namespace Wabbajack.Server.Test
             await Assert.ThrowsAsync<HttpException>(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
             Assert.True(await patcher.Execute() > 1);
 
-            Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
+            Assert.Equal(new Uri("https://test-files.wabbajack.org/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
 
             Assert.Equal("Purged", await AuthorAPI.NoPatch(oldArchive.Hash, "Testing NoPatch"));
             

Wabbajack.Server/Controllers/AuthoredFiles.cs

@@ -114,8 +114,9 @@ namespace Wabbajack.BuildServer.Controllers
             
             await _discord.Send(Channel.Ham,
                 new DiscordMessage {Content = $"{user} has finished uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})"});
-            
-            return Ok($"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{definition.MungedName}");
+
+            var host = Consts.TestMode ? "test-files" : "authored-files";
+            return Ok($"https://{host}.wabbajack.org/{definition.MungedName}");
         }
 
         private async Task<FtpClient> GetBunnyCdnFtpClient()
@@ -172,7 +173,7 @@ namespace Wabbajack.BuildServer.Controllers
             <html><body>
                 <table>
                 {{each $.files }}
-                <tr><td><a href='https://wabbajack.b-cdn.net/{{$.MungedName}}'>{{$.OriginalFileName}}</a></td><td>{{$.Size}}</td><td>{{$.LastTouched}}</td><td>{{$.Finalized}}</td><td>{{$.Author}}</td></tr>
+                <tr><td><a href='https://authored-files.wabbajack.org/{{$.MungedName}}'>{{$.OriginalFileName}}</a></td><td>{{$.Size}}</td><td>{{$.LastTouched}}</td><td>{{$.Finalized}}</td><td>{{$.Author}}</td></tr>
                 {{/each}}
                 </table>
             </body></html>

Wabbajack.Server/Controllers/ModUpgrade.cs

@@ -102,10 +102,11 @@ namespace Wabbajack.BuildServer.Controllers
                 if (patch.PatchSize != 0)
                 {
                     //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch Found");
+                    var host = (await _creds).Username == "wabbajacktest" ? "test-files" : "patches";
                     await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id);
                     return
                         Ok(
-                            $"https://{(await _creds).Username}.b-cdn.net/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
+                            $"https://{host}.wabbajack.org/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
                 }
                 //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed");
 

Wabbajack.Server/DTOs/ValidationData.cs

@@ -1,4 +1,6 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
 using Wabbajack.Common;
 using Wabbajack.Lib;
 using Wabbajack.Lib.ModListRegistry;
@@ -12,6 +14,7 @@ namespace Wabbajack.Server.DTOs
         public List<ModlistMetadata> ModLists { get; set; }
         
         public ConcurrentHashSet<(Game Game, long ModId)> SlowQueriedFor { get; set; } = new ConcurrentHashSet<(Game Game, long ModId)>();
-        public HashSet<Hash> Mirrors { get; set; }
+        public Dictionary<Hash, bool> Mirrors { get; set; }
+        public Lazy<Task<Dictionary<Hash, string>>> AllowedMirrors { get; set; }
     }
 }

Wabbajack.Server/DataLayer/ArchiveDownloads.cs

@@ -248,7 +248,8 @@ namespace Wabbajack.Server.DataLayer
             if (await HaveMirror(hash) && files.Count > 0)
             {
                 var ffile = files.First();
-                var url = new Uri($"https://{(await _mirrorCreds).Username}.b-cdn.net/{hash.ToHex()}");
+                var host = Consts.TestMode ? "test-files" : "mirror";
+                var url = new Uri($"https://{host}.wabbajack.org/{hash.ToHex()}");
                 files.Add(new Archive(
                     new WabbajackCDNDownloader.State(url)) {Hash = hash, Size = ffile.Size, Name = ffile.Name});
             }

Wabbajack.Server/DataLayer/MirroredFiles.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
+using CefSharp.DevTools.Network;
 using Dapper;
 using Wabbajack.Common;
 using Wabbajack.Lib.Downloaders;
@@ -24,10 +25,38 @@ namespace Wabbajack.Server.DataLayer
             };
         }
         
-        public async Task<HashSet<Hash>> GetAllMirroredHashes()
+        public async Task<Dictionary<Hash, bool>> GetAllMirroredHashes()
         {
             await using var conn = await Open();
-            return (await conn.QueryAsync<Hash>("SELECT Hash FROM dbo.MirroredArchives")).ToHashSet();
+            return (await conn.QueryAsync<(Hash, DateTime?)>("SELECT Hash, Uploaded FROM dbo.MirroredArchives"))
+                .GroupBy(d => d.Item1)
+                .ToDictionary(d => d.Key, d => d.First().Item2.HasValue);
+        }
+        
+        public async Task StartMirror((Hash Hash, string Reason) mirror)
+        {
+            await using var conn = await Open();
+            await using var trans = await conn.BeginTransactionAsync();
+
+            if (await conn.QueryFirstOrDefaultAsync<Hash>(@"SELECT Hash FROM dbo.MirroredArchives WHERE Hash = @Hash",
+                new {Hash = mirror.Hash}, trans) != default)
+            {
+                return;
+            }
+
+            await conn.ExecuteAsync(
+                @"INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) VALUES (@Hash, GETUTCDATE(), @Reason)",
+                new {Hash = mirror.Hash, Reason = mirror.Reason}, trans);
+            await trans.CommitAsync();
+
+        }
+
+        public async Task<Dictionary<Hash, string>> GetAllowedMirrors()
+        {
+            await using var conn = await Open();
+            return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, Reason FROM dbo.AllowedMirrorsCache"))
+                .GroupBy(d => d.Item1)
+                .ToDictionary(d => d.Key, d => d.First().Item2);
         }
         
         public async Task UpsertMirroredFile(MirroredFile file)
@@ -70,7 +99,7 @@ namespace Wabbajack.Server.DataLayer
             foreach (var (key, _) in permissions)
             {
                 if (!downloads.TryGetValue(key, out var hash)) continue;
-                if (existing.Contains(hash)) continue;
+                if (existing.ContainsKey(hash)) continue;
 
                 await UpsertMirroredFile(new MirroredFile
                 {
@@ -130,5 +159,20 @@ namespace Wabbajack.Server.DataLayer
 
                 ");
         }
+
+        public async Task AddNexusModWithOpenPerms(Game gameGame, long modId)
+        {
+            await using var conn = await Open();
+
+            await conn.ExecuteAsync(
+                @"INSERT INTO dbo.NexusModsWithOpenPerms(NexusGameID, NexusModID) VALUES(@game, @mod)",
+                new {game = gameGame.MetaData().NexusGameId, modId});
+        }
+
+        public async Task SyncActiveMirroredFiles()
+        {
+            await using var conn = await Open();
+            await conn.ExecuteAsync(@"EXEC dbo.QueueMirroredFiles");
+        }
     }
 }

Wabbajack.Server/DataLayer/ValidationData.cs

@@ -24,6 +24,7 @@ namespace Wabbajack.Server.DataLayer
                 ArchiveStatus = await archiveStatus,
                 ModLists = await modLists,
                 Mirrors = await mirrors,
+                AllowedMirrors = new Lazy<Task<Dictionary<Hash, string>>>(async () => await GetAllowedMirrors()),
             };
         }
         

Wabbajack.Server/Services/ArchiveDownloader.cs

@@ -17,18 +17,20 @@ namespace Wabbajack.Server.Services
         private ArchiveMaintainer _archiveMaintainer;
         private NexusApiClient _nexusClient;
         private DiscordWebHook _discord;
+        private NexusKeyMaintainance _nexus;
 
-        public ArchiveDownloader(ILogger<ArchiveDownloader> logger, AppSettings settings, SqlService sql, ArchiveMaintainer archiveMaintainer, DiscordWebHook discord, QuickSync quickSync) 
+        public ArchiveDownloader(ILogger<ArchiveDownloader> logger, AppSettings settings, SqlService sql, ArchiveMaintainer archiveMaintainer, DiscordWebHook discord, QuickSync quickSync, NexusKeyMaintainance nexus) 
             : base(logger, settings, quickSync, TimeSpan.FromMinutes(10))
         {
             _sql = sql;
             _archiveMaintainer = archiveMaintainer;
             _discord = discord;
+            _nexus = nexus;
         }
 
         public override async Task<int> Execute()
         {
-            _nexusClient ??= await NexusApiClient.Get();
+            _nexusClient ??= await _nexus.GetClient();
             int count = 0;
 
             while (true)

Wabbajack.Server/Services/DiscordFrontend.cs

@@ -20,8 +20,9 @@ namespace Wabbajack.Server.Services
         private DiscordSocketClient _client;
         private SqlService _sql;
         private MetricsKeyCache _keyCache;
+        private ListValidator _listValidator;
 
-        public DiscordFrontend(ILogger<DiscordFrontend> logger, AppSettings settings, QuickSync quickSync, SqlService sql, MetricsKeyCache keyCache)
+        public DiscordFrontend(ILogger<DiscordFrontend> logger, AppSettings settings, QuickSync quickSync, ListValidator listValidator, SqlService sql, MetricsKeyCache keyCache)
         {
             _logger = logger;
             _settings = settings;
@@ -35,6 +36,7 @@ namespace Wabbajack.Server.Services
 
             _sql = sql;
             _keyCache = keyCache;
+            _listValidator = listValidator;
         }
 
         private async Task MessageReceivedAsync(SocketMessage arg)
@@ -86,10 +88,15 @@ namespace Wabbajack.Server.Services
                     else
                     {
                         var deleted = await _sql.PurgeList(parts[2]);
+                        _listValidator.ValidationInfo.TryRemove(parts[2], out var _); 
                         await _quickSync.Notify<ModListDownloader>();
                         await ReplyTo(arg, $"Purged all traces of #{parts[2]} from the server, triggered list downloading. {deleted} records removed");
                     }
                 }
+                else if (parts[1] == "mirror-mod")
+                {
+                    await MirrorModCommand(arg, parts);
+                }
                 else if (parts[1] == "users")
                 {
                     await ReplyTo(arg, $"Wabbajack has {await _keyCache.KeyCount()} known unique users");
@@ -97,6 +104,40 @@ namespace Wabbajack.Server.Services
             }
         }
 
+        private async Task MirrorModCommand(SocketMessage msg, string[] parts)
+        {
+            if (parts.Length != 2)
+            {
+                await ReplyTo(msg, "Command is: mirror-mod <game-name> <mod-id>");
+                return;
+            }
+
+            if (long.TryParse(parts[2], out var modId))
+            {
+                await ReplyTo(msg, $"Got {modId} for a mod-id, expected a integer");
+                return;
+            }
+
+            if (GameRegistry.TryGetByFuzzyName(parts[1], out var game))
+            {
+                var gameNames = GameRegistry.Games.Select(g => g.Value.NexusName)
+                    .Where(g => !string.IsNullOrWhiteSpace(g))
+                    .Select(g => (string)g)
+                    .ToHashSet();
+                var joined = string.Join(", ", gameNames.OrderBy(g => g));
+                await ReplyTo(msg, $"Got {parts[1]} for a game name, expected something like: {joined}");
+            }
+
+            if (game!.NexusGameId == default)
+            {
+                await ReplyTo(msg, $"No NexusGameID found for {game}");
+            }
+
+            await _sql.AddNexusModWithOpenPerms(game.Game, modId);
+            await _quickSync.Notify<MirrorUploader>();
+            await ReplyTo(msg, "Done, and I notified the uploader");
+        }
+
         private async Task PurgeNexusCache(SocketMessage arg, string mod)
         {
             if (Uri.TryCreate(mod, UriKind.Absolute, out var url))

Wabbajack.Server/Services/ListValidator.cs

@@ -68,8 +68,13 @@ namespace Wabbajack.Server.Services
                         var (_, result) = await ValidateArchive(data, archive);
                         if (result == ArchiveStatus.InValid)
                         {
-                            if (data.Mirrors.Contains(archive.Hash))
-                                return (archive, ArchiveStatus.Mirrored);
+                            if (data.Mirrors.TryGetValue(archive.Hash, out var done))
+                                return (archive, done ? ArchiveStatus.Mirrored : ArchiveStatus.Updating);
+                            if ((await data.AllowedMirrors.Value).TryGetValue(archive.Hash, out var reason))
+                            {
+                                await _sql.StartMirror((archive.Hash, reason));
+                                return (archive, ArchiveStatus.Updating);
+                            }
                             return await TryToHeal(data, archive, metadata);
                         }
 
@@ -88,7 +93,7 @@ namespace Wabbajack.Server.Services
                     }
                 });
 
-                var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid);
+                var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid || f.Item2 == ArchiveStatus.Updating);
                 var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated);
                 var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating);
                 var mirroredCount = archives.Count(f => f.Item2 == ArchiveStatus.Mirrored);
@@ -236,7 +241,12 @@ namespace Wabbajack.Server.Services
 
                 return _archives.TryGetPath(foundArchive.Archive.Hash, out var path) ? path : default;
             };
-            
+
+            if (archive.State is NexusDownloader.State)
+            {
+                DownloadDispatcher.GetInstance<NexusDownloader>().Client = await _nexus.GetClient();
+            }
+
             var upgrade = await DownloadDispatcher.FindUpgrade(archive, resolver);
             
             

Wabbajack.Server/Services/MirrorUploader.cs

@@ -25,6 +25,8 @@ namespace Wabbajack.Server.Services
         private ArchiveMaintainer _archives;
         private DiscordWebHook _discord;
 
+        public bool ActiveFileSyncEnabled { get; set; } = true;
+
         public MirrorUploader(ILogger<MirrorUploader> logger, AppSettings settings, SqlService sql, QuickSync quickSync, ArchiveMaintainer archives, DiscordWebHook discord)
             : base(logger, settings, quickSync, TimeSpan.FromHours(1))
         {
@@ -37,9 +39,16 @@ namespace Wabbajack.Server.Services
         {
         
             int uploaded = 0;
+            
+            if (ActiveFileSyncEnabled)
+                await _sql.SyncActiveMirroredFiles();
             TOP:
             var toUpload = await _sql.GetNextMirroredFile();
-            if (toUpload == default) return uploaded;
+            if (toUpload == default)
+            {
+                await DeleteOldMirrorFiles();
+                return uploaded;
+            }
             uploaded += 1;
 
             try
@@ -93,16 +102,21 @@ namespace Wabbajack.Server.Services
                             fs.Position = part.Offset;
                             await fs.ReadAsync(buffer);
                         }
-
-                        using var client = await GetClient(creds);
-                        var name = MakePath(part.Index);
-                        await client.UploadAsync(new MemoryStream(buffer), name);
                         
+                        await CircuitBreaker.WithAutoRetryAllAsync(async () =>{
+                            using var client = await GetClient(creds);
+                            var name = MakePath(part.Index);
+                            await client.UploadAsync(new MemoryStream(buffer), name);
+                        });
+
                     });
 
-                    using (var client = await GetClient(creds))
+                    await CircuitBreaker.WithAutoRetryAllAsync(async () =>
                     {
+                        using var client = await GetClient(creds);
                         _logger.LogInformation($"Finishing mirror upload");
+
+
                         await using var ms = new MemoryStream();
                         await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
                         {
@@ -112,7 +126,7 @@ namespace Wabbajack.Server.Services
                         ms.Position = 0;
                         var remoteName = $"{definition.Hash.ToHex()}/definition.json.gz";
                         await client.UploadAsync(ms, remoteName);
-                    }
+                    });
 
                     await toUpload.Finish(_sql);
                 }
@@ -130,11 +144,64 @@ namespace Wabbajack.Server.Services
             goto TOP;
         }
 
-        private static async Task<FtpClient> GetClient(BunnyCdnFtpInfo creds)
+        private static async Task<FtpClient> GetClient(BunnyCdnFtpInfo creds = null)
         {
-            var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password));
-            await ftpClient.ConnectAsync();
-            return ftpClient;
+            return await CircuitBreaker.WithAutoRetryAllAsync<FtpClient>(async () =>
+            {
+                creds ??= await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
+
+                var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password));
+                ftpClient.DataConnectionType = FtpDataConnectionType.EPSV;
+                await ftpClient.ConnectAsync();
+                return ftpClient;
+            });
+        }
+
+        /// <summary>
+        /// Gets a list of all the Mirrored file hashes that physically exist on the CDN (via FTP lookup)
+        /// </summary>
+        /// <returns></returns>
+        public async Task<HashSet<Hash>> GetHashesOnCDN()
+        {
+            using var ftpClient = await GetClient();
+            var serverFiles = (await ftpClient.GetNameListingAsync("\\"));
+            
+            return serverFiles
+                .Select(f => ((RelativePath)f).FileName)
+                .Select(l =>
+                {
+                    try
+                    {
+                        return Hash.FromHex((string)l);
+                    }
+                    catch (Exception) { return default; }
+                })
+                .Where(h => h != default)
+                .ToHashSet();
+        }
+
+        public async Task DeleteOldMirrorFiles()
+        {
+            var existingHashes = await GetHashesOnCDN();
+            var fromSql = await _sql.GetAllMirroredHashes();
+            
+            foreach (var (hash, _) in fromSql.Where(s => s.Value))
+            {
+                Utils.Log($"Removing {hash} from SQL it's no longer in the CDN");
+                if (!existingHashes.Contains(hash))
+                    await _sql.DeleteMirroredFile(hash);
+            }
+
+            var toDelete = existingHashes.Where(h => !fromSql.ContainsKey(h)).ToArray();
+
+            using var client = await GetClient();
+            foreach (var hash in toDelete)
+            {
+                await _discord.Send(Channel.Spam,
+                    new DiscordMessage {Content = $"Removing mirrored file {hash}, as it's no longer in sql"});
+                Utils.Log($"Removing {hash} from the CDN it's no longer in SQL");
+                await client.DeleteDirectoryAsync(hash.ToHex());
+            }
         }
     }
 }

Wabbajack.Server/Services/ModListDownloader.cs

@@ -112,6 +112,11 @@ namespace Wabbajack.Server.Services
                         }
                     }
 
+                    await _discord.Send(Channel.Ham,
+                        new DiscordMessage
+                        {
+                            Content = $"Ingesting {list.Links.MachineURL} version {modlist.Version}"
+                        });
                     await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist, false);
                 }
                 catch (Exception ex)

Wabbajack.Server/Startup.cs

@@ -141,7 +141,7 @@ namespace Wabbajack.Server
             app.UseService<CDNMirrorList>();
             app.UseService<NexusPermissionsUpdater>();
             app.UseService<MirrorUploader>();
-            app.UseService<MirrorQueueService>();
+            //app.UseService<MirrorQueueService>();
             app.UseService<Watchdog>();
             app.UseService<DiscordFrontend>();
             //app.UseService<AuthoredFilesCleanup>();