Oh yeah, lots off good stuff here

Wabbajack.Common/SemaphoreSlimExtensions.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Reactive.Disposables;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Wabbajack.Common;
+
+public static class SemaphoreSlimExtensions
+{
+    public static async Task<IDisposable> Lock(this SemaphoreSlim slim)
+    {
+        await slim.WaitAsync();
+        return Disposable.Create(() => slim.Release());
+    }
+}

Wabbajack.Server/ApiKeyAuthorizationHandler.cs

@@ -9,9 +9,7 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Wabbajack.DTOs.JsonConverters;
-using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DataModels;
-using Wabbajack.Server.DTOs;
-using Wabbajack.Server.Services;
 
 namespace Wabbajack.BuildServer;
 
@@ -28,23 +26,19 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthentic
     public const string ApiKeyHeaderName = "X-Api-Key";
     private readonly DTOSerializer _dtos;
     private readonly AppSettings _settings;
-    private readonly SqlService _sql;
+    private readonly AuthorKeys _authorKeys;
-
-    private readonly MetricsKeyCache _keyCache;
 
     public ApiKeyAuthenticationHandler(
         IOptionsMonitor<ApiKeyAuthenticationOptions> options,
+        AuthorKeys authorKeys,
         ILoggerFactory logger,
         UrlEncoder encoder,
         ISystemClock clock,
-        MetricsKeyCache keyCache,
         DTOSerializer dtos,
-        AppSettings settings,
+        AppSettings settings) : base(options, logger, encoder, clock)
-        SqlService db) : base(options, logger, encoder, clock)
     {
-        _sql = db;
         _dtos = dtos;
-        _keyCache = keyCache;
+        _authorKeys = authorKeys;
         _settings = settings;
     }
 
@@ -55,7 +49,7 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthentic
         //await LogRequest(metricsKey);
         if (metricsKey != default)
         {
-            await _keyCache.AddKey(metricsKey);
+            /*
             if (await _sql.IsTarKey(metricsKey))
             {
                 await _sql.IngestMetric(new Metric
@@ -68,6 +62,7 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthentic
                 await Task.Delay(TimeSpan.FromSeconds(60));
                 throw new Exception("Error, lipsum timeout of the cross distant cloud.");
             }
+            */
         }
 
         var authorKey = Request.Headers[ApiKeyHeaderName].FirstOrDefault();
@@ -78,10 +73,9 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthentic
 
         if (authorKey == null && metricsKey == null) return AuthenticateResult.NoResult();
         
-
         if (authorKey != null)
         {
-            var owner = await _sql.LoginByApiKey(authorKey);
+            var owner = await _authorKeys.AuthorForKey(authorKey);
             if (owner == null)
                 return AuthenticateResult.Fail("Invalid author key");
 
@@ -99,11 +93,7 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthentic
         }
 
         
-        if (!await _keyCache.IsValidKey(metricsKey))
+        if (!string.IsNullOrWhiteSpace(metricsKey))
-        {
-            return AuthenticateResult.Fail("Invalid Metrics Key");
-        }
-
         {
             var claims = new List<Claim> {new(ClaimTypes.Role, "User")};
 
@@ -115,6 +105,8 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthentic
 
             return AuthenticateResult.Success(ticket);
         }
+
+        return AuthenticateResult.NoResult();
     }
 
     protected override async Task HandleChallengeAsync(AuthenticationProperties properties)

Wabbajack.Server/AppSettings.cs

@@ -9,39 +9,16 @@ public class AppSettings
     {
         config.Bind("WabbajackSettings", this);
     }
-
     public bool TestMode { get; set; }
-    public string AuthorAPIKeyFile { get; set; } = "exported_users";
+    public string AuthorAPIKeyFile { get; set; }
-
-    public string CompressedBodyHeader { get; set; } = "x-compressed-body";
     public string WabbajackBuildServerUri { get; set; } = "https://build.wabbajack.org/";
-
     public string MetricsKeyHeader { get; set; } = "x-metrics-key";
-
-    public string DownloadDir { get; set; }
-    public AbsolutePath DownloadPath => (AbsolutePath) DownloadDir;
-    public string ArchiveDir { get; set; }
-    public AbsolutePath ArchivePath => (AbsolutePath) ArchiveDir;
-
     public string TempFolder { get; set; }
-
     public AbsolutePath TempPath => (AbsolutePath) TempFolder;
-
-    public bool JobScheduler { get; set; }
-    public bool JobRunner { get; set; }
-
-    public bool RunFrontEndJobs { get; set; }
-    public bool RunBackEndJobs { get; set; }
-
-    public bool RunNexusPolling { get; set; }
-    public bool RunDownloader { get; set; }
-
-    public string BunnyCDN_StorageZone { get; set; }
-    public string SqlConnection { get; set; }
-
-    public int MaxJobs { get; set; } = 2;
-
     public string SpamWebHook { get; set; } = null;
     public string HamWebHook { get; set; } = null;
-    public bool ValidateModUpgrades { get; set; } = true;
+    
+    public string AuthoredFilesFolder { get; set; }
+    public string MetricsFolder { get; set; }
+    public string TarLogPath { get; set; }
 }

Wabbajack.Server/Controllers/AuthoredFiles.cs

@@ -1,6 +1,7 @@
 using System;
 using System.IO;
 using System.IO.Compression;
+using System.Linq;
 using System.Net;
 using System.Security.Claims;
 using System.Threading;
@@ -9,15 +10,15 @@ using FluentFTP;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Primitives;
 using Nettle;
 using Wabbajack.Common;
 using Wabbajack.DTOs.CDN;
 using Wabbajack.DTOs.JsonConverters;
 using Wabbajack.Hashing.xxHash64;
-using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DataModels;
 using Wabbajack.Server.DTOs;
 using Wabbajack.Server.Services;
-using Wabbajack.Server.TokenProviders;
 
 namespace Wabbajack.BuildServer.Controllers;
 
@@ -29,7 +30,13 @@ public class AuthoredFiles : ControllerBase
             <html><body>
                 <table>
                 {{each $.files }}
-                <tr><td><a href='https://authored-files.wabbajack.org/{{$.MungedName}}'>{{$.OriginalFileName}}</a></td><td>{{$.Size}}</td><td>{{$.LastTouched}}</td><td>{{$.Finalized}}</td><td>{{$.Author}}</td></tr>
+                <tr>
+                       <td><a href='https://authored-files.wabbajack.org/{{$.Definition.MungedName}}'>{{$.Definition.OriginalFileName}}</a></td>
+                       <td>{{$.HumanSize}}</td>
+                       <td>{{$.Definition.Author}}</td>
+                       <td>{{$.Updated}}</td>
+                       <td><a href='/authored_files/direct_link/{{$.Definition.MungedName}}'>(Slow) HTTP Direct Link</a></td>
+                </tr>
                 {{/each}}
                 </table>
             </body></html>
@@ -38,22 +45,20 @@ public class AuthoredFiles : ControllerBase
 
     private readonly DTOSerializer _dtos;
 
-    private readonly IFtpSiteCredentials _ftpCreds;
     private readonly DiscordWebHook _discord;
     private readonly ILogger<AuthoredFiles> _logger;
     private readonly AppSettings _settings;
-    private readonly SqlService _sql;
+    private readonly AuthorFiles _authoredFiles;
 
 
-    public AuthoredFiles(ILogger<AuthoredFiles> logger, SqlService sql, AppSettings settings, DiscordWebHook discord,
+    public AuthoredFiles(ILogger<AuthoredFiles> logger, AuthorFiles authorFiles, AppSettings settings, DiscordWebHook discord,
-        DTOSerializer dtos, IFtpSiteCredentials ftpCreds)
+        DTOSerializer dtos)
     {
-        _sql = sql;
         _logger = logger;
         _settings = settings;
         _discord = discord;
         _dtos = dtos;
-        _ftpCreds = ftpCreds;
+        _authoredFiles = authorFiles;
     }
     
     [HttpPut]
@@ -61,13 +66,12 @@ public class AuthoredFiles : ControllerBase
     public async Task<IActionResult> UploadFilePart(CancellationToken token, string serverAssignedUniqueId, long index)
     {
         var user = User.FindFirstValue(ClaimTypes.Name);
-        var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId);
+        var definition = await _authoredFiles.ReadDefinitionForServerId(serverAssignedUniqueId);
         if (definition.Author != user)
             return Forbid("File Id does not match authorized user");
         _logger.Log(LogLevel.Information,
             $"Uploading File part {definition.OriginalFileName} - ({index} / {definition.Parts.Length})");
         
-        await _sql.TouchAuthoredFile(definition);
         var part = definition.Parts[index];
 
         await using var ms = new MemoryStream();
@@ -82,7 +86,8 @@ public class AuthoredFiles : ControllerBase
                 $"Hashes don't match for index {index}. Sizes ({ms.Length} vs {part.Size}). Hashes ({hash} vs {part.Hash}");
 
         ms.Position = 0;
-        await UploadAsync(ms, $"{definition.MungedName}/parts/{index}");
+        await using var partStream = await _authoredFiles.CreatePart(definition.MungedName, (int)index);
+        await ms.CopyToAsync(partStream, token);
         return Ok(part.Hash.ToBase64());
     }
 
@@ -96,13 +101,9 @@ public class AuthoredFiles : ControllerBase
 
         _logger.Log(LogLevel.Information, "Creating File upload {originalFileName}", definition.OriginalFileName);
 
-        definition = await _sql.CreateAuthoredFile(definition, user);
+        definition.ServerAssignedUniqueId = Guid.NewGuid().ToString();
-
+        definition.Author = user;
-        using (var client = await GetBunnyCdnFtpClient())
+        await _authoredFiles.WriteDefinition(definition);
-        {
-            await client.CreateDirectoryAsync($"{definition.MungedName}");
-            await client.CreateDirectoryAsync($"{definition.MungedName}/parts");
-        }
         
         await _discord.Send(Channel.Ham,
             new DiscordMessage
@@ -119,22 +120,11 @@ public class AuthoredFiles : ControllerBase
     public async Task<IActionResult> CreateUpload(string serverAssignedUniqueId)
     {
         var user = User.FindFirstValue(ClaimTypes.Name);
-        var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId);
+        var definition = await _authoredFiles.ReadDefinitionForServerId(serverAssignedUniqueId);
         if (definition.Author != user)
             return Forbid("File Id does not match authorized user");
         _logger.Log(LogLevel.Information, $"Finalizing file upload {definition.OriginalFileName}");
 
-        await _sql.Finalize(definition);
-
-        await using var ms = new MemoryStream();
-        await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
-        {
-            await _dtos.Serialize(definition, gz);
-        }
-
-        ms.Position = 0;
-        await UploadAsync(ms, $"{definition.MungedName}/definition.json.gz");
-
         await _discord.Send(Channel.Ham,
             new DiscordMessage
             {
@@ -146,26 +136,14 @@ public class AuthoredFiles : ControllerBase
         return Ok($"https://{host}.wabbajack.org/{definition.MungedName}");
     }
 
-    private async Task<FtpClient> GetBunnyCdnFtpClient()
-    {
-        var info = (await _ftpCreds.Get())[StorageSpace.AuthoredFiles];
-        var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
-        await client.ConnectAsync();
-        return client;
-    }
-
-    private async Task UploadAsync(Stream stream, string path)
-    {
-        using var client = await GetBunnyCdnFtpClient();
-        await client.UploadAsync(stream, path);
-    }
-
     [HttpDelete]
     [Route("{serverAssignedUniqueId}")]
     public async Task<IActionResult> DeleteUpload(string serverAssignedUniqueId)
     {
         var user = User.FindFirstValue(ClaimTypes.Name);
-        var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId);
+        var definition = (await _authoredFiles.AllAuthoredFiles())
+            .First(f => f.Definition.ServerAssignedUniqueId == serverAssignedUniqueId)
+            .Definition;
         if (definition.Author != user)
             return Forbid("File Id does not match authorized user");
         await _discord.Send(Channel.Ham,
@@ -176,33 +154,17 @@ public class AuthoredFiles : ControllerBase
             });
         _logger.Log(LogLevel.Information, $"Deleting upload {definition.OriginalFileName}");
 
-        await DeleteFolderOrSilentlyFail($"{definition.MungedName}");
+        await _authoredFiles.DeleteFile(definition);
-
-        await _sql.DeleteFileDefinition(definition);
         return Ok();
     }
 
-    private async Task DeleteFolderOrSilentlyFail(string path)
-    {
-        try
-        {
-            using var client = await GetBunnyCdnFtpClient();
-            await client.DeleteDirectoryAsync(path);
-        }
-        catch (Exception)
-        {
-            _logger.Log(LogLevel.Information, $"Delete failed for {path}");
-        }
-    }
-
-
     [HttpGet]
     [AllowAnonymous]
     [Route("")]
     public async Task<ContentResult> UploadedFilesGet()
     {
-        var files = await _sql.AllAuthoredFiles();
+        var files = await _authoredFiles.AllAuthoredFiles();
-        var response = HandleGetListTemplate(new {files});
+        var response = HandleGetListTemplate(new {files = files.OrderByDescending(f => f.Updated).ToArray()});
         return new ContentResult
         {
             ContentType = "text/html",
@@ -210,4 +172,20 @@ public class AuthoredFiles : ControllerBase
             Content = response
         };
     }
+    
+    [HttpGet]
+    [AllowAnonymous]
+    [Route("direct_link/{mungedName}")]
+    public async Task DirectLink(string mungedName)
+    {
+        var definition = await _authoredFiles.ReadDefinition(mungedName);
+        Response.Headers.ContentDisposition =
+            new StringValues($"attachment; filename={definition.OriginalFileName}");
+        Response.Headers.ContentType = new StringValues("application/octet-stream");
+        foreach (var part in definition.Parts)
+        {
+            await using var partStream = await _authoredFiles.StreamForPart(mungedName, (int)part.Index);
+            await partStream.CopyToAsync(Response.Body);
+        }
+    }
 }

Wabbajack.Server/Controllers/Heartbeat.cs

@@ -1,13 +1,8 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
-using Nettle;
 using Wabbajack.Server;
-using Wabbajack.Server.DataLayer;
 using Wabbajack.Server.DTOs;
 using Wabbajack.Server.Services;
 
@@ -16,68 +11,18 @@ namespace Wabbajack.BuildServer.Controllers;
 [Route("/heartbeat")]
 public class Heartbeat : ControllerBase
 {
-    private const int MAX_LOG_SIZE = 128;
     private static readonly DateTime _startTime;
-    private static List<string> Log = new();
-
-    private static readonly Func<object, string> HandleGetReport = NettleEngine.GetCompiler().Compile(@"
-            <html><body>
-                <h2>Server Status</h2>
-
-                <h3>Service Overview ({{services.Length}}):</h3>
-                <ul>
-                {{each $.services }}
-                {{if $.IsLate}}
-                <li><a href='/heartbeat/report/services/{{$.Name}}.html'><b>{{$.Name}} - {{$.Time}} - {{$.MaxTime}}</b></a></li>
-                {{else}}
-                <li><a href='/heartbeat/report/services/{{$.Name}}.html'>{{$.Name}} - {{$.Time}} - {{$.MaxTime}}</a></li>
-                {{/if}}
-                {{/each}}
-                </ul>
-
-
-                <h3>Lists ({{lists.Length}}):</h3>
-                <ul>
-                {{each $.lists }}
-                <li><a href='/lists/status/{{$.Name}}.html'>{{$.Name}}</a> - {{$.Time}} {{$.FailMessage}}</li>
-                {{/each}}
-                </ul>
-            </body></html>
-        ");
-
-    private static readonly Func<object, string> HandleGetServiceReport = NettleEngine.GetCompiler().Compile(@"
-            <html><body>
-                <h2>Service Status: {{Name}} {{TimeSinceLastRun}}</h2>
-
-                <h3>Service Overview ({{ActiveWorkQueue.Length}}):</h3>
-                <ul>
-                {{each $.ActiveWorkQueue }}
-                <li>{{$.Name}} {{$.Time}}</li>
-                {{/each}}
-                </ul>
-
-            </body></html>
-        ");
 
     private readonly GlobalInformation _globalInformation;
-    private ILogger<Heartbeat> _logger;
-
-    private readonly QuickSync _quickSync;
-    private SqlService _sql;
-
     static Heartbeat()
     {
         _startTime = DateTime.Now;
     }
 
-
+    public Heartbeat(ILogger<Heartbeat> logger, GlobalInformation globalInformation,
-    public Heartbeat(ILogger<Heartbeat> logger, SqlService sql, GlobalInformation globalInformation,
         QuickSync quickSync)
     {
         _globalInformation = globalInformation;
-        _sql = sql;
-        _logger = logger;
-        _quickSync = quickSync;
     }
 
     [HttpGet]
@@ -86,54 +31,6 @@ public class Heartbeat : ControllerBase
         return Ok(new HeartbeatResult
         {
             Uptime = DateTime.Now - _startTime,
-            LastNexusUpdate = _globalInformation.TimeSinceLastNexusSync
         });
     }
-
-    [HttpGet("report")]
-    public async Task<ContentResult> Report()
-    {
-        var response = HandleGetReport(new
-        {
-            services = (await _quickSync.Report())
-                .Select(s => new
-                {
-                    s.Key.Name, Time = s.Value.LastRunTime, MaxTime = s.Value.Delay,
-                    IsLate = s.Value.LastRunTime > s.Value.Delay
-                })
-                .OrderBy(s => s.Name)
-                .ToArray()
-        });
-        return new ContentResult
-        {
-            ContentType = "text/html",
-            StatusCode = (int) HttpStatusCode.OK,
-            Content = response
-        };
-    }
-
-    [HttpGet("report/services/{serviceName}.html")]
-    public async Task<ContentResult> ReportServiceStatus(string serviceName)
-    {
-        var services = await _quickSync.Report();
-        var info = services.First(kvp => kvp.Key.Name == serviceName);
-
-        var response = HandleGetServiceReport(new
-        {
-            info.Key.Name,
-            TimeSinceLastRun = DateTime.UtcNow - info.Value.LastRunTime,
-            ActiveWorkQueue = info.Value.ActiveWork.Select(p => new
-                {
-                    Name = p.Item1,
-                    Time = DateTime.UtcNow - p.Item2
-                }).OrderByDescending(kp => kp.Time)
-                .ToArray()
-        });
-        return new ContentResult
-        {
-            ContentType = "text/html",
-            StatusCode = (int) HttpStatusCode.OK,
-            Content = response
-        };
-    }
 }

Wabbajack.Server/Controllers/ListDefinitions.cs

@@ -1,83 +0,0 @@
-using System;
-using System.IO;
-using System.IO.Compression;
-using System.Linq;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.Logging;
-using Wabbajack.Common;
-using Wabbajack.DTOs;
-using Wabbajack.DTOs.JsonConverters;
-using Wabbajack.Paths.IO;
-using Wabbajack.Server.DataLayer;
-using Wabbajack.Server.Services;
-
-namespace Wabbajack.BuildServer.Controllers;
-
-[ApiController]
-[Route("/list_definitions")]
-public class ListDefinitions : ControllerBase
-{
-    private readonly DTOSerializer _dtos;
-    private readonly AppSettings _settings;
-    private DiscordWebHook _discord;
-    private readonly ILogger<ListDefinitions> _logger;
-    private SqlService _sql;
-
-    public ListDefinitions(ILogger<ListDefinitions> logger, SqlService sql, DiscordWebHook discord,
-        AppSettings settings,
-        DTOSerializer dtos)
-    {
-        _logger = logger;
-        _sql = sql;
-        _discord = discord;
-        _settings = settings;
-        _dtos = dtos;
-    }
-
-
-    [Route("ingest")]
-    [Authorize(Roles = "User")]
-    [HttpPost]
-    public async Task<IActionResult> PostIngest()
-    {
-        var user = Request.Headers[_settings.MetricsKeyHeader].First();
-        var use_gzip = Request.Headers[_settings.CompressedBodyHeader].Any();
-        _logger.Log(LogLevel.Information, $"Ingesting Modlist Definition for {user}");
-
-        var modlistBytes = await Request.Body.ReadAllAsync();
-
-
-        _logger.LogInformation("Spawning ingestion task");
-        var tsk = Task.Run(async () =>
-        {
-            try
-            {
-                if (use_gzip)
-                {
-                    await using var os = new MemoryStream();
-                    await using var gZipStream =
-                        new GZipStream(new MemoryStream(modlistBytes), CompressionMode.Decompress);
-                    await gZipStream.CopyToAsync(os);
-                    modlistBytes = os.ToArray();
-                }
-
-                var modlist = _dtos.DeserializeAsync<ModList>(new MemoryStream(modlistBytes));
-
-                var file = KnownFolders.EntryPoint.Combine("mod_list_definitions")
-                    .Combine($"{user}_{DateTime.UtcNow.ToFileTimeUtc()}.json");
-                file.Parent.CreateDirectory();
-                await using var stream = file.Open(FileMode.Create, FileAccess.Write);
-                await _dtos.Serialize(modlist, stream);
-                _logger.Log(LogLevel.Information, $"Done Ingesting Modlist Definition for {user}");
-            }
-            catch (Exception ex)
-            {
-                _logger.LogError(ex, "Error ingesting uploaded modlist");
-            }
-        });
-
-        return Accepted(0);
-    }
-}

Wabbajack.Server/Controllers/ListsStatus.cs

@@ -1,184 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.Logging;
-using Nettle;
-using Wabbajack.DTOs;
-using Wabbajack.DTOs.ServerResponses;
-
-namespace Wabbajack.BuildServer.Controllers;
-
-[ApiController]
-[Route("/lists")]
-public class ListsStatus : ControllerBase
-{
-    private static readonly Func<object, string> HandleGetRssFeedTemplate = NettleEngine.GetCompiler().Compile(@"
-<?xml version=""1.0""?>
-<rss version=""2.0"">
-  <channel>
-    <title>{{lst.Name}} - Broken Mods</title>
-    <link>http://build.wabbajack.org/status/{{lst.Name}}.html</link>
-    <description>These are mods that are broken and need updating</description>
-    {{ each $.failed }}
-    <item>
-       <title>{{$.Archive.Name}} {{$.Archive.Hash}} {{$.Archive.State.PrimaryKeyString}}</title>
-       <link>{{$.Archive.Name}}</link>
-    </item>
-    {{/each}}
-  </channel>
-</rss>
-        ");
-
-    private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
-            <html><body>
-                <h2>{{lst.Name}} - {{lst.Checked}} - {{ago}}min ago</h2>
-
-                <h3>Failed ({{failed.Count}}):</h3>
-                <ul>
-                {{each $.failed }}
-                {{if $.HasUrl}}
-                <li><a href='{{$.Url}}'>{{$.Name}}</a></li>
-                {{else}}
-                <li>{{$.Name}}</li>
-                {{/if}}
-                {{/each}}
-                </ul>
-
-
-                <h3>Updated ({{updated.Count}}):</h3>
-                <ul>
-                {{each $.updated }}
-                {{if $.HasUrl}}
-                <li><a href='{{$.Url}}'>{{$.Name}}</a></li>
-                {{else}}
-                <li>{{$.Name}}</li>
-                {{/if}}
-
-                {{/each}}
-                </ul>
-
-                <h3>Mirrored ({{mirrored.Count}}):</h3>
-                <ul>
-                {{each $.mirrored }}
-                {{if $.HasUrl}}
-                <li><a href='{{$.Url}}'>{{$.Name}}</a></li>
-                {{else}}
-                <li>{{$.Name}}</li>
-                {{/if}}
-
-                {{/each}}
-                </ul>
-
-                <h3>Updating ({{updating.Count}}):</h3>
-                <ul>
-                {{each $.updating }}
-                {{if $.HasUrl}}
-                <li><a href='{{$.Url}}'>{{$.Name}}</a></li>
-                {{else}}
-                <li>{{$.Name}}</li>
-                {{/if}}
-                {{/each}}
-                </ul>
-
-                <h3>Passed ({{passed.Count}}):</h3>
-                <ul>
-                {{each $.passed }}
-                {{if $.HasUrl}}
-                <li><a href='{{$.Url}}'>{{$.Name}}</a></li>
-                {{else}}
-                <li>{{$.Name}}</li>
-                {{/if}}
-                {{/each}}
-                </ul>
-            </body></html>
-        ");
-
-    private ILogger<ListsStatus> _logger;
-
-    public ListsStatus(ILogger<ListsStatus> logger)
-    {
-        _logger = logger;
-    }
-
-    [HttpGet]
-    [Route("status.json")]
-    public async Task<IEnumerable<ModListSummary>> HandleGetLists()
-    {
-        throw new NotImplementedException();
-    }
-
-    [HttpGet]
-    [Route("status/{Name}/broken.rss")]
-    public async Task<ContentResult> HandleGetRSSFeed(string Name)
-    {
-        var lst = await DetailedStatus(Name);
-        var response = HandleGetRssFeedTemplate(new
-        {
-            lst,
-            failed = lst.Archives.Where(a => a.IsFailing).ToList(),
-            passed = lst.Archives.Where(a => !a.IsFailing).ToList()
-        });
-        return new ContentResult
-        {
-            ContentType = "application/rss+xml",
-            StatusCode = (int) HttpStatusCode.OK,
-            Content = response
-        };
-    }
-
-    [HttpGet]
-    [Route("status/{Name}.html")]
-    public async Task<ContentResult> HandleGetListHtml(string Name)
-    {
-        var lst = await DetailedStatus(Name);
-
-        var response = HandleGetListTemplate(new
-        {
-            lst,
-            ago = (DateTime.UtcNow - lst.Checked).TotalMinutes,
-            failed = lst.Archives.Where(a => a.IsFailing).ToList(),
-            passed = lst.Archives.Where(a => !a.IsFailing).ToList(),
-            updated = lst.Archives.Where(a => a.ArchiveStatus == ArchiveStatus.Updated).ToList(),
-            updating = lst.Archives.Where(a => a.ArchiveStatus == ArchiveStatus.Updating).ToList(),
-            mirrored = lst.Archives.Where(a => a.ArchiveStatus == ArchiveStatus.Mirrored).ToList()
-        });
-        return new ContentResult
-        {
-            ContentType = "text/html",
-            StatusCode = (int) HttpStatusCode.OK,
-            Content = response
-        };
-    }
-
-    [HttpGet]
-    [Route("status/{Name}.json")]
-    [ResponseCache(Duration = 60 * 5)]
-    public async Task<IActionResult> HandleGetListJson(string Name)
-    {
-        var lst = await DetailedStatus(Name);
-        if (lst == default) return NotFound();
-        return Ok(lst);
-    }
-
-    private async Task<DetailedStatus?> DetailedStatus(string Name)
-    {
-        throw new NotImplementedException();
-    }
-
-    [HttpGet]
-    [Route("status/badge.json")]
-    public async Task<IActionResult> HandleGitHubBadge()
-    {
-        throw new NotImplementedException();
-    }
-
-    [HttpGet]
-    [Route("status/{Name}/badge.json")]
-    public async Task<IActionResult> HandleNamedGitHubBadge(string Name)
-    {
-        throw new NotImplementedException();
-    }
-}

Wabbajack.Server/Controllers/Metrics.cs

@@ -1,17 +1,13 @@
 using System;
-using System.Collections.Generic;
 using System.Linq;
-using System.Net;
 using System.Reflection;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using Nettle;
 using Wabbajack.Common;
-using Wabbajack.Server;
+using Wabbajack.Server.DataModels;
-using Wabbajack.Server.DataLayer;
 using Wabbajack.Server.DTOs;
-using Wabbajack.Server.Services;
 
 namespace Wabbajack.BuildServer.Controllers;
 
@@ -37,17 +33,15 @@ public class MetricsController : ControllerBase
 
     private static Func<object, string> _totalListTemplate;
     private readonly AppSettings _settings;
-    private readonly MetricsKeyCache _keyCache;
     private ILogger<MetricsController> _logger;
-    private readonly SqlService _sql;
+    private readonly Metrics _metricsStore;
 
-    public MetricsController(ILogger<MetricsController> logger, SqlService sql, MetricsKeyCache keyCache,
+    public MetricsController(ILogger<MetricsController> logger, Metrics metricsStore,
         AppSettings settings)
     {
-        _sql = sql;
         _logger = logger;
-        _keyCache = keyCache;
         _settings = settings;
+        _metricsStore = metricsStore;
     }
 
 
@@ -73,169 +67,24 @@ public class MetricsController : ControllerBase
     {
         var date = DateTime.UtcNow;
         var metricsKey = Request.Headers[_settings.MetricsKeyHeader].FirstOrDefault();
-        if (metricsKey != null)
-            await _keyCache.AddKey(metricsKey);
 
         // Used in tests
-        if (value == "Default" || value == "untitled" || subject == "failed_download" || Guid.TryParse(value, out _))
+        if (value is "Default" or "untitled" || subject == "failed_download" || Guid.TryParse(value, out _))
             return new Result {Timestamp = date};
 
-        await Log(date, subject, value, metricsKey);
+        await _metricsStore.Ingest(new Metric
+        {
+            Timestamp = DateTime.UtcNow, 
+            Action = subject, 
+            Subject = subject, 
+            MetricsKey = metricsKey,
+            UserAgent = Request.Headers.UserAgent.FirstOrDefault() ?? "<unknown>",
+        });
         return new Result {Timestamp = date};
     }
 
-    [HttpGet]
-    [Route("report/{subject}")]
-    [ResponseCache(Duration = 60 * 60)]
-    public async Task<IActionResult> MetricsReport(string subject)
-    {
-        var metrics = (await _sql.MetricsReport(subject)).ToList();
-        var labels = metrics.GroupBy(m => m.Date)
-            .OrderBy(m => m.Key)
-            .Select(m => m.Key)
-            .ToArray();
-        var labelStrings = labels.Select(l => l.ToString("MM-dd-yyy")).ToList();
-        var results = metrics
-            .GroupBy(m => m.Subject)
-            .Select(g =>
-            {
-                var indexed = g.ToDictionary(m => m.Date, m => m.Count);
-                return new MetricResult
-                {
-                    SeriesName = g.Key,
-                    Labels = labelStrings,
-                    Values = labels.Select(l => indexed.TryGetValue(l, out var found) ? found : 0).ToList()
-                };
-            });
-        return Ok(results.ToList());
-    }
-
-    [HttpGet]
-    [Route("badge/{name}/total_installs_badge.json")]
-    public async Task<IActionResult> TotalInstallsBadge(string name)
-    {
-        var results = await _sql.TotalInstalls(name);
-
-        Response.ContentType = "application/json";
-
-        return Ok(results == 0
-            ? new Badge($"Modlist {name} not found!", "Error") {color = "red"}
-            : new Badge("Installations: ", "____") {color = "green"});
-    }
-
-    [HttpGet]
-    [Route("badge/{name}/unique_installs_badge.json")]
-    public async Task<IActionResult> UniqueInstallsBadge(string name)
-    {
-        var results = await _sql.UniqueInstalls(name);
-
-        Response.ContentType = "application/json";
-
-        return Ok(results == 0
-            ? new Badge($"Modlist {name} not found!", "Error") {color = "red"}
-            : new Badge("Installations: ", "____") {color = "green"});
-    }
-
-    [HttpGet]
-    [Route("tarlog/{key}")]
-    public async Task<IActionResult> TarLog(string key)
-    {
-        var isTarKey = await _sql.IsTarKey(key);
-
-        var report = new List<(DateTime, string, string)>();
-
-        if (isTarKey) report = await _sql.FullTarReport(key);
-
-        var response = ReportTemplate(new
-        {
-            key,
-            status = isTarKey ? "BANNED" : "NOT BANNED",
-            log = report.Select(entry => new
-            {
-                Timestamp = entry.Item1,
-                Path = entry.Item2,
-                Key = entry.Item3
-            }).ToList()
-        });
-        return new ContentResult
-        {
-            ContentType = "text/html",
-            StatusCode = (int) HttpStatusCode.OK,
-            Content = response
-        };
-    }
-
-    private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
-    {
-        //_logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}");
-        await _sql.IngestMetric(new Metric
-        {
-            Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey
-        });
-    }
-
-
-    [HttpGet("total_installs.html")]
-    [ResponseCache(Duration = 60 * 60)]
-    public async Task<ContentResult> TotalInstalls()
-    {
-        var data = await _sql.GetTotalInstalls();
-        var result = TotalListTemplate(new TotalListTemplateData
-        {
-            Title = "Total Installs",
-            Total = data.Sum(d => d.Item2),
-            Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2})
-                .ToArray()
-        });
-        return new ContentResult
-        {
-            ContentType = "text/html",
-            StatusCode = (int) HttpStatusCode.OK,
-            Content = result
-        };
-    }
-
-    [HttpGet("total_unique_installs.html")]
-    [ResponseCache(Duration = 60 * 60)]
-    public async Task<ContentResult> TotalUniqueInstalls()
-    {
-        var data = await _sql.GetTotalUniqueInstalls();
-        var result = TotalListTemplate(new TotalListTemplateData
-        {
-            Title = "Total Unique Installs",
-            Total = data.Sum(d => d.Item2),
-            Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2})
-                .ToArray()
-        });
-        return new ContentResult
-        {
-            ContentType = "text/html",
-            StatusCode = (int) HttpStatusCode.OK,
-            Content = result
-        };
-    }
-
-    [HttpGet("dump.json")]
-    public async Task<IActionResult> DataDump()
-    {
-        return Ok(await _sql.MetricsDump().ToArrayAsync());
-    }
-
     public class Result
     {
         public DateTime Timestamp { get; set; }
     }
-
-    private class TotalListTemplateData
-    {
-        public string Title { get; set; }
-        public long Total { get; set; }
-        public Item[] Items { get; set; }
-
-        public class Item
-        {
-            public long Count { get; set; }
-            public string Title { get; set; }
-        }
-    }
 }

Wabbajack.Server/Controllers/ModUpgrade.cs

@@ -1,158 +0,0 @@
-using System;
-using System.Linq;
-using System.Security.Claims;
-using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.Logging;
-using Wabbajack.Downloaders;
-using Wabbajack.DTOs.JsonConverters;
-using Wabbajack.DTOs.ServerResponses;
-using Wabbajack.Hashing.xxHash64;
-using Wabbajack.Server.DataLayer;
-using Wabbajack.Server.DTOs;
-using Wabbajack.Server.Services;
-using Wabbajack.Server.TokenProviders;
-
-namespace Wabbajack.BuildServer.Controllers;
-
-[ApiController]
-public class ModUpgrade : ControllerBase
-{
-    private readonly DownloadDispatcher _dispatcher;
-    private readonly DTOSerializer _dtos;
-    private readonly IFtpSiteCredentials _ftpSite;
-    private readonly ILogger<ModUpgrade> _logger;
-    private readonly QuickSync _quickSync;
-    private readonly AppSettings _settings;
-    private readonly SqlService _sql;
-
-    public ModUpgrade(ILogger<ModUpgrade> logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync,
-        AppSettings settings, DTOSerializer dtos,
-        DownloadDispatcher dispatcher, IFtpSiteCredentials ftp)
-    {
-        _logger = logger;
-        _sql = sql;
-        _settings = settings;
-        _quickSync = quickSync;
-        _ftpSite = ftp;
-        _dtos = dtos;
-        _dispatcher = dispatcher;
-    }
-
-    [HttpPost]
-    [Authorize(Roles = "User")]
-    [Route("/mod_upgrade")]
-    public async Task<IActionResult> PostModUpgrade(CancellationToken token)
-    {
-        var isAuthor = User.Claims.Any(c => c.Type == ClaimTypes.Role && c.Value == "Author");
-        var request = await _dtos.DeserializeAsync<ModUpgradeRequest>(Request.Body);
-        if (!isAuthor)
-        {
-            var srcDownload = await _sql.GetArchiveDownload(request!.OldArchive.State.PrimaryKeyString,
-                request.OldArchive.Hash, request.OldArchive.Size);
-            var destDownload = await _sql.GetArchiveDownload(request.NewArchive.State.PrimaryKeyString,
-                request.NewArchive.Hash, request.NewArchive.Size);
-
-            if (srcDownload == default || destDownload == default ||
-                await _sql.FindPatch(srcDownload.Id, destDownload.Id) == default)
-            {
-                if (!await _dispatcher.IsAllowed(request, token))
-                {
-                    _logger.Log(LogLevel.Information,
-                        $"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as upgrade is invalid");
-                    return BadRequest("Invalid mod upgrade");
-                }
-
-                if (_settings.ValidateModUpgrades && !await _sql.HashIsInAModlist(request.OldArchive.Hash))
-                {
-                    _logger.Log(LogLevel.Information,
-                        $"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as src hash is not in a curated modlist");
-                    return BadRequest("Hash is not in a recent modlist");
-                }
-            }
-        }
-
-        try
-        {
-            if (await _dispatcher.Verify(request!.OldArchive, token))
-                //_logger.LogInformation(
-                //    $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), old archive is valid");
-                return NotFound("File is Valid");
-        }
-        catch (Exception)
-        {
-            //_logger.LogInformation(
-            //     $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), due to upgrade failure");
-            return NotFound("File is Valid");
-        }
-
-        var oldDownload = await _sql.GetOrEnqueueArchive(request.OldArchive);
-
-        if (await _sql.IsNoPatch(oldDownload.Archive.Hash)) return BadRequest("File has NoPatch attached");
-
-        var newDownload = await _sql.GetOrEnqueueArchive(request.NewArchive);
-
-        var patch = await _sql.FindOrEnqueuePatch(oldDownload.Id, newDownload.Id);
-        if (patch.Finished.HasValue)
-        {
-            if (patch.PatchSize != 0)
-            {
-                //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch Found");
-                var host = (await _ftpSite.Get())[StorageSpace.Patches].Username == "wabbajacktest"
-                    ? "test-files"
-                    : "patches";
-                await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id);
-                return
-                    Ok(
-                        $"https://{host}.wabbajack.org/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
-            }
-            //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed");
-
-            return NotFound("Patch creation failed");
-        }
-
-        if (!newDownload.DownloadFinished.HasValue)
-            await _quickSync.Notify<ArchiveDownloader>();
-        else
-            await _quickSync.Notify<PatchBuilder>();
-
-        //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found is processing");
-        // Still processing
-        return Accepted();
-    }
-
-    [HttpGet]
-    [Authorize(Roles = "User")]
-    [Route("/mod_upgrade/find/{hashAsHex}")]
-    public async Task<IActionResult> FindUpgrade(string hashAsHex)
-    {
-        var hash = Hash.FromHex(hashAsHex);
-
-        var patches = await _sql.PatchesForSource(hash);
-        return Ok(_dtos.Serialize(patches.Select(p => p.Dest.Archive).ToArray()));
-    }
-
-    [HttpGet]
-    [Authorize(Roles = "Author")]
-    [Route("/mod_upgrade/no_patch/{hashAsHex}/{rationaleAsHex}")]
-    public async Task<IActionResult> PurgePatch(string hashAsHex, string rationaleAsHex)
-    {
-        var hash = Hash.FromHex(hashAsHex);
-        var rationale = Encoding.UTF8.GetString(rationaleAsHex.FromHex());
-        await _sql.PurgePatch(hash, rationale);
-        return Ok("Purged");
-    }
-
-    [HttpGet]
-    [Authorize(Roles = "User")]
-    [Route("/mirror/{hashAsHex}")]
-    public async Task<IActionResult> HaveHash(string hashAsHex)
-    {
-        var result = await _sql.HaveMirror(Hash.FromHex(hashAsHex));
-        if (result) return Ok($"https://{(await _ftpSite.Get())[StorageSpace.Mirrors].Username}.b-cdn.net/{hashAsHex}");
-        return NotFound("Not Mirrored");
-    }
-}

Wabbajack.Server/DTOs/AggregateMetric.cs

@@ -1,10 +0,0 @@
-using System;
-
-namespace Wabbajack.Server.DTOs;
-
-public class AggregateMetric
-{
-    public DateTime Date { get; set; }
-    public string Subject { get; set; }
-    public int Count { get; set; }
-}

Wabbajack.Server/DTOs/ArchiveDownload.cs

@@ -1,30 +0,0 @@
-using System;
-using System.Threading.Tasks;
-using Wabbajack.DTOs;
-using Wabbajack.Server.DataLayer;
-
-namespace Wabbajack.Server.DTOs;
-
-public class ArchiveDownload
-{
-    public Guid Id { get; set; }
-    public Archive Archive { get; set; }
-    public bool? IsFailed { get; set; }
-    public DateTime? DownloadFinished { get; set; }
-    public string FailMessage { get; set; }
-
-    public async Task Fail(SqlService service, string message)
-    {
-        IsFailed = true;
-        DownloadFinished = DateTime.UtcNow;
-        FailMessage = message;
-        await service.UpdatePendingDownload(this);
-    }
-
-    public async Task Finish(SqlService service)
-    {
-        IsFailed = false;
-        DownloadFinished = DateTime.UtcNow;
-        await service.UpdatePendingDownload(this);
-    }
-}

Wabbajack.Server/DTOs/AuthoredFilesSummary.cs

@@ -1,14 +0,0 @@
-using System;
-
-namespace Wabbajack.Server.DTOs;
-
-public class AuthoredFilesSummary
-{
-    public long Size { get; set; }
-    public string OriginalFileName { get; set; }
-    public string Author { get; set; }
-    public DateTime LastTouched { get; set; }
-    public DateTime? Finalized { get; set; }
-    public string MungedName { get; set; }
-    public string ServerAssignedUniqueId { get; set; }
-}

Wabbajack.Server/DTOs/FtpSite.cs

@@ -1,31 +0,0 @@
-using System.Net;
-using System.Threading.Tasks;
-using FluentFTP;
-using Microsoft.Extensions.Logging;
-using Wabbajack.Common;
-
-namespace Wabbajack.Server.DTOs;
-
-public enum StorageSpace
-{
-    AuthoredFiles,
-    Patches,
-    Mirrors
-}
-
-public class FtpSite
-{
-    public string Username { get; set; }
-    public string Password { get; set; }
-    public string Hostname { get; set; }
-
-    public async Task<FtpClient> GetClient(ILogger logger)
-    {
-        return await CircuitBreaker.WithAutoRetryAllAsync(logger, async () =>
-        {
-            var ftpClient = new FtpClient(Hostname, new NetworkCredential(Username, Password));
-            await ftpClient.ConnectAsync();
-            return ftpClient;
-        });
-    }
-}

Wabbajack.Server/DTOs/Metric.cs

@@ -1,4 +1,5 @@
 using System;
+using Microsoft.Extensions.Primitives;
 
 namespace Wabbajack.Server.DTOs;
 
@@ -8,4 +9,5 @@ public class Metric
     public string Action { get; set; }
     public string Subject { get; set; }
     public string MetricsKey { get; set; }
+    public string UserAgent { get; set; }
 }

Wabbajack.Server/DTOs/MetricResult.cs

@@ -1,10 +0,0 @@
-using System.Collections.Generic;
-
-namespace Wabbajack.Server.DTOs;
-
-public class MetricResult
-{
-    public string SeriesName { get; set; }
-    public List<string> Labels { get; set; }
-    public List<int> Values { get; set; }
-}

Wabbajack.Server/DTOs/MirroredFile.cs

@@ -1,29 +0,0 @@
-using System;
-using System.Threading.Tasks;
-using Wabbajack.Hashing.xxHash64;
-using Wabbajack.Server.DataLayer;
-
-namespace Wabbajack.Server.DTOs;
-
-public class MirroredFile
-{
-    public Hash Hash { get; set; }
-    public DateTime Created { get; set; }
-    public DateTime? Uploaded { get; set; }
-    public string Rationale { get; set; }
-
-    public string FailMessage { get; set; }
-
-    public async Task Finish(SqlService sql)
-    {
-        Uploaded = DateTime.UtcNow;
-        await sql.UpsertMirroredFile(this);
-    }
-
-    public async Task Fail(SqlService sql, string message)
-    {
-        Uploaded = DateTime.UtcNow;
-        FailMessage = message;
-        await sql.UpsertMirroredFile(this);
-    }
-}

Wabbajack.Server/DTOs/Patch.cs

@@ -1,32 +0,0 @@
-using System;
-using System.Threading.Tasks;
-using Wabbajack.Server.DataLayer;
-
-namespace Wabbajack.Server.DTOs;
-
-public class Patch
-{
-    public ArchiveDownload Src { get; set; }
-    public ArchiveDownload Dest { get; set; }
-    public long PatchSize { get; set; }
-    public DateTime? Finished { get; set; }
-    public bool? IsFailed { get; set; }
-    public string FailMessage { get; set; }
-
-    public async Task Finish(SqlService sql, long size)
-    {
-        IsFailed = false;
-        Finished = DateTime.UtcNow;
-        PatchSize = size;
-        await sql.FinializePatch(this);
-    }
-
-
-    public async Task Fail(SqlService sql, string msg)
-    {
-        IsFailed = true;
-        Finished = DateTime.UtcNow;
-        FailMessage = msg;
-        await sql.FinializePatch(this);
-    }
-}

Wabbajack.Server/DTOs/ValidationData.cs

@@ -1,17 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Threading.Tasks;
-using Wabbajack.DTOs;
-using Wabbajack.Hashing.xxHash64;
-
-namespace Wabbajack.Server.DTOs;
-
-public class ValidationData
-{
-    public Dictionary<(long Game, long ModId, long FileId), string> NexusFiles { get; set; } = new();
-    public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; }
-    public List<ModlistMetadata> ModLists { get; set; }
-    public Dictionary<Hash, bool> Mirrors { get; set; }
-    public Lazy<Task<Dictionary<Hash, string>>> AllowedMirrors { get; set; }
-    public IEnumerable<AuthoredFilesSummary> AllAuthoredFiles { get; set; }
-}

Wabbajack.Server/DataLayer/ApiKeys.cs

@@ -1,54 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Dapper;
-using Wabbajack.Hashing.xxHash64;
-
-namespace Wabbajack.Server.DataLayer;
-
-public partial class SqlService
-{
-    public async Task<string> LoginByApiKey(string key)
-    {
-        await using var conn = await Open();
-        var result = await conn.QueryAsync<string>(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @ApiKey",
-            new {ApiKey = key});
-        return result.FirstOrDefault();
-    }
-
-    public async Task<string> AddLogin(string name)
-    {
-        var key = NewAPIKey();
-        await using var conn = await Open();
-
-
-        await conn.ExecuteAsync("INSERT INTO dbo.ApiKeys (Owner, ApiKey) VALUES (@Owner, @ApiKey)",
-            new {Owner = name, ApiKey = key});
-        return key;
-    }
-
-
-    public static string NewAPIKey()
-    {
-        var arr = new byte[128];
-        new Random().NextBytes(arr);
-        return arr.ToHex();
-    }
-
-    public async Task<IEnumerable<(string Owner, string Key)>> GetAllUserKeys()
-    {
-        await using var conn = await Open();
-        var result = await conn.QueryAsync<(string Owner, string Key)>("SELECT Owner, ApiKey FROM dbo.ApiKeys");
-        return result;
-    }
-
-
-    public async Task<bool> IsTarKey(string metricsKey)
-    {
-        await using var conn = await Open();
-        var result = await conn.QueryFirstOrDefaultAsync<string>(
-            "SELECT MetricsKey FROM TarKey WHERE MetricsKey = @MetricsKey", new {MetricsKey = metricsKey});
-        return result == metricsKey;
-    }
-}

Wabbajack.Server/DataLayer/ArchiveDownloads.cs

@@ -1,261 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Dapper;
-using Wabbajack.DTOs;
-using Wabbajack.DTOs.DownloadStates;
-using Wabbajack.Hashing.xxHash64;
-using Wabbajack.Server.DTOs;
-
-namespace Wabbajack.Server.DataLayer;
-
-public partial class SqlService
-{
-    public async Task<Guid> AddKnownDownload(Archive a, DateTime downloadFinished)
-    {
-        await using var conn = await Open();
-        var Id = Guid.NewGuid();
-        await conn.ExecuteAsync(
-            "INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader, DownloadFinished, IsFailed) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader, @DownloadFinished, @IsFailed)",
-            new
-            {
-                Id,
-                a.State.PrimaryKeyString,
-                Size = a.Size == 0 ? null : (long?) a.Size,
-                Hash = a.Hash == default ? null : (Hash?) a.Hash,
-                DownloadState = a.State,
-                Downloader = a.State.GetType().ToString(),
-                DownloadFinished = downloadFinished,
-                IsFailed = false
-            });
-        return Id;
-    }
-
-    public async Task<Guid> EnqueueDownload(Archive a)
-    {
-        await using var conn = await Open();
-        var Id = Guid.NewGuid();
-        await conn.ExecuteAsync(
-            "INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)",
-            new
-            {
-                Id,
-                a.State.PrimaryKeyString,
-                Size = a.Size == 0 ? null : (long?) a.Size,
-                Hash = a.Hash == default ? null : (Hash?) a.Hash,
-                DownloadState = a.State,
-                Downloader = a.State.GetType().ToString()
-            });
-        return Id;
-    }
-
-    public async Task<HashSet<(Hash Hash, string PrimaryKeyString)>> GetAllArchiveDownloads()
-    {
-        await using var conn = await Open();
-        return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, PrimaryKeyString FROM ArchiveDownloads"))
-            .ToHashSet();
-    }
-
-    public async Task<HashSet<(Hash Hash, IDownloadState State)>> GetAllArchiveDownloadStates()
-    {
-        await using var conn = await Open();
-        return (await conn.QueryAsync<(Hash, IDownloadState)>("SELECT Hash, DownloadState FROM ArchiveDownloads"))
-            .ToHashSet();
-    }
-
-
-    public async Task<ArchiveDownload> GetArchiveDownload(Guid id)
-    {
-        await using var conn = await Open();
-        var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, IDownloadState, DateTime?)>(
-            "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE Id = @id",
-            new {Id = id});
-        if (result == default)
-            return null;
-
-        return new ArchiveDownload
-        {
-            Id = result.Item1,
-            IsFailed = result.Item4,
-            DownloadFinished = result.Item6,
-            Archive = new Archive {State = result.Item5, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
-        };
-    }
-
-    public async Task<ArchiveDownload> GetArchiveDownload(string primaryKeyString)
-    {
-        await using var conn = await Open();
-        var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, IDownloadState, DateTime?)>(
-            "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND IsFailed = 0",
-            new {PrimaryKeyString = primaryKeyString});
-        if (result == default)
-            return null;
-
-        return new ArchiveDownload
-        {
-            Id = result.Item1,
-            IsFailed = result.Item4,
-            DownloadFinished = result.Item6,
-            Archive = new Archive {State = result.Item5, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
-        };
-    }
-
-    public async Task<ArchiveDownload> GetArchiveDownload(string primaryKeyString, Hash hash, long size)
-    {
-        await using var conn = await Open();
-        var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, IDownloadState, DateTime?)>(
-            "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size",
-            new
-            {
-                PrimaryKeyString = primaryKeyString,
-                Hash = hash,
-                Size = size
-            });
-        if (result == default)
-            return null;
-
-        return new ArchiveDownload
-        {
-            Id = result.Item1,
-            IsFailed = result.Item4,
-            DownloadFinished = result.Item6,
-            Archive = new Archive {State = result.Item5, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
-        };
-    }
-
-
-    public async Task<ArchiveDownload> GetOrEnqueueArchive(Archive a)
-    {
-        await using var conn = await Open();
-        await using var trans = await conn.BeginTransactionAsync();
-        var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, IDownloadState, DateTime?)>(
-            "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size",
-            new
-            {
-                a.State.PrimaryKeyString,
-                a.Hash,
-                a.Size
-            }, trans);
-        if (result.Item1 != default)
-            return new ArchiveDownload
-            {
-                Id = result.Item1,
-                IsFailed = result.Item4,
-                DownloadFinished = result.Item6,
-                Archive = new Archive {State = result.Item5, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
-            };
-
-        var id = Guid.NewGuid();
-        await conn.ExecuteAsync(
-            "INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)",
-            new
-            {
-                Id = id,
-                a.State.PrimaryKeyString,
-                Size = a.Size == 0 ? null : (long?) a.Size,
-                Hash = a.Hash == default ? null : (Hash?) a.Hash,
-                DownloadState = a.State,
-                Downloader = ""
-            }, trans);
-
-        await trans.CommitAsync();
-
-        return new ArchiveDownload {Id = id, Archive = a};
-    }
-
-    public async Task<ArchiveDownload> GetNextPendingDownload(bool ignoreNexus = false)
-    {
-        await using var conn = await Open();
-        (Guid, long?, Hash?, IDownloadState) result;
-
-        if (ignoreNexus)
-            result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, IDownloadState)>(
-                "SELECT TOP(1) Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL AND Downloader != 'NexusDownloader+State'");
-        else
-            result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, IDownloadState)>(
-                "SELECT TOP(1) Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL");
-
-        if (result == default)
-            return null;
-
-        return new ArchiveDownload
-        {
-            Id = result.Item1,
-            Archive = new Archive {State = result.Item4, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
-        };
-    }
-
-    public async Task UpdatePendingDownload(ArchiveDownload ad)
-    {
-        await using var conn = await Open();
-        await conn.ExecuteAsync(
-            "UPDATE dbo.ArchiveDownloads SET IsFailed = @IsFailed, DownloadFinished = @DownloadFinished, Hash = @Hash, Size = @Size, FailMessage = @FailMessage WHERE Id = @Id",
-            new
-            {
-                ad.Id,
-                ad.IsFailed,
-                ad.DownloadFinished,
-                ad.Archive.Size,
-                ad.Archive.Hash,
-                ad.FailMessage
-            });
-    }
-
-    public async Task<int> EnqueueModListFilesForIndexing()
-    {
-        await using var conn = await Open();
-        return await conn.ExecuteAsync(@"
-            INSERT INTO dbo.ArchiveDownloads (Id, PrimaryKeyString, Hash, DownloadState, Size, Downloader)
-            SELECT DISTINCT NEWID(), mla.PrimaryKeyString, mla.Hash, mla.State, mla.Size, SUBSTRING(mla.PrimaryKeyString, 0, CHARINDEX('|', mla.PrimaryKeyString))
-            FROM [dbo].[ModListArchives] mla
-                LEFT JOIN dbo.ArchiveDownloads ad on mla.PrimaryKeyString = ad.PrimaryKeyString AND mla.Hash = ad.Hash
-            WHERE ad.PrimaryKeyString is null");
-    }
-
-    public async Task<List<Archive>> GetGameFiles(Game game, string version)
-    {
-        await using var conn = await Open();
-        var files = (await conn.QueryAsync<(Hash, long, IDownloadState)>(
-                $"SELECT Hash, Size, DownloadState FROM dbo.ArchiveDownloads WHERE PrimaryKeyString like 'GameFileSourceDownloader+State|{game}|{version}|%'"))
-            .Select(f => new Archive
-            {
-                State = f.Item3,
-                Hash = f.Item1,
-                Size = f.Item2
-            }).ToList();
-        return files;
-    }
-
-    public async Task<Archive[]> ResolveDownloadStatesByHash(Hash hash)
-    {
-        await using var conn = await Open();
-        var files = (await conn.QueryAsync<(long, Hash, IDownloadState)>(
-                @"SELECT Size, Hash,  DownloadState from dbo.ArchiveDownloads WHERE Hash = @Hash AND IsFailed = 0 AND DownloadFinished IS NOT NULL ORDER BY DownloadFinished DESC",
-                new {Hash = hash})
-            ).Select(e =>
-                new Archive {State = e.Item3, Size = e.Item1, Hash = e.Item2}
-            ).ToList();
-
-        if (await HaveMirror(hash) && files.Count > 0)
-        {
-            var ffile = files.First();
-            var host = _settings.TestMode ? "test-files" : "mirror";
-            var url = new Uri($"https://{host}.wabbajack.org/{hash.ToHex()}");
-            files.Add(new Archive
-                {State = new WabbajackCDN {Url = url}, Hash = hash, Size = ffile.Size, Name = ffile.Name});
-        }
-
-        return files.ToArray();
-    }
-
-    public async Task<IEnumerable<(Game, string)>> GetAllRegisteredGames()
-    {
-        await using var conn = await Open();
-        var pks = await conn.QueryAsync<string>(
-            @"SELECT PrimaryKeyString FROM dbo.ArchiveDownloads WHERE PrimaryKeyString like 'GameFileSourceDownloader+State|%'");
-        return pks.Select(p => p.Split("|"))
-            .Select(t => (GameRegistry.GetByFuzzyName(t[1]).Game, t[2]))
-            .Distinct();
-    }
-}

Wabbajack.Server/DataLayer/AuthoredFiles.cs

@@ -1,76 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Dapper;
-using Wabbajack.DTOs.CDN;
-using Wabbajack.Server.DTOs;
-
-namespace Wabbajack.Server.DataLayer;
-
-public partial class SqlService
-{
-    public async Task TouchAuthoredFile(FileDefinition definition, DateTime? date = null)
-    {
-        await using var conn = await Open();
-        if (date == null)
-            await conn.ExecuteAsync(
-                "UPDATE AuthoredFiles SET LastTouched = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid",
-                new {Uid = definition.ServerAssignedUniqueId});
-        else
-            await conn.ExecuteAsync(
-                "UPDATE AuthoredFiles SET LastTouched = @Date WHERE ServerAssignedUniqueId = @Uid",
-                new {Uid = definition.ServerAssignedUniqueId, Date = date});
-    }
-
-    public async Task<FileDefinition> CreateAuthoredFile(FileDefinition definition, string login)
-    {
-        definition.Author = login;
-        var uid = Guid.NewGuid().ToString();
-        await using var conn = await Open();
-        definition.ServerAssignedUniqueId = uid;
-        await conn.ExecuteAsync(
-            "INSERT INTO dbo.AuthoredFiles (ServerAssignedUniqueId, LastTouched, CDNFileDefinition) VALUES (@Uid, GETUTCDATE(), @CdnFile)",
-            new
-            {
-                Uid = uid,
-                CdnFile = definition
-            });
-        return definition;
-    }
-
-    public async Task Finalize(FileDefinition definition)
-    {
-        await using var conn = await Open();
-        await conn.ExecuteAsync(
-            "UPDATE AuthoredFiles SET LastTouched = GETUTCDATE(), Finalized = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid",
-            new
-            {
-                Uid = definition.ServerAssignedUniqueId
-            });
-    }
-
-    public async Task<FileDefinition> GetCDNFileDefinition(string serverAssignedUniqueId)
-    {
-        await using var conn = await Open();
-        return (await conn.QueryAsync<FileDefinition>(
-            "SELECT CDNFileDefinition FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid",
-            new {Uid = serverAssignedUniqueId})).First();
-    }
-
-    public async Task DeleteFileDefinition(FileDefinition definition)
-    {
-        await using var conn = await Open();
-        await conn.ExecuteAsync(
-            "DELETE FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid",
-            new {Uid = definition.ServerAssignedUniqueId});
-    }
-
-    public async Task<IEnumerable<AuthoredFilesSummary>> AllAuthoredFiles()
-    {
-        await using var conn = await Open();
-        var results = await conn.QueryAsync<AuthoredFilesSummary>(
-            "SELECT CONVERT(NVARCHAR(50), ServerAssignedUniqueId) as ServerAssignedUniqueId, Size, OriginalFileName, Author, LastTouched, Finalized, MungedName from dbo.AuthoredFilesSummaries ORDER BY LastTouched DESC");
-        return results;
-    }
-}

Wabbajack.Server/DataLayer/Mappers.cs

@@ -1,109 +0,0 @@
-using System;
-using System.Data;
-using Dapper;
-using Wabbajack.DTOs;
-using Wabbajack.DTOs.CDN;
-using Wabbajack.DTOs.DownloadStates;
-using Wabbajack.DTOs.JsonConverters;
-using Wabbajack.Hashing.xxHash64;
-using Wabbajack.Paths;
-
-namespace Wabbajack.Server.DataLayer;
-
-public partial class SqlService
-{
-    private static DTOSerializer _dtoStatic;
-
-    static SqlService()
-    {
-        SqlMapper.AddTypeHandler(new HashMapper());
-        SqlMapper.AddTypeHandler(new RelativePathMapper());
-        SqlMapper.AddTypeHandler(new JsonMapper<IDownloadState>());
-        SqlMapper.AddTypeHandler(new JsonMapper<FileDefinition>());
-        SqlMapper.AddTypeHandler(new JsonMapper<ModlistMetadata>());
-        SqlMapper.AddTypeHandler(new VersionMapper());
-        SqlMapper.AddTypeHandler(new GameMapper());
-        SqlMapper.AddTypeHandler(new DateTimeHandler());
-    }
-
-    /// <summary>
-    ///     Needed to make sure dates are all in UTC format
-    /// </summary>
-    private class DateTimeHandler : SqlMapper.TypeHandler<DateTime>
-    {
-        public override void SetValue(IDbDataParameter parameter, DateTime value)
-        {
-            parameter.Value = value;
-        }
-
-        public override DateTime Parse(object value)
-        {
-            return DateTime.SpecifyKind((DateTime) value, DateTimeKind.Utc);
-        }
-    }
-
-    private class JsonMapper<T> : SqlMapper.TypeHandler<T>
-    {
-        public override void SetValue(IDbDataParameter parameter, T value)
-        {
-            parameter.Value = _dtoStatic.Serialize(value);
-        }
-
-        public override T Parse(object value)
-        {
-            return _dtoStatic.Deserialize<T>((string) value)!;
-        }
-    }
-
-    private class RelativePathMapper : SqlMapper.TypeHandler<RelativePath>
-    {
-        public override void SetValue(IDbDataParameter parameter, RelativePath value)
-        {
-            parameter.Value = value.ToString();
-        }
-
-        public override RelativePath Parse(object value)
-        {
-            return (RelativePath) (string) value;
-        }
-    }
-
-    private class HashMapper : SqlMapper.TypeHandler<Hash>
-    {
-        public override void SetValue(IDbDataParameter parameter, Hash value)
-        {
-            parameter.Value = (long) value;
-        }
-
-        public override Hash Parse(object value)
-        {
-            return Hash.FromLong((long) value);
-        }
-    }
-
-    private class VersionMapper : SqlMapper.TypeHandler<Version>
-    {
-        public override void SetValue(IDbDataParameter parameter, Version value)
-        {
-            parameter.Value = value.ToString();
-        }
-
-        public override Version Parse(object value)
-        {
-            return Version.Parse((string) value);
-        }
-    }
-
-    private class GameMapper : SqlMapper.TypeHandler<Game>
-    {
-        public override void SetValue(IDbDataParameter parameter, Game value)
-        {
-            parameter.Value = value.ToString();
-        }
-
-        public override Game Parse(object value)
-        {
-            return GameRegistry.GetByFuzzyName((string) value).Game;
-        }
-    }
-}

Wabbajack.Server/DataLayer/Metrics.cs

@@ -1,189 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Dapper;
-using Wabbajack.Server.DTOs;
-
-namespace Wabbajack.Server.DataLayer;
-
-public partial class SqlService
-{
-    public async Task IngestMetric(Metric metric)
-    {
-        await using var conn = await Open();
-        await conn.ExecuteAsync(
-            @"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)",
-            metric);
-    }
-
-    public async Task IngestAccess(string ip, string log)
-    {
-        await using var conn = await Open();
-        await conn.ExecuteAsync(@"INSERT INTO dbo.AccessLog (Timestamp, Action, Ip) VALUES (@Timestamp, @Action, @Ip)",
-            new
-            {
-                Timestamp = DateTime.UtcNow,
-                Ip = ip,
-                Action = log
-            });
-    }
-
-    public async Task<IEnumerable<AggregateMetric>> MetricsReport(string action)
-    {
-        await using var conn = await Open();
-        return (await conn.QueryAsync<AggregateMetric>(@"
-                select
-                datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)) as Date, 
-                GroupingSubject as Subject,
-                count(*) as Count 
-                from dbo.metrics where 
-                Action = @Action
-                AND GroupingSubject in  (select DISTINCT GroupingSubject from dbo.Metrics
-                 WHERE action = @Action
-                 AND MetricsKey is not null 
-                AND Subject != 'Default'
-                AND Subject != 'untitled'
-                AND TRY_CONVERT(uniqueidentifier, Subject) is null
-                 AND Timestamp >= DATEADD(DAY, -1, GETUTCDATE()))
-                group by
-                 datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)),
-                 GroupingSubject
-                 Order by  datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)) asc",
-                new {Action = action}))
-            .ToList();
-    }
-
-    public async Task<List<(DateTime, string, string)>> FullTarReport(string key)
-    {
-        await using var conn = await Open();
-        return (await conn.QueryAsync<(DateTime, string, string)>(@"
-                                SELECT u.Timestamp, u.Path, u.MetricsKey FROM
-                                (SELECT al.Timestamp, JSON_VALUE(al.Action, '$.Path') as Path, al.MetricsKey FROM dbo.AccessLog al
-                                WHERE al.MetricsKey = @MetricsKey
-                                UNION ALL
-                                SELECT m.Timestamp, m.Action + ' ' + m.Subject as Path, m.MetricsKey FROM dbo.Metrics m
-                                WHERE m.MetricsKey = @MetricsKey
-                                AND m.Action != 'TarKey') u
-                                ORDER BY u.Timestamp Desc",
-            new {MetricsKey = key})).ToList();
-    }
-
-    public async Task<bool> ValidMetricsKey(string metricsKey)
-    {
-        await using var conn = await Open();
-        return await conn.QuerySingleOrDefaultAsync<string>(
-            "SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey",
-            new {MetricsKey = metricsKey}) != default;
-    }
-
-    public async Task AddMetricsKey(string metricsKey)
-    {
-        await using var conn = await Open();
-        await using var trans = conn.BeginTransaction();
-
-        if (await conn.QuerySingleOrDefaultAsync<string>(
-                "SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey",
-                new {MetricsKey = metricsKey}, trans) != default)
-            return;
-
-        await conn.ExecuteAsync("INSERT INTO dbo.MetricsKeys (MetricsKey) VALUES (@MetricsKey)",
-            new {MetricsKey = metricsKey}, trans);
-    }
-
-    public async Task<string[]> AllKeys()
-    {
-        await using var conn = await Open();
-        return (await conn.QueryAsync<string>("SELECT MetricsKey from dbo.MetricsKeys")).ToArray();
-    }
-
-
-    public async Task<long> UniqueInstalls(string machineUrl)
-    {
-        await using var conn = await Open();
-        return await conn.QueryFirstAsync<long>(
-            @"SELECT COUNT(*) FROM (
-                        SELECT DISTINCT MetricsKey from dbo.Metrics where Action = 'finish_install' and GroupingSubject in (
-                        SELECT JSON_VALUE(Metadata, '$.title') FROM dbo.ModLists
-                        WHERE JSON_VALUE(Metadata, '$.links.machineURL') = @MachineURL)) s",
-            new {MachineURL = machineUrl});
-    }
-
-    public async Task<long> TotalInstalls(string machineUrl)
-    {
-        await using var conn = await Open();
-        return await conn.QueryFirstAsync<long>(
-            @"SELECT COUNT(*) from dbo.Metrics where Action = 'finish_install' and GroupingSubject in (
-                        SELECT JSON_VALUE(Metadata, '$.title') FROM dbo.ModLists
-                        WHERE JSON_VALUE(Metadata, '$.links.machineURL') = @MachineURL)",
-            new {MachineURL = machineUrl});
-    }
-
-    public async Task<IEnumerable<(string, long)>> GetTotalInstalls()
-    {
-        await using var conn = await Open();
-        return await conn.QueryAsync<(string, long)>(
-            @"SELECT GroupingSubject, Count(*) as Count
-                        From dbo.Metrics
-                        WHERE 
-
-                        GroupingSubject in  (select DISTINCT GroupingSubject from dbo.Metrics
-                            WHERE action = 'finish_install'
-                            AND MetricsKey is not null)
-	                        group by GroupingSubject
-	                        order by Count(*) desc");
-    }
-
-    public async Task<IEnumerable<(string, long)>> GetTotalUniqueInstalls()
-    {
-        await using var conn = await Open();
-        return await conn.QueryAsync<(string, long)>(
-            @"Select GroupingSubject, Count(*) as Count
-                        FROM
-                        (select DISTINCT MetricsKey, GroupingSubject
-                        From dbo.Metrics
-                        WHERE 
-                        GroupingSubject in  (select DISTINCT GroupingSubject from dbo.Metrics
-                            WHERE action = 'finish_install'
-                            AND MetricsKey is not null)) m
-                        GROUP BY GroupingSubject
-                        Order by Count(*) desc
-                        ");
-    }
-
-    public async IAsyncEnumerable<MetricRow> MetricsDump()
-    {
-        var keys = new Dictionary<string, long>();
-
-        await using var conn = await Open();
-        foreach (var row in await conn.QueryAsync<(long, DateTime, string, string, string, string)>(
-            @"select Id, Timestamp, Action, Subject, MetricsKey, GroupingSubject from dbo.metrics WHERE MetricsKey is not null"))
-        {
-            if (!keys.TryGetValue(row.Item5, out var keyid))
-            {
-                keyid = keys.Count;
-                keys[row.Item5] = keyid;
-            }
-
-            yield return new MetricRow
-            {
-                Id = row.Item1,
-                Timestamp = row.Item2,
-                Action = row.Item3,
-                Subject = row.Item4,
-                MetricsKey = keyid,
-                GroupingSubject = row.Item6
-            };
-        }
-    }
-
-    public class MetricRow
-    {
-        public string Action;
-        public string GroupingSubject;
-        public long Id;
-        public long MetricsKey;
-        public string Subject;
-        public DateTime Timestamp;
-    }
-}

Wabbajack.Server/DataLayer/MirroredFiles.cs

@@ -1,147 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Dapper;
-using Wabbajack.DTOs;
-using Wabbajack.Hashing.xxHash64;
-using Wabbajack.Server.DTOs;
-
-namespace Wabbajack.Server.DataLayer;
-
-public partial class SqlService
-{
-    public async Task<MirroredFile> GetNextMirroredFile()
-    {
-        await using var conn = await Open();
-        var result = await conn.QueryFirstOrDefaultAsync<(Hash, DateTime, DateTime, string, string)>(
-            "SELECT Hash, Created, Uploaded, Rationale, FailMessage from dbo.MirroredArchives WHERE Uploaded IS NULL");
-        if (result == default) return null;
-        return new MirroredFile
-        {
-            Hash = result.Item1, Created = result.Item2, Uploaded = result.Item3, Rationale = result.Item4,
-            FailMessage = result.Item5
-        };
-    }
-
-    public async Task<Dictionary<Hash, bool>> GetAllMirroredHashes()
-    {
-        await using var conn = await Open();
-        return (await conn.QueryAsync<(Hash, DateTime?)>("SELECT Hash, Uploaded FROM dbo.MirroredArchives"))
-            .GroupBy(d => d.Item1)
-            .ToDictionary(d => d.Key, d => d.First().Item2.HasValue);
-    }
-
-    public async Task StartMirror((Hash Hash, string Reason) mirror)
-    {
-        await using var conn = await Open();
-        await using var trans = await conn.BeginTransactionAsync();
-
-        if (await conn.QueryFirstOrDefaultAsync<Hash>(@"SELECT Hash FROM dbo.MirroredArchives WHERE Hash = @Hash",
-                new {mirror.Hash}, trans) != default)
-            return;
-
-        await conn.ExecuteAsync(
-            @"INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) VALUES (@Hash, GETUTCDATE(), @Reason)",
-            new {mirror.Hash, mirror.Reason}, trans);
-        await trans.CommitAsync();
-    }
-
-    public async Task<Dictionary<Hash, string>> GetAllowedMirrors()
-    {
-        await using var conn = await Open();
-        return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, Reason FROM dbo.AllowedMirrorsCache"))
-            .GroupBy(d => d.Item1)
-            .ToDictionary(d => d.Key, d => d.First().Item2);
-    }
-
-    public async Task UpsertMirroredFile(MirroredFile file)
-    {
-        await using var conn = await Open();
-        await using var trans = await conn.BeginTransactionAsync();
-
-        await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash", new {file.Hash}, trans);
-        await conn.ExecuteAsync(
-            "INSERT INTO dbo.MirroredArchives (Hash, Created, Uploaded, Rationale, FailMessage) VALUES (@Hash, @Created, @Uploaded, @Rationale, @FailMessage)",
-            new
-            {
-                file.Hash,
-                file.Created,
-                file.Uploaded,
-                file.Rationale,
-                file.FailMessage
-            }, trans);
-        await trans.CommitAsync();
-    }
-
-    public async Task DeleteMirroredFile(Hash hash)
-    {
-        await using var conn = await Open();
-        await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash",
-            new {Hash = hash});
-    }
-
-    public async Task<bool> HaveMirror(Hash hash)
-    {
-        await using var conn = await Open();
-
-        return await conn.QueryFirstOrDefaultAsync<Hash>("SELECT Hash FROM dbo.MirroredArchives WHERE Hash = @Hash",
-            new {Hash = hash}) != default;
-    }
-
-    public async Task QueueMirroredFiles()
-    {
-        await using var conn = await Open();
-
-        await conn.ExecuteAsync(@"
-
-                INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
-
-                SELECT hs.Hash, GETUTCDATE(), 'File has re-upload permissions on the Nexus' FROM
-                (SELECT DISTINCT ad.Hash FROM dbo.NexusModPermissions p
-                INNER JOIN GameMetadata md on md.NexusGameId = p.NexusGameID
-                INNER JOIN dbo.ArchiveDownloads ad on ad.PrimaryKeyString like 'NexusDownloader+State|'+md.WabbajackName+'|'+CAST(p.ModID as nvarchar)+'|%'
-                WHERE p.Permissions = 1
-                AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
-                ) hs
-
-                INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
-                SELECT DISTINCT Hash, GETUTCDATE(), 'File is hosted on GitHub'
-                FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%github.com/%'
-                AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
-
-
-                INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
-                SELECT DISTINCT Hash, GETUTCDATE(), 'File license allows uploading to any Non-nexus site'
-                FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%enbdev.com/%'
-                AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
-
-                INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
-                SELECT DISTINCT Hash, GETUTCDATE(), 'DynDOLOD file' /*, Name*/
-                from dbo.ModListArchives mla WHERE Name like '%DynDoLOD%standalone%'
-                and Hash not in (select Hash from dbo.MirroredArchives)
-
-                INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
-                SELECT DISTINCT Hash, GETUTCDATE(), 'Distribution allowed by author' /*, Name*/
-                from dbo.ModListArchives mla WHERE Name like '%particle%patch%'
-                and Hash not in (select Hash from dbo.MirroredArchives)
-
-
-                ");
-    }
-
-    public async Task AddNexusModWithOpenPerms(Game gameGame, long modId)
-    {
-        await using var conn = await Open();
-
-        await conn.ExecuteAsync(
-            @"INSERT INTO dbo.NexusModsWithOpenPerms(NexusGameID, NexusModID) VALUES(@game, @mod)",
-            new {game = gameGame.MetaData().NexusGameId, modId});
-    }
-
-    public async Task SyncActiveMirroredFiles()
-    {
-        await using var conn = await Open();
-        await conn.ExecuteAsync(@"EXEC dbo.QueueMirroredFiles");
-    }
-}

Wabbajack.Server/DataLayer/ModLists.cs

@@ -1,117 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Dapper;
-using Wabbajack.DTOs;
-using Wabbajack.DTOs.DownloadStates;
-using Wabbajack.Hashing.xxHash64;
-
-namespace Wabbajack.Server.DataLayer;
-
-public partial class SqlService
-{
-    public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist, bool brokenDownload)
-    {
-        await using var conn = await Open();
-        await using var tran = await conn.BeginTransactionAsync();
-
-        await conn.ExecuteAsync(@"DELETE FROM dbo.ModLists Where MachineUrl = @MachineUrl",
-            new {MachineUrl = metadata.Links.MachineURL}, tran);
-
-        var archives = modlist.Archives;
-        var directives = modlist.Directives;
-        modlist.Archives = Array.Empty<Archive>();
-        modlist.Directives = Array.Empty<Directive>();
-
-        await conn.ExecuteAsync(
-            @"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList, BrokenDownload) VALUES (@MachineUrl, @Hash, @Metadata, @ModList, @BrokenDownload)",
-            new
-            {
-                MachineUrl = metadata.Links.MachineURL,
-                Hash = hash,
-                MetaData = _dtos.Serialize(metadata),
-                ModList = _dtos.Serialize(modlist),
-                BrokenDownload = brokenDownload
-            }, tran);
-
-        var entries = archives.Select(a =>
-            new
-            {
-                MachineUrl = metadata.Links.MachineURL,
-                a.Name,
-                a.Hash,
-                a.Size,
-                State = _dtos.Serialize(a.State),
-                a.State.PrimaryKeyString
-            }).ToArray();
-
-        await conn.ExecuteAsync(@"DELETE FROM dbo.ModListArchives WHERE MachineURL = @machineURL",
-            new {MachineUrl = metadata.Links.MachineURL}, tran);
-
-        foreach (var entry in entries)
-            await conn.ExecuteAsync(
-                "INSERT INTO dbo.ModListArchives (MachineURL, Name, Hash, Size, PrimaryKeyString, State) VALUES (@MachineURL, @Name, @Hash, @Size, @PrimaryKeyString, @State)",
-                entry, tran);
-
-        await tran.CommitAsync();
-    }
-
-    public async Task<bool> HaveIndexedModlist(string machineUrl, Hash hash)
-    {
-        await using var conn = await Open();
-        var result = await conn.QueryFirstOrDefaultAsync<string>(
-            "SELECT MachineURL from dbo.Modlists WHERE MachineURL = @MachineUrl AND Hash = @Hash",
-            new {MachineUrl = machineUrl, Hash = hash});
-        return result != null;
-    }
-
-    public async Task<bool> HashIsInAModlist(Hash hash)
-    {
-        await using var conn = await Open();
-        var result = await conn.QueryFirstOrDefaultAsync<bool>(
-            "SELECT Hash FROM dbo.ModListArchives Where Hash = @Hash",
-            new {Hash = hash});
-        return result;
-    }
-
-    public async Task<List<Archive>> ModListArchives(string machineURL)
-    {
-        await using var conn = await Open();
-        var archives = await conn.QueryAsync<(string, Hash, long, IDownloadState)>(
-            "SELECT Name, Hash, Size, State FROM dbo.ModListArchives WHERE MachineUrl = @MachineUrl",
-            new {MachineUrl = machineURL});
-        return archives.Select(t => new Archive
-        {
-            State = t.Item4,
-            Name = string.IsNullOrWhiteSpace(t.Item1) ? t.Item4.PrimaryKeyString : t.Item1,
-            Size = t.Item3,
-            Hash = t.Item2
-        }).ToList();
-    }
-
-    public async Task<List<Archive>> ModListArchives()
-    {
-        await using var conn = await Open();
-        var archives =
-            await conn.QueryAsync<(string, Hash, long, IDownloadState)>(
-                "SELECT Name, Hash, Size, State FROM dbo.ModListArchives");
-        return archives.Select(t => new Archive
-        {
-            State = t.Item4,
-            Name = string.IsNullOrWhiteSpace(t.Item1) ? t.Item4.PrimaryKeyString : t.Item1,
-            Size = t.Item3,
-            Hash = t.Item2
-        }).ToList();
-    }
-
-    public async Task<int> PurgeList(string machineURL)
-    {
-        await using var conn = await Open();
-        var ret1 = await conn.ExecuteAsync(@" delete from dbo.ModListArchives where MachineURL = @machineURL",
-            new {machineURL});
-        var ret2 = await conn.ExecuteAsync(@" delete from dbo.ModLists where MachineURL = @machineURL",
-            new {machineURL});
-        return ret1 + ret2;
-    }
-}

Wabbajack.Server/DataLayer/Nexus.cs

@@ -1,156 +0,0 @@
-using System;
-using System.Linq;
-using System.Threading.Tasks;
-using Dapper;
-using Wabbajack.DTOs;
-using Wabbajack.Networking.NexusApi.DTOs;
-
-namespace Wabbajack.Server.DataLayer;
-
-/// <summary>
-///     SQL routines that read/write cached information from the Nexus
-/// </summary>
-public partial class SqlService
-{
-    public async Task<long> DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date)
-    {
-        await using var conn = await Open();
-        var deleted = await conn.ExecuteScalarAsync<long>(
-            @"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
-                      SELECT @@ROWCOUNT AS Deleted",
-            new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
-        return deleted;
-    }
-
-    public async Task<long> DeleteNexusModFilesUpdatedBeforeDate(Game game, long modId, DateTime date)
-    {
-        await using var conn = await Open();
-        var deleted = await conn.ExecuteScalarAsync<long>(
-            @"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
-                      SELECT @@ROWCOUNT AS Deleted",
-            new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
-        return deleted;
-    }
-
-    public async Task<ModInfo?> GetNexusModInfoString(Game game, long modId)
-    {
-        await using var conn = await Open();
-        var result = await conn.QueryFirstOrDefaultAsync<string>(
-            "SELECT Data FROM dbo.NexusModInfos WHERE Game = @Game AND @ModId = ModId",
-            new {Game = game.MetaData().NexusGameId, ModId = modId});
-        return result == null ? null : _dtos.Deserialize<ModInfo>(result);
-    }
-
-    public async Task AddNexusModInfo(Game game, long modId, DateTime lastCheckedUtc, ModInfo data)
-    {
-        await using var conn = await Open();
-
-        await conn.ExecuteAsync(
-            @"MERGE dbo.NexusModInfos AS Target
-                      USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
-                      ON Target.Game = Source.Game AND Target.ModId = Source.ModId
-                      WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
-                      WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
-            new
-            {
-                Game = game.MetaData().NexusGameId,
-                ModId = modId,
-                LastChecked = lastCheckedUtc,
-                Data = _dtos.Serialize(data)
-            });
-    }
-
-    public async Task AddNexusModFiles(Game game, long modId, DateTime lastCheckedUtc, ModFiles data)
-    {
-        await using var conn = await Open();
-
-        await conn.ExecuteAsync(
-            @"MERGE dbo.NexusModFiles AS Target
-                      USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
-                      ON Target.Game = Source.Game AND Target.ModId = Source.ModId
-                      WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
-                      WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
-            new
-            {
-                Game = game.MetaData().NexusGameId,
-                ModId = modId,
-                LastChecked = lastCheckedUtc,
-                Data = _dtos.Serialize(data)
-            });
-    }
-
-    public async Task AddNexusModFileSlow(Game game, long modId, long fileId, DateTime lastCheckedUtc)
-    {
-        await using var conn = await Open();
-
-        await conn.ExecuteAsync(
-            @"MERGE dbo.NexusModFilesSlow AS Target
-                      USING (SELECT @GameId GameId, @ModId ModId, @LastChecked LastChecked, @FileId FileId) AS Source
-                      ON Target.GameId = Source.GameId AND Target.ModId = Source.ModId AND Target.FileId = Source.FileId
-                      WHEN MATCHED THEN UPDATE SET Target.LastChecked = @LastChecked
-                      WHEN NOT MATCHED THEN INSERT (GameId, ModId, LastChecked, FileId) VALUES (@GameId, @ModId, @LastChecked, @FileId);",
-            new
-            {
-                GameId = game.MetaData().NexusGameId,
-                ModId = modId,
-                FileId = fileId,
-                LastChecked = lastCheckedUtc
-            });
-    }
-
-    public async Task<ModFiles?> GetModFiles(Game game, long modId)
-    {
-        await using var conn = await Open();
-        var result = await conn.QueryFirstOrDefaultAsync<string>(
-            "SELECT Data FROM dbo.NexusModFiles WHERE Game = @Game AND @ModId = ModId",
-            new {Game = game.MetaData().NexusGameId, ModId = modId});
-        return result == null ? null : _dtos.Deserialize<ModFiles>(result);
-    }
-
-    public async Task PurgeNexusCache(long modId)
-    {
-        await using var conn = await Open();
-        await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId});
-        await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId});
-        await conn.ExecuteAsync("DELETE FROM dbo.NexusModPermissions WHERE ModId = @ModId", new {ModId = modId});
-        await conn.ExecuteAsync("DELETE FROM dbo.NexusModFile WHERE ModId = @ModID", new {ModId = modId});
-    }
-
-    public async Task UpdateGameMetadata()
-    {
-        await using var conn = await Open();
-        var existing = (await conn.QueryAsync<string>("SELECT WabbajackName FROM dbo.GameMetadata")).ToHashSet();
-
-        var missing = GameRegistry.Games.Values.Where(g => !existing.Contains(g.Game.ToString())).ToList();
-        foreach (var add in missing.Where(g => g.NexusGameId != 0))
-            await conn.ExecuteAsync(
-                "INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, @WabbajackName)",
-                new {add.NexusGameId, WabbajackName = add.Game.ToString()});
-    }
-
-    public async Task<ModFile?> GetModFile(Game game, long modId, long fileId)
-    {
-        await using var conn = await Open();
-        var result = await conn.QueryFirstOrDefaultAsync<string>(
-            "SELECT Data FROM dbo.NexusModFile WHERE Game = @Game AND @ModId = ModId AND @FileId = FileId",
-            new {Game = game.MetaData().NexusGameId, ModId = modId, FileId = fileId});
-        return result == null ? null : _dtos.Deserialize<ModFile>(result);
-    }
-
-    public async Task AddNexusModFile(Game game, long modId, long fileId, DateTime lastCheckedUtc, ModFile data)
-    {
-        await using var conn = await Open();
-
-        await conn.ExecuteAsync(
-            @"INSERT INTO dbo.NexusModFile (Game, ModId, FileId, LastChecked, Data)
-                     VALUES (@Game, @ModId, @FileId, @LastChecked, @Data)",
-            new
-            {
-                Game = game.MetaData().NexusGameId,
-                ModId = modId,
-                FileId = fileId,
-                LastChecked = lastCheckedUtc,
-                Data = _dtos.Serialize(data)
-            });
-    }
-}

Wabbajack.Server/DataLayer/NexusKeys.cs

@@ -1,51 +0,0 @@
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Dapper;
-
-namespace Wabbajack.Server.DataLayer;
-
-public partial class SqlService
-{
-    public async Task SetNexusAPIKey(string key, long daily, long hourly)
-    {
-        await using var conn = await Open();
-        await using var trans = await conn.BeginTransactionAsync();
-        await conn.ExecuteAsync(@"DELETE FROM NexusKeys WHERE ApiKey = @ApiKey", new {ApiKey = key}, trans);
-        await conn.ExecuteAsync(
-            @"INSERT INTO NexusKeys (ApiKey, DailyRemain, HourlyRemain) VALUES (@ApiKey, @DailyRemain, @HourlyRemain)",
-            new {ApiKey = key, DailyRemain = daily, HourlyRemain = hourly}, trans);
-        await trans.CommitAsync();
-    }
-
-
-    public async Task DeleteNexusAPIKey(string key)
-    {
-        await using var conn = await Open();
-        await conn.ExecuteAsync(@"DELETE FROM NexusKeys WHERE ApiKey = @ApiKey", new {ApiKey = key});
-    }
-
-    public async Task<List<string>> GetNexusApiKeys(int threshold = 1500)
-    {
-        await using var conn = await Open();
-        return (await conn.QueryAsync<string>(
-            @"SELECT ApiKey FROM NexusKeys WHERE DailyRemain >= @Threshold ORDER BY DailyRemain DESC",
-            new {Threshold = threshold})).ToList();
-    }
-
-    public async Task<List<(string Key, int Daily, int Hourly)>> GetNexusApiKeysWithCounts(int threshold = 1500)
-    {
-        await using var conn = await Open();
-        return (await conn.QueryAsync<(string, int, int)>(
-            @"SELECT ApiKey, DailyRemain, HourlyRemain FROM NexusKeys WHERE DailyRemain >= @Threshold ORDER BY DailyRemain DESC",
-            new {Threshold = threshold})).ToList();
-    }
-
-
-    public async Task<bool> HaveKey(string key)
-    {
-        await using var conn = await Open();
-        return (await conn.QueryAsync<string>(@"SELECT ApiKey FROM NexusKeys WHERE ApiKey = @ApiKey",
-            new {ApiKey = key})).Any();
-    }
-}

Wabbajack.Server/DataLayer/NonNexusModlistArchives.cs

@@ -1,43 +0,0 @@
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Dapper;
-using Wabbajack.DTOs;
-using Wabbajack.DTOs.DownloadStates;
-
-namespace Wabbajack.Server.DataLayer;
-
-public partial class SqlService
-{
-    public async Task<List<Archive>> GetNonNexusModlistArchives()
-    {
-        await using var conn = await Open();
-        var results = await conn.QueryAsync<(Hashing.xxHash64.Hash Hash, long Size, string State)>(
-            @"SELECT Hash, Size, State FROM dbo.ModListArchives WHERE PrimaryKeyString NOT LIKE 'NexusDownloader+State|%'");
-        return results.Select(r => new Archive
-        {
-            State = _dtos.Deserialize<IDownloadState>(r.State)!,
-            Size = r.Size,
-            Hash = r.Hash
-        }).ToList();
-    }
-
-    public async Task UpdateNonNexusModlistArchivesStatus(IEnumerable<(Archive Archive, bool IsValid)> results)
-    {
-        await using var conn = await Open();
-        var trans = await conn.BeginTransactionAsync();
-        await conn.ExecuteAsync("DELETE FROM dbo.ModlistArchiveStatus;", transaction: trans);
-
-        foreach (var itm in results.DistinctBy(itm => (itm.Archive.Hash, itm.Archive.State.PrimaryKeyString)))
-            await conn.ExecuteAsync(
-                @"INSERT INTO dbo.ModlistArchiveStatus (PrimaryKeyStringHash, PrimaryKeyString, Hash, IsValid) 
-               VALUES (HASHBYTES('SHA2_256', @PrimaryKeyString), @PrimaryKeyString, @Hash, @IsValid)", new
-                {
-                    itm.Archive.State.PrimaryKeyString,
-                    itm.Archive.Hash,
-                    itm.IsValid
-                }, trans);
-
-        await trans.CommitAsync();
-    }
-}

Wabbajack.Server/DataLayer/Patches.cs

@@ -1,259 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Dapper;
-using Wabbajack.Hashing.xxHash64;
-using Wabbajack.Server.DTOs;
-
-namespace Wabbajack.Server.DataLayer;
-
-public partial class SqlService
-{
-    /// <summary>
-    ///     Adds a patch record
-    /// </summary>
-    /// <param name="patch"></param>
-    /// <returns></returns>
-    public async Task<bool> AddPatch(Patch patch)
-    {
-        await using var conn = await Open();
-        await using var trans = conn.BeginTransaction();
-
-        if (await conn.QuerySingleOrDefaultAsync<(Guid, Guid)>(
-                "Select SrcID, DestID FROM dbo.Patches where SrcID = @SrcId and DestID = @DestId",
-                new {SrcId = patch.Src.Id, DestId = patch.Dest.Id}, trans) != default)
-            return false;
-
-        await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)",
-            new {SrcId = patch.Src.Id, DestId = patch.Dest.Id}, trans);
-        await trans.CommitAsync();
-        return true;
-    }
-
-    /// <summary>
-    ///     Adds a patch record
-    /// </summary>
-    /// <param name="patch"></param>
-    /// <returns></returns>
-    public async Task FinializePatch(Patch patch)
-    {
-        await using var conn = await Open();
-        await conn.ExecuteAsync(
-            "UPDATE dbo.Patches SET PatchSize = @PatchSize, Finished = @Finished, IsFailed = @IsFailed, FailMessage = @FailMessage WHERE SrcId = @SrcId AND DestID = @DestId",
-            new
-            {
-                SrcId = patch.Src.Id,
-                DestId = patch.Dest.Id,
-                patch.PatchSize,
-                patch.Finished,
-                patch.IsFailed,
-                patch.FailMessage
-            });
-    }
-
-    public async Task<Patch> FindPatch(Guid src, Guid dest)
-    {
-        await using var conn = await Open();
-        var patch = await conn.QueryFirstOrDefaultAsync<(long, DateTime?, bool?, string)>(
-            @"SELECT p.PatchSize, p.Finished, p.IsFailed, p.FailMessage 
-                      FROM dbo.Patches p
-                      LEFT JOIN dbo.ArchiveDownloads src ON p.SrcId = src.Id
-                      LEFT JOIN dbo.ArchiveDownloads dest ON p.SrcId = dest.Id
-                      WHERE SrcId = @SrcId 
-                        AND DestId = @DestId
-                        AND src.DownloadFinished IS NOT NULL
-                        AND dest.DownloadFinished IS NOT NULL",
-            new
-            {
-                SrcId = src,
-                DestId = dest
-            });
-        if (patch == default)
-            return default;
-
-        return new Patch
-        {
-            Src = await GetArchiveDownload(src),
-            Dest = await GetArchiveDownload(dest),
-            PatchSize = patch.Item1,
-            Finished = patch.Item2,
-            IsFailed = patch.Item3,
-            FailMessage = patch.Item4
-        };
-    }
-
-    public async Task<Patch> FindOrEnqueuePatch(Guid src, Guid dest)
-    {
-        await using var conn = await Open();
-        var trans = await conn.BeginTransactionAsync();
-        var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
-            "SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestId",
-            new
-            {
-                SrcId = src,
-                DestId = dest
-            }, trans);
-        if (patch == default)
-        {
-            await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)",
-                new {SrcId = src, DestId = dest}, trans);
-            await trans.CommitAsync();
-            return new Patch {Src = await GetArchiveDownload(src), Dest = await GetArchiveDownload(dest)};
-        }
-
-        return new Patch
-        {
-            Src = await GetArchiveDownload(src),
-            Dest = await GetArchiveDownload(dest),
-            PatchSize = patch.Item3,
-            Finished = patch.Item4,
-            IsFailed = patch.Item5,
-            FailMessage = patch.Item6
-        };
-    }
-
-    public async Task<Patch> GetPendingPatch()
-    {
-        await using var conn = await Open();
-        var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
-            @"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage FROM dbo.Patches p 
-                      LEFT JOIN dbo.ArchiveDownloads src ON src.Id = p.SrcId
-                      LEFT JOIN dbo.ArchiveDownloads dest ON dest.Id = p.DestId
-                      WHERE p.Finished is NULL AND src.IsFailed = 0 AND dest.IsFailed = 0 ");
-        if (patch == default)
-            return default;
-
-        return new Patch
-        {
-            Src = await GetArchiveDownload(patch.Item1),
-            Dest = await GetArchiveDownload(patch.Item2),
-            PatchSize = patch.Item3,
-            Finished = patch.Item4,
-            IsFailed = patch.Item5,
-            FailMessage = patch.Item6
-        };
-    }
-
-    public async Task<List<Patch>> PatchesForSource(Guid sourceDownload)
-    {
-        await using var conn = await Open();
-        var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
-            "SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId",
-            new {SrcId = sourceDownload});
-
-        return await AsPatches(patches);
-    }
-
-    public async Task<List<Patch>> PatchesForSource(Hash sourceHash)
-    {
-        await using var conn = await Open();
-        var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
-            @"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage 
-                     FROM dbo.Patches p
-                     LEFT JOIN dbo.ArchiveDownloads a ON p.SrcId = a.Id 
-                     
-                     WHERE a.Hash = @Hash AND p.Finished IS NOT NULL AND p.IsFailed = 0", new {Hash = sourceHash});
-
-        return await AsPatches(patches);
-    }
-
-    public async Task MarkPatchUsage(Guid srcId, Guid destId)
-    {
-        await using var conn = await Open();
-        await conn.ExecuteAsync(
-            @"UPDATE dbo.Patches SET Downloads = Downloads + 1, LastUsed = GETUTCDATE() WHERE SrcId = @srcId AND DestID = @destId",
-            new {SrcId = srcId, DestId = destId});
-    }
-
-    public async Task<List<Patch>> GetOldPatches()
-    {
-        await using var conn = await Open();
-        var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
-            @"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
-                        FROM dbo.Patches p
-                        LEFT JOIN dbo.ArchiveDownloads a ON p.SrcId = a.Id
-                        WHERE a.Hash not in (SELECT Hash FROM dbo.ModListArchives)");
-
-        return await AsPatches(patches);
-    }
-
-    private async Task<List<Patch>> AsPatches(IEnumerable<(Guid, Guid, long, DateTime?, bool?, string)> patches)
-    {
-        var results = new List<Patch>();
-        foreach (var (srcId, destId, patchSize, finished, isFailed, failMessage) in patches)
-            results.Add(new Patch
-            {
-                Src = await GetArchiveDownload(srcId),
-                Dest = await GetArchiveDownload(destId),
-                PatchSize = patchSize,
-                Finished = finished,
-                IsFailed = isFailed,
-                FailMessage = failMessage
-            });
-
-        return results;
-    }
-
-
-    public async Task DeletePatch(Patch patch)
-    {
-        await using var conn = await Open();
-        await conn.ExecuteAsync(@"DELETE FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestID",
-            new
-            {
-                SrcId = patch.Src.Id,
-                DestId = patch.Dest.Id
-            });
-    }
-
-    public async Task<HashSet<(Hash, Hash)>> AllPatchHashes()
-    {
-        await using var conn = await Open();
-        return (await conn.QueryAsync<(Hash, Hash)>(@"SELECT a1.Hash, a2.Hash
-                      FROM dbo.Patches p
-                      LEFT JOIN dbo.ArchiveDownloads a1 ON a1.Id = p.SrcId
-                      LEFT JOIN dbo.ArchiveDownloads a2 on a2.Id = p.DestId
-                      WHERE p.Finished IS NOT NULL")).ToHashSet();
-    }
-
-    public async Task DeletePatchesForHashPair((Hash, Hash) sqlFile)
-    {
-        await using var conn = await Open();
-        await conn.ExecuteAsync(@"DELETE p
-                      FROM dbo.Patches p
-                      LEFT JOIN dbo.ArchiveDownloads a1 ON a1.Id = p.SrcId
-                      LEFT JOIN dbo.ArchiveDownloads a2 on a2.Id = p.DestId
-                      WHERE a1.Hash = @SrcHash
-                      AND a2.Hash = @DestHash", new
-        {
-            SrcHash = sqlFile.Item1,
-            DestHash = sqlFile.Item2
-        });
-    }
-
-    public async Task PurgePatch(Hash hash, string rationale)
-    {
-        await using var conn = await Open();
-        await using var tx = await conn.BeginTransactionAsync();
-
-        await conn.ExecuteAsync(
-            "DELETE p FROM dbo.Patches p LEFT JOIN dbo.ArchiveDownloads ad ON ad.Id = p.SrcId WHERE ad.Hash = @Hash ",
-            new {Hash = hash}, tx);
-        await conn.ExecuteAsync(
-            "INSERT INTO dbo.NoPatch (Hash, Created, Rationale) VALUES (@Hash, GETUTCDATE(), @Rationale)",
-            new
-            {
-                Hash = hash,
-                Rationale = rationale
-            }, tx);
-        await tx.CommitAsync();
-    }
-
-    public async Task<bool> IsNoPatch(Hash hash)
-    {
-        await using var conn = await Open();
-        return await conn.QueryFirstOrDefaultAsync<Hash>("SELECT Hash FROM NoPatch WHERE Hash = @Hash",
-            new {Hash = hash}) != default;
-    }
-}

Wabbajack.Server/DataLayer/SqlService.cs

@@ -1,30 +0,0 @@
-using System.Data.SqlClient;
-using System.Threading.Tasks;
-using Wabbajack.BuildServer;
-using Wabbajack.Downloaders;
-using Wabbajack.DTOs.JsonConverters;
-
-namespace Wabbajack.Server.DataLayer;
-
-public partial class SqlService
-{
-    private readonly DownloadDispatcher _dispatcher;
-    private readonly DTOSerializer _dtos;
-    private readonly AppSettings _settings;
-
-    public SqlService(AppSettings settings, DTOSerializer dtos, DownloadDispatcher dispatcher)
-    {
-        _settings = settings;
-        _dtos = dtos;
-        _dispatcher = dispatcher;
-        // Ugly hack, but the SQL mappers need it
-        _dtoStatic = dtos;
-    }
-
-    public async Task<SqlConnection> Open()
-    {
-        var conn = new SqlConnection(_settings.SqlConnection);
-        await conn.OpenAsync();
-        return conn;
-    }
-}

Wabbajack.Server/DataLayer/ValidationData.cs

@@ -1,56 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Dapper;
-using Wabbajack.DTOs;
-using Wabbajack.Hashing.xxHash64;
-using Wabbajack.Server.DTOs;
-
-namespace Wabbajack.Server.DataLayer;
-
-public partial class SqlService
-{
-    public async Task<ValidationData> GetValidationData()
-    {
-        var archiveStatus = AllModListArchivesStatus();
-        var modLists = AllModLists();
-        var mirrors = GetAllMirroredHashes();
-        var authoredFiles = AllAuthoredFiles();
-        var nexusFiles = await AllNexusFiles();
-        return new ValidationData
-        {
-            NexusFiles = nexusFiles.ToDictionary(nf => (nf.NexusGameId, nf.ModId, nf.FileId), nf => nf.category),
-            ArchiveStatus = await archiveStatus,
-            ModLists = await modLists,
-            Mirrors = await mirrors,
-            AllowedMirrors = new Lazy<Task<Dictionary<Hash, string>>>(async () => await GetAllowedMirrors()),
-            AllAuthoredFiles = await authoredFiles
-        };
-    }
-
-    public async Task<Dictionary<(string PrimaryKeyString, Hash Hash), bool>> AllModListArchivesStatus()
-    {
-        await using var conn = await Open();
-        var results =
-            await conn.QueryAsync<(string, Hash, bool)>(
-                @"SELECT PrimaryKeyString, Hash, IsValid FROM dbo.ModListArchiveStatus");
-        return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3);
-    }
-
-    public async Task<HashSet<(long NexusGameId, long ModId, long FileId, string category)>> AllNexusFiles()
-    {
-        await using var conn = await Open();
-        var results =
-            await conn.QueryAsync<(long, long, long, string)>(
-                @"SELECT Game, ModId, FileId, JSON_VALUE(Data, '$.category_name') FROM dbo.NexusModFile");
-        return results.ToHashSet();
-    }
-
-    public async Task<List<ModlistMetadata>> AllModLists()
-    {
-        await using var conn = await Open();
-        var results = await conn.QueryAsync<ModlistMetadata>(@"SELECT Metadata FROM dbo.ModLists");
-        return results.ToList();
-    }
-}

Wabbajack.Server/DataModels/AuthorFiles.cs

@@ -0,0 +1,110 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer;
+using Wabbajack.Common;
+using Wabbajack.DTOs.CDN;
+using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+
+
+namespace Wabbajack.Server.DataModels;
+
+public class AuthorFiles
+{
+    private readonly ILogger<AuthorFiles> _logger;
+    private readonly AppSettings _settings;
+    private readonly DTOSerializer _dtos;
+    private Dictionary<string, FileDefinition> _byServerId = new();
+
+    public AbsolutePath AuthorFilesLocation => _settings.AuthoredFilesFolder.ToAbsolutePath();
+
+    public AuthorFiles(ILogger<AuthorFiles> logger, AppSettings settings, DTOSerializer dtos)
+    {
+        _logger = logger;
+        _settings = settings;
+        _dtos = dtos;
+    }
+
+    public IEnumerable<AbsolutePath> AllDefinitions => AuthorFilesLocation.EnumerateFiles("definition.json.gz");
+
+    public async Task<FileDefinitionMetadata[]> AllAuthoredFiles()
+    {
+        var defs = new List<FileDefinitionMetadata>();
+        foreach (var file in AllDefinitions)
+        {
+            defs.Add(new FileDefinitionMetadata
+            {
+                Definition = await ReadDefinition(file),
+                Updated = file.LastModifiedUtc()
+            });
+        }
+
+        _byServerId = defs.ToDictionary(f => f.Definition.ServerAssignedUniqueId!, f => f.Definition);
+        return defs.ToArray();
+    }
+
+    public async Task<Stream> StreamForPart(string mungedName, int part)
+    {
+        return AuthorFilesLocation.Combine(mungedName, "parts", part.ToString()).Open(FileMode.Open);
+    }
+    
+    public async Task<Stream> CreatePart(string mungedName, int part)
+    {
+        return AuthorFilesLocation.Combine(mungedName, "parts", part.ToString()).Open(FileMode.Create, FileAccess.Write, FileShare.None);
+    }
+
+    public async Task WriteDefinition(FileDefinition definition)
+    {
+        var path = AuthorFilesLocation.Combine(definition.MungedName, "definition.json.gz");
+        path.Parent.CreateDirectory();
+        path.Parent.Combine("parts").CreateDirectory();
+        
+        await using var ms = new MemoryStream();
+        await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
+        {
+            await _dtos.Serialize(definition, gz);
+        }
+
+        await path.WriteAllBytesAsync(ms.ToArray());
+    }
+
+    public async Task<FileDefinition> ReadDefinition(string mungedName)
+    {
+        return await ReadDefinition(AuthorFilesLocation.Combine(mungedName, "definition.json.gz"));
+    }
+    
+    private async Task<FileDefinition> ReadDefinition(AbsolutePath file)
+    {
+        var gz = new GZipStream(new MemoryStream(await file.ReadAllBytesAsync()), CompressionMode.Decompress);
+        var definition = (await _dtos.DeserializeAsync<FileDefinition>(gz))!;
+        return definition;
+    }
+
+    public class FileDefinitionMetadata
+    {
+        public FileDefinition Definition { get; set; }
+        public DateTime Updated { get; set; }
+        public string HumanSize => Definition.Size.ToFileSizeString();
+    }
+
+    public async Task DeleteFile(FileDefinition definition)
+    {
+        var folder = AuthorFilesLocation.Combine(definition.MungedName);
+        folder.DeleteDirectory();
+    }
+
+    public async Task<FileDefinition> ReadDefinitionForServerId(string serverAssignedUniqueId)
+    {
+        if (_byServerId.TryGetValue(serverAssignedUniqueId, out var found))
+            return found;
+        await AllAuthoredFiles();
+        return _byServerId[serverAssignedUniqueId];
+    }
+}

Wabbajack.Server/DataModels/AuthorKeys.cs

@@ -0,0 +1,28 @@
+using System.Threading.Tasks;
+using Wabbajack.BuildServer;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+
+namespace Wabbajack.Server.DataModels;
+
+public class AuthorKeys
+{
+    private readonly AppSettings _settings;
+    private AbsolutePath AuthorKeysPath => _settings.AuthorAPIKeyFile.ToAbsolutePath();
+
+    public AuthorKeys(AppSettings settings)
+    {
+        _settings = settings;
+    }
+
+    public async Task<string?> AuthorForKey(string key)
+    {
+        await foreach (var line in AuthorKeysPath.ReadAllLinesAsync())
+        {
+            var parts = line.Split("\t");
+            if (parts[0].Trim() == key)
+                return parts[1].Trim();
+        }
+        return null;
+    }
+}

Wabbajack.Server/DataModels/Metrics.cs

@@ -0,0 +1,39 @@
+using System;
+using System.IO;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Toolkit.HighPerformance;
+using Wabbajack.BuildServer;
+using Wabbajack.Common;
+using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+using Wabbajack.Server.DTOs;
+
+namespace Wabbajack.Server.DataModels;
+
+public class Metrics
+{
+    private readonly AppSettings _settings;
+    public SemaphoreSlim _lock = new(1);
+    private readonly DTOSerializer _dtos;
+
+    public Metrics(AppSettings settings, DTOSerializer dtos)
+    {
+        _settings = settings;
+        _dtos = dtos;
+    }
+
+    public async Task Ingest(Metric metric)
+    {
+        using var _ = await _lock.Lock();
+        var data = Encoding.UTF8.GetBytes(_dtos.Serialize(metric));
+        var metricsFile = _settings.MetricsFolder.ToAbsolutePath().Combine(DateTime.Now.ToString("yyyy_MM_dd") + ".json");
+        await using var fs = metricsFile.Open(FileMode.Append, FileAccess.Write, FileShare.Read);
+        fs.Write(data);
+        fs.Write(Encoding.UTF8.GetBytes("\n"));
+    }
+    
+    
+}

Wabbajack.Server/Program.cs

@@ -22,16 +22,7 @@ public class Program
                 webBuilder.UseStartup<Startup>()
                     .UseKestrel(options =>
                     {
-                        options.Listen(IPAddress.Any, testMode ? 8080 : 80);
+                        options.Listen(IPAddress.Any, 5000);
-                        if (!testMode)
-                            options.Listen(IPAddress.Any, 443, listenOptions =>
-                            {
-                                using var store = new X509Store(StoreName.My);
-                                store.Open(OpenFlags.ReadOnly);
-                                var cert = store.Certificates.Find(X509FindType.FindBySubjectName,
-                                    "build.wabbajack.org", true)[0];
-                                listenOptions.UseHttps(cert);
-                            });
                         options.Limits.MaxRequestBodySize = null;
                     });
             });

Wabbajack.Server/Services/ArchiveDownloader.cs

@@ -1,150 +0,0 @@
-using System;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.Extensions.Logging;
-using Wabbajack.BuildServer;
-using Wabbajack.Common;
-using Wabbajack.Downloaders;
-using Wabbajack.DTOs.DownloadStates;
-using Wabbajack.Networking.NexusApi;
-using Wabbajack.Paths.IO;
-using Wabbajack.Server.DataLayer;
-using Wabbajack.Server.DTOs;
-
-namespace Wabbajack.Server.Services;
-
-public class ArchiveDownloader : AbstractService<ArchiveDownloader, int>
-{
-    private readonly DownloadDispatcher _dispatcher;
-    private readonly TemporaryFileManager _manager;
-    private readonly ArchiveMaintainer _archiveMaintainer;
-    private readonly DiscordWebHook _discord;
-    private NexusApi _nexusClient;
-    private readonly SqlService _sql;
-
-    public ArchiveDownloader(ILogger<ArchiveDownloader> logger, AppSettings settings, SqlService sql,
-        ArchiveMaintainer archiveMaintainer,
-        DiscordWebHook discord, QuickSync quickSync, DownloadDispatcher dispatcher, TemporaryFileManager manager)
-        : base(logger, settings, quickSync, TimeSpan.FromMinutes(10))
-    {
-        _sql = sql;
-        _archiveMaintainer = archiveMaintainer;
-        _discord = discord;
-        _dispatcher = dispatcher;
-        _manager = manager;
-    }
-
-    public override async Task<int> Execute()
-    {
-        var count = 0;
-
-        while (true)
-        {
-            var (_, header) = await _nexusClient.Validate();
-            var ignoreNexus = header.DailyRemaining < 100 && header.HourlyRemaining < 10;
-            //var ignoreNexus = true;
-            if (ignoreNexus)
-                _logger.LogWarning(
-                    $"Ignoring Nexus Downloads due to low hourly api limit (Daily: {header.DailyRemaining}, Hourly:{header.HourlyRemaining})");
-            else
-                _logger.LogInformation(
-                    $"Looking for any download (Daily: {header.DailyRemaining}, Hourly:{header.HourlyRemaining})");
-
-            var nextDownload = await _sql.GetNextPendingDownload(ignoreNexus);
-
-            if (nextDownload == default)
-                break;
-
-            _logger.LogInformation($"Checking for previously archived {nextDownload.Archive.Hash}");
-
-            if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash))
-            {
-                await nextDownload.Finish(_sql);
-                continue;
-            }
-
-            if (nextDownload.Archive.State is Manual or GameFileSource)
-            {
-                await nextDownload.Finish(_sql);
-                continue;
-            }
-
-            try
-            {
-                _logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}");
-                ReportStarting(nextDownload.Archive.State.PrimaryKeyString);
-                await _discord.Send(Channel.Spam,
-                    new DiscordMessage
-                    {
-                        Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"
-                    });
-
-                await _dispatcher.PrepareAll(new[] {nextDownload.Archive.State});
-
-                await using var tempPath = _manager.CreateFile();
-                if (await _dispatcher.Download(nextDownload.Archive, tempPath.Path, CancellationToken.None) == default)
-                {
-                    _logger.LogError(
-                        $"Downloader returned false for {nextDownload.Archive.State.PrimaryKeyString}");
-                    await nextDownload.Fail(_sql, "Downloader returned false");
-                    continue;
-                }
-
-                var hash = await tempPath.Path.Hash();
-
-                if (hash == default || nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)
-                {
-                    _logger.Log(LogLevel.Warning,
-                        $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size()}");
-                    await nextDownload.Fail(_sql, "Invalid Hash");
-                    continue;
-                }
-
-                if (nextDownload.Archive.Size != default &&
-                    tempPath.Path.Size() != nextDownload.Archive.Size)
-                {
-                    await nextDownload.Fail(_sql, "Invalid Size");
-                    continue;
-                }
-
-                nextDownload.Archive.Hash = hash;
-                nextDownload.Archive.Size = tempPath.Path.Size();
-
-                _logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}");
-                await _archiveMaintainer.Ingest(tempPath.Path);
-
-                _logger.Log(LogLevel.Information,
-                    $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
-                await nextDownload.Finish(_sql);
-
-                await _discord.Send(Channel.Spam,
-                    new DiscordMessage
-                    {
-                        Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}"
-                    });
-            }
-            catch (Exception ex)
-            {
-                _logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}");
-                await nextDownload.Fail(_sql, ex.ToString());
-                await _discord.Send(Channel.Spam,
-                    new DiscordMessage
-                    {
-                        Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}"
-                    });
-            }
-            finally
-            {
-                ReportEnding(nextDownload.Archive.State.PrimaryKeyString);
-            }
-
-            count++;
-        }
-
-        if (count > 0)
-            // Wake the Patch builder up in case it needs to build a patch now
-            await _quickSync.Notify<PatchBuilder>();
-
-        return count;
-    }
-}

Wabbajack.Server/Services/ArchiveMaintainer.cs

@@ -1,66 +0,0 @@
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using Wabbajack.BuildServer;
-using Wabbajack.Hashing.xxHash64;
-using Wabbajack.Paths;
-using Wabbajack.Paths.IO;
-using Wabbajack.VFS;
-
-namespace Wabbajack.Server.Services;
-
-/// <summary>
-///     Maintains a concurrent cache of all the files we've downloaded, indexed by Hash.
-/// </summary>
-public class ArchiveMaintainer
-{
-    private readonly ILogger<ArchiveMaintainer> _logger;
-    private readonly AppSettings _settings;
-
-    public ArchiveMaintainer(ILogger<ArchiveMaintainer> logger, AppSettings settings)
-    {
-        _settings = settings;
-        _logger = logger;
-        _logger.Log(LogLevel.Information, "Creating Archive Maintainer");
-    }
-
-    private AbsolutePath ArchivePath(Hash hash)
-    {
-        return _settings.ArchivePath.Combine(hash.ToHex());
-    }
-
-    public async Task Ingest(AbsolutePath file)
-    {
-        var hash = await file.Hash(CancellationToken.None);
-        if (hash == default) return;
-
-        var newPath = ArchivePath(hash);
-        if (HaveArchive(hash))
-        {
-            file.Delete();
-            return;
-        }
-
-        await file.MoveToAsync(newPath, true, CancellationToken.None);
-    }
-
-    public bool HaveArchive(Hash hash)
-    {
-        return ArchivePath(hash).FileExists();
-    }
-
-    public bool TryGetPath(Hash hash, out AbsolutePath path)
-    {
-        path = ArchivePath(hash);
-        return path.FileExists();
-    }
-}
-
-public static class ArchiveMaintainerExtensions
-{
-    public static IServiceCollection UseArchiveMaintainer(this IServiceCollection b)
-    {
-        return b.AddSingleton<ArchiveMaintainer>();
-    }
-}

Wabbajack.Server/Services/DiscordFrontend.cs

@@ -1,189 +0,0 @@
-using System;
-using System.Linq;
-using System.Threading.Tasks;
-using Discord;
-using Discord.WebSocket;
-using Microsoft.Extensions.Logging;
-using Wabbajack.BuildServer;
-using Wabbajack.DTOs;
-using Wabbajack.Server.DataLayer;
-using Wabbajack.Server.TokenProviders;
-
-namespace Wabbajack.Server.Services;
-
-public class DiscordFrontend : IStartable
-{
-    private readonly IDiscordToken _token;
-    private readonly DiscordSocketClient _client;
-    private readonly MetricsKeyCache _keyCache;
-    private readonly ILogger<DiscordFrontend> _logger;
-    private readonly QuickSync _quickSync;
-    private AppSettings _settings;
-    private readonly SqlService _sql;
-
-    public DiscordFrontend(ILogger<DiscordFrontend> logger, AppSettings settings, QuickSync quickSync, SqlService sql,
-        MetricsKeyCache keyCache, IDiscordToken token)
-    {
-        _logger = logger;
-        _settings = settings;
-        _quickSync = quickSync;
-
-        _client = new DiscordSocketClient();
-
-        _client.Log += LogAsync;
-        _client.Ready += ReadyAsync;
-        _client.MessageReceived += MessageReceivedAsync;
-
-        _sql = sql;
-        _keyCache = keyCache;
-        _token = token;
-    }
-
-    public async Task Start()
-    {
-        await _client.LoginAsync(TokenType.Bot, await _token.Get());
-        await _client.StartAsync();
-    }
-
-    private async Task MessageReceivedAsync(SocketMessage arg)
-    {
-        _logger.LogInformation(arg.Content);
-        if (arg.Content.StartsWith("!dervenin"))
-        {
-            var parts = arg.Content.Split(" ", StringSplitOptions.RemoveEmptyEntries);
-            if (parts[0] != "!dervenin")
-                return;
-
-            if (parts.Length == 1) await ReplyTo(arg, "Wat?");
-
-            if (parts[1] == "purge-nexus-cache")
-            {
-                if (parts.Length != 3)
-                {
-                    await ReplyTo(arg, "Welp you did that wrong, gotta give me a mod-id or url");
-                    return;
-                }
-
-                await PurgeNexusCache(arg, parts[2]);
-            }
-            else if (parts[1] == "quick-sync")
-            {
-                var options = await _quickSync.Report();
-                if (parts.Length != 3)
-                {
-                    var optionsStr = string.Join(", ", options.Select(o => o.Key.Name));
-                    await ReplyTo(arg, $"Can't expect me to quicksync the whole damn world! Try: {optionsStr}");
-                }
-                else
-                {
-                    foreach (var pair in options.Where(o => o.Key.Name == parts[2]))
-                    {
-                        await _quickSync.Notify(pair.Key);
-                        await ReplyTo(arg, $"Notified {pair.Key}");
-                    }
-                }
-            }
-            else if (parts[1] == "purge-list")
-            {
-                if (parts.Length != 3)
-                {
-                    await ReplyTo(arg, "Yeah, I'm not gonna purge the whole server...");
-                }
-                else
-                {
-                    var deleted = await _sql.PurgeList(parts[2]);
-                    await _quickSync.Notify<ModListDownloader>();
-                    await ReplyTo(arg,
-                        $"Purged all traces of #{parts[2]} from the server, triggered list downloading. {deleted} records removed");
-                }
-            }
-            else if (parts[1] == "mirror-mod")
-            {
-                await MirrorModCommand(arg, parts);
-            }
-            else if (parts[1] == "users")
-            {
-                await ReplyTo(arg, $"Wabbajack has {await _keyCache.KeyCount()} known unique users");
-            }
-        }
-    }
-
-    private async Task MirrorModCommand(SocketMessage msg, string[] parts)
-    {
-        if (parts.Length != 2)
-        {
-            await ReplyTo(msg, "Command is: mirror-mod <game-name> <mod-id>");
-            return;
-        }
-
-        if (long.TryParse(parts[2], out var modId))
-        {
-            await ReplyTo(msg, $"Got {modId} for a mod-id, expected a integer");
-            return;
-        }
-
-        if (GameRegistry.TryGetByFuzzyName(parts[1], out var game))
-        {
-            var gameNames = GameRegistry.Games.Select(g => g.Value.NexusName)
-                .Where(g => !string.IsNullOrWhiteSpace(g))
-                .Select(g => (string) g)
-                .ToHashSet();
-            var joined = string.Join(", ", gameNames.OrderBy(g => g));
-            await ReplyTo(msg, $"Got {parts[1]} for a game name, expected something like: {joined}");
-        }
-
-        if (game!.NexusGameId == default) await ReplyTo(msg, $"No NexusGameID found for {game}");
-
-        await _sql.AddNexusModWithOpenPerms(game.Game, modId);
-        await _quickSync.Notify<MirrorUploader>();
-        await ReplyTo(msg, "Done, and I notified the uploader");
-    }
-
-    private async Task PurgeNexusCache(SocketMessage arg, string mod)
-    {
-        if (Uri.TryCreate(mod, UriKind.Absolute, out var url))
-            mod = url.AbsolutePath.Split("/", StringSplitOptions.RemoveEmptyEntries).Last();
-
-        if (int.TryParse(mod, out var mod_id))
-        {
-            await _sql.PurgeNexusCache(mod_id);
-            await ReplyTo(arg, $"It is done, {mod_id} has been purged, list validation has been triggered");
-        }
-    }
-
-    private async Task ReplyTo(SocketMessage socketMessage, string message)
-    {
-        await socketMessage.Channel.SendMessageAsync(message);
-    }
-
-    private async Task ReadyAsync()
-    {
-    }
-
-    private async Task LogAsync(LogMessage arg)
-    {
-        switch (arg.Severity)
-        {
-            case LogSeverity.Info:
-                _logger.LogInformation(arg.Message);
-                break;
-            case LogSeverity.Warning:
-                _logger.LogWarning(arg.Message);
-                break;
-            case LogSeverity.Critical:
-                _logger.LogCritical(arg.Message);
-                break;
-            case LogSeverity.Error:
-                _logger.LogError(arg.Exception, arg.Message);
-                break;
-            case LogSeverity.Verbose:
-                _logger.LogTrace(arg.Message);
-                break;
-            case LogSeverity.Debug:
-                _logger.LogDebug(arg.Message);
-                break;
-            default:
-                throw new ArgumentOutOfRangeException();
-        }
-    }
-}

Wabbajack.Server/Services/MetricsKeyCache.cs

@@ -1,61 +0,0 @@
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Microsoft.Extensions.Logging;
-using Wabbajack.Common;
-using Wabbajack.Server.DataLayer;
-
-namespace Wabbajack.Server.Services;
-
-public class MetricsKeyCache : IStartable
-{
-    private HashSet<string> _knownKeys = new();
-    private readonly AsyncLock _lock = new();
-    private ILogger<MetricsKeyCache> _logger;
-    private readonly SqlService _sql;
-
-    public MetricsKeyCache(ILogger<MetricsKeyCache> logger, SqlService sql)
-    {
-        _logger = logger;
-        _sql = sql;
-    }
-
-    public async Task Start()
-    {
-        _knownKeys = _sql.AllKeys().Result.ToHashSet();
-    }
-
-    public async Task<bool> IsValidKey(string key)
-    {
-        using (var _ = await _lock.WaitAsync())
-        {
-            if (_knownKeys.Contains(key)) return true;
-        }
-
-        if (await _sql.ValidMetricsKey(key))
-        {
-            using var _ = await _lock.WaitAsync();
-            _knownKeys.Add(key);
-            return true;
-        }
-
-        return false;
-    }
-
-    public async Task AddKey(string key)
-    {
-        using (var _ = await _lock.WaitAsync())
-        {
-            if (_knownKeys.Contains(key)) return;
-            _knownKeys.Add(key);
-        }
-
-        await _sql.AddMetricsKey(key);
-    }
-
-    public async Task<long> KeyCount()
-    {
-        using var _ = await _lock.WaitAsync();
-        return _knownKeys.Count;
-    }
-}

Wabbajack.Server/Services/MirrorQueueService.cs

@@ -1,27 +0,0 @@
-using System;
-using System.Threading.Tasks;
-using Microsoft.Extensions.Logging;
-using Wabbajack.BuildServer;
-using Wabbajack.Server.DataLayer;
-
-namespace Wabbajack.Server.Services;
-
-public class MirrorQueueService : AbstractService<MirrorQueueService, int>
-{
-    private DiscordWebHook _discord;
-    private readonly SqlService _sql;
-
-    public MirrorQueueService(ILogger<MirrorQueueService> logger, AppSettings settings, QuickSync quickSync,
-        DiscordWebHook discordWebHook, SqlService sqlService) :
-        base(logger, settings, quickSync, TimeSpan.FromMinutes(5))
-    {
-        _discord = discordWebHook;
-        _sql = sqlService;
-    }
-
-    public override async Task<int> Execute()
-    {
-        await _sql.QueueMirroredFiles();
-        return 1;
-    }
-}

Wabbajack.Server/Services/MirrorUploader.cs

@@ -1,227 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.IO.Compression;
-using System.Linq;
-using System.Net;
-using System.Threading.Tasks;
-using FluentFTP;
-using FluentFTP.Helpers;
-using Microsoft.Extensions.Logging;
-using Wabbajack.BuildServer;
-using Wabbajack.Common;
-using Wabbajack.DTOs.JsonConverters;
-using Wabbajack.Hashing.xxHash64;
-using Wabbajack.Networking.WabbajackClientApi;
-using Wabbajack.Paths;
-using Wabbajack.Paths.IO;
-using Wabbajack.RateLimiter;
-using Wabbajack.Server.DataLayer;
-using Wabbajack.Server.DTOs;
-using Wabbajack.Server.TokenProviders;
-
-namespace Wabbajack.Server.Services;
-
-public class MirrorUploader : AbstractService<MirrorUploader, int>
-{
-    private readonly IFtpSiteCredentials _credentials;
-    private readonly DTOSerializer _dtos;
-    private readonly IFtpSiteCredentials _ftpCreds;
-    private readonly ParallelOptions _parallelOptions;
-    private readonly Client _wjClient;
-    private readonly ArchiveMaintainer _archives;
-    private readonly DiscordWebHook _discord;
-    private readonly SqlService _sql;
-
-    public MirrorUploader(ILogger<MirrorUploader> logger, AppSettings settings, SqlService sql, QuickSync quickSync,
-        ArchiveMaintainer archives,
-        DiscordWebHook discord, IFtpSiteCredentials credentials, Client wjClient, ParallelOptions parallelOptions,
-        DTOSerializer dtos,
-        IFtpSiteCredentials ftpCreds)
-        : base(logger, settings, quickSync, TimeSpan.FromHours(1))
-    {
-        _sql = sql;
-        _archives = archives;
-        _discord = discord;
-        _credentials = credentials;
-        _wjClient = wjClient;
-        _parallelOptions = parallelOptions;
-        _dtos = dtos;
-        _ftpCreds = ftpCreds;
-    }
-
-    public bool ActiveFileSyncEnabled { get; set; } = true;
-
-    public override async Task<int> Execute()
-    {
-        var uploaded = 0;
-
-        if (ActiveFileSyncEnabled)
-            await _sql.SyncActiveMirroredFiles();
-        TOP:
-        var toUpload = await _sql.GetNextMirroredFile();
-        if (toUpload == default)
-        {
-            await DeleteOldMirrorFiles();
-            return uploaded;
-        }
-
-        uploaded += 1;
-
-        try
-        {
-            var creds = (await _credentials.Get())[StorageSpace.Mirrors];
-
-            if (_archives.TryGetPath(toUpload.Hash, out var path))
-            {
-                _logger.LogInformation($"Uploading mirror file {toUpload.Hash} {path.Size().FileSizeToString()}");
-
-                var exists = false;
-                using (var client = await GetClient(creds))
-                {
-                    exists = await client.FileExistsAsync($"{toUpload.Hash.ToHex()}/definition.json.gz");
-                }
-
-                if (exists)
-                {
-                    _logger.LogInformation($"Skipping {toUpload.Hash} it's already on the server");
-                    await toUpload.Finish(_sql);
-                    goto TOP;
-                }
-
-                await _discord.Send(Channel.Spam,
-                    new DiscordMessage
-                    {
-                        Content = $"Uploading {toUpload.Hash} - {toUpload.Created} because {toUpload.Rationale}"
-                    });
-
-                var definition = await _wjClient.GenerateFileDefinition(path);
-
-                using (var client = await GetClient(creds))
-                {
-                    await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}");
-                    await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}/parts");
-                }
-
-                string MakePath(long idx)
-                {
-                    return $"{definition.Hash.ToHex()}/parts/{idx}";
-                }
-
-                await definition.Parts.PDoAll(new Resource<MirrorUploader>(), async part =>
-                {
-                    _logger.LogInformation("Uploading mirror part ({index}/{length})", part.Index,
-                        definition.Parts.Length);
-
-                    var buffer = new byte[part.Size];
-                    await using (var fs = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read))
-                    {
-                        fs.Position = part.Offset;
-                        await fs.ReadAsync(buffer);
-                    }
-
-                    await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () =>
-                    {
-                        using var client = await GetClient(creds);
-                        var name = MakePath(part.Index);
-                        await client.UploadAsync(new MemoryStream(buffer), name);
-                    });
-                });
-
-                await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () =>
-                {
-                    using var client = await GetClient(creds);
-                    _logger.LogInformation("Finishing mirror upload");
-
-
-                    await using var ms = new MemoryStream();
-                    await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
-                    {
-                        await _dtos.Serialize(definition, gz);
-                    }
-
-                    ms.Position = 0;
-                    var remoteName = $"{definition.Hash.ToHex()}/definition.json.gz";
-                    await client.UploadAsync(ms, remoteName);
-                });
-
-                await toUpload.Finish(_sql);
-            }
-            else
-            {
-                await toUpload.Fail(_sql, "Archive not found");
-            }
-        }
-        catch (Exception ex)
-        {
-            _logger.LogInformation($"{toUpload.Created} {toUpload.Uploaded}");
-            _logger.LogError(ex, "Error uploading");
-            await toUpload.Fail(_sql, ex.ToString());
-        }
-
-        goto TOP;
-    }
-
-    private async Task<FtpClient> GetClient(FtpSite? creds = null)
-    {
-        return await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () =>
-        {
-            creds ??= (await _ftpCreds.Get())[StorageSpace.Mirrors];
-
-            var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password));
-            ftpClient.DataConnectionType = FtpDataConnectionType.EPSV;
-            await ftpClient.ConnectAsync();
-            return ftpClient;
-        });
-    }
-
-    /// <summary>
-    ///     Gets a list of all the Mirrored file hashes that physically exist on the CDN (via FTP lookup)
-    /// </summary>
-    /// <returns></returns>
-    public async Task<HashSet<Hash>> GetHashesOnCDN()
-    {
-        using var ftpClient = await GetClient();
-        var serverFiles = await ftpClient.GetNameListingAsync("\\");
-
-        return serverFiles
-            .Select(f => ((RelativePath) f).FileName)
-            .Select(l =>
-            {
-                try
-                {
-                    return Hash.FromHex((string) l);
-                }
-                catch (Exception)
-                {
-                    return default;
-                }
-            })
-            .Where(h => h != default)
-            .ToHashSet();
-    }
-
-    public async Task DeleteOldMirrorFiles()
-    {
-        var existingHashes = await GetHashesOnCDN();
-        var fromSql = await _sql.GetAllMirroredHashes();
-
-        foreach (var (hash, _) in fromSql.Where(s => s.Value))
-        {
-            _logger.LogInformation("Removing {hash} from SQL it's no longer in the CDN", hash);
-            if (!existingHashes.Contains(hash))
-                await _sql.DeleteMirroredFile(hash);
-        }
-
-        var toDelete = existingHashes.Where(h => !fromSql.ContainsKey(h)).ToArray();
-
-        using var client = await GetClient();
-        foreach (var hash in toDelete)
-        {
-            await _discord.Send(Channel.Spam,
-                new DiscordMessage {Content = $"Removing mirrored file {hash}, as it's no longer in sql"});
-            _logger.LogInformation("Removing {hash} from the CDN it's no longer in SQL", hash);
-            await client.DeleteDirectoryAsync(hash.ToHex());
-        }
-    }
-}

Wabbajack.Server/Services/ModListDownloader.cs

@@ -1,140 +0,0 @@
-using System;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Builder;
-using Microsoft.Extensions.Logging;
-using Wabbajack.BuildServer;
-using Wabbajack.Common;
-using Wabbajack.Downloaders;
-using Wabbajack.DTOs;
-using Wabbajack.DTOs.JsonConverters;
-using Wabbajack.Installer;
-using Wabbajack.Networking.WabbajackClientApi;
-using Wabbajack.Paths.IO;
-using Wabbajack.Server.DataLayer;
-using Wabbajack.Server.DTOs;
-
-namespace Wabbajack.Server.Services;
-
-public class ModListDownloader : AbstractService<ModListDownloader, int>
-{
-    private readonly DownloadDispatcher _dispatcher;
-    private readonly DTOSerializer _dtos;
-    private readonly TemporaryFileManager _manager;
-    private readonly Client _wjClient;
-    private readonly DiscordWebHook _discord;
-    private readonly ArchiveMaintainer _maintainer;
-    private readonly SqlService _sql;
-
-    public ModListDownloader(ILogger<ModListDownloader> logger, AppSettings settings, ArchiveMaintainer maintainer,
-        SqlService sql, DiscordWebHook discord, QuickSync quickSync, Client wjClient, TemporaryFileManager manager,
-        DownloadDispatcher dispatcher, DTOSerializer dtos)
-        : base(logger, settings, quickSync, TimeSpan.FromMinutes(1))
-    {
-        _logger = logger;
-        _settings = settings;
-        _maintainer = maintainer;
-        _sql = sql;
-        _discord = discord;
-        _wjClient = wjClient;
-        _manager = manager;
-        _dispatcher = dispatcher;
-        _dtos = dtos;
-    }
-
-
-    public override async Task<int> Execute()
-    {
-        var downloaded = 0;
-        var lists = await _wjClient.LoadLists();
-
-        foreach (var list in lists)
-            try
-            {
-                ReportStarting(list.Links.MachineURL);
-                if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
-                    continue;
-
-
-                if (!_maintainer.HaveArchive(list.DownloadMetadata!.Hash))
-                {
-                    _logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
-                    await _discord.Send(Channel.Ham,
-                        new DiscordMessage
-                        {
-                            Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
-                        });
-                    var tf = _manager.CreateFile();
-                    var state = _dispatcher.Parse(new Uri(list.Links.Download));
-                    if (state == null)
-                    {
-                        _logger.Log(LogLevel.Error,
-                            $"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}");
-                        continue;
-                    }
-
-                    downloaded += 1;
-                    await _dispatcher.Download(new Archive {State = state, Name = $"{list.Links.MachineURL}.wabbajack"},
-                        tf.Path, CancellationToken.None);
-                    var hash = await tf.Path.Hash();
-                    if (hash != list.DownloadMetadata.Hash)
-                    {
-                        _logger.Log(LogLevel.Error,
-                            $"Downloaded modlist {list.Links.MachineURL} {list.DownloadMetadata.Hash} didn't match metadata hash of {hash}");
-                        await _sql.IngestModList(list.DownloadMetadata.Hash, list, new ModList(), true);
-                        continue;
-                    }
-
-                    await _maintainer.Ingest(tf.Path);
-                }
-
-                _maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath);
-                ModList modlist;
-
-                modlist = await StandardInstaller.LoadFromFile(_dtos, modlistPath);
-
-                await _discord.Send(Channel.Ham,
-                    new DiscordMessage
-                    {
-                        Content = $"Ingesting {list.Links.MachineURL} version {modlist.Version}"
-                    });
-                await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist, false);
-            }
-            catch (Exception ex)
-            {
-                _logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
-                await _discord.Send(Channel.Ham,
-                    new DiscordMessage
-                    {
-                        Content =
-                            $"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash} - {ex.Message}"
-                    });
-            }
-            finally
-            {
-                ReportEnding(list.Links.MachineURL);
-            }
-
-        _logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");
-        if (downloaded > 0)
-            await _discord.Send(Channel.Ham,
-                new DiscordMessage {Content = $"Downloaded {downloaded} new lists"});
-
-        var fc = await _sql.EnqueueModListFilesForIndexing();
-        _logger.Log(LogLevel.Information, $"Enqueing {fc} files for downloading");
-        if (fc > 0)
-            await _discord.Send(Channel.Ham,
-                new DiscordMessage {Content = $"Enqueing {fc} files for downloading"});
-
-        return downloaded;
-    }
-}
-
-public static class ModListDownloaderExtensions
-{
-    public static void UseModListDownloader(this IApplicationBuilder b)
-    {
-        var poll = (ModListDownloader) b.ApplicationServices.GetService(typeof(ModListDownloader));
-        poll.Start();
-    }
-}

Wabbajack.Server/Services/NexusPoll.cs

@@ -1,97 +0,0 @@
-using System;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Builder;
-using Microsoft.Extensions.Logging;
-using Wabbajack.BuildServer;
-using Wabbajack.Common;
-using Wabbajack.DTOs;
-using Wabbajack.Networking.NexusApi;
-using Wabbajack.Server.DataLayer;
-
-namespace Wabbajack.Server.Services;
-
-public class NexusPoll
-{
-    private readonly NexusApi _api;
-    private readonly ParallelOptions _parallelOptions;
-    private readonly GlobalInformation _globalInformation;
-    private readonly ILogger<NexusPoll> _logger;
-    private readonly AppSettings _settings;
-    private readonly SqlService _sql;
-
-    public NexusPoll(ILogger<NexusPoll> logger, AppSettings settings, SqlService service,
-        GlobalInformation globalInformation, ParallelOptions parallelOptions, NexusApi api)
-    {
-        _sql = service;
-        _settings = settings;
-        _globalInformation = globalInformation;
-        _logger = logger;
-        _parallelOptions = parallelOptions;
-        _api = api;
-    }
-
-    public async Task UpdateNexusCacheAPI(CancellationToken token)
-    {
-        using var _ = _logger.BeginScope("Nexus Update via API");
-        _logger.Log(LogLevel.Information, "Starting Nexus Update via API");
-
-        var purged = await GameRegistry.Games.Values
-            .Where(game => game.NexusName != null)
-            .SelectMany(async game =>
-            {
-                var (mods, _) = await _api.GetUpdates(game.Game, token);
-
-                return mods.Select(mod => new {Game = game, Mod = mod});
-            })
-            .Select(async row =>
-            {
-                var a = row.Mod.LatestFileUpdate.AsUnixTime();
-                // Mod activity could hide files
-                var b = row.Mod.LastestModActivity.AsUnixTime();
-
-                var t = a > b ? a : b;
-
-                long purgeCount = 0;
-                purgeCount += await _sql.DeleteNexusModInfosUpdatedBeforeDate(row.Game.Game, row.Mod.ModId, t.Date);
-                purgeCount += await _sql.DeleteNexusModFilesUpdatedBeforeDate(row.Game.Game, row.Mod.ModId, t.Date);
-                return purgeCount;
-            })
-            .SumAsync(x => x);
-
-        _logger.Log(LogLevel.Information, "Purged {count} cache entries", purged);
-        _globalInformation.LastNexusSyncUTC = DateTime.UtcNow;
-    }
-
-    public void Start()
-    {
-        if (!_settings.RunBackEndJobs) return;
-
-        Task.Run(async () =>
-        {
-            while (true)
-            {
-                try
-                {
-                    await UpdateNexusCacheAPI(CancellationToken.None);
-                }
-                catch (Exception ex)
-                {
-                    _logger.LogError(ex, "Error getting API feed from Nexus");
-                }
-
-                await Task.Delay(_globalInformation.NexusAPIPollRate);
-            }
-        });
-    }
-}
-
-public static class NexusPollExtensions
-{
-    public static void UseNexusPoll(this IApplicationBuilder b)
-    {
-        var poll = (NexusPoll) b.ApplicationServices.GetService(typeof(NexusPoll));
-        poll.Start();
-    }
-}

Wabbajack.Server/Services/NonNexusDownloadValidator.cs

@@ -1,85 +0,0 @@
-using System;
-using System.Linq;
-using System.Threading.Tasks;
-using Microsoft.Extensions.Logging;
-using Wabbajack.BuildServer;
-using Wabbajack.Downloaders;
-using Wabbajack.Server.DataLayer;
-
-namespace Wabbajack.Server.Services;
-
-public class NonNexusDownloadValidator : AbstractService<NonNexusDownloadValidator, int>
-{
-    private readonly DownloadDispatcher _dispatcher;
-    private readonly ParallelOptions _parallelOptions;
-    private readonly SqlService _sql;
-
-    public NonNexusDownloadValidator(ILogger<NonNexusDownloadValidator> logger, AppSettings settings, SqlService sql,
-        QuickSync quickSync, DownloadDispatcher dispatcher, ParallelOptions parallelOptions)
-        : base(logger, settings, quickSync, TimeSpan.FromHours(2))
-    {
-        _sql = sql;
-        _dispatcher = dispatcher;
-        _parallelOptions = parallelOptions;
-    }
-
-    public override async Task<int> Execute()
-    {
-        var archives = await _sql.GetNonNexusModlistArchives();
-        _logger.Log(LogLevel.Information, $"Validating {archives.Count} non-Nexus archives");
-        await _dispatcher.PrepareAll(archives.Select(a => a.State));
-        var random = new Random();
-
-        /*
-        var results = await archives.PMap(_parallelOptions, async archive =>
-        {
-            try
-            {
-                await Task.Delay(random.Next(1000, 5000));
-                
-                var token = new CancellationTokenSource();
-                token.CancelAfter(TimeSpan.FromMinutes(10));
-                
-                ReportStarting(archive.State.PrimaryKeyString);
-                bool isValid = false;
-                switch (archive.State)
-                {
-                    //case WabbajackCDNDownloader.State _: 
-                    //case GoogleDriveDownloader.State _: // Let's try validating Google again 2/10/2021
-                    case GameFileSource _:
-                        isValid = true;
-                        break;
-                    case Manual _:
-                    case ModDB _:
-                    case Http h when h.Url.ToString().StartsWith("https://wabbajack"):
-                        isValid = true;
-                        break;
-                    default:
-                        isValid = await _dispatcher.Verify(archive, token.Token);
-                        break;
-                }
-                return (Archive: archive, IsValid: isValid);
-            }
-            catch (Exception ex)
-            {
-                _logger.Log(LogLevel.Warning, $"Error for {archive.Name} {archive.State.PrimaryKeyString} {ex}");
-                return (Archive: archive, IsValid: false);
-            }
-            finally
-            {
-                ReportEnding(archive.State.PrimaryKeyString);
-            }
-
-        }).ToList();
-
-        await _sql.UpdateNonNexusModlistArchivesStatus(results);
-        var failed = results.Count(r => !r.IsValid);
-        var passed = results.Count() - failed;
-        foreach(var (archive, _) in results.Where(f => !f.IsValid))
-            _logger.Log(LogLevel.Warning, $"Validation failed for {archive.Name} from {archive.State.PrimaryKeyString}");
-        
-        _logger.Log(LogLevel.Information, $"Non-nexus validation completed {failed} out of {passed} failed");
-        */
-        return default;
-    }
-}

Wabbajack.Server/Services/PatchBuilder.cs

@@ -1,237 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Net;
-using System.Threading.Tasks;
-using FluentFTP;
-using FluentFTP.Helpers;
-using Microsoft.Extensions.Logging;
-using Wabbajack.BuildServer;
-using Wabbajack.Common;
-using Wabbajack.Compiler.PatchCache;
-using Wabbajack.Hashing.xxHash64;
-using Wabbajack.Paths;
-using Wabbajack.Paths.IO;
-using Wabbajack.Server.DataLayer;
-using Wabbajack.Server.DTOs;
-using Wabbajack.Server.TokenProviders;
-
-namespace Wabbajack.Server.Services;
-
-public class PatchBuilder : AbstractService<PatchBuilder, int>
-{
-    private readonly IFtpSiteCredentials _ftpCreds;
-    private readonly TemporaryFileManager _manager;
-    private readonly DiscordWebHook _discordWebHook;
-    private readonly ArchiveMaintainer _maintainer;
-    private readonly SqlService _sql;
-
-    public PatchBuilder(ILogger<PatchBuilder> logger, SqlService sql, AppSettings settings,
-        ArchiveMaintainer maintainer,
-        DiscordWebHook discordWebHook, QuickSync quickSync, TemporaryFileManager manager, IFtpSiteCredentials ftpCreds)
-        : base(logger, settings, quickSync, TimeSpan.FromMinutes(1))
-    {
-        _discordWebHook = discordWebHook;
-        _sql = sql;
-        _maintainer = maintainer;
-        _manager = manager;
-        _ftpCreds = ftpCreds;
-    }
-
-    public bool NoCleaning { get; set; }
-
-    public override async Task<int> Execute()
-    {
-        var count = 0;
-        while (true)
-        {
-            count++;
-
-            var patch = await _sql.GetPendingPatch();
-            if (patch == default) break;
-
-            try
-            {
-                _logger.LogInformation(
-                    $"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}");
-                await _discordWebHook.Send(Channel.Spam,
-                    new DiscordMessage
-                    {
-                        Content =
-                            $"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
-                    });
-
-                if (patch.Src.Archive.Hash == patch.Dest.Archive.Hash && patch.Src.Archive.State.PrimaryKeyString ==
-                    patch.Dest.Archive.State.PrimaryKeyString)
-                {
-                    await patch.Fail(_sql, "Hashes match");
-                    continue;
-                }
-
-                if (patch.Src.Archive.Size > 2_500_000_000 || patch.Dest.Archive.Size > 2_500_000_000)
-                {
-                    await patch.Fail(_sql, "Too large to patch");
-                    continue;
-                }
-
-                _maintainer.TryGetPath(patch.Src.Archive.Hash, out var srcPath);
-                _maintainer.TryGetPath(patch.Dest.Archive.Hash, out var destPath);
-
-                await using var sigFile = _manager.CreateFile();
-                await using var patchFile = _manager.CreateFile();
-                await using var srcStream = srcPath.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
-                await using var destStream = destPath.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
-                await using var sigStream = sigFile.Path.Open(FileMode.Create, FileAccess.ReadWrite);
-                await using var patchOutput = patchFile.Path.Open(FileMode.Create, FileAccess.ReadWrite);
-                OctoDiff.Create(destStream, srcStream, sigStream, patchOutput);
-                await patchOutput.DisposeAsync();
-                var size = patchFile.Path.Size();
-
-                await UploadToCDN(patchFile.Path, PatchName(patch));
-
-
-                await patch.Finish(_sql, size);
-                await _discordWebHook.Send(Channel.Spam,
-                    new DiscordMessage
-                    {
-                        Content =
-                            $"Built {size.ToFileSizeString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
-                    });
-            }
-            catch (Exception ex)
-            {
-                _logger.LogError(ex, "Error while building patch");
-                await patch.Fail(_sql, ex.ToString());
-                await _discordWebHook.Send(Channel.Spam,
-                    new DiscordMessage
-                    {
-                        Content =
-                            $"Failure building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
-                    });
-            }
-        }
-
-        if (count > 0)
-        {
-        }
-
-        if (!NoCleaning)
-            await CleanupOldPatches();
-
-        return count;
-    }
-
-    private static string PatchName(Patch patch)
-    {
-        return PatchName(patch.Src.Archive.Hash, patch.Dest.Archive.Hash);
-    }
-
-    private static string PatchName(Hash oldHash, Hash newHash)
-    {
-        return $"{oldHash.ToHex()}_{newHash.ToHex()}";
-    }
-
-    private async Task CleanupOldPatches()
-    {
-        var patches = await _sql.GetOldPatches();
-        using var client = await GetBunnyCdnFtpClient();
-
-        foreach (var patch in patches)
-        {
-            _logger.LogInformation($"Cleaning patch {patch.Src.Archive.Hash} -> {patch.Dest.Archive.Hash}");
-
-            await _discordWebHook.Send(Channel.Spam,
-                new DiscordMessage
-                {
-                    Content =
-                        $"Removing {patch.PatchSize.FileSizeToString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString} due it no longer being required by curated lists"
-                });
-
-            if (!await DeleteFromCDN(client, PatchName(patch)))
-                _logger.LogWarning($"Patch file didn't exist {PatchName(patch)}");
-
-            await _sql.DeletePatch(patch);
-
-            var pendingPatch = await _sql.GetPendingPatch();
-            if (pendingPatch != default) break;
-        }
-
-        var files = await client.GetListingAsync("\\");
-        _logger.LogInformation($"Found {files.Length} on the CDN");
-
-        var sqlFiles = await _sql.AllPatchHashes();
-        _logger.LogInformation($"Found {sqlFiles.Count} in SQL");
-
-        HashSet<(Hash, Hash)> NamesToPairs(IEnumerable<FtpListItem> ftpFiles)
-        {
-            return ftpFiles.Select(f => f.Name).Where(f => f.Contains("_")).Select(p =>
-            {
-                try
-                {
-                    var lst = p.Split("_", StringSplitOptions.RemoveEmptyEntries).Select(Hash.FromHex).ToArray();
-                    return (lst[0], lst[1]);
-                }
-                catch (ArgumentException)
-                {
-                    return default;
-                }
-                catch (FormatException)
-                {
-                    return default;
-                }
-            }).Where(f => f != default).ToHashSet();
-        }
-
-        var oldHashPairs = NamesToPairs(files.Where(f => DateTime.UtcNow - f.Modified > TimeSpan.FromDays(2)));
-        foreach (var (oldHash, newHash) in oldHashPairs.Where(o => !sqlFiles.Contains(o)))
-        {
-            _logger.LogInformation($"Removing CDN File entry for {oldHash} -> {newHash} it's not SQL");
-            await client.DeleteFileAsync(PatchName(oldHash, newHash));
-        }
-
-        var hashPairs = NamesToPairs(files);
-        foreach (var sqlFile in sqlFiles.Where(s => !hashPairs.Contains(s)))
-        {
-            _logger.LogInformation("Removing SQL File entry for {from} -> {to} it's not on the CDN", sqlFile.Item1,
-                sqlFile.Item2);
-            await _sql.DeletePatchesForHashPair(sqlFile);
-        }
-    }
-
-    private async Task UploadToCDN(AbsolutePath patchFile, string patchName)
-    {
-        for (var times = 0; times < 5; times++)
-            try
-            {
-                _logger.Log(LogLevel.Information,
-                    $"Uploading {patchFile.Size().ToFileSizeString()} patch file to CDN {patchName}");
-                using var client = await GetBunnyCdnFtpClient();
-
-                await client.UploadFileAsync(patchFile.ToString(), patchName);
-                return;
-            }
-            catch (Exception ex)
-            {
-                _logger.LogError(ex, $"Error uploading {patchFile} to CDN");
-            }
-
-        _logger.Log(LogLevel.Error, $"Couldn't upload {patchFile} to {patchName}");
-    }
-
-    private async Task<bool> DeleteFromCDN(FtpClient client, string patchName)
-    {
-        if (!await client.FileExistsAsync(patchName))
-            return false;
-        await client.DeleteFileAsync(patchName);
-        return true;
-    }
-
-    private async Task<FtpClient> GetBunnyCdnFtpClient()
-    {
-        var info = (await _ftpCreds.Get())[StorageSpace.Patches];
-        var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
-        await client.ConnectAsync();
-        return client;
-    }
-}

Wabbajack.Server/Startup.cs

@@ -1,5 +1,6 @@
 using System.Diagnostics;
 using System.IO;
+using System.Net.Http;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
@@ -12,7 +13,8 @@ using Microsoft.Extensions.FileProviders;
 using Microsoft.Extensions.Hosting;
 using Newtonsoft.Json;
 using Wabbajack.BuildServer;
-using Wabbajack.Server.DataLayer;
+using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Server.DataModels;
 using Wabbajack.Server.Services;
 
 namespace Wabbajack.Server;
@@ -51,20 +53,15 @@ public class Startup
 
         services.AddSingleton<AppSettings>();
         services.AddSingleton<QuickSync>();
-        services.AddSingleton<SqlService>();
         services.AddSingleton<GlobalInformation>();
-        services.AddSingleton<NexusPoll>();
-        services.AddSingleton<ArchiveMaintainer>();
-        services.AddSingleton<ModListDownloader>();
-        services.AddSingleton<NonNexusDownloadValidator>();
-        services.AddSingleton<ArchiveDownloader>();
         services.AddSingleton<DiscordWebHook>();
-        services.AddSingleton<PatchBuilder>();
-        services.AddSingleton<MirrorUploader>();
-        services.AddSingleton<MirrorQueueService>();
         services.AddSingleton<Watchdog>();
-        services.AddSingleton<DiscordFrontend>();
+        services.AddSingleton<Metrics>();
-        services.AddSingleton<MetricsKeyCache>();
+        services.AddSingleton<HttpClient>();
+        services.AddSingleton<AuthorFiles>();
+        services.AddSingleton<AuthorKeys>();
+        services.AddDTOSerializer();
+        services.AddDTOConverters();
         services.AddResponseCompression(options =>
         {
             options.Providers.Add<BrotliCompressionProvider>();
@@ -82,9 +79,6 @@ public class Startup
     {
         if (env.IsDevelopment()) app.UseDeveloperExceptionPage();
 
-        if (this is not TestStartup)
-            app.UseHttpsRedirection();
-
         app.UseDeveloperExceptionPage();
 
         var provider = new FileExtensionContentTypeProvider();
@@ -98,18 +92,10 @@ public class Startup
 
         app.UseAuthentication();
         app.UseAuthorization();
-        app.UseNexusPoll();
-        app.UseModListDownloader();
         app.UseResponseCompression();
 
-        app.UseService<NonNexusDownloadValidator>();
-        app.UseService<ArchiveDownloader>();
         app.UseService<DiscordWebHook>();
-        app.UseService<PatchBuilder>();
-        app.UseService<MirrorUploader>();
         app.UseService<Watchdog>();
-        app.UseService<DiscordFrontend>();
-        app.UseService<MetricsKeyCache>();
 
         app.Use(next =>
         {

Wabbajack.Server/TokenProviders/IFtpSiteCredentials.cs

@@ -1,9 +0,0 @@
-using System.Collections.Generic;
-using Wabbajack.Networking.Http.Interfaces;
-using Wabbajack.Server.DTOs;
-
-namespace Wabbajack.Server.TokenProviders;
-
-public interface IFtpSiteCredentials : ITokenProvider<Dictionary<StorageSpace, FtpSite>>
-{
-}

Wabbajack.Server/Wabbajack.Server.csproj

@@ -54,6 +54,10 @@
     <ItemGroup>
         <Compile Remove="Controllers\UploadedFiles.cs" />
         <Compile Remove="Services\ListValidator.cs" />
+        <Compile Remove="Controllers\ModFiles.cs" />
+        <Compile Remove="Controllers\Users.cs" />
+        <Compile Remove="Controllers\NexusCache.cs" />
+        <Compile Remove="Controllers\AuthorControls.cs" />
     </ItemGroup>
 
     <ItemGroup>

Wabbajack.Server/appsettings.json

@@ -7,15 +7,10 @@
     }
   },
   "WabbajackSettings": {
-    "DownloadDir": "c:\\tmp\\downloads",
+    "TempFolder": "c:\\tmp\\server_temp",
-    "ArchiveDir": "w:\\archives",
+    "MetricsFolder": "c:\\tmp\\server_metrics",
-    "TempFolder": "c:\\tmp",
+    "AuthoredFilesFolder": "c:\\tmp\\server_authored_files",
-    "JobRunner": true,
+    "AuthorAPIKeyFile": "c:\\tmp\\author_keys.txt"
-    "JobScheduler": false,
-    "RunFrontEndJobs": true,
-    "RunBackEndJobs": false,
-    "BunnyCDN_StorageZone": "wabbajacktest",
-    "SQLConnection": "Data Source=.\\SQLEXPRESS;Integrated Security=True;Initial Catalog=wabbajack_prod;MultipleActiveResultSets=true"
   },
   "AllowedHosts": "*"
 }