From 2cb17aa69a77d7949f3779d7526217e63a08677f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 27 Nov 2021 11:31:35 -0700 Subject: [PATCH] Oh yeah, lots off good stuff here --- Wabbajack.Common/SemaphoreSlimExtensions.cs | 15 + .../ApiKeyAuthorizationHandler.cs | 34 +-- Wabbajack.Server/AppSettings.cs | 33 +-- Wabbajack.Server/Controllers/AuthoredFiles.cs | 114 +++----- Wabbajack.Server/Controllers/Heartbeat.cs | 105 +------ .../Controllers/ListDefinitions.cs | 83 ------ Wabbajack.Server/Controllers/ListsStatus.cs | 184 ------------ Wabbajack.Server/Controllers/Metrics.cs | 177 +----------- Wabbajack.Server/Controllers/ModUpgrade.cs | 158 ----------- Wabbajack.Server/DTOs/AggregateMetric.cs | 10 - Wabbajack.Server/DTOs/ArchiveDownload.cs | 30 -- Wabbajack.Server/DTOs/AuthoredFilesSummary.cs | 14 - Wabbajack.Server/DTOs/FtpSite.cs | 31 --- Wabbajack.Server/DTOs/Metric.cs | 2 + Wabbajack.Server/DTOs/MetricResult.cs | 10 - Wabbajack.Server/DTOs/MirroredFile.cs | 29 -- Wabbajack.Server/DTOs/Patch.cs | 32 --- Wabbajack.Server/DTOs/ValidationData.cs | 17 -- Wabbajack.Server/DataLayer/ApiKeys.cs | 54 ---- .../DataLayer/ArchiveDownloads.cs | 261 ------------------ Wabbajack.Server/DataLayer/AuthoredFiles.cs | 76 ----- Wabbajack.Server/DataLayer/Mappers.cs | 109 -------- Wabbajack.Server/DataLayer/Metrics.cs | 189 ------------- Wabbajack.Server/DataLayer/MirroredFiles.cs | 147 ---------- Wabbajack.Server/DataLayer/ModLists.cs | 117 -------- Wabbajack.Server/DataLayer/Nexus.cs | 156 ----------- Wabbajack.Server/DataLayer/NexusKeys.cs | 51 ---- .../DataLayer/NonNexusModlistArchives.cs | 43 --- Wabbajack.Server/DataLayer/Patches.cs | 259 ----------------- Wabbajack.Server/DataLayer/SqlService.cs | 30 -- Wabbajack.Server/DataLayer/ValidationData.cs | 56 ---- Wabbajack.Server/DataModels/AuthorFiles.cs | 110 ++++++++ Wabbajack.Server/DataModels/AuthorKeys.cs | 28 ++ Wabbajack.Server/DataModels/Metrics.cs | 39 +++ Wabbajack.Server/Program.cs | 11 +- .../Services/ArchiveDownloader.cs | 150 ---------- .../Services/ArchiveMaintainer.cs | 66 ----- Wabbajack.Server/Services/DiscordFrontend.cs | 189 ------------- Wabbajack.Server/Services/MetricsKeyCache.cs | 61 ---- .../Services/MirrorQueueService.cs | 27 -- Wabbajack.Server/Services/MirrorUploader.cs | 227 --------------- .../Services/ModListDownloader.cs | 140 ---------- Wabbajack.Server/Services/NexusPoll.cs | 97 ------- .../Services/NonNexusDownloadValidator.cs | 85 ------ Wabbajack.Server/Services/PatchBuilder.cs | 237 ---------------- Wabbajack.Server/Startup.cs | 32 +-- .../TokenProviders/IFtpSiteCredentials.cs | 9 - Wabbajack.Server/Wabbajack.Server.csproj | 4 + Wabbajack.Server/appsettings.json | 13 +- 49 files changed, 290 insertions(+), 3861 deletions(-) create mode 100644 Wabbajack.Common/SemaphoreSlimExtensions.cs delete mode 100644 Wabbajack.Server/Controllers/ListDefinitions.cs delete mode 100644 Wabbajack.Server/Controllers/ListsStatus.cs delete mode 100644 Wabbajack.Server/Controllers/ModUpgrade.cs delete mode 100644 Wabbajack.Server/DTOs/AggregateMetric.cs delete mode 100644 Wabbajack.Server/DTOs/ArchiveDownload.cs delete mode 100644 Wabbajack.Server/DTOs/AuthoredFilesSummary.cs delete mode 100644 Wabbajack.Server/DTOs/FtpSite.cs delete mode 100644 Wabbajack.Server/DTOs/MetricResult.cs delete mode 100644 Wabbajack.Server/DTOs/MirroredFile.cs delete mode 100644 Wabbajack.Server/DTOs/Patch.cs delete mode 100644 Wabbajack.Server/DTOs/ValidationData.cs delete mode 100644 Wabbajack.Server/DataLayer/ApiKeys.cs delete mode 100644 Wabbajack.Server/DataLayer/ArchiveDownloads.cs delete mode 100644 Wabbajack.Server/DataLayer/AuthoredFiles.cs delete mode 100644 Wabbajack.Server/DataLayer/Mappers.cs delete mode 100644 Wabbajack.Server/DataLayer/Metrics.cs delete mode 100644 Wabbajack.Server/DataLayer/MirroredFiles.cs delete mode 100644 Wabbajack.Server/DataLayer/ModLists.cs delete mode 100644 Wabbajack.Server/DataLayer/Nexus.cs delete mode 100644 Wabbajack.Server/DataLayer/NexusKeys.cs delete mode 100644 Wabbajack.Server/DataLayer/NonNexusModlistArchives.cs delete mode 100644 Wabbajack.Server/DataLayer/Patches.cs delete mode 100644 Wabbajack.Server/DataLayer/SqlService.cs delete mode 100644 Wabbajack.Server/DataLayer/ValidationData.cs create mode 100644 Wabbajack.Server/DataModels/AuthorFiles.cs create mode 100644 Wabbajack.Server/DataModels/AuthorKeys.cs create mode 100644 Wabbajack.Server/DataModels/Metrics.cs delete mode 100644 Wabbajack.Server/Services/ArchiveDownloader.cs delete mode 100644 Wabbajack.Server/Services/ArchiveMaintainer.cs delete mode 100644 Wabbajack.Server/Services/DiscordFrontend.cs delete mode 100644 Wabbajack.Server/Services/MetricsKeyCache.cs delete mode 100644 Wabbajack.Server/Services/MirrorQueueService.cs delete mode 100644 Wabbajack.Server/Services/MirrorUploader.cs delete mode 100644 Wabbajack.Server/Services/ModListDownloader.cs delete mode 100644 Wabbajack.Server/Services/NexusPoll.cs delete mode 100644 Wabbajack.Server/Services/NonNexusDownloadValidator.cs delete mode 100644 Wabbajack.Server/Services/PatchBuilder.cs delete mode 100644 Wabbajack.Server/TokenProviders/IFtpSiteCredentials.cs diff --git a/Wabbajack.Common/SemaphoreSlimExtensions.cs b/Wabbajack.Common/SemaphoreSlimExtensions.cs new file mode 100644 index 00000000..032233a4 --- /dev/null +++ b/Wabbajack.Common/SemaphoreSlimExtensions.cs @@ -0,0 +1,15 @@ +using System; +using System.Reactive.Disposables; +using System.Threading; +using System.Threading.Tasks; + +namespace Wabbajack.Common; + +public static class SemaphoreSlimExtensions +{ + public static async Task Lock(this SemaphoreSlim slim) + { + await slim.WaitAsync(); + return Disposable.Create(() => slim.Release()); + } +} \ No newline at end of file diff --git a/Wabbajack.Server/ApiKeyAuthorizationHandler.cs b/Wabbajack.Server/ApiKeyAuthorizationHandler.cs index 18f7012f..ad4835fb 100644 --- a/Wabbajack.Server/ApiKeyAuthorizationHandler.cs +++ b/Wabbajack.Server/ApiKeyAuthorizationHandler.cs @@ -9,9 +9,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Wabbajack.DTOs.JsonConverters; -using Wabbajack.Server.DataLayer; -using Wabbajack.Server.DTOs; -using Wabbajack.Server.Services; +using Wabbajack.Server.DataModels; namespace Wabbajack.BuildServer; @@ -28,23 +26,19 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler options, + AuthorKeys authorKeys, ILoggerFactory logger, UrlEncoder encoder, ISystemClock clock, - MetricsKeyCache keyCache, DTOSerializer dtos, - AppSettings settings, - SqlService db) : base(options, logger, encoder, clock) + AppSettings settings) : base(options, logger, encoder, clock) { - _sql = db; _dtos = dtos; - _keyCache = keyCache; + _authorKeys = authorKeys; _settings = settings; } @@ -55,7 +49,7 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler {new(ClaimTypes.Role, "User")}; @@ -115,6 +105,8 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler (AbsolutePath) DownloadDir; - public string ArchiveDir { get; set; } - public AbsolutePath ArchivePath => (AbsolutePath) ArchiveDir; - public string TempFolder { get; set; } - public AbsolutePath TempPath => (AbsolutePath) TempFolder; - - public bool JobScheduler { get; set; } - public bool JobRunner { get; set; } - - public bool RunFrontEndJobs { get; set; } - public bool RunBackEndJobs { get; set; } - - public bool RunNexusPolling { get; set; } - public bool RunDownloader { get; set; } - - public string BunnyCDN_StorageZone { get; set; } - public string SqlConnection { get; set; } - - public int MaxJobs { get; set; } = 2; - public string SpamWebHook { get; set; } = null; public string HamWebHook { get; set; } = null; - public bool ValidateModUpgrades { get; set; } = true; + + public string AuthoredFilesFolder { get; set; } + public string MetricsFolder { get; set; } + public string TarLogPath { get; set; } } \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/AuthoredFiles.cs b/Wabbajack.Server/Controllers/AuthoredFiles.cs index 21286085..d216a392 100644 --- a/Wabbajack.Server/Controllers/AuthoredFiles.cs +++ b/Wabbajack.Server/Controllers/AuthoredFiles.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.IO.Compression; +using System.Linq; using System.Net; using System.Security.Claims; using System.Threading; @@ -9,15 +10,15 @@ using FluentFTP; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; using Nettle; using Wabbajack.Common; using Wabbajack.DTOs.CDN; using Wabbajack.DTOs.JsonConverters; using Wabbajack.Hashing.xxHash64; -using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DataModels; using Wabbajack.Server.DTOs; using Wabbajack.Server.Services; -using Wabbajack.Server.TokenProviders; namespace Wabbajack.BuildServer.Controllers; @@ -29,7 +30,13 @@ public class AuthoredFiles : ControllerBase {{each $.files }} - + + + + + + + {{/each}}
{{$.OriginalFileName}}{{$.Size}}{{$.LastTouched}}{{$.Finalized}}{{$.Author}}
{{$.Definition.OriginalFileName}}{{$.HumanSize}}{{$.Definition.Author}}{{$.Updated}}(Slow) HTTP Direct Link
@@ -38,36 +45,33 @@ public class AuthoredFiles : ControllerBase private readonly DTOSerializer _dtos; - private readonly IFtpSiteCredentials _ftpCreds; private readonly DiscordWebHook _discord; private readonly ILogger _logger; private readonly AppSettings _settings; - private readonly SqlService _sql; + private readonly AuthorFiles _authoredFiles; - public AuthoredFiles(ILogger logger, SqlService sql, AppSettings settings, DiscordWebHook discord, - DTOSerializer dtos, IFtpSiteCredentials ftpCreds) + public AuthoredFiles(ILogger logger, AuthorFiles authorFiles, AppSettings settings, DiscordWebHook discord, + DTOSerializer dtos) { - _sql = sql; _logger = logger; _settings = settings; _discord = discord; _dtos = dtos; - _ftpCreds = ftpCreds; + _authoredFiles = authorFiles; } - + [HttpPut] [Route("{serverAssignedUniqueId}/part/{index}")] public async Task UploadFilePart(CancellationToken token, string serverAssignedUniqueId, long index) { var user = User.FindFirstValue(ClaimTypes.Name); - var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId); + var definition = await _authoredFiles.ReadDefinitionForServerId(serverAssignedUniqueId); if (definition.Author != user) return Forbid("File Id does not match authorized user"); _logger.Log(LogLevel.Information, $"Uploading File part {definition.OriginalFileName} - ({index} / {definition.Parts.Length})"); - - await _sql.TouchAuthoredFile(definition); + var part = definition.Parts[index]; await using var ms = new MemoryStream(); @@ -82,7 +86,8 @@ public class AuthoredFiles : ControllerBase $"Hashes don't match for index {index}. Sizes ({ms.Length} vs {part.Size}). Hashes ({hash} vs {part.Hash}"); ms.Position = 0; - await UploadAsync(ms, $"{definition.MungedName}/parts/{index}"); + await using var partStream = await _authoredFiles.CreatePart(definition.MungedName, (int)index); + await ms.CopyToAsync(partStream, token); return Ok(part.Hash.ToBase64()); } @@ -96,14 +101,10 @@ public class AuthoredFiles : ControllerBase _logger.Log(LogLevel.Information, "Creating File upload {originalFileName}", definition.OriginalFileName); - definition = await _sql.CreateAuthoredFile(definition, user); - - using (var client = await GetBunnyCdnFtpClient()) - { - await client.CreateDirectoryAsync($"{definition.MungedName}"); - await client.CreateDirectoryAsync($"{definition.MungedName}/parts"); - } - + definition.ServerAssignedUniqueId = Guid.NewGuid().ToString(); + definition.Author = user; + await _authoredFiles.WriteDefinition(definition); + await _discord.Send(Channel.Ham, new DiscordMessage { @@ -119,22 +120,11 @@ public class AuthoredFiles : ControllerBase public async Task CreateUpload(string serverAssignedUniqueId) { var user = User.FindFirstValue(ClaimTypes.Name); - var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId); + var definition = await _authoredFiles.ReadDefinitionForServerId(serverAssignedUniqueId); if (definition.Author != user) return Forbid("File Id does not match authorized user"); _logger.Log(LogLevel.Information, $"Finalizing file upload {definition.OriginalFileName}"); - await _sql.Finalize(definition); - - await using var ms = new MemoryStream(); - await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true)) - { - await _dtos.Serialize(definition, gz); - } - - ms.Position = 0; - await UploadAsync(ms, $"{definition.MungedName}/definition.json.gz"); - await _discord.Send(Channel.Ham, new DiscordMessage { @@ -146,26 +136,14 @@ public class AuthoredFiles : ControllerBase return Ok($"https://{host}.wabbajack.org/{definition.MungedName}"); } - private async Task GetBunnyCdnFtpClient() - { - var info = (await _ftpCreds.Get())[StorageSpace.AuthoredFiles]; - var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)}; - await client.ConnectAsync(); - return client; - } - - private async Task UploadAsync(Stream stream, string path) - { - using var client = await GetBunnyCdnFtpClient(); - await client.UploadAsync(stream, path); - } - [HttpDelete] [Route("{serverAssignedUniqueId}")] public async Task DeleteUpload(string serverAssignedUniqueId) { var user = User.FindFirstValue(ClaimTypes.Name); - var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId); + var definition = (await _authoredFiles.AllAuthoredFiles()) + .First(f => f.Definition.ServerAssignedUniqueId == serverAssignedUniqueId) + .Definition; if (definition.Author != user) return Forbid("File Id does not match authorized user"); await _discord.Send(Channel.Ham, @@ -176,33 +154,17 @@ public class AuthoredFiles : ControllerBase }); _logger.Log(LogLevel.Information, $"Deleting upload {definition.OriginalFileName}"); - await DeleteFolderOrSilentlyFail($"{definition.MungedName}"); - - await _sql.DeleteFileDefinition(definition); + await _authoredFiles.DeleteFile(definition); return Ok(); } - private async Task DeleteFolderOrSilentlyFail(string path) - { - try - { - using var client = await GetBunnyCdnFtpClient(); - await client.DeleteDirectoryAsync(path); - } - catch (Exception) - { - _logger.Log(LogLevel.Information, $"Delete failed for {path}"); - } - } - - [HttpGet] [AllowAnonymous] [Route("")] public async Task UploadedFilesGet() { - var files = await _sql.AllAuthoredFiles(); - var response = HandleGetListTemplate(new {files}); + var files = await _authoredFiles.AllAuthoredFiles(); + var response = HandleGetListTemplate(new {files = files.OrderByDescending(f => f.Updated).ToArray()}); return new ContentResult { ContentType = "text/html", @@ -210,4 +172,20 @@ public class AuthoredFiles : ControllerBase Content = response }; } + + [HttpGet] + [AllowAnonymous] + [Route("direct_link/{mungedName}")] + public async Task DirectLink(string mungedName) + { + var definition = await _authoredFiles.ReadDefinition(mungedName); + Response.Headers.ContentDisposition = + new StringValues($"attachment; filename={definition.OriginalFileName}"); + Response.Headers.ContentType = new StringValues("application/octet-stream"); + foreach (var part in definition.Parts) + { + await using var partStream = await _authoredFiles.StreamForPart(mungedName, (int)part.Index); + await partStream.CopyToAsync(Response.Body); + } + } } \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Heartbeat.cs b/Wabbajack.Server/Controllers/Heartbeat.cs index cec45d86..41e2d4bf 100644 --- a/Wabbajack.Server/Controllers/Heartbeat.cs +++ b/Wabbajack.Server/Controllers/Heartbeat.cs @@ -1,13 +1,8 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; -using Nettle; using Wabbajack.Server; -using Wabbajack.Server.DataLayer; using Wabbajack.Server.DTOs; using Wabbajack.Server.Services; @@ -16,68 +11,18 @@ namespace Wabbajack.BuildServer.Controllers; [Route("/heartbeat")] public class Heartbeat : ControllerBase { - private const int MAX_LOG_SIZE = 128; private static readonly DateTime _startTime; - private static List Log = new(); - - private static readonly Func HandleGetReport = NettleEngine.GetCompiler().Compile(@" - -

Server Status

- -

Service Overview ({{services.Length}}):

- - - -

Lists ({{lists.Length}}):

-
    - {{each $.lists }} -
  • {{$.Name}} - {{$.Time}} {{$.FailMessage}}
    • - {{/each}} -
- - "); - - private static readonly Func HandleGetServiceReport = NettleEngine.GetCompiler().Compile(@" - -

Service Status: {{Name}} {{TimeSinceLastRun}}

- -

Service Overview ({{ActiveWorkQueue.Length}}):

-
    - {{each $.ActiveWorkQueue }} -
  • {{$.Name}} {{$.Time}}
    • - {{/each}} -
- - - "); private readonly GlobalInformation _globalInformation; - private ILogger _logger; - - private readonly QuickSync _quickSync; - private SqlService _sql; - static Heartbeat() { _startTime = DateTime.Now; } - - public Heartbeat(ILogger logger, SqlService sql, GlobalInformation globalInformation, + public Heartbeat(ILogger logger, GlobalInformation globalInformation, QuickSync quickSync) { _globalInformation = globalInformation; - _sql = sql; - _logger = logger; - _quickSync = quickSync; } [HttpGet] @@ -86,54 +31,6 @@ public class Heartbeat : ControllerBase return Ok(new HeartbeatResult { Uptime = DateTime.Now - _startTime, - LastNexusUpdate = _globalInformation.TimeSinceLastNexusSync }); } - - [HttpGet("report")] - public async Task Report() - { - var response = HandleGetReport(new - { - services = (await _quickSync.Report()) - .Select(s => new - { - s.Key.Name, Time = s.Value.LastRunTime, MaxTime = s.Value.Delay, - IsLate = s.Value.LastRunTime > s.Value.Delay - }) - .OrderBy(s => s.Name) - .ToArray() - }); - return new ContentResult - { - ContentType = "text/html", - StatusCode = (int) HttpStatusCode.OK, - Content = response - }; - } - - [HttpGet("report/services/{serviceName}.html")] - public async Task ReportServiceStatus(string serviceName) - { - var services = await _quickSync.Report(); - var info = services.First(kvp => kvp.Key.Name == serviceName); - - var response = HandleGetServiceReport(new - { - info.Key.Name, - TimeSinceLastRun = DateTime.UtcNow - info.Value.LastRunTime, - ActiveWorkQueue = info.Value.ActiveWork.Select(p => new - { - Name = p.Item1, - Time = DateTime.UtcNow - p.Item2 - }).OrderByDescending(kp => kp.Time) - .ToArray() - }); - return new ContentResult - { - ContentType = "text/html", - StatusCode = (int) HttpStatusCode.OK, - Content = response - }; - } } \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/ListDefinitions.cs b/Wabbajack.Server/Controllers/ListDefinitions.cs deleted file mode 100644 index 62c51401..00000000 --- a/Wabbajack.Server/Controllers/ListDefinitions.cs +++ /dev/null @@ -1,83 +0,0 @@ -using System; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Wabbajack.Common; -using Wabbajack.DTOs; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.Paths.IO; -using Wabbajack.Server.DataLayer; -using Wabbajack.Server.Services; - -namespace Wabbajack.BuildServer.Controllers; - -[ApiController] -[Route("/list_definitions")] -public class ListDefinitions : ControllerBase -{ - private readonly DTOSerializer _dtos; - private readonly AppSettings _settings; - private DiscordWebHook _discord; - private readonly ILogger _logger; - private SqlService _sql; - - public ListDefinitions(ILogger logger, SqlService sql, DiscordWebHook discord, - AppSettings settings, - DTOSerializer dtos) - { - _logger = logger; - _sql = sql; - _discord = discord; - _settings = settings; - _dtos = dtos; - } - - - [Route("ingest")] - [Authorize(Roles = "User")] - [HttpPost] - public async Task PostIngest() - { - var user = Request.Headers[_settings.MetricsKeyHeader].First(); - var use_gzip = Request.Headers[_settings.CompressedBodyHeader].Any(); - _logger.Log(LogLevel.Information, $"Ingesting Modlist Definition for {user}"); - - var modlistBytes = await Request.Body.ReadAllAsync(); - - - _logger.LogInformation("Spawning ingestion task"); - var tsk = Task.Run(async () => - { - try - { - if (use_gzip) - { - await using var os = new MemoryStream(); - await using var gZipStream = - new GZipStream(new MemoryStream(modlistBytes), CompressionMode.Decompress); - await gZipStream.CopyToAsync(os); - modlistBytes = os.ToArray(); - } - - var modlist = _dtos.DeserializeAsync(new MemoryStream(modlistBytes)); - - var file = KnownFolders.EntryPoint.Combine("mod_list_definitions") - .Combine($"{user}_{DateTime.UtcNow.ToFileTimeUtc()}.json"); - file.Parent.CreateDirectory(); - await using var stream = file.Open(FileMode.Create, FileAccess.Write); - await _dtos.Serialize(modlist, stream); - _logger.Log(LogLevel.Information, $"Done Ingesting Modlist Definition for {user}"); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error ingesting uploaded modlist"); - } - }); - - return Accepted(0); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/ListsStatus.cs b/Wabbajack.Server/Controllers/ListsStatus.cs deleted file mode 100644 index dedadc5f..00000000 --- a/Wabbajack.Server/Controllers/ListsStatus.cs +++ /dev/null @@ -1,184 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Nettle; -using Wabbajack.DTOs; -using Wabbajack.DTOs.ServerResponses; - -namespace Wabbajack.BuildServer.Controllers; - -[ApiController] -[Route("/lists")] -public class ListsStatus : ControllerBase -{ - private static readonly Func HandleGetRssFeedTemplate = NettleEngine.GetCompiler().Compile(@" - - - - {{lst.Name}} - Broken Mods - http://build.wabbajack.org/status/{{lst.Name}}.html - These are mods that are broken and need updating - {{ each $.failed }} - - {{$.Archive.Name}} {{$.Archive.Hash}} {{$.Archive.State.PrimaryKeyString}} - {{$.Archive.Name}} - - {{/each}} - - - "); - - private static readonly Func HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@" - -

{{lst.Name}} - {{lst.Checked}} - {{ago}}min ago

- -

Failed ({{failed.Count}}):

-
    - {{each $.failed }} - {{if $.HasUrl}} -
  • {{$.Name}}
    • - {{else}} -
  • {{$.Name}}
    • - {{/if}} - {{/each}} -
- - -

Updated ({{updated.Count}}):

-
    - {{each $.updated }} - {{if $.HasUrl}} -
  • {{$.Name}}
    • - {{else}} -
  • {{$.Name}}
    • - {{/if}} - - {{/each}} -
- -

Mirrored ({{mirrored.Count}}):

-
    - {{each $.mirrored }} - {{if $.HasUrl}} -
  • {{$.Name}}
    • - {{else}} -
  • {{$.Name}}
    • - {{/if}} - - {{/each}} -
- -

Updating ({{updating.Count}}):

-
    - {{each $.updating }} - {{if $.HasUrl}} -
  • {{$.Name}}
    • - {{else}} -
  • {{$.Name}}
    • - {{/if}} - {{/each}} -
- -

Passed ({{passed.Count}}):

-
    - {{each $.passed }} - {{if $.HasUrl}} -
  • {{$.Name}}
    • - {{else}} -
  • {{$.Name}}
    • - {{/if}} - {{/each}} -
- - "); - - private ILogger _logger; - - public ListsStatus(ILogger logger) - { - _logger = logger; - } - - [HttpGet] - [Route("status.json")] - public async Task> HandleGetLists() - { - throw new NotImplementedException(); - } - - [HttpGet] - [Route("status/{Name}/broken.rss")] - public async Task HandleGetRSSFeed(string Name) - { - var lst = await DetailedStatus(Name); - var response = HandleGetRssFeedTemplate(new - { - lst, - failed = lst.Archives.Where(a => a.IsFailing).ToList(), - passed = lst.Archives.Where(a => !a.IsFailing).ToList() - }); - return new ContentResult - { - ContentType = "application/rss+xml", - StatusCode = (int) HttpStatusCode.OK, - Content = response - }; - } - - [HttpGet] - [Route("status/{Name}.html")] - public async Task HandleGetListHtml(string Name) - { - var lst = await DetailedStatus(Name); - - var response = HandleGetListTemplate(new - { - lst, - ago = (DateTime.UtcNow - lst.Checked).TotalMinutes, - failed = lst.Archives.Where(a => a.IsFailing).ToList(), - passed = lst.Archives.Where(a => !a.IsFailing).ToList(), - updated = lst.Archives.Where(a => a.ArchiveStatus == ArchiveStatus.Updated).ToList(), - updating = lst.Archives.Where(a => a.ArchiveStatus == ArchiveStatus.Updating).ToList(), - mirrored = lst.Archives.Where(a => a.ArchiveStatus == ArchiveStatus.Mirrored).ToList() - }); - return new ContentResult - { - ContentType = "text/html", - StatusCode = (int) HttpStatusCode.OK, - Content = response - }; - } - - [HttpGet] - [Route("status/{Name}.json")] - [ResponseCache(Duration = 60 * 5)] - public async Task HandleGetListJson(string Name) - { - var lst = await DetailedStatus(Name); - if (lst == default) return NotFound(); - return Ok(lst); - } - - private async Task DetailedStatus(string Name) - { - throw new NotImplementedException(); - } - - [HttpGet] - [Route("status/badge.json")] - public async Task HandleGitHubBadge() - { - throw new NotImplementedException(); - } - - [HttpGet] - [Route("status/{Name}/badge.json")] - public async Task HandleNamedGitHubBadge(string Name) - { - throw new NotImplementedException(); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Metrics.cs b/Wabbajack.Server/Controllers/Metrics.cs index 8099166d..fd5acac7 100644 --- a/Wabbajack.Server/Controllers/Metrics.cs +++ b/Wabbajack.Server/Controllers/Metrics.cs @@ -1,17 +1,13 @@ using System; -using System.Collections.Generic; using System.Linq; -using System.Net; using System.Reflection; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Nettle; using Wabbajack.Common; -using Wabbajack.Server; -using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DataModels; using Wabbajack.Server.DTOs; -using Wabbajack.Server.Services; namespace Wabbajack.BuildServer.Controllers; @@ -37,17 +33,15 @@ public class MetricsController : ControllerBase private static Func _totalListTemplate; private readonly AppSettings _settings; - private readonly MetricsKeyCache _keyCache; private ILogger _logger; - private readonly SqlService _sql; + private readonly Metrics _metricsStore; - public MetricsController(ILogger logger, SqlService sql, MetricsKeyCache keyCache, + public MetricsController(ILogger logger, Metrics metricsStore, AppSettings settings) { - _sql = sql; _logger = logger; - _keyCache = keyCache; _settings = settings; + _metricsStore = metricsStore; } @@ -73,169 +67,24 @@ public class MetricsController : ControllerBase { var date = DateTime.UtcNow; var metricsKey = Request.Headers[_settings.MetricsKeyHeader].FirstOrDefault(); - if (metricsKey != null) - await _keyCache.AddKey(metricsKey); // Used in tests - if (value == "Default" || value == "untitled" || subject == "failed_download" || Guid.TryParse(value, out _)) + if (value is "Default" or "untitled" || subject == "failed_download" || Guid.TryParse(value, out _)) return new Result {Timestamp = date}; - await Log(date, subject, value, metricsKey); + await _metricsStore.Ingest(new Metric + { + Timestamp = DateTime.UtcNow, + Action = subject, + Subject = subject, + MetricsKey = metricsKey, + UserAgent = Request.Headers.UserAgent.FirstOrDefault() ?? "", + }); return new Result {Timestamp = date}; } - [HttpGet] - [Route("report/{subject}")] - [ResponseCache(Duration = 60 * 60)] - public async Task MetricsReport(string subject) - { - var metrics = (await _sql.MetricsReport(subject)).ToList(); - var labels = metrics.GroupBy(m => m.Date) - .OrderBy(m => m.Key) - .Select(m => m.Key) - .ToArray(); - var labelStrings = labels.Select(l => l.ToString("MM-dd-yyy")).ToList(); - var results = metrics - .GroupBy(m => m.Subject) - .Select(g => - { - var indexed = g.ToDictionary(m => m.Date, m => m.Count); - return new MetricResult - { - SeriesName = g.Key, - Labels = labelStrings, - Values = labels.Select(l => indexed.TryGetValue(l, out var found) ? found : 0).ToList() - }; - }); - return Ok(results.ToList()); - } - - [HttpGet] - [Route("badge/{name}/total_installs_badge.json")] - public async Task TotalInstallsBadge(string name) - { - var results = await _sql.TotalInstalls(name); - - Response.ContentType = "application/json"; - - return Ok(results == 0 - ? new Badge($"Modlist {name} not found!", "Error") {color = "red"} - : new Badge("Installations: ", "____") {color = "green"}); - } - - [HttpGet] - [Route("badge/{name}/unique_installs_badge.json")] - public async Task UniqueInstallsBadge(string name) - { - var results = await _sql.UniqueInstalls(name); - - Response.ContentType = "application/json"; - - return Ok(results == 0 - ? new Badge($"Modlist {name} not found!", "Error") {color = "red"} - : new Badge("Installations: ", "____") {color = "green"}); - } - - [HttpGet] - [Route("tarlog/{key}")] - public async Task TarLog(string key) - { - var isTarKey = await _sql.IsTarKey(key); - - var report = new List<(DateTime, string, string)>(); - - if (isTarKey) report = await _sql.FullTarReport(key); - - var response = ReportTemplate(new - { - key, - status = isTarKey ? "BANNED" : "NOT BANNED", - log = report.Select(entry => new - { - Timestamp = entry.Item1, - Path = entry.Item2, - Key = entry.Item3 - }).ToList() - }); - return new ContentResult - { - ContentType = "text/html", - StatusCode = (int) HttpStatusCode.OK, - Content = response - }; - } - - private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null) - { - //_logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}"); - await _sql.IngestMetric(new Metric - { - Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey - }); - } - - - [HttpGet("total_installs.html")] - [ResponseCache(Duration = 60 * 60)] - public async Task TotalInstalls() - { - var data = await _sql.GetTotalInstalls(); - var result = TotalListTemplate(new TotalListTemplateData - { - Title = "Total Installs", - Total = data.Sum(d => d.Item2), - Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2}) - .ToArray() - }); - return new ContentResult - { - ContentType = "text/html", - StatusCode = (int) HttpStatusCode.OK, - Content = result - }; - } - - [HttpGet("total_unique_installs.html")] - [ResponseCache(Duration = 60 * 60)] - public async Task TotalUniqueInstalls() - { - var data = await _sql.GetTotalUniqueInstalls(); - var result = TotalListTemplate(new TotalListTemplateData - { - Title = "Total Unique Installs", - Total = data.Sum(d => d.Item2), - Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2}) - .ToArray() - }); - return new ContentResult - { - ContentType = "text/html", - StatusCode = (int) HttpStatusCode.OK, - Content = result - }; - } - - [HttpGet("dump.json")] - public async Task DataDump() - { - return Ok(await _sql.MetricsDump().ToArrayAsync()); - } - public class Result { public DateTime Timestamp { get; set; } } - - private class TotalListTemplateData - { - public string Title { get; set; } - public long Total { get; set; } - public Item[] Items { get; set; } - - public class Item - { - public long Count { get; set; } - public string Title { get; set; } - } - } } \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/ModUpgrade.cs b/Wabbajack.Server/Controllers/ModUpgrade.cs deleted file mode 100644 index b6fd1fbf..00000000 --- a/Wabbajack.Server/Controllers/ModUpgrade.cs +++ /dev/null @@ -1,158 +0,0 @@ -using System; -using System.Linq; -using System.Security.Claims; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; -using Wabbajack.Downloaders; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.DTOs.ServerResponses; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Server.DataLayer; -using Wabbajack.Server.DTOs; -using Wabbajack.Server.Services; -using Wabbajack.Server.TokenProviders; - -namespace Wabbajack.BuildServer.Controllers; - -[ApiController] -public class ModUpgrade : ControllerBase -{ - private readonly DownloadDispatcher _dispatcher; - private readonly DTOSerializer _dtos; - private readonly IFtpSiteCredentials _ftpSite; - private readonly ILogger _logger; - private readonly QuickSync _quickSync; - private readonly AppSettings _settings; - private readonly SqlService _sql; - - public ModUpgrade(ILogger logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync, - AppSettings settings, DTOSerializer dtos, - DownloadDispatcher dispatcher, IFtpSiteCredentials ftp) - { - _logger = logger; - _sql = sql; - _settings = settings; - _quickSync = quickSync; - _ftpSite = ftp; - _dtos = dtos; - _dispatcher = dispatcher; - } - - [HttpPost] - [Authorize(Roles = "User")] - [Route("/mod_upgrade")] - public async Task PostModUpgrade(CancellationToken token) - { - var isAuthor = User.Claims.Any(c => c.Type == ClaimTypes.Role && c.Value == "Author"); - var request = await _dtos.DeserializeAsync(Request.Body); - if (!isAuthor) - { - var srcDownload = await _sql.GetArchiveDownload(request!.OldArchive.State.PrimaryKeyString, - request.OldArchive.Hash, request.OldArchive.Size); - var destDownload = await _sql.GetArchiveDownload(request.NewArchive.State.PrimaryKeyString, - request.NewArchive.Hash, request.NewArchive.Size); - - if (srcDownload == default || destDownload == default || - await _sql.FindPatch(srcDownload.Id, destDownload.Id) == default) - { - if (!await _dispatcher.IsAllowed(request, token)) - { - _logger.Log(LogLevel.Information, - $"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as upgrade is invalid"); - return BadRequest("Invalid mod upgrade"); - } - - if (_settings.ValidateModUpgrades && !await _sql.HashIsInAModlist(request.OldArchive.Hash)) - { - _logger.Log(LogLevel.Information, - $"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as src hash is not in a curated modlist"); - return BadRequest("Hash is not in a recent modlist"); - } - } - } - - try - { - if (await _dispatcher.Verify(request!.OldArchive, token)) - //_logger.LogInformation( - // $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), old archive is valid"); - return NotFound("File is Valid"); - } - catch (Exception) - { - //_logger.LogInformation( - // $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), due to upgrade failure"); - return NotFound("File is Valid"); - } - - var oldDownload = await _sql.GetOrEnqueueArchive(request.OldArchive); - - if (await _sql.IsNoPatch(oldDownload.Archive.Hash)) return BadRequest("File has NoPatch attached"); - - var newDownload = await _sql.GetOrEnqueueArchive(request.NewArchive); - - var patch = await _sql.FindOrEnqueuePatch(oldDownload.Id, newDownload.Id); - if (patch.Finished.HasValue) - { - if (patch.PatchSize != 0) - { - //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch Found"); - var host = (await _ftpSite.Get())[StorageSpace.Patches].Username == "wabbajacktest" - ? "test-files" - : "patches"; - await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id); - return - Ok( - $"https://{host}.wabbajack.org/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); - } - //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed"); - - return NotFound("Patch creation failed"); - } - - if (!newDownload.DownloadFinished.HasValue) - await _quickSync.Notify(); - else - await _quickSync.Notify(); - - //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found is processing"); - // Still processing - return Accepted(); - } - - [HttpGet] - [Authorize(Roles = "User")] - [Route("/mod_upgrade/find/{hashAsHex}")] - public async Task FindUpgrade(string hashAsHex) - { - var hash = Hash.FromHex(hashAsHex); - - var patches = await _sql.PatchesForSource(hash); - return Ok(_dtos.Serialize(patches.Select(p => p.Dest.Archive).ToArray())); - } - - [HttpGet] - [Authorize(Roles = "Author")] - [Route("/mod_upgrade/no_patch/{hashAsHex}/{rationaleAsHex}")] - public async Task PurgePatch(string hashAsHex, string rationaleAsHex) - { - var hash = Hash.FromHex(hashAsHex); - var rationale = Encoding.UTF8.GetString(rationaleAsHex.FromHex()); - await _sql.PurgePatch(hash, rationale); - return Ok("Purged"); - } - - [HttpGet] - [Authorize(Roles = "User")] - [Route("/mirror/{hashAsHex}")] - public async Task HaveHash(string hashAsHex) - { - var result = await _sql.HaveMirror(Hash.FromHex(hashAsHex)); - if (result) return Ok($"https://{(await _ftpSite.Get())[StorageSpace.Mirrors].Username}.b-cdn.net/{hashAsHex}"); - return NotFound("Not Mirrored"); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DTOs/AggregateMetric.cs b/Wabbajack.Server/DTOs/AggregateMetric.cs deleted file mode 100644 index b62487c9..00000000 --- a/Wabbajack.Server/DTOs/AggregateMetric.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System; - -namespace Wabbajack.Server.DTOs; - -public class AggregateMetric -{ - public DateTime Date { get; set; } - public string Subject { get; set; } - public int Count { get; set; } -} \ No newline at end of file diff --git a/Wabbajack.Server/DTOs/ArchiveDownload.cs b/Wabbajack.Server/DTOs/ArchiveDownload.cs deleted file mode 100644 index 70fe5673..00000000 --- a/Wabbajack.Server/DTOs/ArchiveDownload.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; -using System.Threading.Tasks; -using Wabbajack.DTOs; -using Wabbajack.Server.DataLayer; - -namespace Wabbajack.Server.DTOs; - -public class ArchiveDownload -{ - public Guid Id { get; set; } - public Archive Archive { get; set; } - public bool? IsFailed { get; set; } - public DateTime? DownloadFinished { get; set; } - public string FailMessage { get; set; } - - public async Task Fail(SqlService service, string message) - { - IsFailed = true; - DownloadFinished = DateTime.UtcNow; - FailMessage = message; - await service.UpdatePendingDownload(this); - } - - public async Task Finish(SqlService service) - { - IsFailed = false; - DownloadFinished = DateTime.UtcNow; - await service.UpdatePendingDownload(this); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DTOs/AuthoredFilesSummary.cs b/Wabbajack.Server/DTOs/AuthoredFilesSummary.cs deleted file mode 100644 index 715759d0..00000000 --- a/Wabbajack.Server/DTOs/AuthoredFilesSummary.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; - -namespace Wabbajack.Server.DTOs; - -public class AuthoredFilesSummary -{ - public long Size { get; set; } - public string OriginalFileName { get; set; } - public string Author { get; set; } - public DateTime LastTouched { get; set; } - public DateTime? Finalized { get; set; } - public string MungedName { get; set; } - public string ServerAssignedUniqueId { get; set; } -} \ No newline at end of file diff --git a/Wabbajack.Server/DTOs/FtpSite.cs b/Wabbajack.Server/DTOs/FtpSite.cs deleted file mode 100644 index d813fa97..00000000 --- a/Wabbajack.Server/DTOs/FtpSite.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System.Net; -using System.Threading.Tasks; -using FluentFTP; -using Microsoft.Extensions.Logging; -using Wabbajack.Common; - -namespace Wabbajack.Server.DTOs; - -public enum StorageSpace -{ - AuthoredFiles, - Patches, - Mirrors -} - -public class FtpSite -{ - public string Username { get; set; } - public string Password { get; set; } - public string Hostname { get; set; } - - public async Task GetClient(ILogger logger) - { - return await CircuitBreaker.WithAutoRetryAllAsync(logger, async () => - { - var ftpClient = new FtpClient(Hostname, new NetworkCredential(Username, Password)); - await ftpClient.ConnectAsync(); - return ftpClient; - }); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DTOs/Metric.cs b/Wabbajack.Server/DTOs/Metric.cs index 2e413523..12234904 100644 --- a/Wabbajack.Server/DTOs/Metric.cs +++ b/Wabbajack.Server/DTOs/Metric.cs @@ -1,4 +1,5 @@ using System; +using Microsoft.Extensions.Primitives; namespace Wabbajack.Server.DTOs; @@ -8,4 +9,5 @@ public class Metric public string Action { get; set; } public string Subject { get; set; } public string MetricsKey { get; set; } + public string UserAgent { get; set; } } \ No newline at end of file diff --git a/Wabbajack.Server/DTOs/MetricResult.cs b/Wabbajack.Server/DTOs/MetricResult.cs deleted file mode 100644 index 18585c77..00000000 --- a/Wabbajack.Server/DTOs/MetricResult.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System.Collections.Generic; - -namespace Wabbajack.Server.DTOs; - -public class MetricResult -{ - public string SeriesName { get; set; } - public List Labels { get; set; } - public List Values { get; set; } -} \ No newline at end of file diff --git a/Wabbajack.Server/DTOs/MirroredFile.cs b/Wabbajack.Server/DTOs/MirroredFile.cs deleted file mode 100644 index 3c163bb8..00000000 --- a/Wabbajack.Server/DTOs/MirroredFile.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System; -using System.Threading.Tasks; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Server.DataLayer; - -namespace Wabbajack.Server.DTOs; - -public class MirroredFile -{ - public Hash Hash { get; set; } - public DateTime Created { get; set; } - public DateTime? Uploaded { get; set; } - public string Rationale { get; set; } - - public string FailMessage { get; set; } - - public async Task Finish(SqlService sql) - { - Uploaded = DateTime.UtcNow; - await sql.UpsertMirroredFile(this); - } - - public async Task Fail(SqlService sql, string message) - { - Uploaded = DateTime.UtcNow; - FailMessage = message; - await sql.UpsertMirroredFile(this); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DTOs/Patch.cs b/Wabbajack.Server/DTOs/Patch.cs deleted file mode 100644 index 551c933d..00000000 --- a/Wabbajack.Server/DTOs/Patch.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System; -using System.Threading.Tasks; -using Wabbajack.Server.DataLayer; - -namespace Wabbajack.Server.DTOs; - -public class Patch -{ - public ArchiveDownload Src { get; set; } - public ArchiveDownload Dest { get; set; } - public long PatchSize { get; set; } - public DateTime? Finished { get; set; } - public bool? IsFailed { get; set; } - public string FailMessage { get; set; } - - public async Task Finish(SqlService sql, long size) - { - IsFailed = false; - Finished = DateTime.UtcNow; - PatchSize = size; - await sql.FinializePatch(this); - } - - - public async Task Fail(SqlService sql, string msg) - { - IsFailed = true; - Finished = DateTime.UtcNow; - FailMessage = msg; - await sql.FinializePatch(this); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DTOs/ValidationData.cs b/Wabbajack.Server/DTOs/ValidationData.cs deleted file mode 100644 index 40273c39..00000000 --- a/Wabbajack.Server/DTOs/ValidationData.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Wabbajack.DTOs; -using Wabbajack.Hashing.xxHash64; - -namespace Wabbajack.Server.DTOs; - -public class ValidationData -{ - public Dictionary<(long Game, long ModId, long FileId), string> NexusFiles { get; set; } = new(); - public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; } - public List ModLists { get; set; } - public Dictionary Mirrors { get; set; } - public Lazy>> AllowedMirrors { get; set; } - public IEnumerable AllAuthoredFiles { get; set; } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataLayer/ApiKeys.cs b/Wabbajack.Server/DataLayer/ApiKeys.cs deleted file mode 100644 index d08ec838..00000000 --- a/Wabbajack.Server/DataLayer/ApiKeys.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using Wabbajack.Hashing.xxHash64; - -namespace Wabbajack.Server.DataLayer; - -public partial class SqlService -{ - public async Task LoginByApiKey(string key) - { - await using var conn = await Open(); - var result = await conn.QueryAsync(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @ApiKey", - new {ApiKey = key}); - return result.FirstOrDefault(); - } - - public async Task AddLogin(string name) - { - var key = NewAPIKey(); - await using var conn = await Open(); - - - await conn.ExecuteAsync("INSERT INTO dbo.ApiKeys (Owner, ApiKey) VALUES (@Owner, @ApiKey)", - new {Owner = name, ApiKey = key}); - return key; - } - - - public static string NewAPIKey() - { - var arr = new byte[128]; - new Random().NextBytes(arr); - return arr.ToHex(); - } - - public async Task> GetAllUserKeys() - { - await using var conn = await Open(); - var result = await conn.QueryAsync<(string Owner, string Key)>("SELECT Owner, ApiKey FROM dbo.ApiKeys"); - return result; - } - - - public async Task IsTarKey(string metricsKey) - { - await using var conn = await Open(); - var result = await conn.QueryFirstOrDefaultAsync( - "SELECT MetricsKey FROM TarKey WHERE MetricsKey = @MetricsKey", new {MetricsKey = metricsKey}); - return result == metricsKey; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs deleted file mode 100644 index 02fa8be0..00000000 --- a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs +++ /dev/null @@ -1,261 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using Wabbajack.DTOs; -using Wabbajack.DTOs.DownloadStates; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Server.DTOs; - -namespace Wabbajack.Server.DataLayer; - -public partial class SqlService -{ - public async Task AddKnownDownload(Archive a, DateTime downloadFinished) - { - await using var conn = await Open(); - var Id = Guid.NewGuid(); - await conn.ExecuteAsync( - "INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader, DownloadFinished, IsFailed) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader, @DownloadFinished, @IsFailed)", - new - { - Id, - a.State.PrimaryKeyString, - Size = a.Size == 0 ? null : (long?) a.Size, - Hash = a.Hash == default ? null : (Hash?) a.Hash, - DownloadState = a.State, - Downloader = a.State.GetType().ToString(), - DownloadFinished = downloadFinished, - IsFailed = false - }); - return Id; - } - - public async Task EnqueueDownload(Archive a) - { - await using var conn = await Open(); - var Id = Guid.NewGuid(); - await conn.ExecuteAsync( - "INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)", - new - { - Id, - a.State.PrimaryKeyString, - Size = a.Size == 0 ? null : (long?) a.Size, - Hash = a.Hash == default ? null : (Hash?) a.Hash, - DownloadState = a.State, - Downloader = a.State.GetType().ToString() - }); - return Id; - } - - public async Task> GetAllArchiveDownloads() - { - await using var conn = await Open(); - return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, PrimaryKeyString FROM ArchiveDownloads")) - .ToHashSet(); - } - - public async Task> GetAllArchiveDownloadStates() - { - await using var conn = await Open(); - return (await conn.QueryAsync<(Hash, IDownloadState)>("SELECT Hash, DownloadState FROM ArchiveDownloads")) - .ToHashSet(); - } - - - public async Task GetArchiveDownload(Guid id) - { - await using var conn = await Open(); - var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, IDownloadState, DateTime?)>( - "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE Id = @id", - new {Id = id}); - if (result == default) - return null; - - return new ArchiveDownload - { - Id = result.Item1, - IsFailed = result.Item4, - DownloadFinished = result.Item6, - Archive = new Archive {State = result.Item5, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default} - }; - } - - public async Task GetArchiveDownload(string primaryKeyString) - { - await using var conn = await Open(); - var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, IDownloadState, DateTime?)>( - "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND IsFailed = 0", - new {PrimaryKeyString = primaryKeyString}); - if (result == default) - return null; - - return new ArchiveDownload - { - Id = result.Item1, - IsFailed = result.Item4, - DownloadFinished = result.Item6, - Archive = new Archive {State = result.Item5, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default} - }; - } - - public async Task GetArchiveDownload(string primaryKeyString, Hash hash, long size) - { - await using var conn = await Open(); - var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, IDownloadState, DateTime?)>( - "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size", - new - { - PrimaryKeyString = primaryKeyString, - Hash = hash, - Size = size - }); - if (result == default) - return null; - - return new ArchiveDownload - { - Id = result.Item1, - IsFailed = result.Item4, - DownloadFinished = result.Item6, - Archive = new Archive {State = result.Item5, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default} - }; - } - - - public async Task GetOrEnqueueArchive(Archive a) - { - await using var conn = await Open(); - await using var trans = await conn.BeginTransactionAsync(); - var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, IDownloadState, DateTime?)>( - "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size", - new - { - a.State.PrimaryKeyString, - a.Hash, - a.Size - }, trans); - if (result.Item1 != default) - return new ArchiveDownload - { - Id = result.Item1, - IsFailed = result.Item4, - DownloadFinished = result.Item6, - Archive = new Archive {State = result.Item5, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default} - }; - - var id = Guid.NewGuid(); - await conn.ExecuteAsync( - "INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)", - new - { - Id = id, - a.State.PrimaryKeyString, - Size = a.Size == 0 ? null : (long?) a.Size, - Hash = a.Hash == default ? null : (Hash?) a.Hash, - DownloadState = a.State, - Downloader = "" - }, trans); - - await trans.CommitAsync(); - - return new ArchiveDownload {Id = id, Archive = a}; - } - - public async Task GetNextPendingDownload(bool ignoreNexus = false) - { - await using var conn = await Open(); - (Guid, long?, Hash?, IDownloadState) result; - - if (ignoreNexus) - result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, IDownloadState)>( - "SELECT TOP(1) Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL AND Downloader != 'NexusDownloader+State'"); - else - result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, IDownloadState)>( - "SELECT TOP(1) Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL"); - - if (result == default) - return null; - - return new ArchiveDownload - { - Id = result.Item1, - Archive = new Archive {State = result.Item4, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default} - }; - } - - public async Task UpdatePendingDownload(ArchiveDownload ad) - { - await using var conn = await Open(); - await conn.ExecuteAsync( - "UPDATE dbo.ArchiveDownloads SET IsFailed = @IsFailed, DownloadFinished = @DownloadFinished, Hash = @Hash, Size = @Size, FailMessage = @FailMessage WHERE Id = @Id", - new - { - ad.Id, - ad.IsFailed, - ad.DownloadFinished, - ad.Archive.Size, - ad.Archive.Hash, - ad.FailMessage - }); - } - - public async Task EnqueueModListFilesForIndexing() - { - await using var conn = await Open(); - return await conn.ExecuteAsync(@" - INSERT INTO dbo.ArchiveDownloads (Id, PrimaryKeyString, Hash, DownloadState, Size, Downloader) - SELECT DISTINCT NEWID(), mla.PrimaryKeyString, mla.Hash, mla.State, mla.Size, SUBSTRING(mla.PrimaryKeyString, 0, CHARINDEX('|', mla.PrimaryKeyString)) - FROM [dbo].[ModListArchives] mla - LEFT JOIN dbo.ArchiveDownloads ad on mla.PrimaryKeyString = ad.PrimaryKeyString AND mla.Hash = ad.Hash - WHERE ad.PrimaryKeyString is null"); - } - - public async Task> GetGameFiles(Game game, string version) - { - await using var conn = await Open(); - var files = (await conn.QueryAsync<(Hash, long, IDownloadState)>( - $"SELECT Hash, Size, DownloadState FROM dbo.ArchiveDownloads WHERE PrimaryKeyString like 'GameFileSourceDownloader+State|{game}|{version}|%'")) - .Select(f => new Archive - { - State = f.Item3, - Hash = f.Item1, - Size = f.Item2 - }).ToList(); - return files; - } - - public async Task ResolveDownloadStatesByHash(Hash hash) - { - await using var conn = await Open(); - var files = (await conn.QueryAsync<(long, Hash, IDownloadState)>( - @"SELECT Size, Hash, DownloadState from dbo.ArchiveDownloads WHERE Hash = @Hash AND IsFailed = 0 AND DownloadFinished IS NOT NULL ORDER BY DownloadFinished DESC", - new {Hash = hash}) - ).Select(e => - new Archive {State = e.Item3, Size = e.Item1, Hash = e.Item2} - ).ToList(); - - if (await HaveMirror(hash) && files.Count > 0) - { - var ffile = files.First(); - var host = _settings.TestMode ? "test-files" : "mirror"; - var url = new Uri($"https://{host}.wabbajack.org/{hash.ToHex()}"); - files.Add(new Archive - {State = new WabbajackCDN {Url = url}, Hash = hash, Size = ffile.Size, Name = ffile.Name}); - } - - return files.ToArray(); - } - - public async Task> GetAllRegisteredGames() - { - await using var conn = await Open(); - var pks = await conn.QueryAsync( - @"SELECT PrimaryKeyString FROM dbo.ArchiveDownloads WHERE PrimaryKeyString like 'GameFileSourceDownloader+State|%'"); - return pks.Select(p => p.Split("|")) - .Select(t => (GameRegistry.GetByFuzzyName(t[1]).Game, t[2])) - .Distinct(); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataLayer/AuthoredFiles.cs b/Wabbajack.Server/DataLayer/AuthoredFiles.cs deleted file mode 100644 index 433d3431..00000000 --- a/Wabbajack.Server/DataLayer/AuthoredFiles.cs +++ /dev/null @@ -1,76 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using Wabbajack.DTOs.CDN; -using Wabbajack.Server.DTOs; - -namespace Wabbajack.Server.DataLayer; - -public partial class SqlService -{ - public async Task TouchAuthoredFile(FileDefinition definition, DateTime? date = null) - { - await using var conn = await Open(); - if (date == null) - await conn.ExecuteAsync( - "UPDATE AuthoredFiles SET LastTouched = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid", - new {Uid = definition.ServerAssignedUniqueId}); - else - await conn.ExecuteAsync( - "UPDATE AuthoredFiles SET LastTouched = @Date WHERE ServerAssignedUniqueId = @Uid", - new {Uid = definition.ServerAssignedUniqueId, Date = date}); - } - - public async Task CreateAuthoredFile(FileDefinition definition, string login) - { - definition.Author = login; - var uid = Guid.NewGuid().ToString(); - await using var conn = await Open(); - definition.ServerAssignedUniqueId = uid; - await conn.ExecuteAsync( - "INSERT INTO dbo.AuthoredFiles (ServerAssignedUniqueId, LastTouched, CDNFileDefinition) VALUES (@Uid, GETUTCDATE(), @CdnFile)", - new - { - Uid = uid, - CdnFile = definition - }); - return definition; - } - - public async Task Finalize(FileDefinition definition) - { - await using var conn = await Open(); - await conn.ExecuteAsync( - "UPDATE AuthoredFiles SET LastTouched = GETUTCDATE(), Finalized = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid", - new - { - Uid = definition.ServerAssignedUniqueId - }); - } - - public async Task GetCDNFileDefinition(string serverAssignedUniqueId) - { - await using var conn = await Open(); - return (await conn.QueryAsync( - "SELECT CDNFileDefinition FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid", - new {Uid = serverAssignedUniqueId})).First(); - } - - public async Task DeleteFileDefinition(FileDefinition definition) - { - await using var conn = await Open(); - await conn.ExecuteAsync( - "DELETE FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid", - new {Uid = definition.ServerAssignedUniqueId}); - } - - public async Task> AllAuthoredFiles() - { - await using var conn = await Open(); - var results = await conn.QueryAsync( - "SELECT CONVERT(NVARCHAR(50), ServerAssignedUniqueId) as ServerAssignedUniqueId, Size, OriginalFileName, Author, LastTouched, Finalized, MungedName from dbo.AuthoredFilesSummaries ORDER BY LastTouched DESC"); - return results; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataLayer/Mappers.cs b/Wabbajack.Server/DataLayer/Mappers.cs deleted file mode 100644 index b0392535..00000000 --- a/Wabbajack.Server/DataLayer/Mappers.cs +++ /dev/null @@ -1,109 +0,0 @@ -using System; -using System.Data; -using Dapper; -using Wabbajack.DTOs; -using Wabbajack.DTOs.CDN; -using Wabbajack.DTOs.DownloadStates; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Paths; - -namespace Wabbajack.Server.DataLayer; - -public partial class SqlService -{ - private static DTOSerializer _dtoStatic; - - static SqlService() - { - SqlMapper.AddTypeHandler(new HashMapper()); - SqlMapper.AddTypeHandler(new RelativePathMapper()); - SqlMapper.AddTypeHandler(new JsonMapper()); - SqlMapper.AddTypeHandler(new JsonMapper()); - SqlMapper.AddTypeHandler(new JsonMapper()); - SqlMapper.AddTypeHandler(new VersionMapper()); - SqlMapper.AddTypeHandler(new GameMapper()); - SqlMapper.AddTypeHandler(new DateTimeHandler()); - } - - /// - /// Needed to make sure dates are all in UTC format - /// - private class DateTimeHandler : SqlMapper.TypeHandler - { - public override void SetValue(IDbDataParameter parameter, DateTime value) - { - parameter.Value = value; - } - - public override DateTime Parse(object value) - { - return DateTime.SpecifyKind((DateTime) value, DateTimeKind.Utc); - } - } - - private class JsonMapper : SqlMapper.TypeHandler - { - public override void SetValue(IDbDataParameter parameter, T value) - { - parameter.Value = _dtoStatic.Serialize(value); - } - - public override T Parse(object value) - { - return _dtoStatic.Deserialize((string) value)!; - } - } - - private class RelativePathMapper : SqlMapper.TypeHandler - { - public override void SetValue(IDbDataParameter parameter, RelativePath value) - { - parameter.Value = value.ToString(); - } - - public override RelativePath Parse(object value) - { - return (RelativePath) (string) value; - } - } - - private class HashMapper : SqlMapper.TypeHandler - { - public override void SetValue(IDbDataParameter parameter, Hash value) - { - parameter.Value = (long) value; - } - - public override Hash Parse(object value) - { - return Hash.FromLong((long) value); - } - } - - private class VersionMapper : SqlMapper.TypeHandler - { - public override void SetValue(IDbDataParameter parameter, Version value) - { - parameter.Value = value.ToString(); - } - - public override Version Parse(object value) - { - return Version.Parse((string) value); - } - } - - private class GameMapper : SqlMapper.TypeHandler - { - public override void SetValue(IDbDataParameter parameter, Game value) - { - parameter.Value = value.ToString(); - } - - public override Game Parse(object value) - { - return GameRegistry.GetByFuzzyName((string) value).Game; - } - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataLayer/Metrics.cs b/Wabbajack.Server/DataLayer/Metrics.cs deleted file mode 100644 index 6b33bc44..00000000 --- a/Wabbajack.Server/DataLayer/Metrics.cs +++ /dev/null @@ -1,189 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using Wabbajack.Server.DTOs; - -namespace Wabbajack.Server.DataLayer; - -public partial class SqlService -{ - public async Task IngestMetric(Metric metric) - { - await using var conn = await Open(); - await conn.ExecuteAsync( - @"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", - metric); - } - - public async Task IngestAccess(string ip, string log) - { - await using var conn = await Open(); - await conn.ExecuteAsync(@"INSERT INTO dbo.AccessLog (Timestamp, Action, Ip) VALUES (@Timestamp, @Action, @Ip)", - new - { - Timestamp = DateTime.UtcNow, - Ip = ip, - Action = log - }); - } - - public async Task> MetricsReport(string action) - { - await using var conn = await Open(); - return (await conn.QueryAsync(@" - select - datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)) as Date, - GroupingSubject as Subject, - count(*) as Count - from dbo.metrics where - Action = @Action - AND GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics - WHERE action = @Action - AND MetricsKey is not null - AND Subject != 'Default' - AND Subject != 'untitled' - AND TRY_CONVERT(uniqueidentifier, Subject) is null - AND Timestamp >= DATEADD(DAY, -1, GETUTCDATE())) - group by - datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)), - GroupingSubject - Order by datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)) asc", - new {Action = action})) - .ToList(); - } - - public async Task> FullTarReport(string key) - { - await using var conn = await Open(); - return (await conn.QueryAsync<(DateTime, string, string)>(@" - SELECT u.Timestamp, u.Path, u.MetricsKey FROM - (SELECT al.Timestamp, JSON_VALUE(al.Action, '$.Path') as Path, al.MetricsKey FROM dbo.AccessLog al - WHERE al.MetricsKey = @MetricsKey - UNION ALL - SELECT m.Timestamp, m.Action + ' ' + m.Subject as Path, m.MetricsKey FROM dbo.Metrics m - WHERE m.MetricsKey = @MetricsKey - AND m.Action != 'TarKey') u - ORDER BY u.Timestamp Desc", - new {MetricsKey = key})).ToList(); - } - - public async Task ValidMetricsKey(string metricsKey) - { - await using var conn = await Open(); - return await conn.QuerySingleOrDefaultAsync( - "SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey", - new {MetricsKey = metricsKey}) != default; - } - - public async Task AddMetricsKey(string metricsKey) - { - await using var conn = await Open(); - await using var trans = conn.BeginTransaction(); - - if (await conn.QuerySingleOrDefaultAsync( - "SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey", - new {MetricsKey = metricsKey}, trans) != default) - return; - - await conn.ExecuteAsync("INSERT INTO dbo.MetricsKeys (MetricsKey) VALUES (@MetricsKey)", - new {MetricsKey = metricsKey}, trans); - } - - public async Task AllKeys() - { - await using var conn = await Open(); - return (await conn.QueryAsync("SELECT MetricsKey from dbo.MetricsKeys")).ToArray(); - } - - - public async Task UniqueInstalls(string machineUrl) - { - await using var conn = await Open(); - return await conn.QueryFirstAsync( - @"SELECT COUNT(*) FROM ( - SELECT DISTINCT MetricsKey from dbo.Metrics where Action = 'finish_install' and GroupingSubject in ( - SELECT JSON_VALUE(Metadata, '$.title') FROM dbo.ModLists - WHERE JSON_VALUE(Metadata, '$.links.machineURL') = @MachineURL)) s", - new {MachineURL = machineUrl}); - } - - public async Task TotalInstalls(string machineUrl) - { - await using var conn = await Open(); - return await conn.QueryFirstAsync( - @"SELECT COUNT(*) from dbo.Metrics where Action = 'finish_install' and GroupingSubject in ( - SELECT JSON_VALUE(Metadata, '$.title') FROM dbo.ModLists - WHERE JSON_VALUE(Metadata, '$.links.machineURL') = @MachineURL)", - new {MachineURL = machineUrl}); - } - - public async Task> GetTotalInstalls() - { - await using var conn = await Open(); - return await conn.QueryAsync<(string, long)>( - @"SELECT GroupingSubject, Count(*) as Count - From dbo.Metrics - WHERE - - GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics - WHERE action = 'finish_install' - AND MetricsKey is not null) - group by GroupingSubject - order by Count(*) desc"); - } - - public async Task> GetTotalUniqueInstalls() - { - await using var conn = await Open(); - return await conn.QueryAsync<(string, long)>( - @"Select GroupingSubject, Count(*) as Count - FROM - (select DISTINCT MetricsKey, GroupingSubject - From dbo.Metrics - WHERE - GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics - WHERE action = 'finish_install' - AND MetricsKey is not null)) m - GROUP BY GroupingSubject - Order by Count(*) desc - "); - } - - public async IAsyncEnumerable MetricsDump() - { - var keys = new Dictionary(); - - await using var conn = await Open(); - foreach (var row in await conn.QueryAsync<(long, DateTime, string, string, string, string)>( - @"select Id, Timestamp, Action, Subject, MetricsKey, GroupingSubject from dbo.metrics WHERE MetricsKey is not null")) - { - if (!keys.TryGetValue(row.Item5, out var keyid)) - { - keyid = keys.Count; - keys[row.Item5] = keyid; - } - - yield return new MetricRow - { - Id = row.Item1, - Timestamp = row.Item2, - Action = row.Item3, - Subject = row.Item4, - MetricsKey = keyid, - GroupingSubject = row.Item6 - }; - } - } - - public class MetricRow - { - public string Action; - public string GroupingSubject; - public long Id; - public long MetricsKey; - public string Subject; - public DateTime Timestamp; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataLayer/MirroredFiles.cs b/Wabbajack.Server/DataLayer/MirroredFiles.cs deleted file mode 100644 index 5ec3355e..00000000 --- a/Wabbajack.Server/DataLayer/MirroredFiles.cs +++ /dev/null @@ -1,147 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using Wabbajack.DTOs; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Server.DTOs; - -namespace Wabbajack.Server.DataLayer; - -public partial class SqlService -{ - public async Task GetNextMirroredFile() - { - await using var conn = await Open(); - var result = await conn.QueryFirstOrDefaultAsync<(Hash, DateTime, DateTime, string, string)>( - "SELECT Hash, Created, Uploaded, Rationale, FailMessage from dbo.MirroredArchives WHERE Uploaded IS NULL"); - if (result == default) return null; - return new MirroredFile - { - Hash = result.Item1, Created = result.Item2, Uploaded = result.Item3, Rationale = result.Item4, - FailMessage = result.Item5 - }; - } - - public async Task> GetAllMirroredHashes() - { - await using var conn = await Open(); - return (await conn.QueryAsync<(Hash, DateTime?)>("SELECT Hash, Uploaded FROM dbo.MirroredArchives")) - .GroupBy(d => d.Item1) - .ToDictionary(d => d.Key, d => d.First().Item2.HasValue); - } - - public async Task StartMirror((Hash Hash, string Reason) mirror) - { - await using var conn = await Open(); - await using var trans = await conn.BeginTransactionAsync(); - - if (await conn.QueryFirstOrDefaultAsync(@"SELECT Hash FROM dbo.MirroredArchives WHERE Hash = @Hash", - new {mirror.Hash}, trans) != default) - return; - - await conn.ExecuteAsync( - @"INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) VALUES (@Hash, GETUTCDATE(), @Reason)", - new {mirror.Hash, mirror.Reason}, trans); - await trans.CommitAsync(); - } - - public async Task> GetAllowedMirrors() - { - await using var conn = await Open(); - return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, Reason FROM dbo.AllowedMirrorsCache")) - .GroupBy(d => d.Item1) - .ToDictionary(d => d.Key, d => d.First().Item2); - } - - public async Task UpsertMirroredFile(MirroredFile file) - { - await using var conn = await Open(); - await using var trans = await conn.BeginTransactionAsync(); - - await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash", new {file.Hash}, trans); - await conn.ExecuteAsync( - "INSERT INTO dbo.MirroredArchives (Hash, Created, Uploaded, Rationale, FailMessage) VALUES (@Hash, @Created, @Uploaded, @Rationale, @FailMessage)", - new - { - file.Hash, - file.Created, - file.Uploaded, - file.Rationale, - file.FailMessage - }, trans); - await trans.CommitAsync(); - } - - public async Task DeleteMirroredFile(Hash hash) - { - await using var conn = await Open(); - await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash", - new {Hash = hash}); - } - - public async Task HaveMirror(Hash hash) - { - await using var conn = await Open(); - - return await conn.QueryFirstOrDefaultAsync("SELECT Hash FROM dbo.MirroredArchives WHERE Hash = @Hash", - new {Hash = hash}) != default; - } - - public async Task QueueMirroredFiles() - { - await using var conn = await Open(); - - await conn.ExecuteAsync(@" - - INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) - - SELECT hs.Hash, GETUTCDATE(), 'File has re-upload permissions on the Nexus' FROM - (SELECT DISTINCT ad.Hash FROM dbo.NexusModPermissions p - INNER JOIN GameMetadata md on md.NexusGameId = p.NexusGameID - INNER JOIN dbo.ArchiveDownloads ad on ad.PrimaryKeyString like 'NexusDownloader+State|'+md.WabbajackName+'|'+CAST(p.ModID as nvarchar)+'|%' - WHERE p.Permissions = 1 - AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives) - ) hs - - INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) - SELECT DISTINCT Hash, GETUTCDATE(), 'File is hosted on GitHub' - FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%github.com/%' - AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives) - - - INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) - SELECT DISTINCT Hash, GETUTCDATE(), 'File license allows uploading to any Non-nexus site' - FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%enbdev.com/%' - AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives) - - INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) - SELECT DISTINCT Hash, GETUTCDATE(), 'DynDOLOD file' /*, Name*/ - from dbo.ModListArchives mla WHERE Name like '%DynDoLOD%standalone%' - and Hash not in (select Hash from dbo.MirroredArchives) - - INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) - SELECT DISTINCT Hash, GETUTCDATE(), 'Distribution allowed by author' /*, Name*/ - from dbo.ModListArchives mla WHERE Name like '%particle%patch%' - and Hash not in (select Hash from dbo.MirroredArchives) - - - "); - } - - public async Task AddNexusModWithOpenPerms(Game gameGame, long modId) - { - await using var conn = await Open(); - - await conn.ExecuteAsync( - @"INSERT INTO dbo.NexusModsWithOpenPerms(NexusGameID, NexusModID) VALUES(@game, @mod)", - new {game = gameGame.MetaData().NexusGameId, modId}); - } - - public async Task SyncActiveMirroredFiles() - { - await using var conn = await Open(); - await conn.ExecuteAsync(@"EXEC dbo.QueueMirroredFiles"); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataLayer/ModLists.cs b/Wabbajack.Server/DataLayer/ModLists.cs deleted file mode 100644 index 30279cc8..00000000 --- a/Wabbajack.Server/DataLayer/ModLists.cs +++ /dev/null @@ -1,117 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using Wabbajack.DTOs; -using Wabbajack.DTOs.DownloadStates; -using Wabbajack.Hashing.xxHash64; - -namespace Wabbajack.Server.DataLayer; - -public partial class SqlService -{ - public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist, bool brokenDownload) - { - await using var conn = await Open(); - await using var tran = await conn.BeginTransactionAsync(); - - await conn.ExecuteAsync(@"DELETE FROM dbo.ModLists Where MachineUrl = @MachineUrl", - new {MachineUrl = metadata.Links.MachineURL}, tran); - - var archives = modlist.Archives; - var directives = modlist.Directives; - modlist.Archives = Array.Empty(); - modlist.Directives = Array.Empty(); - - await conn.ExecuteAsync( - @"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList, BrokenDownload) VALUES (@MachineUrl, @Hash, @Metadata, @ModList, @BrokenDownload)", - new - { - MachineUrl = metadata.Links.MachineURL, - Hash = hash, - MetaData = _dtos.Serialize(metadata), - ModList = _dtos.Serialize(modlist), - BrokenDownload = brokenDownload - }, tran); - - var entries = archives.Select(a => - new - { - MachineUrl = metadata.Links.MachineURL, - a.Name, - a.Hash, - a.Size, - State = _dtos.Serialize(a.State), - a.State.PrimaryKeyString - }).ToArray(); - - await conn.ExecuteAsync(@"DELETE FROM dbo.ModListArchives WHERE MachineURL = @machineURL", - new {MachineUrl = metadata.Links.MachineURL}, tran); - - foreach (var entry in entries) - await conn.ExecuteAsync( - "INSERT INTO dbo.ModListArchives (MachineURL, Name, Hash, Size, PrimaryKeyString, State) VALUES (@MachineURL, @Name, @Hash, @Size, @PrimaryKeyString, @State)", - entry, tran); - - await tran.CommitAsync(); - } - - public async Task HaveIndexedModlist(string machineUrl, Hash hash) - { - await using var conn = await Open(); - var result = await conn.QueryFirstOrDefaultAsync( - "SELECT MachineURL from dbo.Modlists WHERE MachineURL = @MachineUrl AND Hash = @Hash", - new {MachineUrl = machineUrl, Hash = hash}); - return result != null; - } - - public async Task HashIsInAModlist(Hash hash) - { - await using var conn = await Open(); - var result = await conn.QueryFirstOrDefaultAsync( - "SELECT Hash FROM dbo.ModListArchives Where Hash = @Hash", - new {Hash = hash}); - return result; - } - - public async Task> ModListArchives(string machineURL) - { - await using var conn = await Open(); - var archives = await conn.QueryAsync<(string, Hash, long, IDownloadState)>( - "SELECT Name, Hash, Size, State FROM dbo.ModListArchives WHERE MachineUrl = @MachineUrl", - new {MachineUrl = machineURL}); - return archives.Select(t => new Archive - { - State = t.Item4, - Name = string.IsNullOrWhiteSpace(t.Item1) ? t.Item4.PrimaryKeyString : t.Item1, - Size = t.Item3, - Hash = t.Item2 - }).ToList(); - } - - public async Task> ModListArchives() - { - await using var conn = await Open(); - var archives = - await conn.QueryAsync<(string, Hash, long, IDownloadState)>( - "SELECT Name, Hash, Size, State FROM dbo.ModListArchives"); - return archives.Select(t => new Archive - { - State = t.Item4, - Name = string.IsNullOrWhiteSpace(t.Item1) ? t.Item4.PrimaryKeyString : t.Item1, - Size = t.Item3, - Hash = t.Item2 - }).ToList(); - } - - public async Task PurgeList(string machineURL) - { - await using var conn = await Open(); - var ret1 = await conn.ExecuteAsync(@" delete from dbo.ModListArchives where MachineURL = @machineURL", - new {machineURL}); - var ret2 = await conn.ExecuteAsync(@" delete from dbo.ModLists where MachineURL = @machineURL", - new {machineURL}); - return ret1 + ret2; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataLayer/Nexus.cs b/Wabbajack.Server/DataLayer/Nexus.cs deleted file mode 100644 index c6ac82a2..00000000 --- a/Wabbajack.Server/DataLayer/Nexus.cs +++ /dev/null @@ -1,156 +0,0 @@ -using System; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using Wabbajack.DTOs; -using Wabbajack.Networking.NexusApi.DTOs; - -namespace Wabbajack.Server.DataLayer; - -/// -/// SQL routines that read/write cached information from the Nexus -/// -public partial class SqlService -{ - public async Task DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date) - { - await using var conn = await Open(); - var deleted = await conn.ExecuteScalarAsync( - @"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date - SELECT @@ROWCOUNT AS Deleted", - new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date}); - return deleted; - } - - public async Task DeleteNexusModFilesUpdatedBeforeDate(Game game, long modId, DateTime date) - { - await using var conn = await Open(); - var deleted = await conn.ExecuteScalarAsync( - @"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date - SELECT @@ROWCOUNT AS Deleted", - new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date}); - return deleted; - } - - public async Task GetNexusModInfoString(Game game, long modId) - { - await using var conn = await Open(); - var result = await conn.QueryFirstOrDefaultAsync( - "SELECT Data FROM dbo.NexusModInfos WHERE Game = @Game AND @ModId = ModId", - new {Game = game.MetaData().NexusGameId, ModId = modId}); - return result == null ? null : _dtos.Deserialize(result); - } - - public async Task AddNexusModInfo(Game game, long modId, DateTime lastCheckedUtc, ModInfo data) - { - await using var conn = await Open(); - - await conn.ExecuteAsync( - @"MERGE dbo.NexusModInfos AS Target - USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source - ON Target.Game = Source.Game AND Target.ModId = Source.ModId - WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked - WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);", - new - { - Game = game.MetaData().NexusGameId, - ModId = modId, - LastChecked = lastCheckedUtc, - Data = _dtos.Serialize(data) - }); - } - - public async Task AddNexusModFiles(Game game, long modId, DateTime lastCheckedUtc, ModFiles data) - { - await using var conn = await Open(); - - await conn.ExecuteAsync( - @"MERGE dbo.NexusModFiles AS Target - USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source - ON Target.Game = Source.Game AND Target.ModId = Source.ModId - WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked - WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);", - new - { - Game = game.MetaData().NexusGameId, - ModId = modId, - LastChecked = lastCheckedUtc, - Data = _dtos.Serialize(data) - }); - } - - public async Task AddNexusModFileSlow(Game game, long modId, long fileId, DateTime lastCheckedUtc) - { - await using var conn = await Open(); - - await conn.ExecuteAsync( - @"MERGE dbo.NexusModFilesSlow AS Target - USING (SELECT @GameId GameId, @ModId ModId, @LastChecked LastChecked, @FileId FileId) AS Source - ON Target.GameId = Source.GameId AND Target.ModId = Source.ModId AND Target.FileId = Source.FileId - WHEN MATCHED THEN UPDATE SET Target.LastChecked = @LastChecked - WHEN NOT MATCHED THEN INSERT (GameId, ModId, LastChecked, FileId) VALUES (@GameId, @ModId, @LastChecked, @FileId);", - new - { - GameId = game.MetaData().NexusGameId, - ModId = modId, - FileId = fileId, - LastChecked = lastCheckedUtc - }); - } - - public async Task GetModFiles(Game game, long modId) - { - await using var conn = await Open(); - var result = await conn.QueryFirstOrDefaultAsync( - "SELECT Data FROM dbo.NexusModFiles WHERE Game = @Game AND @ModId = ModId", - new {Game = game.MetaData().NexusGameId, ModId = modId}); - return result == null ? null : _dtos.Deserialize(result); - } - - public async Task PurgeNexusCache(long modId) - { - await using var conn = await Open(); - await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId}); - await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId}); - await conn.ExecuteAsync("DELETE FROM dbo.NexusModPermissions WHERE ModId = @ModId", new {ModId = modId}); - await conn.ExecuteAsync("DELETE FROM dbo.NexusModFile WHERE ModId = @ModID", new {ModId = modId}); - } - - public async Task UpdateGameMetadata() - { - await using var conn = await Open(); - var existing = (await conn.QueryAsync("SELECT WabbajackName FROM dbo.GameMetadata")).ToHashSet(); - - var missing = GameRegistry.Games.Values.Where(g => !existing.Contains(g.Game.ToString())).ToList(); - foreach (var add in missing.Where(g => g.NexusGameId != 0)) - await conn.ExecuteAsync( - "INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, @WabbajackName)", - new {add.NexusGameId, WabbajackName = add.Game.ToString()}); - } - - public async Task GetModFile(Game game, long modId, long fileId) - { - await using var conn = await Open(); - var result = await conn.QueryFirstOrDefaultAsync( - "SELECT Data FROM dbo.NexusModFile WHERE Game = @Game AND @ModId = ModId AND @FileId = FileId", - new {Game = game.MetaData().NexusGameId, ModId = modId, FileId = fileId}); - return result == null ? null : _dtos.Deserialize(result); - } - - public async Task AddNexusModFile(Game game, long modId, long fileId, DateTime lastCheckedUtc, ModFile data) - { - await using var conn = await Open(); - - await conn.ExecuteAsync( - @"INSERT INTO dbo.NexusModFile (Game, ModId, FileId, LastChecked, Data) - VALUES (@Game, @ModId, @FileId, @LastChecked, @Data)", - new - { - Game = game.MetaData().NexusGameId, - ModId = modId, - FileId = fileId, - LastChecked = lastCheckedUtc, - Data = _dtos.Serialize(data) - }); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataLayer/NexusKeys.cs b/Wabbajack.Server/DataLayer/NexusKeys.cs deleted file mode 100644 index fea22bcc..00000000 --- a/Wabbajack.Server/DataLayer/NexusKeys.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Dapper; - -namespace Wabbajack.Server.DataLayer; - -public partial class SqlService -{ - public async Task SetNexusAPIKey(string key, long daily, long hourly) - { - await using var conn = await Open(); - await using var trans = await conn.BeginTransactionAsync(); - await conn.ExecuteAsync(@"DELETE FROM NexusKeys WHERE ApiKey = @ApiKey", new {ApiKey = key}, trans); - await conn.ExecuteAsync( - @"INSERT INTO NexusKeys (ApiKey, DailyRemain, HourlyRemain) VALUES (@ApiKey, @DailyRemain, @HourlyRemain)", - new {ApiKey = key, DailyRemain = daily, HourlyRemain = hourly}, trans); - await trans.CommitAsync(); - } - - - public async Task DeleteNexusAPIKey(string key) - { - await using var conn = await Open(); - await conn.ExecuteAsync(@"DELETE FROM NexusKeys WHERE ApiKey = @ApiKey", new {ApiKey = key}); - } - - public async Task> GetNexusApiKeys(int threshold = 1500) - { - await using var conn = await Open(); - return (await conn.QueryAsync( - @"SELECT ApiKey FROM NexusKeys WHERE DailyRemain >= @Threshold ORDER BY DailyRemain DESC", - new {Threshold = threshold})).ToList(); - } - - public async Task> GetNexusApiKeysWithCounts(int threshold = 1500) - { - await using var conn = await Open(); - return (await conn.QueryAsync<(string, int, int)>( - @"SELECT ApiKey, DailyRemain, HourlyRemain FROM NexusKeys WHERE DailyRemain >= @Threshold ORDER BY DailyRemain DESC", - new {Threshold = threshold})).ToList(); - } - - - public async Task HaveKey(string key) - { - await using var conn = await Open(); - return (await conn.QueryAsync(@"SELECT ApiKey FROM NexusKeys WHERE ApiKey = @ApiKey", - new {ApiKey = key})).Any(); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataLayer/NonNexusModlistArchives.cs b/Wabbajack.Server/DataLayer/NonNexusModlistArchives.cs deleted file mode 100644 index abf13659..00000000 --- a/Wabbajack.Server/DataLayer/NonNexusModlistArchives.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using Wabbajack.DTOs; -using Wabbajack.DTOs.DownloadStates; - -namespace Wabbajack.Server.DataLayer; - -public partial class SqlService -{ - public async Task> GetNonNexusModlistArchives() - { - await using var conn = await Open(); - var results = await conn.QueryAsync<(Hashing.xxHash64.Hash Hash, long Size, string State)>( - @"SELECT Hash, Size, State FROM dbo.ModListArchives WHERE PrimaryKeyString NOT LIKE 'NexusDownloader+State|%'"); - return results.Select(r => new Archive - { - State = _dtos.Deserialize(r.State)!, - Size = r.Size, - Hash = r.Hash - }).ToList(); - } - - public async Task UpdateNonNexusModlistArchivesStatus(IEnumerable<(Archive Archive, bool IsValid)> results) - { - await using var conn = await Open(); - var trans = await conn.BeginTransactionAsync(); - await conn.ExecuteAsync("DELETE FROM dbo.ModlistArchiveStatus;", transaction: trans); - - foreach (var itm in results.DistinctBy(itm => (itm.Archive.Hash, itm.Archive.State.PrimaryKeyString))) - await conn.ExecuteAsync( - @"INSERT INTO dbo.ModlistArchiveStatus (PrimaryKeyStringHash, PrimaryKeyString, Hash, IsValid) - VALUES (HASHBYTES('SHA2_256', @PrimaryKeyString), @PrimaryKeyString, @Hash, @IsValid)", new - { - itm.Archive.State.PrimaryKeyString, - itm.Archive.Hash, - itm.IsValid - }, trans); - - await trans.CommitAsync(); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataLayer/Patches.cs b/Wabbajack.Server/DataLayer/Patches.cs deleted file mode 100644 index 55432aa4..00000000 --- a/Wabbajack.Server/DataLayer/Patches.cs +++ /dev/null @@ -1,259 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Server.DTOs; - -namespace Wabbajack.Server.DataLayer; - -public partial class SqlService -{ - /// - /// Adds a patch record - /// - /// - /// - public async Task AddPatch(Patch patch) - { - await using var conn = await Open(); - await using var trans = conn.BeginTransaction(); - - if (await conn.QuerySingleOrDefaultAsync<(Guid, Guid)>( - "Select SrcID, DestID FROM dbo.Patches where SrcID = @SrcId and DestID = @DestId", - new {SrcId = patch.Src.Id, DestId = patch.Dest.Id}, trans) != default) - return false; - - await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)", - new {SrcId = patch.Src.Id, DestId = patch.Dest.Id}, trans); - await trans.CommitAsync(); - return true; - } - - /// - /// Adds a patch record - /// - /// - /// - public async Task FinializePatch(Patch patch) - { - await using var conn = await Open(); - await conn.ExecuteAsync( - "UPDATE dbo.Patches SET PatchSize = @PatchSize, Finished = @Finished, IsFailed = @IsFailed, FailMessage = @FailMessage WHERE SrcId = @SrcId AND DestID = @DestId", - new - { - SrcId = patch.Src.Id, - DestId = patch.Dest.Id, - patch.PatchSize, - patch.Finished, - patch.IsFailed, - patch.FailMessage - }); - } - - public async Task FindPatch(Guid src, Guid dest) - { - await using var conn = await Open(); - var patch = await conn.QueryFirstOrDefaultAsync<(long, DateTime?, bool?, string)>( - @"SELECT p.PatchSize, p.Finished, p.IsFailed, p.FailMessage - FROM dbo.Patches p - LEFT JOIN dbo.ArchiveDownloads src ON p.SrcId = src.Id - LEFT JOIN dbo.ArchiveDownloads dest ON p.SrcId = dest.Id - WHERE SrcId = @SrcId - AND DestId = @DestId - AND src.DownloadFinished IS NOT NULL - AND dest.DownloadFinished IS NOT NULL", - new - { - SrcId = src, - DestId = dest - }); - if (patch == default) - return default; - - return new Patch - { - Src = await GetArchiveDownload(src), - Dest = await GetArchiveDownload(dest), - PatchSize = patch.Item1, - Finished = patch.Item2, - IsFailed = patch.Item3, - FailMessage = patch.Item4 - }; - } - - public async Task FindOrEnqueuePatch(Guid src, Guid dest) - { - await using var conn = await Open(); - var trans = await conn.BeginTransactionAsync(); - var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>( - "SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestId", - new - { - SrcId = src, - DestId = dest - }, trans); - if (patch == default) - { - await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)", - new {SrcId = src, DestId = dest}, trans); - await trans.CommitAsync(); - return new Patch {Src = await GetArchiveDownload(src), Dest = await GetArchiveDownload(dest)}; - } - - return new Patch - { - Src = await GetArchiveDownload(src), - Dest = await GetArchiveDownload(dest), - PatchSize = patch.Item3, - Finished = patch.Item4, - IsFailed = patch.Item5, - FailMessage = patch.Item6 - }; - } - - public async Task GetPendingPatch() - { - await using var conn = await Open(); - var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>( - @"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage FROM dbo.Patches p - LEFT JOIN dbo.ArchiveDownloads src ON src.Id = p.SrcId - LEFT JOIN dbo.ArchiveDownloads dest ON dest.Id = p.DestId - WHERE p.Finished is NULL AND src.IsFailed = 0 AND dest.IsFailed = 0 "); - if (patch == default) - return default; - - return new Patch - { - Src = await GetArchiveDownload(patch.Item1), - Dest = await GetArchiveDownload(patch.Item2), - PatchSize = patch.Item3, - Finished = patch.Item4, - IsFailed = patch.Item5, - FailMessage = patch.Item6 - }; - } - - public async Task> PatchesForSource(Guid sourceDownload) - { - await using var conn = await Open(); - var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>( - "SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId", - new {SrcId = sourceDownload}); - - return await AsPatches(patches); - } - - public async Task> PatchesForSource(Hash sourceHash) - { - await using var conn = await Open(); - var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>( - @"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage - FROM dbo.Patches p - LEFT JOIN dbo.ArchiveDownloads a ON p.SrcId = a.Id - - WHERE a.Hash = @Hash AND p.Finished IS NOT NULL AND p.IsFailed = 0", new {Hash = sourceHash}); - - return await AsPatches(patches); - } - - public async Task MarkPatchUsage(Guid srcId, Guid destId) - { - await using var conn = await Open(); - await conn.ExecuteAsync( - @"UPDATE dbo.Patches SET Downloads = Downloads + 1, LastUsed = GETUTCDATE() WHERE SrcId = @srcId AND DestID = @destId", - new {SrcId = srcId, DestId = destId}); - } - - public async Task> GetOldPatches() - { - await using var conn = await Open(); - var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>( - @"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage - FROM dbo.Patches p - LEFT JOIN dbo.ArchiveDownloads a ON p.SrcId = a.Id - WHERE a.Hash not in (SELECT Hash FROM dbo.ModListArchives)"); - - return await AsPatches(patches); - } - - private async Task> AsPatches(IEnumerable<(Guid, Guid, long, DateTime?, bool?, string)> patches) - { - var results = new List(); - foreach (var (srcId, destId, patchSize, finished, isFailed, failMessage) in patches) - results.Add(new Patch - { - Src = await GetArchiveDownload(srcId), - Dest = await GetArchiveDownload(destId), - PatchSize = patchSize, - Finished = finished, - IsFailed = isFailed, - FailMessage = failMessage - }); - - return results; - } - - - public async Task DeletePatch(Patch patch) - { - await using var conn = await Open(); - await conn.ExecuteAsync(@"DELETE FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestID", - new - { - SrcId = patch.Src.Id, - DestId = patch.Dest.Id - }); - } - - public async Task> AllPatchHashes() - { - await using var conn = await Open(); - return (await conn.QueryAsync<(Hash, Hash)>(@"SELECT a1.Hash, a2.Hash - FROM dbo.Patches p - LEFT JOIN dbo.ArchiveDownloads a1 ON a1.Id = p.SrcId - LEFT JOIN dbo.ArchiveDownloads a2 on a2.Id = p.DestId - WHERE p.Finished IS NOT NULL")).ToHashSet(); - } - - public async Task DeletePatchesForHashPair((Hash, Hash) sqlFile) - { - await using var conn = await Open(); - await conn.ExecuteAsync(@"DELETE p - FROM dbo.Patches p - LEFT JOIN dbo.ArchiveDownloads a1 ON a1.Id = p.SrcId - LEFT JOIN dbo.ArchiveDownloads a2 on a2.Id = p.DestId - WHERE a1.Hash = @SrcHash - AND a2.Hash = @DestHash", new - { - SrcHash = sqlFile.Item1, - DestHash = sqlFile.Item2 - }); - } - - public async Task PurgePatch(Hash hash, string rationale) - { - await using var conn = await Open(); - await using var tx = await conn.BeginTransactionAsync(); - - await conn.ExecuteAsync( - "DELETE p FROM dbo.Patches p LEFT JOIN dbo.ArchiveDownloads ad ON ad.Id = p.SrcId WHERE ad.Hash = @Hash ", - new {Hash = hash}, tx); - await conn.ExecuteAsync( - "INSERT INTO dbo.NoPatch (Hash, Created, Rationale) VALUES (@Hash, GETUTCDATE(), @Rationale)", - new - { - Hash = hash, - Rationale = rationale - }, tx); - await tx.CommitAsync(); - } - - public async Task IsNoPatch(Hash hash) - { - await using var conn = await Open(); - return await conn.QueryFirstOrDefaultAsync("SELECT Hash FROM NoPatch WHERE Hash = @Hash", - new {Hash = hash}) != default; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataLayer/SqlService.cs b/Wabbajack.Server/DataLayer/SqlService.cs deleted file mode 100644 index 2164ba99..00000000 --- a/Wabbajack.Server/DataLayer/SqlService.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System.Data.SqlClient; -using System.Threading.Tasks; -using Wabbajack.BuildServer; -using Wabbajack.Downloaders; -using Wabbajack.DTOs.JsonConverters; - -namespace Wabbajack.Server.DataLayer; - -public partial class SqlService -{ - private readonly DownloadDispatcher _dispatcher; - private readonly DTOSerializer _dtos; - private readonly AppSettings _settings; - - public SqlService(AppSettings settings, DTOSerializer dtos, DownloadDispatcher dispatcher) - { - _settings = settings; - _dtos = dtos; - _dispatcher = dispatcher; - // Ugly hack, but the SQL mappers need it - _dtoStatic = dtos; - } - - public async Task Open() - { - var conn = new SqlConnection(_settings.SqlConnection); - await conn.OpenAsync(); - return conn; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataLayer/ValidationData.cs b/Wabbajack.Server/DataLayer/ValidationData.cs deleted file mode 100644 index 6e4a70d5..00000000 --- a/Wabbajack.Server/DataLayer/ValidationData.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using Wabbajack.DTOs; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Server.DTOs; - -namespace Wabbajack.Server.DataLayer; - -public partial class SqlService -{ - public async Task GetValidationData() - { - var archiveStatus = AllModListArchivesStatus(); - var modLists = AllModLists(); - var mirrors = GetAllMirroredHashes(); - var authoredFiles = AllAuthoredFiles(); - var nexusFiles = await AllNexusFiles(); - return new ValidationData - { - NexusFiles = nexusFiles.ToDictionary(nf => (nf.NexusGameId, nf.ModId, nf.FileId), nf => nf.category), - ArchiveStatus = await archiveStatus, - ModLists = await modLists, - Mirrors = await mirrors, - AllowedMirrors = new Lazy>>(async () => await GetAllowedMirrors()), - AllAuthoredFiles = await authoredFiles - }; - } - - public async Task> AllModListArchivesStatus() - { - await using var conn = await Open(); - var results = - await conn.QueryAsync<(string, Hash, bool)>( - @"SELECT PrimaryKeyString, Hash, IsValid FROM dbo.ModListArchiveStatus"); - return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3); - } - - public async Task> AllNexusFiles() - { - await using var conn = await Open(); - var results = - await conn.QueryAsync<(long, long, long, string)>( - @"SELECT Game, ModId, FileId, JSON_VALUE(Data, '$.category_name') FROM dbo.NexusModFile"); - return results.ToHashSet(); - } - - public async Task> AllModLists() - { - await using var conn = await Open(); - var results = await conn.QueryAsync(@"SELECT Metadata FROM dbo.ModLists"); - return results.ToList(); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/DataModels/AuthorFiles.cs b/Wabbajack.Server/DataModels/AuthorFiles.cs new file mode 100644 index 00000000..ae04d42c --- /dev/null +++ b/Wabbajack.Server/DataModels/AuthorFiles.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.BuildServer; +using Wabbajack.Common; +using Wabbajack.DTOs.CDN; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; + + +namespace Wabbajack.Server.DataModels; + +public class AuthorFiles +{ + private readonly ILogger _logger; + private readonly AppSettings _settings; + private readonly DTOSerializer _dtos; + private Dictionary _byServerId = new(); + + public AbsolutePath AuthorFilesLocation => _settings.AuthoredFilesFolder.ToAbsolutePath(); + + public AuthorFiles(ILogger logger, AppSettings settings, DTOSerializer dtos) + { + _logger = logger; + _settings = settings; + _dtos = dtos; + } + + public IEnumerable AllDefinitions => AuthorFilesLocation.EnumerateFiles("definition.json.gz"); + + public async Task AllAuthoredFiles() + { + var defs = new List(); + foreach (var file in AllDefinitions) + { + defs.Add(new FileDefinitionMetadata + { + Definition = await ReadDefinition(file), + Updated = file.LastModifiedUtc() + }); + } + + _byServerId = defs.ToDictionary(f => f.Definition.ServerAssignedUniqueId!, f => f.Definition); + return defs.ToArray(); + } + + public async Task StreamForPart(string mungedName, int part) + { + return AuthorFilesLocation.Combine(mungedName, "parts", part.ToString()).Open(FileMode.Open); + } + + public async Task CreatePart(string mungedName, int part) + { + return AuthorFilesLocation.Combine(mungedName, "parts", part.ToString()).Open(FileMode.Create, FileAccess.Write, FileShare.None); + } + + public async Task WriteDefinition(FileDefinition definition) + { + var path = AuthorFilesLocation.Combine(definition.MungedName, "definition.json.gz"); + path.Parent.CreateDirectory(); + path.Parent.Combine("parts").CreateDirectory(); + + await using var ms = new MemoryStream(); + await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true)) + { + await _dtos.Serialize(definition, gz); + } + + await path.WriteAllBytesAsync(ms.ToArray()); + } + + public async Task ReadDefinition(string mungedName) + { + return await ReadDefinition(AuthorFilesLocation.Combine(mungedName, "definition.json.gz")); + } + + private async Task ReadDefinition(AbsolutePath file) + { + var gz = new GZipStream(new MemoryStream(await file.ReadAllBytesAsync()), CompressionMode.Decompress); + var definition = (await _dtos.DeserializeAsync(gz))!; + return definition; + } + + public class FileDefinitionMetadata + { + public FileDefinition Definition { get; set; } + public DateTime Updated { get; set; } + public string HumanSize => Definition.Size.ToFileSizeString(); + } + + public async Task DeleteFile(FileDefinition definition) + { + var folder = AuthorFilesLocation.Combine(definition.MungedName); + folder.DeleteDirectory(); + } + + public async Task ReadDefinitionForServerId(string serverAssignedUniqueId) + { + if (_byServerId.TryGetValue(serverAssignedUniqueId, out var found)) + return found; + await AllAuthoredFiles(); + return _byServerId[serverAssignedUniqueId]; + } +} \ No newline at end of file diff --git a/Wabbajack.Server/DataModels/AuthorKeys.cs b/Wabbajack.Server/DataModels/AuthorKeys.cs new file mode 100644 index 00000000..03a0e49e --- /dev/null +++ b/Wabbajack.Server/DataModels/AuthorKeys.cs @@ -0,0 +1,28 @@ +using System.Threading.Tasks; +using Wabbajack.BuildServer; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; + +namespace Wabbajack.Server.DataModels; + +public class AuthorKeys +{ + private readonly AppSettings _settings; + private AbsolutePath AuthorKeysPath => _settings.AuthorAPIKeyFile.ToAbsolutePath(); + + public AuthorKeys(AppSettings settings) + { + _settings = settings; + } + + public async Task AuthorForKey(string key) + { + await foreach (var line in AuthorKeysPath.ReadAllLinesAsync()) + { + var parts = line.Split("\t"); + if (parts[0].Trim() == key) + return parts[1].Trim(); + } + return null; + } +} \ No newline at end of file diff --git a/Wabbajack.Server/DataModels/Metrics.cs b/Wabbajack.Server/DataModels/Metrics.cs new file mode 100644 index 00000000..d8b10285 --- /dev/null +++ b/Wabbajack.Server/DataModels/Metrics.cs @@ -0,0 +1,39 @@ +using System; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Toolkit.HighPerformance; +using Wabbajack.BuildServer; +using Wabbajack.Common; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; +using Wabbajack.Server.DTOs; + +namespace Wabbajack.Server.DataModels; + +public class Metrics +{ + private readonly AppSettings _settings; + public SemaphoreSlim _lock = new(1); + private readonly DTOSerializer _dtos; + + public Metrics(AppSettings settings, DTOSerializer dtos) + { + _settings = settings; + _dtos = dtos; + } + + public async Task Ingest(Metric metric) + { + using var _ = await _lock.Lock(); + var data = Encoding.UTF8.GetBytes(_dtos.Serialize(metric)); + var metricsFile = _settings.MetricsFolder.ToAbsolutePath().Combine(DateTime.Now.ToString("yyyy_MM_dd") + ".json"); + await using var fs = metricsFile.Open(FileMode.Append, FileAccess.Write, FileShare.Read); + fs.Write(data); + fs.Write(Encoding.UTF8.GetBytes("\n")); + } + + +} \ No newline at end of file diff --git a/Wabbajack.Server/Program.cs b/Wabbajack.Server/Program.cs index cb102dfe..1b515133 100644 --- a/Wabbajack.Server/Program.cs +++ b/Wabbajack.Server/Program.cs @@ -22,16 +22,7 @@ public class Program webBuilder.UseStartup() .UseKestrel(options => { - options.Listen(IPAddress.Any, testMode ? 8080 : 80); - if (!testMode) - options.Listen(IPAddress.Any, 443, listenOptions => - { - using var store = new X509Store(StoreName.My); - store.Open(OpenFlags.ReadOnly); - var cert = store.Certificates.Find(X509FindType.FindBySubjectName, - "build.wabbajack.org", true)[0]; - listenOptions.UseHttps(cert); - }); + options.Listen(IPAddress.Any, 5000); options.Limits.MaxRequestBodySize = null; }); }); diff --git a/Wabbajack.Server/Services/ArchiveDownloader.cs b/Wabbajack.Server/Services/ArchiveDownloader.cs deleted file mode 100644 index 60f8bb66..00000000 --- a/Wabbajack.Server/Services/ArchiveDownloader.cs +++ /dev/null @@ -1,150 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Wabbajack.BuildServer; -using Wabbajack.Common; -using Wabbajack.Downloaders; -using Wabbajack.DTOs.DownloadStates; -using Wabbajack.Networking.NexusApi; -using Wabbajack.Paths.IO; -using Wabbajack.Server.DataLayer; -using Wabbajack.Server.DTOs; - -namespace Wabbajack.Server.Services; - -public class ArchiveDownloader : AbstractService -{ - private readonly DownloadDispatcher _dispatcher; - private readonly TemporaryFileManager _manager; - private readonly ArchiveMaintainer _archiveMaintainer; - private readonly DiscordWebHook _discord; - private NexusApi _nexusClient; - private readonly SqlService _sql; - - public ArchiveDownloader(ILogger logger, AppSettings settings, SqlService sql, - ArchiveMaintainer archiveMaintainer, - DiscordWebHook discord, QuickSync quickSync, DownloadDispatcher dispatcher, TemporaryFileManager manager) - : base(logger, settings, quickSync, TimeSpan.FromMinutes(10)) - { - _sql = sql; - _archiveMaintainer = archiveMaintainer; - _discord = discord; - _dispatcher = dispatcher; - _manager = manager; - } - - public override async Task Execute() - { - var count = 0; - - while (true) - { - var (_, header) = await _nexusClient.Validate(); - var ignoreNexus = header.DailyRemaining < 100 && header.HourlyRemaining < 10; - //var ignoreNexus = true; - if (ignoreNexus) - _logger.LogWarning( - $"Ignoring Nexus Downloads due to low hourly api limit (Daily: {header.DailyRemaining}, Hourly:{header.HourlyRemaining})"); - else - _logger.LogInformation( - $"Looking for any download (Daily: {header.DailyRemaining}, Hourly:{header.HourlyRemaining})"); - - var nextDownload = await _sql.GetNextPendingDownload(ignoreNexus); - - if (nextDownload == default) - break; - - _logger.LogInformation($"Checking for previously archived {nextDownload.Archive.Hash}"); - - if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash)) - { - await nextDownload.Finish(_sql); - continue; - } - - if (nextDownload.Archive.State is Manual or GameFileSource) - { - await nextDownload.Finish(_sql); - continue; - } - - try - { - _logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"); - ReportStarting(nextDownload.Archive.State.PrimaryKeyString); - await _discord.Send(Channel.Spam, - new DiscordMessage - { - Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}" - }); - - await _dispatcher.PrepareAll(new[] {nextDownload.Archive.State}); - - await using var tempPath = _manager.CreateFile(); - if (await _dispatcher.Download(nextDownload.Archive, tempPath.Path, CancellationToken.None) == default) - { - _logger.LogError( - $"Downloader returned false for {nextDownload.Archive.State.PrimaryKeyString}"); - await nextDownload.Fail(_sql, "Downloader returned false"); - continue; - } - - var hash = await tempPath.Path.Hash(); - - if (hash == default || nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash) - { - _logger.Log(LogLevel.Warning, - $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size()}"); - await nextDownload.Fail(_sql, "Invalid Hash"); - continue; - } - - if (nextDownload.Archive.Size != default && - tempPath.Path.Size() != nextDownload.Archive.Size) - { - await nextDownload.Fail(_sql, "Invalid Size"); - continue; - } - - nextDownload.Archive.Hash = hash; - nextDownload.Archive.Size = tempPath.Path.Size(); - - _logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}"); - await _archiveMaintainer.Ingest(tempPath.Path); - - _logger.Log(LogLevel.Information, - $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}"); - await nextDownload.Finish(_sql); - - await _discord.Send(Channel.Spam, - new DiscordMessage - { - Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}" - }); - } - catch (Exception ex) - { - _logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}"); - await nextDownload.Fail(_sql, ex.ToString()); - await _discord.Send(Channel.Spam, - new DiscordMessage - { - Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}" - }); - } - finally - { - ReportEnding(nextDownload.Archive.State.PrimaryKeyString); - } - - count++; - } - - if (count > 0) - // Wake the Patch builder up in case it needs to build a patch now - await _quickSync.Notify(); - - return count; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Services/ArchiveMaintainer.cs b/Wabbajack.Server/Services/ArchiveMaintainer.cs deleted file mode 100644 index 434c12c1..00000000 --- a/Wabbajack.Server/Services/ArchiveMaintainer.cs +++ /dev/null @@ -1,66 +0,0 @@ -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Wabbajack.BuildServer; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; -using Wabbajack.VFS; - -namespace Wabbajack.Server.Services; - -/// -/// Maintains a concurrent cache of all the files we've downloaded, indexed by Hash. -/// -public class ArchiveMaintainer -{ - private readonly ILogger _logger; - private readonly AppSettings _settings; - - public ArchiveMaintainer(ILogger logger, AppSettings settings) - { - _settings = settings; - _logger = logger; - _logger.Log(LogLevel.Information, "Creating Archive Maintainer"); - } - - private AbsolutePath ArchivePath(Hash hash) - { - return _settings.ArchivePath.Combine(hash.ToHex()); - } - - public async Task Ingest(AbsolutePath file) - { - var hash = await file.Hash(CancellationToken.None); - if (hash == default) return; - - var newPath = ArchivePath(hash); - if (HaveArchive(hash)) - { - file.Delete(); - return; - } - - await file.MoveToAsync(newPath, true, CancellationToken.None); - } - - public bool HaveArchive(Hash hash) - { - return ArchivePath(hash).FileExists(); - } - - public bool TryGetPath(Hash hash, out AbsolutePath path) - { - path = ArchivePath(hash); - return path.FileExists(); - } -} - -public static class ArchiveMaintainerExtensions -{ - public static IServiceCollection UseArchiveMaintainer(this IServiceCollection b) - { - return b.AddSingleton(); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Services/DiscordFrontend.cs b/Wabbajack.Server/Services/DiscordFrontend.cs deleted file mode 100644 index b5c0b2a0..00000000 --- a/Wabbajack.Server/Services/DiscordFrontend.cs +++ /dev/null @@ -1,189 +0,0 @@ -using System; -using System.Linq; -using System.Threading.Tasks; -using Discord; -using Discord.WebSocket; -using Microsoft.Extensions.Logging; -using Wabbajack.BuildServer; -using Wabbajack.DTOs; -using Wabbajack.Server.DataLayer; -using Wabbajack.Server.TokenProviders; - -namespace Wabbajack.Server.Services; - -public class DiscordFrontend : IStartable -{ - private readonly IDiscordToken _token; - private readonly DiscordSocketClient _client; - private readonly MetricsKeyCache _keyCache; - private readonly ILogger _logger; - private readonly QuickSync _quickSync; - private AppSettings _settings; - private readonly SqlService _sql; - - public DiscordFrontend(ILogger logger, AppSettings settings, QuickSync quickSync, SqlService sql, - MetricsKeyCache keyCache, IDiscordToken token) - { - _logger = logger; - _settings = settings; - _quickSync = quickSync; - - _client = new DiscordSocketClient(); - - _client.Log += LogAsync; - _client.Ready += ReadyAsync; - _client.MessageReceived += MessageReceivedAsync; - - _sql = sql; - _keyCache = keyCache; - _token = token; - } - - public async Task Start() - { - await _client.LoginAsync(TokenType.Bot, await _token.Get()); - await _client.StartAsync(); - } - - private async Task MessageReceivedAsync(SocketMessage arg) - { - _logger.LogInformation(arg.Content); - if (arg.Content.StartsWith("!dervenin")) - { - var parts = arg.Content.Split(" ", StringSplitOptions.RemoveEmptyEntries); - if (parts[0] != "!dervenin") - return; - - if (parts.Length == 1) await ReplyTo(arg, "Wat?"); - - if (parts[1] == "purge-nexus-cache") - { - if (parts.Length != 3) - { - await ReplyTo(arg, "Welp you did that wrong, gotta give me a mod-id or url"); - return; - } - - await PurgeNexusCache(arg, parts[2]); - } - else if (parts[1] == "quick-sync") - { - var options = await _quickSync.Report(); - if (parts.Length != 3) - { - var optionsStr = string.Join(", ", options.Select(o => o.Key.Name)); - await ReplyTo(arg, $"Can't expect me to quicksync the whole damn world! Try: {optionsStr}"); - } - else - { - foreach (var pair in options.Where(o => o.Key.Name == parts[2])) - { - await _quickSync.Notify(pair.Key); - await ReplyTo(arg, $"Notified {pair.Key}"); - } - } - } - else if (parts[1] == "purge-list") - { - if (parts.Length != 3) - { - await ReplyTo(arg, "Yeah, I'm not gonna purge the whole server..."); - } - else - { - var deleted = await _sql.PurgeList(parts[2]); - await _quickSync.Notify(); - await ReplyTo(arg, - $"Purged all traces of #{parts[2]} from the server, triggered list downloading. {deleted} records removed"); - } - } - else if (parts[1] == "mirror-mod") - { - await MirrorModCommand(arg, parts); - } - else if (parts[1] == "users") - { - await ReplyTo(arg, $"Wabbajack has {await _keyCache.KeyCount()} known unique users"); - } - } - } - - private async Task MirrorModCommand(SocketMessage msg, string[] parts) - { - if (parts.Length != 2) - { - await ReplyTo(msg, "Command is: mirror-mod "); - return; - } - - if (long.TryParse(parts[2], out var modId)) - { - await ReplyTo(msg, $"Got {modId} for a mod-id, expected a integer"); - return; - } - - if (GameRegistry.TryGetByFuzzyName(parts[1], out var game)) - { - var gameNames = GameRegistry.Games.Select(g => g.Value.NexusName) - .Where(g => !string.IsNullOrWhiteSpace(g)) - .Select(g => (string) g) - .ToHashSet(); - var joined = string.Join(", ", gameNames.OrderBy(g => g)); - await ReplyTo(msg, $"Got {parts[1]} for a game name, expected something like: {joined}"); - } - - if (game!.NexusGameId == default) await ReplyTo(msg, $"No NexusGameID found for {game}"); - - await _sql.AddNexusModWithOpenPerms(game.Game, modId); - await _quickSync.Notify(); - await ReplyTo(msg, "Done, and I notified the uploader"); - } - - private async Task PurgeNexusCache(SocketMessage arg, string mod) - { - if (Uri.TryCreate(mod, UriKind.Absolute, out var url)) - mod = url.AbsolutePath.Split("/", StringSplitOptions.RemoveEmptyEntries).Last(); - - if (int.TryParse(mod, out var mod_id)) - { - await _sql.PurgeNexusCache(mod_id); - await ReplyTo(arg, $"It is done, {mod_id} has been purged, list validation has been triggered"); - } - } - - private async Task ReplyTo(SocketMessage socketMessage, string message) - { - await socketMessage.Channel.SendMessageAsync(message); - } - - private async Task ReadyAsync() - { - } - - private async Task LogAsync(LogMessage arg) - { - switch (arg.Severity) - { - case LogSeverity.Info: - _logger.LogInformation(arg.Message); - break; - case LogSeverity.Warning: - _logger.LogWarning(arg.Message); - break; - case LogSeverity.Critical: - _logger.LogCritical(arg.Message); - break; - case LogSeverity.Error: - _logger.LogError(arg.Exception, arg.Message); - break; - case LogSeverity.Verbose: - _logger.LogTrace(arg.Message); - break; - case LogSeverity.Debug: - _logger.LogDebug(arg.Message); - break; - default: - throw new ArgumentOutOfRangeException(); - } - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Services/MetricsKeyCache.cs b/Wabbajack.Server/Services/MetricsKeyCache.cs deleted file mode 100644 index 6b030c7f..00000000 --- a/Wabbajack.Server/Services/MetricsKeyCache.cs +++ /dev/null @@ -1,61 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Wabbajack.Common; -using Wabbajack.Server.DataLayer; - -namespace Wabbajack.Server.Services; - -public class MetricsKeyCache : IStartable -{ - private HashSet _knownKeys = new(); - private readonly AsyncLock _lock = new(); - private ILogger _logger; - private readonly SqlService _sql; - - public MetricsKeyCache(ILogger logger, SqlService sql) - { - _logger = logger; - _sql = sql; - } - - public async Task Start() - { - _knownKeys = _sql.AllKeys().Result.ToHashSet(); - } - - public async Task IsValidKey(string key) - { - using (var _ = await _lock.WaitAsync()) - { - if (_knownKeys.Contains(key)) return true; - } - - if (await _sql.ValidMetricsKey(key)) - { - using var _ = await _lock.WaitAsync(); - _knownKeys.Add(key); - return true; - } - - return false; - } - - public async Task AddKey(string key) - { - using (var _ = await _lock.WaitAsync()) - { - if (_knownKeys.Contains(key)) return; - _knownKeys.Add(key); - } - - await _sql.AddMetricsKey(key); - } - - public async Task KeyCount() - { - using var _ = await _lock.WaitAsync(); - return _knownKeys.Count; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Services/MirrorQueueService.cs b/Wabbajack.Server/Services/MirrorQueueService.cs deleted file mode 100644 index 291d652c..00000000 --- a/Wabbajack.Server/Services/MirrorQueueService.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Wabbajack.BuildServer; -using Wabbajack.Server.DataLayer; - -namespace Wabbajack.Server.Services; - -public class MirrorQueueService : AbstractService -{ - private DiscordWebHook _discord; - private readonly SqlService _sql; - - public MirrorQueueService(ILogger logger, AppSettings settings, QuickSync quickSync, - DiscordWebHook discordWebHook, SqlService sqlService) : - base(logger, settings, quickSync, TimeSpan.FromMinutes(5)) - { - _discord = discordWebHook; - _sql = sqlService; - } - - public override async Task Execute() - { - await _sql.QueueMirroredFiles(); - return 1; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Services/MirrorUploader.cs b/Wabbajack.Server/Services/MirrorUploader.cs deleted file mode 100644 index b31135ff..00000000 --- a/Wabbajack.Server/Services/MirrorUploader.cs +++ /dev/null @@ -1,227 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Net; -using System.Threading.Tasks; -using FluentFTP; -using FluentFTP.Helpers; -using Microsoft.Extensions.Logging; -using Wabbajack.BuildServer; -using Wabbajack.Common; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Networking.WabbajackClientApi; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; -using Wabbajack.RateLimiter; -using Wabbajack.Server.DataLayer; -using Wabbajack.Server.DTOs; -using Wabbajack.Server.TokenProviders; - -namespace Wabbajack.Server.Services; - -public class MirrorUploader : AbstractService -{ - private readonly IFtpSiteCredentials _credentials; - private readonly DTOSerializer _dtos; - private readonly IFtpSiteCredentials _ftpCreds; - private readonly ParallelOptions _parallelOptions; - private readonly Client _wjClient; - private readonly ArchiveMaintainer _archives; - private readonly DiscordWebHook _discord; - private readonly SqlService _sql; - - public MirrorUploader(ILogger logger, AppSettings settings, SqlService sql, QuickSync quickSync, - ArchiveMaintainer archives, - DiscordWebHook discord, IFtpSiteCredentials credentials, Client wjClient, ParallelOptions parallelOptions, - DTOSerializer dtos, - IFtpSiteCredentials ftpCreds) - : base(logger, settings, quickSync, TimeSpan.FromHours(1)) - { - _sql = sql; - _archives = archives; - _discord = discord; - _credentials = credentials; - _wjClient = wjClient; - _parallelOptions = parallelOptions; - _dtos = dtos; - _ftpCreds = ftpCreds; - } - - public bool ActiveFileSyncEnabled { get; set; } = true; - - public override async Task Execute() - { - var uploaded = 0; - - if (ActiveFileSyncEnabled) - await _sql.SyncActiveMirroredFiles(); - TOP: - var toUpload = await _sql.GetNextMirroredFile(); - if (toUpload == default) - { - await DeleteOldMirrorFiles(); - return uploaded; - } - - uploaded += 1; - - try - { - var creds = (await _credentials.Get())[StorageSpace.Mirrors]; - - if (_archives.TryGetPath(toUpload.Hash, out var path)) - { - _logger.LogInformation($"Uploading mirror file {toUpload.Hash} {path.Size().FileSizeToString()}"); - - var exists = false; - using (var client = await GetClient(creds)) - { - exists = await client.FileExistsAsync($"{toUpload.Hash.ToHex()}/definition.json.gz"); - } - - if (exists) - { - _logger.LogInformation($"Skipping {toUpload.Hash} it's already on the server"); - await toUpload.Finish(_sql); - goto TOP; - } - - await _discord.Send(Channel.Spam, - new DiscordMessage - { - Content = $"Uploading {toUpload.Hash} - {toUpload.Created} because {toUpload.Rationale}" - }); - - var definition = await _wjClient.GenerateFileDefinition(path); - - using (var client = await GetClient(creds)) - { - await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}"); - await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}/parts"); - } - - string MakePath(long idx) - { - return $"{definition.Hash.ToHex()}/parts/{idx}"; - } - - await definition.Parts.PDoAll(new Resource(), async part => - { - _logger.LogInformation("Uploading mirror part ({index}/{length})", part.Index, - definition.Parts.Length); - - var buffer = new byte[part.Size]; - await using (var fs = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read)) - { - fs.Position = part.Offset; - await fs.ReadAsync(buffer); - } - - await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () => - { - using var client = await GetClient(creds); - var name = MakePath(part.Index); - await client.UploadAsync(new MemoryStream(buffer), name); - }); - }); - - await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () => - { - using var client = await GetClient(creds); - _logger.LogInformation("Finishing mirror upload"); - - - await using var ms = new MemoryStream(); - await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true)) - { - await _dtos.Serialize(definition, gz); - } - - ms.Position = 0; - var remoteName = $"{definition.Hash.ToHex()}/definition.json.gz"; - await client.UploadAsync(ms, remoteName); - }); - - await toUpload.Finish(_sql); - } - else - { - await toUpload.Fail(_sql, "Archive not found"); - } - } - catch (Exception ex) - { - _logger.LogInformation($"{toUpload.Created} {toUpload.Uploaded}"); - _logger.LogError(ex, "Error uploading"); - await toUpload.Fail(_sql, ex.ToString()); - } - - goto TOP; - } - - private async Task GetClient(FtpSite? creds = null) - { - return await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () => - { - creds ??= (await _ftpCreds.Get())[StorageSpace.Mirrors]; - - var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password)); - ftpClient.DataConnectionType = FtpDataConnectionType.EPSV; - await ftpClient.ConnectAsync(); - return ftpClient; - }); - } - - /// - /// Gets a list of all the Mirrored file hashes that physically exist on the CDN (via FTP lookup) - /// - /// - public async Task> GetHashesOnCDN() - { - using var ftpClient = await GetClient(); - var serverFiles = await ftpClient.GetNameListingAsync("\\"); - - return serverFiles - .Select(f => ((RelativePath) f).FileName) - .Select(l => - { - try - { - return Hash.FromHex((string) l); - } - catch (Exception) - { - return default; - } - }) - .Where(h => h != default) - .ToHashSet(); - } - - public async Task DeleteOldMirrorFiles() - { - var existingHashes = await GetHashesOnCDN(); - var fromSql = await _sql.GetAllMirroredHashes(); - - foreach (var (hash, _) in fromSql.Where(s => s.Value)) - { - _logger.LogInformation("Removing {hash} from SQL it's no longer in the CDN", hash); - if (!existingHashes.Contains(hash)) - await _sql.DeleteMirroredFile(hash); - } - - var toDelete = existingHashes.Where(h => !fromSql.ContainsKey(h)).ToArray(); - - using var client = await GetClient(); - foreach (var hash in toDelete) - { - await _discord.Send(Channel.Spam, - new DiscordMessage {Content = $"Removing mirrored file {hash}, as it's no longer in sql"}); - _logger.LogInformation("Removing {hash} from the CDN it's no longer in SQL", hash); - await client.DeleteDirectoryAsync(hash.ToHex()); - } - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Services/ModListDownloader.cs b/Wabbajack.Server/Services/ModListDownloader.cs deleted file mode 100644 index b8856a35..00000000 --- a/Wabbajack.Server/Services/ModListDownloader.cs +++ /dev/null @@ -1,140 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Builder; -using Microsoft.Extensions.Logging; -using Wabbajack.BuildServer; -using Wabbajack.Common; -using Wabbajack.Downloaders; -using Wabbajack.DTOs; -using Wabbajack.DTOs.JsonConverters; -using Wabbajack.Installer; -using Wabbajack.Networking.WabbajackClientApi; -using Wabbajack.Paths.IO; -using Wabbajack.Server.DataLayer; -using Wabbajack.Server.DTOs; - -namespace Wabbajack.Server.Services; - -public class ModListDownloader : AbstractService -{ - private readonly DownloadDispatcher _dispatcher; - private readonly DTOSerializer _dtos; - private readonly TemporaryFileManager _manager; - private readonly Client _wjClient; - private readonly DiscordWebHook _discord; - private readonly ArchiveMaintainer _maintainer; - private readonly SqlService _sql; - - public ModListDownloader(ILogger logger, AppSettings settings, ArchiveMaintainer maintainer, - SqlService sql, DiscordWebHook discord, QuickSync quickSync, Client wjClient, TemporaryFileManager manager, - DownloadDispatcher dispatcher, DTOSerializer dtos) - : base(logger, settings, quickSync, TimeSpan.FromMinutes(1)) - { - _logger = logger; - _settings = settings; - _maintainer = maintainer; - _sql = sql; - _discord = discord; - _wjClient = wjClient; - _manager = manager; - _dispatcher = dispatcher; - _dtos = dtos; - } - - - public override async Task Execute() - { - var downloaded = 0; - var lists = await _wjClient.LoadLists(); - - foreach (var list in lists) - try - { - ReportStarting(list.Links.MachineURL); - if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash)) - continue; - - - if (!_maintainer.HaveArchive(list.DownloadMetadata!.Hash)) - { - _logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}"); - await _discord.Send(Channel.Ham, - new DiscordMessage - { - Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}" - }); - var tf = _manager.CreateFile(); - var state = _dispatcher.Parse(new Uri(list.Links.Download)); - if (state == null) - { - _logger.Log(LogLevel.Error, - $"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}"); - continue; - } - - downloaded += 1; - await _dispatcher.Download(new Archive {State = state, Name = $"{list.Links.MachineURL}.wabbajack"}, - tf.Path, CancellationToken.None); - var hash = await tf.Path.Hash(); - if (hash != list.DownloadMetadata.Hash) - { - _logger.Log(LogLevel.Error, - $"Downloaded modlist {list.Links.MachineURL} {list.DownloadMetadata.Hash} didn't match metadata hash of {hash}"); - await _sql.IngestModList(list.DownloadMetadata.Hash, list, new ModList(), true); - continue; - } - - await _maintainer.Ingest(tf.Path); - } - - _maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath); - ModList modlist; - - modlist = await StandardInstaller.LoadFromFile(_dtos, modlistPath); - - await _discord.Send(Channel.Ham, - new DiscordMessage - { - Content = $"Ingesting {list.Links.MachineURL} version {modlist.Version}" - }); - await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist, false); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}"); - await _discord.Send(Channel.Ham, - new DiscordMessage - { - Content = - $"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash} - {ex.Message}" - }); - } - finally - { - ReportEnding(list.Links.MachineURL); - } - - _logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists"); - if (downloaded > 0) - await _discord.Send(Channel.Ham, - new DiscordMessage {Content = $"Downloaded {downloaded} new lists"}); - - var fc = await _sql.EnqueueModListFilesForIndexing(); - _logger.Log(LogLevel.Information, $"Enqueing {fc} files for downloading"); - if (fc > 0) - await _discord.Send(Channel.Ham, - new DiscordMessage {Content = $"Enqueing {fc} files for downloading"}); - - return downloaded; - } -} - -public static class ModListDownloaderExtensions -{ - public static void UseModListDownloader(this IApplicationBuilder b) - { - var poll = (ModListDownloader) b.ApplicationServices.GetService(typeof(ModListDownloader)); - poll.Start(); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Services/NexusPoll.cs b/Wabbajack.Server/Services/NexusPoll.cs deleted file mode 100644 index 7d8e322c..00000000 --- a/Wabbajack.Server/Services/NexusPoll.cs +++ /dev/null @@ -1,97 +0,0 @@ -using System; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Builder; -using Microsoft.Extensions.Logging; -using Wabbajack.BuildServer; -using Wabbajack.Common; -using Wabbajack.DTOs; -using Wabbajack.Networking.NexusApi; -using Wabbajack.Server.DataLayer; - -namespace Wabbajack.Server.Services; - -public class NexusPoll -{ - private readonly NexusApi _api; - private readonly ParallelOptions _parallelOptions; - private readonly GlobalInformation _globalInformation; - private readonly ILogger _logger; - private readonly AppSettings _settings; - private readonly SqlService _sql; - - public NexusPoll(ILogger logger, AppSettings settings, SqlService service, - GlobalInformation globalInformation, ParallelOptions parallelOptions, NexusApi api) - { - _sql = service; - _settings = settings; - _globalInformation = globalInformation; - _logger = logger; - _parallelOptions = parallelOptions; - _api = api; - } - - public async Task UpdateNexusCacheAPI(CancellationToken token) - { - using var _ = _logger.BeginScope("Nexus Update via API"); - _logger.Log(LogLevel.Information, "Starting Nexus Update via API"); - - var purged = await GameRegistry.Games.Values - .Where(game => game.NexusName != null) - .SelectMany(async game => - { - var (mods, _) = await _api.GetUpdates(game.Game, token); - - return mods.Select(mod => new {Game = game, Mod = mod}); - }) - .Select(async row => - { - var a = row.Mod.LatestFileUpdate.AsUnixTime(); - // Mod activity could hide files - var b = row.Mod.LastestModActivity.AsUnixTime(); - - var t = a > b ? a : b; - - long purgeCount = 0; - purgeCount += await _sql.DeleteNexusModInfosUpdatedBeforeDate(row.Game.Game, row.Mod.ModId, t.Date); - purgeCount += await _sql.DeleteNexusModFilesUpdatedBeforeDate(row.Game.Game, row.Mod.ModId, t.Date); - return purgeCount; - }) - .SumAsync(x => x); - - _logger.Log(LogLevel.Information, "Purged {count} cache entries", purged); - _globalInformation.LastNexusSyncUTC = DateTime.UtcNow; - } - - public void Start() - { - if (!_settings.RunBackEndJobs) return; - - Task.Run(async () => - { - while (true) - { - try - { - await UpdateNexusCacheAPI(CancellationToken.None); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error getting API feed from Nexus"); - } - - await Task.Delay(_globalInformation.NexusAPIPollRate); - } - }); - } -} - -public static class NexusPollExtensions -{ - public static void UseNexusPoll(this IApplicationBuilder b) - { - var poll = (NexusPoll) b.ApplicationServices.GetService(typeof(NexusPoll)); - poll.Start(); - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Services/NonNexusDownloadValidator.cs b/Wabbajack.Server/Services/NonNexusDownloadValidator.cs deleted file mode 100644 index 7394b3c0..00000000 --- a/Wabbajack.Server/Services/NonNexusDownloadValidator.cs +++ /dev/null @@ -1,85 +0,0 @@ -using System; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Wabbajack.BuildServer; -using Wabbajack.Downloaders; -using Wabbajack.Server.DataLayer; - -namespace Wabbajack.Server.Services; - -public class NonNexusDownloadValidator : AbstractService -{ - private readonly DownloadDispatcher _dispatcher; - private readonly ParallelOptions _parallelOptions; - private readonly SqlService _sql; - - public NonNexusDownloadValidator(ILogger logger, AppSettings settings, SqlService sql, - QuickSync quickSync, DownloadDispatcher dispatcher, ParallelOptions parallelOptions) - : base(logger, settings, quickSync, TimeSpan.FromHours(2)) - { - _sql = sql; - _dispatcher = dispatcher; - _parallelOptions = parallelOptions; - } - - public override async Task Execute() - { - var archives = await _sql.GetNonNexusModlistArchives(); - _logger.Log(LogLevel.Information, $"Validating {archives.Count} non-Nexus archives"); - await _dispatcher.PrepareAll(archives.Select(a => a.State)); - var random = new Random(); - - /* - var results = await archives.PMap(_parallelOptions, async archive => - { - try - { - await Task.Delay(random.Next(1000, 5000)); - - var token = new CancellationTokenSource(); - token.CancelAfter(TimeSpan.FromMinutes(10)); - - ReportStarting(archive.State.PrimaryKeyString); - bool isValid = false; - switch (archive.State) - { - //case WabbajackCDNDownloader.State _: - //case GoogleDriveDownloader.State _: // Let's try validating Google again 2/10/2021 - case GameFileSource _: - isValid = true; - break; - case Manual _: - case ModDB _: - case Http h when h.Url.ToString().StartsWith("https://wabbajack"): - isValid = true; - break; - default: - isValid = await _dispatcher.Verify(archive, token.Token); - break; - } - return (Archive: archive, IsValid: isValid); - } - catch (Exception ex) - { - _logger.Log(LogLevel.Warning, $"Error for {archive.Name} {archive.State.PrimaryKeyString} {ex}"); - return (Archive: archive, IsValid: false); - } - finally - { - ReportEnding(archive.State.PrimaryKeyString); - } - - }).ToList(); - - await _sql.UpdateNonNexusModlistArchivesStatus(results); - var failed = results.Count(r => !r.IsValid); - var passed = results.Count() - failed; - foreach(var (archive, _) in results.Where(f => !f.IsValid)) - _logger.Log(LogLevel.Warning, $"Validation failed for {archive.Name} from {archive.State.PrimaryKeyString}"); - - _logger.Log(LogLevel.Information, $"Non-nexus validation completed {failed} out of {passed} failed"); - */ - return default; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs deleted file mode 100644 index 5ecb78d4..00000000 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ /dev/null @@ -1,237 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Threading.Tasks; -using FluentFTP; -using FluentFTP.Helpers; -using Microsoft.Extensions.Logging; -using Wabbajack.BuildServer; -using Wabbajack.Common; -using Wabbajack.Compiler.PatchCache; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; -using Wabbajack.Server.DataLayer; -using Wabbajack.Server.DTOs; -using Wabbajack.Server.TokenProviders; - -namespace Wabbajack.Server.Services; - -public class PatchBuilder : AbstractService -{ - private readonly IFtpSiteCredentials _ftpCreds; - private readonly TemporaryFileManager _manager; - private readonly DiscordWebHook _discordWebHook; - private readonly ArchiveMaintainer _maintainer; - private readonly SqlService _sql; - - public PatchBuilder(ILogger logger, SqlService sql, AppSettings settings, - ArchiveMaintainer maintainer, - DiscordWebHook discordWebHook, QuickSync quickSync, TemporaryFileManager manager, IFtpSiteCredentials ftpCreds) - : base(logger, settings, quickSync, TimeSpan.FromMinutes(1)) - { - _discordWebHook = discordWebHook; - _sql = sql; - _maintainer = maintainer; - _manager = manager; - _ftpCreds = ftpCreds; - } - - public bool NoCleaning { get; set; } - - public override async Task Execute() - { - var count = 0; - while (true) - { - count++; - - var patch = await _sql.GetPendingPatch(); - if (patch == default) break; - - try - { - _logger.LogInformation( - $"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"); - await _discordWebHook.Send(Channel.Spam, - new DiscordMessage - { - Content = - $"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}" - }); - - if (patch.Src.Archive.Hash == patch.Dest.Archive.Hash && patch.Src.Archive.State.PrimaryKeyString == - patch.Dest.Archive.State.PrimaryKeyString) - { - await patch.Fail(_sql, "Hashes match"); - continue; - } - - if (patch.Src.Archive.Size > 2_500_000_000 || patch.Dest.Archive.Size > 2_500_000_000) - { - await patch.Fail(_sql, "Too large to patch"); - continue; - } - - _maintainer.TryGetPath(patch.Src.Archive.Hash, out var srcPath); - _maintainer.TryGetPath(patch.Dest.Archive.Hash, out var destPath); - - await using var sigFile = _manager.CreateFile(); - await using var patchFile = _manager.CreateFile(); - await using var srcStream = srcPath.Open(FileMode.Open, FileAccess.Read, FileShare.Read); - await using var destStream = destPath.Open(FileMode.Open, FileAccess.Read, FileShare.Read); - await using var sigStream = sigFile.Path.Open(FileMode.Create, FileAccess.ReadWrite); - await using var patchOutput = patchFile.Path.Open(FileMode.Create, FileAccess.ReadWrite); - OctoDiff.Create(destStream, srcStream, sigStream, patchOutput); - await patchOutput.DisposeAsync(); - var size = patchFile.Path.Size(); - - await UploadToCDN(patchFile.Path, PatchName(patch)); - - - await patch.Finish(_sql, size); - await _discordWebHook.Send(Channel.Spam, - new DiscordMessage - { - Content = - $"Built {size.ToFileSizeString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}" - }); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error while building patch"); - await patch.Fail(_sql, ex.ToString()); - await _discordWebHook.Send(Channel.Spam, - new DiscordMessage - { - Content = - $"Failure building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}" - }); - } - } - - if (count > 0) - { - } - - if (!NoCleaning) - await CleanupOldPatches(); - - return count; - } - - private static string PatchName(Patch patch) - { - return PatchName(patch.Src.Archive.Hash, patch.Dest.Archive.Hash); - } - - private static string PatchName(Hash oldHash, Hash newHash) - { - return $"{oldHash.ToHex()}_{newHash.ToHex()}"; - } - - private async Task CleanupOldPatches() - { - var patches = await _sql.GetOldPatches(); - using var client = await GetBunnyCdnFtpClient(); - - foreach (var patch in patches) - { - _logger.LogInformation($"Cleaning patch {patch.Src.Archive.Hash} -> {patch.Dest.Archive.Hash}"); - - await _discordWebHook.Send(Channel.Spam, - new DiscordMessage - { - Content = - $"Removing {patch.PatchSize.FileSizeToString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString} due it no longer being required by curated lists" - }); - - if (!await DeleteFromCDN(client, PatchName(patch))) - _logger.LogWarning($"Patch file didn't exist {PatchName(patch)}"); - - await _sql.DeletePatch(patch); - - var pendingPatch = await _sql.GetPendingPatch(); - if (pendingPatch != default) break; - } - - var files = await client.GetListingAsync("\\"); - _logger.LogInformation($"Found {files.Length} on the CDN"); - - var sqlFiles = await _sql.AllPatchHashes(); - _logger.LogInformation($"Found {sqlFiles.Count} in SQL"); - - HashSet<(Hash, Hash)> NamesToPairs(IEnumerable ftpFiles) - { - return ftpFiles.Select(f => f.Name).Where(f => f.Contains("_")).Select(p => - { - try - { - var lst = p.Split("_", StringSplitOptions.RemoveEmptyEntries).Select(Hash.FromHex).ToArray(); - return (lst[0], lst[1]); - } - catch (ArgumentException) - { - return default; - } - catch (FormatException) - { - return default; - } - }).Where(f => f != default).ToHashSet(); - } - - var oldHashPairs = NamesToPairs(files.Where(f => DateTime.UtcNow - f.Modified > TimeSpan.FromDays(2))); - foreach (var (oldHash, newHash) in oldHashPairs.Where(o => !sqlFiles.Contains(o))) - { - _logger.LogInformation($"Removing CDN File entry for {oldHash} -> {newHash} it's not SQL"); - await client.DeleteFileAsync(PatchName(oldHash, newHash)); - } - - var hashPairs = NamesToPairs(files); - foreach (var sqlFile in sqlFiles.Where(s => !hashPairs.Contains(s))) - { - _logger.LogInformation("Removing SQL File entry for {from} -> {to} it's not on the CDN", sqlFile.Item1, - sqlFile.Item2); - await _sql.DeletePatchesForHashPair(sqlFile); - } - } - - private async Task UploadToCDN(AbsolutePath patchFile, string patchName) - { - for (var times = 0; times < 5; times++) - try - { - _logger.Log(LogLevel.Information, - $"Uploading {patchFile.Size().ToFileSizeString()} patch file to CDN {patchName}"); - using var client = await GetBunnyCdnFtpClient(); - - await client.UploadFileAsync(patchFile.ToString(), patchName); - return; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error uploading {patchFile} to CDN"); - } - - _logger.Log(LogLevel.Error, $"Couldn't upload {patchFile} to {patchName}"); - } - - private async Task DeleteFromCDN(FtpClient client, string patchName) - { - if (!await client.FileExistsAsync(patchName)) - return false; - await client.DeleteFileAsync(patchName); - return true; - } - - private async Task GetBunnyCdnFtpClient() - { - var info = (await _ftpCreds.Get())[StorageSpace.Patches]; - var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)}; - await client.ConnectAsync(); - return client; - } -} \ No newline at end of file diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index eeeda31d..d5496dc8 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.IO; +using System.Net.Http; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -12,7 +13,8 @@ using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Hosting; using Newtonsoft.Json; using Wabbajack.BuildServer; -using Wabbajack.Server.DataLayer; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Server.DataModels; using Wabbajack.Server.Services; namespace Wabbajack.Server; @@ -51,20 +53,15 @@ public class Startup services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddDTOSerializer(); + services.AddDTOConverters(); services.AddResponseCompression(options => { options.Providers.Add(); @@ -82,9 +79,6 @@ public class Startup { if (env.IsDevelopment()) app.UseDeveloperExceptionPage(); - if (this is not TestStartup) - app.UseHttpsRedirection(); - app.UseDeveloperExceptionPage(); var provider = new FileExtensionContentTypeProvider(); @@ -98,18 +92,10 @@ public class Startup app.UseAuthentication(); app.UseAuthorization(); - app.UseNexusPoll(); - app.UseModListDownloader(); app.UseResponseCompression(); - app.UseService(); - app.UseService(); app.UseService(); - app.UseService(); - app.UseService(); app.UseService(); - app.UseService(); - app.UseService(); app.Use(next => { diff --git a/Wabbajack.Server/TokenProviders/IFtpSiteCredentials.cs b/Wabbajack.Server/TokenProviders/IFtpSiteCredentials.cs deleted file mode 100644 index 74d73d40..00000000 --- a/Wabbajack.Server/TokenProviders/IFtpSiteCredentials.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System.Collections.Generic; -using Wabbajack.Networking.Http.Interfaces; -using Wabbajack.Server.DTOs; - -namespace Wabbajack.Server.TokenProviders; - -public interface IFtpSiteCredentials : ITokenProvider> -{ -} \ No newline at end of file diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index 9631b1d3..73db7441 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -54,6 +54,10 @@ + + + + diff --git a/Wabbajack.Server/appsettings.json b/Wabbajack.Server/appsettings.json index 6ef56c22..409d360a 100644 --- a/Wabbajack.Server/appsettings.json +++ b/Wabbajack.Server/appsettings.json @@ -7,15 +7,10 @@ } }, "WabbajackSettings": { - "DownloadDir": "c:\\tmp\\downloads", - "ArchiveDir": "w:\\archives", - "TempFolder": "c:\\tmp", - "JobRunner": true, - "JobScheduler": false, - "RunFrontEndJobs": true, - "RunBackEndJobs": false, - "BunnyCDN_StorageZone": "wabbajacktest", - "SQLConnection": "Data Source=.\\SQLEXPRESS;Integrated Security=True;Initial Catalog=wabbajack_prod;MultipleActiveResultSets=true" + "TempFolder": "c:\\tmp\\server_temp", + "MetricsFolder": "c:\\tmp\\server_metrics", + "AuthoredFilesFolder": "c:\\tmp\\server_authored_files", + "AuthorAPIKeyFile": "c:\\tmp\\author_keys.txt" }, "AllowedHosts": "*" }