Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Oh yeah, lots off good stuff here

Browse Source
This commit is contained in:
Timothy Baldridge 2021-11-27 11:31:35 -07:00
parent 0cbd1d6f29
commit 2cb17aa69a
49 changed files with 290 additions and 3861 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Wabbajack.Common/SemaphoreSlimExtensions.cs Normal file
View File

@ -0,0 +1,15 @@
using System;
using System.Reactive.Disposables;
using System.Threading;
using System.Threading.Tasks;
namespace Wabbajack.Common;
public static class SemaphoreSlimExtensions
{
public static async Task<IDisposable> Lock(this SemaphoreSlim slim)
{
await slim.WaitAsync();
return Disposable.Create(() => slim.Release());
}
}

34
Wabbajack.Server/ApiKeyAuthorizationHandler.cs
View File

@ -9,9 +9,7 @@ using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using Wabbajack.Server.DataModels;
namespace Wabbajack.BuildServer;
@ -28,23 +26,19 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthentic
public const string ApiKeyHeaderName = "X-Api-Key";
private readonly DTOSerializer _dtos;
private readonly AppSettings _settings;
private readonly SqlService _sql;
private readonly MetricsKeyCache _keyCache;
private readonly AuthorKeys _authorKeys;
public ApiKeyAuthenticationHandler(
IOptionsMonitor<ApiKeyAuthenticationOptions> options,
AuthorKeys authorKeys,
ILoggerFactory logger,
UrlEncoder encoder,
ISystemClock clock,
MetricsKeyCache keyCache,
DTOSerializer dtos,
AppSettings settings,
SqlService db) : base(options, logger, encoder, clock)
AppSettings settings) : base(options, logger, encoder, clock)
{
_sql = db;
_dtos = dtos;
_keyCache = keyCache;
_authorKeys = authorKeys;
_settings = settings;
}
@ -55,7 +49,7 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthentic
//await LogRequest(metricsKey);
if (metricsKey != default)
{
await _keyCache.AddKey(metricsKey);
/*
if (await _sql.IsTarKey(metricsKey))
{
await _sql.IngestMetric(new Metric
@ -68,6 +62,7 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthentic
await Task.Delay(TimeSpan.FromSeconds(60));
throw new Exception("Error, lipsum timeout of the cross distant cloud.");
}
*/
}
var authorKey = Request.Headers[ApiKeyHeaderName].FirstOrDefault();
@ -77,11 +72,10 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthentic
if (authorKey == null && metricsKey == null) return AuthenticateResult.NoResult();
if (authorKey != null)
{
var owner = await _sql.LoginByApiKey(authorKey);
var owner = await _authorKeys.AuthorForKey(authorKey);
if (owner == null)
return AuthenticateResult.Fail("Invalid author key");
@ -98,12 +92,8 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthentic
return AuthenticateResult.Success(ticket);
}
if (!await _keyCache.IsValidKey(metricsKey))
{
return AuthenticateResult.Fail("Invalid Metrics Key");
}
if (!string.IsNullOrWhiteSpace(metricsKey))
{
var claims = new List<Claim> {new(ClaimTypes.Role, "User")};
@ -115,6 +105,8 @@ public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthentic
return AuthenticateResult.Success(ticket);
}
return AuthenticateResult.NoResult();
}
protected override async Task HandleChallengeAsync(AuthenticationProperties properties)

33
Wabbajack.Server/AppSettings.cs
View File

@ -9,39 +9,16 @@ public class AppSettings
{
config.Bind("WabbajackSettings", this);
}
public bool TestMode { get; set; }
public string AuthorAPIKeyFile { get; set; } = "exported_users";
public string CompressedBodyHeader { get; set; } = "x-compressed-body";
public string AuthorAPIKeyFile { get; set; }
public string WabbajackBuildServerUri { get; set; } = "https://build.wabbajack.org/";
public string MetricsKeyHeader { get; set; } = "x-metrics-key";
public string DownloadDir { get; set; }
public AbsolutePath DownloadPath => (AbsolutePath) DownloadDir;
public string ArchiveDir { get; set; }
public AbsolutePath ArchivePath => (AbsolutePath) ArchiveDir;
public string TempFolder { get; set; }
public AbsolutePath TempPath => (AbsolutePath) TempFolder;
public bool JobScheduler { get; set; }
public bool JobRunner { get; set; }
public bool RunFrontEndJobs { get; set; }
public bool RunBackEndJobs { get; set; }
public bool RunNexusPolling { get; set; }
public bool RunDownloader { get; set; }
public string BunnyCDN_StorageZone { get; set; }
public string SqlConnection { get; set; }
public int MaxJobs { get; set; } = 2;
public string SpamWebHook { get; set; } = null;
public string HamWebHook { get; set; } = null;
public bool ValidateModUpgrades { get; set; } = true;
public string AuthoredFilesFolder { get; set; }
public string MetricsFolder { get; set; }
public string TarLogPath { get; set; }
}

114
Wabbajack.Server/Controllers/AuthoredFiles.cs
View File

@ -1,6 +1,7 @@
using System;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Security.Claims;
using System.Threading;
@ -9,15 +10,15 @@ using FluentFTP;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Primitives;
using Nettle;
using Wabbajack.Common;
using Wabbajack.DTOs.CDN;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DataModels;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using Wabbajack.Server.TokenProviders;
namespace Wabbajack.BuildServer.Controllers;
@ -29,7 +30,13 @@ public class AuthoredFiles : ControllerBase
<html><body>
<table>
{{each $.files }}
<tr><td><a href='https://authored-files.wabbajack.org/{{$.MungedName}}'>{{$.OriginalFileName}}</a></td><td>{{$.Size}}</td><td>{{$.LastTouched}}</td><td>{{$.Finalized}}</td><td>{{$.Author}}</td></tr>
<tr>
<td><a href='https://authored-files.wabbajack.org/{{$.Definition.MungedName}}'>{{$.Definition.OriginalFileName}}</a></td>
<td>{{$.HumanSize}}</td>
<td>{{$.Definition.Author}}</td>
<td>{{$.Updated}}</td>
<td><a href='/authored_files/direct_link/{{$.Definition.MungedName}}'>(Slow) HTTP Direct Link</a></td>
</tr>
{{/each}}
</table>
</body></html>
@ -38,36 +45,33 @@ public class AuthoredFiles : ControllerBase
private readonly DTOSerializer _dtos;
private readonly IFtpSiteCredentials _ftpCreds;
private readonly DiscordWebHook _discord;
private readonly ILogger<AuthoredFiles> _logger;
private readonly AppSettings _settings;
private readonly SqlService _sql;
private readonly AuthorFiles _authoredFiles;
public AuthoredFiles(ILogger<AuthoredFiles> logger, SqlService sql, AppSettings settings, DiscordWebHook discord,
DTOSerializer dtos, IFtpSiteCredentials ftpCreds)
public AuthoredFiles(ILogger<AuthoredFiles> logger, AuthorFiles authorFiles, AppSettings settings, DiscordWebHook discord,
DTOSerializer dtos)
{
_sql = sql;
_logger = logger;
_settings = settings;
_discord = discord;
_dtos = dtos;
_ftpCreds = ftpCreds;
_authoredFiles = authorFiles;
}
[HttpPut]
[Route("{serverAssignedUniqueId}/part/{index}")]
public async Task<IActionResult> UploadFilePart(CancellationToken token, string serverAssignedUniqueId, long index)
{
var user = User.FindFirstValue(ClaimTypes.Name);
var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId);
var definition = await _authoredFiles.ReadDefinitionForServerId(serverAssignedUniqueId);
if (definition.Author != user)
return Forbid("File Id does not match authorized user");
_logger.Log(LogLevel.Information,
$"Uploading File part {definition.OriginalFileName} - ({index} / {definition.Parts.Length})");
await _sql.TouchAuthoredFile(definition);
var part = definition.Parts[index];
await using var ms = new MemoryStream();
@ -82,7 +86,8 @@ public class AuthoredFiles : ControllerBase
$"Hashes don't match for index {index}. Sizes ({ms.Length} vs {part.Size}). Hashes ({hash} vs {part.Hash}");
ms.Position = 0;
await UploadAsync(ms, $"{definition.MungedName}/parts/{index}");
await using var partStream = await _authoredFiles.CreatePart(definition.MungedName, (int)index);
await ms.CopyToAsync(partStream, token);
return Ok(part.Hash.ToBase64());
}
@ -96,14 +101,10 @@ public class AuthoredFiles : ControllerBase
_logger.Log(LogLevel.Information, "Creating File upload {originalFileName}", definition.OriginalFileName);
definition = await _sql.CreateAuthoredFile(definition, user);
using (var client = await GetBunnyCdnFtpClient())
{
await client.CreateDirectoryAsync($"{definition.MungedName}");
await client.CreateDirectoryAsync($"{definition.MungedName}/parts");
}
definition.ServerAssignedUniqueId = Guid.NewGuid().ToString();
definition.Author = user;
await _authoredFiles.WriteDefinition(definition);
await _discord.Send(Channel.Ham,
new DiscordMessage
{
@ -119,22 +120,11 @@ public class AuthoredFiles : ControllerBase
public async Task<IActionResult> CreateUpload(string serverAssignedUniqueId)
{
var user = User.FindFirstValue(ClaimTypes.Name);
var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId);
var definition = await _authoredFiles.ReadDefinitionForServerId(serverAssignedUniqueId);
if (definition.Author != user)
return Forbid("File Id does not match authorized user");
_logger.Log(LogLevel.Information, $"Finalizing file upload {definition.OriginalFileName}");
await _sql.Finalize(definition);
await using var ms = new MemoryStream();
await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
{
await _dtos.Serialize(definition, gz);
}
ms.Position = 0;
await UploadAsync(ms, $"{definition.MungedName}/definition.json.gz");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
@ -146,26 +136,14 @@ public class AuthoredFiles : ControllerBase
return Ok($"https://{host}.wabbajack.org/{definition.MungedName}");
}
private async Task<FtpClient> GetBunnyCdnFtpClient()
{
var info = (await _ftpCreds.Get())[StorageSpace.AuthoredFiles];
var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
await client.ConnectAsync();
return client;
}
private async Task UploadAsync(Stream stream, string path)
{
using var client = await GetBunnyCdnFtpClient();
await client.UploadAsync(stream, path);
}
[HttpDelete]
[Route("{serverAssignedUniqueId}")]
public async Task<IActionResult> DeleteUpload(string serverAssignedUniqueId)
{
var user = User.FindFirstValue(ClaimTypes.Name);
var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId);
var definition = (await _authoredFiles.AllAuthoredFiles())
.First(f => f.Definition.ServerAssignedUniqueId == serverAssignedUniqueId)
.Definition;
if (definition.Author != user)
return Forbid("File Id does not match authorized user");
await _discord.Send(Channel.Ham,
@ -176,33 +154,17 @@ public class AuthoredFiles : ControllerBase
});
_logger.Log(LogLevel.Information, $"Deleting upload {definition.OriginalFileName}");
await DeleteFolderOrSilentlyFail($"{definition.MungedName}");
await _sql.DeleteFileDefinition(definition);
await _authoredFiles.DeleteFile(definition);
return Ok();
}
private async Task DeleteFolderOrSilentlyFail(string path)
{
try
{
using var client = await GetBunnyCdnFtpClient();
await client.DeleteDirectoryAsync(path);
}
catch (Exception)
{
_logger.Log(LogLevel.Information, $"Delete failed for {path}");
}
}
[HttpGet]
[AllowAnonymous]
[Route("")]
public async Task<ContentResult> UploadedFilesGet()
{
var files = await _sql.AllAuthoredFiles();
var response = HandleGetListTemplate(new {files});
var files = await _authoredFiles.AllAuthoredFiles();
var response = HandleGetListTemplate(new {files = files.OrderByDescending(f => f.Updated).ToArray()});
return new ContentResult
{
ContentType = "text/html",
@ -210,4 +172,20 @@ public class AuthoredFiles : ControllerBase
Content = response
};
}
[HttpGet]
[AllowAnonymous]
[Route("direct_link/{mungedName}")]
public async Task DirectLink(string mungedName)
{
var definition = await _authoredFiles.ReadDefinition(mungedName);
Response.Headers.ContentDisposition =
new StringValues($"attachment; filename={definition.OriginalFileName}");
Response.Headers.ContentType = new StringValues("application/octet-stream");
foreach (var part in definition.Parts)
{
await using var partStream = await _authoredFiles.StreamForPart(mungedName, (int)part.Index);
await partStream.CopyToAsync(Response.Body);
}
}
}

105
Wabbajack.Server/Controllers/Heartbeat.cs
View File

@ -1,13 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
using Wabbajack.Server;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
@ -16,68 +11,18 @@ namespace Wabbajack.BuildServer.Controllers;
[Route("/heartbeat")]
public class Heartbeat : ControllerBase
{
private const int MAX_LOG_SIZE = 128;
private static readonly DateTime _startTime;
private static List<string> Log = new();
private static readonly Func<object, string> HandleGetReport = NettleEngine.GetCompiler().Compile(@"
<html><body>
<h2>Server Status</h2>
<h3>Service Overview ({{services.Length}}):</h3>
<ul>
{{each $.services }}
{{if $.IsLate}}
<li><a href='/heartbeat/report/services/{{$.Name}}.html'><b>{{$.Name}} - {{$.Time}} - {{$.MaxTime}}</b></a></li>
{{else}}
<li><a href='/heartbeat/report/services/{{$.Name}}.html'>{{$.Name}} - {{$.Time}} - {{$.MaxTime}}</a></li>
{{/if}}
{{/each}}
</ul>
<h3>Lists ({{lists.Length}}):</h3>
<ul>
{{each $.lists }}
<li><a href='/lists/status/{{$.Name}}.html'>{{$.Name}}</a> - {{$.Time}} {{$.FailMessage}}</li>
{{/each}}
</ul>
</body></html>
");
private static readonly Func<object, string> HandleGetServiceReport = NettleEngine.GetCompiler().Compile(@"
<html><body>
<h2>Service Status: {{Name}} {{TimeSinceLastRun}}</h2>
<h3>Service Overview ({{ActiveWorkQueue.Length}}):</h3>
<ul>
{{each $.ActiveWorkQueue }}
<li>{{$.Name}} {{$.Time}}</li>
{{/each}}
</ul>
</body></html>
");
private readonly GlobalInformation _globalInformation;
private ILogger<Heartbeat> _logger;
private readonly QuickSync _quickSync;
private SqlService _sql;
static Heartbeat()
{
_startTime = DateTime.Now;
}
public Heartbeat(ILogger<Heartbeat> logger, SqlService sql, GlobalInformation globalInformation,
public Heartbeat(ILogger<Heartbeat> logger, GlobalInformation globalInformation,
QuickSync quickSync)
{
_globalInformation = globalInformation;
_sql = sql;
_logger = logger;
_quickSync = quickSync;
}
[HttpGet]
@ -86,54 +31,6 @@ public class Heartbeat : ControllerBase
return Ok(new HeartbeatResult
{
Uptime = DateTime.Now - _startTime,
LastNexusUpdate = _globalInformation.TimeSinceLastNexusSync
});
}
[HttpGet("report")]
public async Task<ContentResult> Report()
{
var response = HandleGetReport(new
{
services = (await _quickSync.Report())
.Select(s => new
{
s.Key.Name, Time = s.Value.LastRunTime, MaxTime = s.Value.Delay,
IsLate = s.Value.LastRunTime > s.Value.Delay
})
.OrderBy(s => s.Name)
.ToArray()
});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
[HttpGet("report/services/{serviceName}.html")]
public async Task<ContentResult> ReportServiceStatus(string serviceName)
{
var services = await _quickSync.Report();
var info = services.First(kvp => kvp.Key.Name == serviceName);
var response = HandleGetServiceReport(new
{
info.Key.Name,
TimeSinceLastRun = DateTime.UtcNow - info.Value.LastRunTime,
ActiveWorkQueue = info.Value.ActiveWork.Select(p => new
{
Name = p.Item1,
Time = DateTime.UtcNow - p.Item2
}).OrderByDescending(kp => kp.Time)
.ToArray()
});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
}

83
Wabbajack.Server/Controllers/ListDefinitions.cs
View File

@ -1,83 +0,0 @@
using System;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.DTOs;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Paths.IO;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers;
[ApiController]
[Route("/list_definitions")]
public class ListDefinitions : ControllerBase
{
private readonly DTOSerializer _dtos;
private readonly AppSettings _settings;
private DiscordWebHook _discord;
private readonly ILogger<ListDefinitions> _logger;
private SqlService _sql;
public ListDefinitions(ILogger<ListDefinitions> logger, SqlService sql, DiscordWebHook discord,
AppSettings settings,
DTOSerializer dtos)
{
_logger = logger;
_sql = sql;
_discord = discord;
_settings = settings;
_dtos = dtos;
}
[Route("ingest")]
[Authorize(Roles = "User")]
[HttpPost]
public async Task<IActionResult> PostIngest()
{
var user = Request.Headers[_settings.MetricsKeyHeader].First();
var use_gzip = Request.Headers[_settings.CompressedBodyHeader].Any();
_logger.Log(LogLevel.Information, $"Ingesting Modlist Definition for {user}");
var modlistBytes = await Request.Body.ReadAllAsync();
_logger.LogInformation("Spawning ingestion task");
var tsk = Task.Run(async () =>
{
try
{
if (use_gzip)
{
await using var os = new MemoryStream();
await using var gZipStream =
new GZipStream(new MemoryStream(modlistBytes), CompressionMode.Decompress);
await gZipStream.CopyToAsync(os);
modlistBytes = os.ToArray();
}
var modlist = _dtos.DeserializeAsync<ModList>(new MemoryStream(modlistBytes));
var file = KnownFolders.EntryPoint.Combine("mod_list_definitions")
.Combine($"{user}_{DateTime.UtcNow.ToFileTimeUtc()}.json");
file.Parent.CreateDirectory();
await using var stream = file.Open(FileMode.Create, FileAccess.Write);
await _dtos.Serialize(modlist, stream);
_logger.Log(LogLevel.Information, $"Done Ingesting Modlist Definition for {user}");
}
catch (Exception ex)
{
_logger.LogError(ex, "Error ingesting uploaded modlist");
}
});
return Accepted(0);
}
}

184
Wabbajack.Server/Controllers/ListsStatus.cs
View File

@ -1,184 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
using Wabbajack.DTOs;
using Wabbajack.DTOs.ServerResponses;
namespace Wabbajack.BuildServer.Controllers;
[ApiController]
[Route("/lists")]
public class ListsStatus : ControllerBase
{
private static readonly Func<object, string> HandleGetRssFeedTemplate = NettleEngine.GetCompiler().Compile(@"
<?xml version=""1.0""?>
<rss version=""2.0"">
<channel>
<title>{{lst.Name}} - Broken Mods</title>
<link>http://build.wabbajack.org/status/{{lst.Name}}.html</link>
<description>These are mods that are broken and need updating</description>
{{ each $.failed }}
<item>
<title>{{$.Archive.Name}} {{$.Archive.Hash}} {{$.Archive.State.PrimaryKeyString}}</title>
<link>{{$.Archive.Name}}</link>
</item>
{{/each}}
</channel>
</rss>
");
private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
<html><body>
<h2>{{lst.Name}} - {{lst.Checked}} - {{ago}}min ago</h2>
<h3>Failed ({{failed.Count}}):</h3>
<ul>
{{each $.failed }}
{{if $.HasUrl}}
<li><a href='{{$.Url}}'>{{$.Name}}</a></li>
{{else}}
<li>{{$.Name}}</li>
{{/if}}
{{/each}}
</ul>
<h3>Updated ({{updated.Count}}):</h3>
<ul>
{{each $.updated }}
{{if $.HasUrl}}
<li><a href='{{$.Url}}'>{{$.Name}}</a></li>
{{else}}
<li>{{$.Name}}</li>
{{/if}}
{{/each}}
</ul>
<h3>Mirrored ({{mirrored.Count}}):</h3>
<ul>
{{each $.mirrored }}
{{if $.HasUrl}}
<li><a href='{{$.Url}}'>{{$.Name}}</a></li>
{{else}}
<li>{{$.Name}}</li>
{{/if}}
{{/each}}
</ul>
<h3>Updating ({{updating.Count}}):</h3>
<ul>
{{each $.updating }}
{{if $.HasUrl}}
<li><a href='{{$.Url}}'>{{$.Name}}</a></li>
{{else}}
<li>{{$.Name}}</li>
{{/if}}
{{/each}}
</ul>
<h3>Passed ({{passed.Count}}):</h3>
<ul>
{{each $.passed }}
{{if $.HasUrl}}
<li><a href='{{$.Url}}'>{{$.Name}}</a></li>
{{else}}
<li>{{$.Name}}</li>
{{/if}}
{{/each}}
</ul>
</body></html>
");
private ILogger<ListsStatus> _logger;
public ListsStatus(ILogger<ListsStatus> logger)
{
_logger = logger;
}
[HttpGet]
[Route("status.json")]
public async Task<IEnumerable<ModListSummary>> HandleGetLists()
{
throw new NotImplementedException();
}
[HttpGet]
[Route("status/{Name}/broken.rss")]
public async Task<ContentResult> HandleGetRSSFeed(string Name)
{
var lst = await DetailedStatus(Name);
var response = HandleGetRssFeedTemplate(new
{
lst,
failed = lst.Archives.Where(a => a.IsFailing).ToList(),
passed = lst.Archives.Where(a => !a.IsFailing).ToList()
});
return new ContentResult
{
ContentType = "application/rss+xml",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
[HttpGet]
[Route("status/{Name}.html")]
public async Task<ContentResult> HandleGetListHtml(string Name)
{
var lst = await DetailedStatus(Name);
var response = HandleGetListTemplate(new
{
lst,
ago = (DateTime.UtcNow - lst.Checked).TotalMinutes,
failed = lst.Archives.Where(a => a.IsFailing).ToList(),
passed = lst.Archives.Where(a => !a.IsFailing).ToList(),
updated = lst.Archives.Where(a => a.ArchiveStatus == ArchiveStatus.Updated).ToList(),
updating = lst.Archives.Where(a => a.ArchiveStatus == ArchiveStatus.Updating).ToList(),
mirrored = lst.Archives.Where(a => a.ArchiveStatus == ArchiveStatus.Mirrored).ToList()
});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
[HttpGet]
[Route("status/{Name}.json")]
[ResponseCache(Duration = 60 * 5)]
public async Task<IActionResult> HandleGetListJson(string Name)
{
var lst = await DetailedStatus(Name);
if (lst == default) return NotFound();
return Ok(lst);
}
private async Task<DetailedStatus?> DetailedStatus(string Name)
{
throw new NotImplementedException();
}
[HttpGet]
[Route("status/badge.json")]
public async Task<IActionResult> HandleGitHubBadge()
{
throw new NotImplementedException();
}
[HttpGet]
[Route("status/{Name}/badge.json")]
public async Task<IActionResult> HandleNamedGitHubBadge(string Name)
{
throw new NotImplementedException();
}
}

177
Wabbajack.Server/Controllers/Metrics.cs
View File

@ -1,17 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
using Wabbajack.Common;
using Wabbajack.Server;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DataModels;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers;
@ -37,17 +33,15 @@ public class MetricsController : ControllerBase
private static Func<object, string> _totalListTemplate;
private readonly AppSettings _settings;
private readonly MetricsKeyCache _keyCache;
private ILogger<MetricsController> _logger;
private readonly SqlService _sql;
private readonly Metrics _metricsStore;
public MetricsController(ILogger<MetricsController> logger, SqlService sql, MetricsKeyCache keyCache,
public MetricsController(ILogger<MetricsController> logger, Metrics metricsStore,
AppSettings settings)
{
_sql = sql;
_logger = logger;
_keyCache = keyCache;
_settings = settings;
_metricsStore = metricsStore;
}
@ -73,169 +67,24 @@ public class MetricsController : ControllerBase
{
var date = DateTime.UtcNow;
var metricsKey = Request.Headers[_settings.MetricsKeyHeader].FirstOrDefault();
if (metricsKey != null)
await _keyCache.AddKey(metricsKey);
// Used in tests
if (value == "Default" || value == "untitled" || subject == "failed_download" || Guid.TryParse(value, out _))
if (value is "Default" or "untitled" || subject == "failed_download" || Guid.TryParse(value, out _))
return new Result {Timestamp = date};
await Log(date, subject, value, metricsKey);
await _metricsStore.Ingest(new Metric
{
Timestamp = DateTime.UtcNow,
Action = subject,
Subject = subject,
MetricsKey = metricsKey,
UserAgent = Request.Headers.UserAgent.FirstOrDefault() ?? "<unknown>",
});
return new Result {Timestamp = date};
}
[HttpGet]
[Route("report/{subject}")]
[ResponseCache(Duration = 60 * 60)]
public async Task<IActionResult> MetricsReport(string subject)
{
var metrics = (await _sql.MetricsReport(subject)).ToList();
var labels = metrics.GroupBy(m => m.Date)
.OrderBy(m => m.Key)
.Select(m => m.Key)
.ToArray();
var labelStrings = labels.Select(l => l.ToString("MM-dd-yyy")).ToList();
var results = metrics
.GroupBy(m => m.Subject)
.Select(g =>
{
var indexed = g.ToDictionary(m => m.Date, m => m.Count);
return new MetricResult
{
SeriesName = g.Key,
Labels = labelStrings,
Values = labels.Select(l => indexed.TryGetValue(l, out var found) ? found : 0).ToList()
};
});
return Ok(results.ToList());
}
[HttpGet]
[Route("badge/{name}/total_installs_badge.json")]
public async Task<IActionResult> TotalInstallsBadge(string name)
{
var results = await _sql.TotalInstalls(name);
Response.ContentType = "application/json";
return Ok(results == 0
? new Badge($"Modlist {name} not found!", "Error") {color = "red"}
: new Badge("Installations: ", "____") {color = "green"});
}
[HttpGet]
[Route("badge/{name}/unique_installs_badge.json")]
public async Task<IActionResult> UniqueInstallsBadge(string name)
{
var results = await _sql.UniqueInstalls(name);
Response.ContentType = "application/json";
return Ok(results == 0
? new Badge($"Modlist {name} not found!", "Error") {color = "red"}
: new Badge("Installations: ", "____") {color = "green"});
}
[HttpGet]
[Route("tarlog/{key}")]
public async Task<IActionResult> TarLog(string key)
{
var isTarKey = await _sql.IsTarKey(key);
var report = new List<(DateTime, string, string)>();
if (isTarKey) report = await _sql.FullTarReport(key);
var response = ReportTemplate(new
{
key,
status = isTarKey ? "BANNED" : "NOT BANNED",
log = report.Select(entry => new
{
Timestamp = entry.Item1,
Path = entry.Item2,
Key = entry.Item3
}).ToList()
});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
{
//_logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}");
await _sql.IngestMetric(new Metric
{
Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey
});
}
[HttpGet("total_installs.html")]
[ResponseCache(Duration = 60 * 60)]
public async Task<ContentResult> TotalInstalls()
{
var data = await _sql.GetTotalInstalls();
var result = TotalListTemplate(new TotalListTemplateData
{
Title = "Total Installs",
Total = data.Sum(d => d.Item2),
Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2})
.ToArray()
});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = result
};
}
[HttpGet("total_unique_installs.html")]
[ResponseCache(Duration = 60 * 60)]
public async Task<ContentResult> TotalUniqueInstalls()
{
var data = await _sql.GetTotalUniqueInstalls();
var result = TotalListTemplate(new TotalListTemplateData
{
Title = "Total Unique Installs",
Total = data.Sum(d => d.Item2),
Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2})
.ToArray()
});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = result
};
}
[HttpGet("dump.json")]
public async Task<IActionResult> DataDump()
{
return Ok(await _sql.MetricsDump().ToArrayAsync());
}
public class Result
{
public DateTime Timestamp { get; set; }
}
private class TotalListTemplateData
{
public string Title { get; set; }
public long Total { get; set; }
public Item[] Items { get; set; }
public class Item
{
public long Count { get; set; }
public string Title { get; set; }
}
}
}

158
Wabbajack.Server/Controllers/ModUpgrade.cs
View File

@ -1,158 +0,0 @@
using System;
using System.Linq;
using System.Security.Claims;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Downloaders;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.DTOs.ServerResponses;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using Wabbajack.Server.TokenProviders;
namespace Wabbajack.BuildServer.Controllers;
[ApiController]
public class ModUpgrade : ControllerBase
{
private readonly DownloadDispatcher _dispatcher;
private readonly DTOSerializer _dtos;
private readonly IFtpSiteCredentials _ftpSite;
private readonly ILogger<ModUpgrade> _logger;
private readonly QuickSync _quickSync;
private readonly AppSettings _settings;
private readonly SqlService _sql;
public ModUpgrade(ILogger<ModUpgrade> logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync,
AppSettings settings, DTOSerializer dtos,
DownloadDispatcher dispatcher, IFtpSiteCredentials ftp)
{
_logger = logger;
_sql = sql;
_settings = settings;
_quickSync = quickSync;
_ftpSite = ftp;
_dtos = dtos;
_dispatcher = dispatcher;
}
[HttpPost]
[Authorize(Roles = "User")]
[Route("/mod_upgrade")]
public async Task<IActionResult> PostModUpgrade(CancellationToken token)
{
var isAuthor = User.Claims.Any(c => c.Type == ClaimTypes.Role && c.Value == "Author");
var request = await _dtos.DeserializeAsync<ModUpgradeRequest>(Request.Body);
if (!isAuthor)
{
var srcDownload = await _sql.GetArchiveDownload(request!.OldArchive.State.PrimaryKeyString,
request.OldArchive.Hash, request.OldArchive.Size);
var destDownload = await _sql.GetArchiveDownload(request.NewArchive.State.PrimaryKeyString,
request.NewArchive.Hash, request.NewArchive.Size);
if (srcDownload == default || destDownload == default ||
await _sql.FindPatch(srcDownload.Id, destDownload.Id) == default)
{
if (!await _dispatcher.IsAllowed(request, token))
{
_logger.Log(LogLevel.Information,
$"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as upgrade is invalid");
return BadRequest("Invalid mod upgrade");
}
if (_settings.ValidateModUpgrades && !await _sql.HashIsInAModlist(request.OldArchive.Hash))
{
_logger.Log(LogLevel.Information,
$"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as src hash is not in a curated modlist");
return BadRequest("Hash is not in a recent modlist");
}
}
}
try
{
if (await _dispatcher.Verify(request!.OldArchive, token))
//_logger.LogInformation(
// $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), old archive is valid");
return NotFound("File is Valid");
}
catch (Exception)
{
//_logger.LogInformation(
// $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), due to upgrade failure");
return NotFound("File is Valid");
}
var oldDownload = await _sql.GetOrEnqueueArchive(request.OldArchive);
if (await _sql.IsNoPatch(oldDownload.Archive.Hash)) return BadRequest("File has NoPatch attached");
var newDownload = await _sql.GetOrEnqueueArchive(request.NewArchive);
var patch = await _sql.FindOrEnqueuePatch(oldDownload.Id, newDownload.Id);
if (patch.Finished.HasValue)
{
if (patch.PatchSize != 0)
{
//_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch Found");
var host = (await _ftpSite.Get())[StorageSpace.Patches].Username == "wabbajacktest"
? "test-files"
: "patches";
await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id);
return
Ok(
$"https://{host}.wabbajack.org/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}");
}
//_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed");
return NotFound("Patch creation failed");
}
if (!newDownload.DownloadFinished.HasValue)
await _quickSync.Notify<ArchiveDownloader>();
else
await _quickSync.Notify<PatchBuilder>();
//_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found is processing");
// Still processing
return Accepted();
}
[HttpGet]
[Authorize(Roles = "User")]
[Route("/mod_upgrade/find/{hashAsHex}")]
public async Task<IActionResult> FindUpgrade(string hashAsHex)
{
var hash = Hash.FromHex(hashAsHex);
var patches = await _sql.PatchesForSource(hash);
return Ok(_dtos.Serialize(patches.Select(p => p.Dest.Archive).ToArray()));
}
[HttpGet]
[Authorize(Roles = "Author")]
[Route("/mod_upgrade/no_patch/{hashAsHex}/{rationaleAsHex}")]
public async Task<IActionResult> PurgePatch(string hashAsHex, string rationaleAsHex)
{
var hash = Hash.FromHex(hashAsHex);
var rationale = Encoding.UTF8.GetString(rationaleAsHex.FromHex());
await _sql.PurgePatch(hash, rationale);
return Ok("Purged");
}
[HttpGet]
[Authorize(Roles = "User")]
[Route("/mirror/{hashAsHex}")]
public async Task<IActionResult> HaveHash(string hashAsHex)
{
var result = await _sql.HaveMirror(Hash.FromHex(hashAsHex));
if (result) return Ok($"https://{(await _ftpSite.Get())[StorageSpace.Mirrors].Username}.b-cdn.net/{hashAsHex}");
return NotFound("Not Mirrored");
}
}

10
Wabbajack.Server/DTOs/AggregateMetric.cs
View File

@ -1,10 +0,0 @@
using System;
namespace Wabbajack.Server.DTOs;
public class AggregateMetric
{
public DateTime Date { get; set; }
public string Subject { get; set; }
public int Count { get; set; }
}

30
Wabbajack.Server/DTOs/ArchiveDownload.cs
View File

@ -1,30 +0,0 @@
using System;
using System.Threading.Tasks;
using Wabbajack.DTOs;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.DTOs;
public class ArchiveDownload
{
public Guid Id { get; set; }
public Archive Archive { get; set; }
public bool? IsFailed { get; set; }
public DateTime? DownloadFinished { get; set; }
public string FailMessage { get; set; }
public async Task Fail(SqlService service, string message)
{
IsFailed = true;
DownloadFinished = DateTime.UtcNow;
FailMessage = message;
await service.UpdatePendingDownload(this);
}
public async Task Finish(SqlService service)
{
IsFailed = false;
DownloadFinished = DateTime.UtcNow;
await service.UpdatePendingDownload(this);
}
}

14
Wabbajack.Server/DTOs/AuthoredFilesSummary.cs
View File

@ -1,14 +0,0 @@
using System;
namespace Wabbajack.Server.DTOs;
public class AuthoredFilesSummary
{
public long Size { get; set; }
public string OriginalFileName { get; set; }
public string Author { get; set; }
public DateTime LastTouched { get; set; }
public DateTime? Finalized { get; set; }
public string MungedName { get; set; }
public string ServerAssignedUniqueId { get; set; }
}

31
Wabbajack.Server/DTOs/FtpSite.cs
View File

@ -1,31 +0,0 @@
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
namespace Wabbajack.Server.DTOs;
public enum StorageSpace
{
AuthoredFiles,
Patches,
Mirrors
}
public class FtpSite
{
public string Username { get; set; }
public string Password { get; set; }
public string Hostname { get; set; }
public async Task<FtpClient> GetClient(ILogger logger)
{
return await CircuitBreaker.WithAutoRetryAllAsync(logger, async () =>
{
var ftpClient = new FtpClient(Hostname, new NetworkCredential(Username, Password));
await ftpClient.ConnectAsync();
return ftpClient;
});
}
}

2
Wabbajack.Server/DTOs/Metric.cs
View File

@ -1,4 +1,5 @@
using System;
using Microsoft.Extensions.Primitives;
namespace Wabbajack.Server.DTOs;
@ -8,4 +9,5 @@ public class Metric
public string Action { get; set; }
public string Subject { get; set; }
public string MetricsKey { get; set; }
public string UserAgent { get; set; }
}

10
Wabbajack.Server/DTOs/MetricResult.cs
View File

@ -1,10 +0,0 @@
using System.Collections.Generic;
namespace Wabbajack.Server.DTOs;
public class MetricResult
{
public string SeriesName { get; set; }
public List<string> Labels { get; set; }
public List<int> Values { get; set; }
}

29
Wabbajack.Server/DTOs/MirroredFile.cs
View File

@ -1,29 +0,0 @@
using System;
using System.Threading.Tasks;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.DTOs;
public class MirroredFile
{
public Hash Hash { get; set; }
public DateTime Created { get; set; }
public DateTime? Uploaded { get; set; }
public string Rationale { get; set; }
public string FailMessage { get; set; }
public async Task Finish(SqlService sql)
{
Uploaded = DateTime.UtcNow;
await sql.UpsertMirroredFile(this);
}
public async Task Fail(SqlService sql, string message)
{
Uploaded = DateTime.UtcNow;
FailMessage = message;
await sql.UpsertMirroredFile(this);
}
}

32
Wabbajack.Server/DTOs/Patch.cs
View File

@ -1,32 +0,0 @@
using System;
using System.Threading.Tasks;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.DTOs;
public class Patch
{
public ArchiveDownload Src { get; set; }
public ArchiveDownload Dest { get; set; }
public long PatchSize { get; set; }
public DateTime? Finished { get; set; }
public bool? IsFailed { get; set; }
public string FailMessage { get; set; }
public async Task Finish(SqlService sql, long size)
{
IsFailed = false;
Finished = DateTime.UtcNow;
PatchSize = size;
await sql.FinializePatch(this);
}
public async Task Fail(SqlService sql, string msg)
{
IsFailed = true;
Finished = DateTime.UtcNow;
FailMessage = msg;
await sql.FinializePatch(this);
}
}

17
Wabbajack.Server/DTOs/ValidationData.cs
View File

@ -1,17 +0,0 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Wabbajack.DTOs;
using Wabbajack.Hashing.xxHash64;
namespace Wabbajack.Server.DTOs;
public class ValidationData
{
public Dictionary<(long Game, long ModId, long FileId), string> NexusFiles { get; set; } = new();
public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; }
public List<ModlistMetadata> ModLists { get; set; }
public Dictionary<Hash, bool> Mirrors { get; set; }
public Lazy<Task<Dictionary<Hash, string>>> AllowedMirrors { get; set; }
public IEnumerable<AuthoredFilesSummary> AllAuthoredFiles { get; set; }
}

54
Wabbajack.Server/DataLayer/ApiKeys.cs
View File

@ -1,54 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Hashing.xxHash64;
namespace Wabbajack.Server.DataLayer;
public partial class SqlService
{
public async Task<string> LoginByApiKey(string key)
{
await using var conn = await Open();
var result = await conn.QueryAsync<string>(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @ApiKey",
new {ApiKey = key});
return result.FirstOrDefault();
}
public async Task<string> AddLogin(string name)
{
var key = NewAPIKey();
await using var conn = await Open();
await conn.ExecuteAsync("INSERT INTO dbo.ApiKeys (Owner, ApiKey) VALUES (@Owner, @ApiKey)",
new {Owner = name, ApiKey = key});
return key;
}
public static string NewAPIKey()
{
var arr = new byte[128];
new Random().NextBytes(arr);
return arr.ToHex();
}
public async Task<IEnumerable<(string Owner, string Key)>> GetAllUserKeys()
{
await using var conn = await Open();
var result = await conn.QueryAsync<(string Owner, string Key)>("SELECT Owner, ApiKey FROM dbo.ApiKeys");
return result;
}
public async Task<bool> IsTarKey(string metricsKey)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT MetricsKey FROM TarKey WHERE MetricsKey = @MetricsKey", new {MetricsKey = metricsKey});
return result == metricsKey;
}
}

261
Wabbajack.Server/DataLayer/ArchiveDownloads.cs
View File

@ -1,261 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer;
public partial class SqlService
{
public async Task<Guid> AddKnownDownload(Archive a, DateTime downloadFinished)
{
await using var conn = await Open();
var Id = Guid.NewGuid();
await conn.ExecuteAsync(
"INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader, DownloadFinished, IsFailed) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader, @DownloadFinished, @IsFailed)",
new
{
Id,
a.State.PrimaryKeyString,
Size = a.Size == 0 ? null : (long?) a.Size,
Hash = a.Hash == default ? null : (Hash?) a.Hash,
DownloadState = a.State,
Downloader = a.State.GetType().ToString(),
DownloadFinished = downloadFinished,
IsFailed = false
});
return Id;
}
public async Task<Guid> EnqueueDownload(Archive a)
{
await using var conn = await Open();
var Id = Guid.NewGuid();
await conn.ExecuteAsync(
"INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)",
new
{
Id,
a.State.PrimaryKeyString,
Size = a.Size == 0 ? null : (long?) a.Size,
Hash = a.Hash == default ? null : (Hash?) a.Hash,
DownloadState = a.State,
Downloader = a.State.GetType().ToString()
});
return Id;
}
public async Task<HashSet<(Hash Hash, string PrimaryKeyString)>> GetAllArchiveDownloads()
{
await using var conn = await Open();
return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, PrimaryKeyString FROM ArchiveDownloads"))
.ToHashSet();
}
public async Task<HashSet<(Hash Hash, IDownloadState State)>> GetAllArchiveDownloadStates()
{
await using var conn = await Open();
return (await conn.QueryAsync<(Hash, IDownloadState)>("SELECT Hash, DownloadState FROM ArchiveDownloads"))
.ToHashSet();
}
public async Task<ArchiveDownload> GetArchiveDownload(Guid id)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, IDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE Id = @id",
new {Id = id});
if (result == default)
return null;
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive {State = result.Item5, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
public async Task<ArchiveDownload> GetArchiveDownload(string primaryKeyString)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, IDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND IsFailed = 0",
new {PrimaryKeyString = primaryKeyString});
if (result == default)
return null;
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive {State = result.Item5, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
public async Task<ArchiveDownload> GetArchiveDownload(string primaryKeyString, Hash hash, long size)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, IDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size",
new
{
PrimaryKeyString = primaryKeyString,
Hash = hash,
Size = size
});
if (result == default)
return null;
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive {State = result.Item5, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
public async Task<ArchiveDownload> GetOrEnqueueArchive(Archive a)
{
await using var conn = await Open();
await using var trans = await conn.BeginTransactionAsync();
var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, IDownloadState, DateTime?)>(
"SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND Hash = @Hash AND Size = @Size",
new
{
a.State.PrimaryKeyString,
a.Hash,
a.Size
}, trans);
if (result.Item1 != default)
return new ArchiveDownload
{
Id = result.Item1,
IsFailed = result.Item4,
DownloadFinished = result.Item6,
Archive = new Archive {State = result.Item5, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
var id = Guid.NewGuid();
await conn.ExecuteAsync(
"INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader)",
new
{
Id = id,
a.State.PrimaryKeyString,
Size = a.Size == 0 ? null : (long?) a.Size,
Hash = a.Hash == default ? null : (Hash?) a.Hash,
DownloadState = a.State,
Downloader = ""
}, trans);
await trans.CommitAsync();
return new ArchiveDownload {Id = id, Archive = a};
}
public async Task<ArchiveDownload> GetNextPendingDownload(bool ignoreNexus = false)
{
await using var conn = await Open();
(Guid, long?, Hash?, IDownloadState) result;
if (ignoreNexus)
result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, IDownloadState)>(
"SELECT TOP(1) Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL AND Downloader != 'NexusDownloader+State'");
else
result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, IDownloadState)>(
"SELECT TOP(1) Id, Size, Hash, DownloadState FROM dbo.ArchiveDownloads WHERE DownloadFinished is NULL");
if (result == default)
return null;
return new ArchiveDownload
{
Id = result.Item1,
Archive = new Archive {State = result.Item4, Size = result.Item2 ?? 0, Hash = result.Item3 ?? default}
};
}
public async Task UpdatePendingDownload(ArchiveDownload ad)
{
await using var conn = await Open();
await conn.ExecuteAsync(
"UPDATE dbo.ArchiveDownloads SET IsFailed = @IsFailed, DownloadFinished = @DownloadFinished, Hash = @Hash, Size = @Size, FailMessage = @FailMessage WHERE Id = @Id",
new
{
ad.Id,
ad.IsFailed,
ad.DownloadFinished,
ad.Archive.Size,
ad.Archive.Hash,
ad.FailMessage
});
}
public async Task<int> EnqueueModListFilesForIndexing()
{
await using var conn = await Open();
return await conn.ExecuteAsync(@"
INSERT INTO dbo.ArchiveDownloads (Id, PrimaryKeyString, Hash, DownloadState, Size, Downloader)
SELECT DISTINCT NEWID(), mla.PrimaryKeyString, mla.Hash, mla.State, mla.Size, SUBSTRING(mla.PrimaryKeyString, 0, CHARINDEX('|', mla.PrimaryKeyString))
FROM [dbo].[ModListArchives] mla
LEFT JOIN dbo.ArchiveDownloads ad on mla.PrimaryKeyString = ad.PrimaryKeyString AND mla.Hash = ad.Hash
WHERE ad.PrimaryKeyString is null");
}
public async Task<List<Archive>> GetGameFiles(Game game, string version)
{
await using var conn = await Open();
var files = (await conn.QueryAsync<(Hash, long, IDownloadState)>(
$"SELECT Hash, Size, DownloadState FROM dbo.ArchiveDownloads WHERE PrimaryKeyString like 'GameFileSourceDownloader+State|{game}|{version}|%'"))
.Select(f => new Archive
{
State = f.Item3,
Hash = f.Item1,
Size = f.Item2
}).ToList();
return files;
}
public async Task<Archive[]> ResolveDownloadStatesByHash(Hash hash)
{
await using var conn = await Open();
var files = (await conn.QueryAsync<(long, Hash, IDownloadState)>(
@"SELECT Size, Hash, DownloadState from dbo.ArchiveDownloads WHERE Hash = @Hash AND IsFailed = 0 AND DownloadFinished IS NOT NULL ORDER BY DownloadFinished DESC",
new {Hash = hash})
).Select(e =>
new Archive {State = e.Item3, Size = e.Item1, Hash = e.Item2}
).ToList();
if (await HaveMirror(hash) && files.Count > 0)
{
var ffile = files.First();
var host = _settings.TestMode ? "test-files" : "mirror";
var url = new Uri($"https://{host}.wabbajack.org/{hash.ToHex()}");
files.Add(new Archive
{State = new WabbajackCDN {Url = url}, Hash = hash, Size = ffile.Size, Name = ffile.Name});
}
return files.ToArray();
}
public async Task<IEnumerable<(Game, string)>> GetAllRegisteredGames()
{
await using var conn = await Open();
var pks = await conn.QueryAsync<string>(
@"SELECT PrimaryKeyString FROM dbo.ArchiveDownloads WHERE PrimaryKeyString like 'GameFileSourceDownloader+State|%'");
return pks.Select(p => p.Split("|"))
.Select(t => (GameRegistry.GetByFuzzyName(t[1]).Game, t[2]))
.Distinct();
}
}

76
Wabbajack.Server/DataLayer/AuthoredFiles.cs
View File

@ -1,76 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.DTOs.CDN;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer;
public partial class SqlService
{
public async Task TouchAuthoredFile(FileDefinition definition, DateTime? date = null)
{
await using var conn = await Open();
if (date == null)
await conn.ExecuteAsync(
"UPDATE AuthoredFiles SET LastTouched = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid",
new {Uid = definition.ServerAssignedUniqueId});
else
await conn.ExecuteAsync(
"UPDATE AuthoredFiles SET LastTouched = @Date WHERE ServerAssignedUniqueId = @Uid",
new {Uid = definition.ServerAssignedUniqueId, Date = date});
}
public async Task<FileDefinition> CreateAuthoredFile(FileDefinition definition, string login)
{
definition.Author = login;
var uid = Guid.NewGuid().ToString();
await using var conn = await Open();
definition.ServerAssignedUniqueId = uid;
await conn.ExecuteAsync(
"INSERT INTO dbo.AuthoredFiles (ServerAssignedUniqueId, LastTouched, CDNFileDefinition) VALUES (@Uid, GETUTCDATE(), @CdnFile)",
new
{
Uid = uid,
CdnFile = definition
});
return definition;
}
public async Task Finalize(FileDefinition definition)
{
await using var conn = await Open();
await conn.ExecuteAsync(
"UPDATE AuthoredFiles SET LastTouched = GETUTCDATE(), Finalized = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid",
new
{
Uid = definition.ServerAssignedUniqueId
});
}
public async Task<FileDefinition> GetCDNFileDefinition(string serverAssignedUniqueId)
{
await using var conn = await Open();
return (await conn.QueryAsync<FileDefinition>(
"SELECT CDNFileDefinition FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid",
new {Uid = serverAssignedUniqueId})).First();
}
public async Task DeleteFileDefinition(FileDefinition definition)
{
await using var conn = await Open();
await conn.ExecuteAsync(
"DELETE FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid",
new {Uid = definition.ServerAssignedUniqueId});
}
public async Task<IEnumerable<AuthoredFilesSummary>> AllAuthoredFiles()
{
await using var conn = await Open();
var results = await conn.QueryAsync<AuthoredFilesSummary>(
"SELECT CONVERT(NVARCHAR(50), ServerAssignedUniqueId) as ServerAssignedUniqueId, Size, OriginalFileName, Author, LastTouched, Finalized, MungedName from dbo.AuthoredFilesSummaries ORDER BY LastTouched DESC");
return results;
}
}

109
Wabbajack.Server/DataLayer/Mappers.cs
View File

@ -1,109 +0,0 @@
using System;
using System.Data;
using Dapper;
using Wabbajack.DTOs;
using Wabbajack.DTOs.CDN;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
namespace Wabbajack.Server.DataLayer;
public partial class SqlService
{
private static DTOSerializer _dtoStatic;
static SqlService()
{
SqlMapper.AddTypeHandler(new HashMapper());
SqlMapper.AddTypeHandler(new RelativePathMapper());
SqlMapper.AddTypeHandler(new JsonMapper<IDownloadState>());
SqlMapper.AddTypeHandler(new JsonMapper<FileDefinition>());
SqlMapper.AddTypeHandler(new JsonMapper<ModlistMetadata>());
SqlMapper.AddTypeHandler(new VersionMapper());
SqlMapper.AddTypeHandler(new GameMapper());
SqlMapper.AddTypeHandler(new DateTimeHandler());
}
/// <summary>
/// Needed to make sure dates are all in UTC format
/// </summary>
private class DateTimeHandler : SqlMapper.TypeHandler<DateTime>
{
public override void SetValue(IDbDataParameter parameter, DateTime value)
{
parameter.Value = value;
}
public override DateTime Parse(object value)
{
return DateTime.SpecifyKind((DateTime) value, DateTimeKind.Utc);
}
}
private class JsonMapper<T> : SqlMapper.TypeHandler<T>
{
public override void SetValue(IDbDataParameter parameter, T value)
{
parameter.Value = _dtoStatic.Serialize(value);
}
public override T Parse(object value)
{
return _dtoStatic.Deserialize<T>((string) value)!;
}
}
private class RelativePathMapper : SqlMapper.TypeHandler<RelativePath>
{
public override void SetValue(IDbDataParameter parameter, RelativePath value)
{
parameter.Value = value.ToString();
}
public override RelativePath Parse(object value)
{
return (RelativePath) (string) value;
}
}
private class HashMapper : SqlMapper.TypeHandler<Hash>
{
public override void SetValue(IDbDataParameter parameter, Hash value)
{
parameter.Value = (long) value;
}
public override Hash Parse(object value)
{
return Hash.FromLong((long) value);
}
}
private class VersionMapper : SqlMapper.TypeHandler<Version>
{
public override void SetValue(IDbDataParameter parameter, Version value)
{
parameter.Value = value.ToString();
}
public override Version Parse(object value)
{
return Version.Parse((string) value);
}
}
private class GameMapper : SqlMapper.TypeHandler<Game>
{
public override void SetValue(IDbDataParameter parameter, Game value)
{
parameter.Value = value.ToString();
}
public override Game Parse(object value)
{
return GameRegistry.GetByFuzzyName((string) value).Game;
}
}
}

189
Wabbajack.Server/DataLayer/Metrics.cs
View File

@ -1,189 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer;
public partial class SqlService
{
public async Task IngestMetric(Metric metric)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)",
metric);
}
public async Task IngestAccess(string ip, string log)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"INSERT INTO dbo.AccessLog (Timestamp, Action, Ip) VALUES (@Timestamp, @Action, @Ip)",
new
{
Timestamp = DateTime.UtcNow,
Ip = ip,
Action = log
});
}
public async Task<IEnumerable<AggregateMetric>> MetricsReport(string action)
{
await using var conn = await Open();
return (await conn.QueryAsync<AggregateMetric>(@"
select
datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)) as Date,
GroupingSubject as Subject,
count(*) as Count
from dbo.metrics where
Action = @Action
AND GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics
WHERE action = @Action
AND MetricsKey is not null
AND Subject != 'Default'
AND Subject != 'untitled'
AND TRY_CONVERT(uniqueidentifier, Subject) is null
AND Timestamp >= DATEADD(DAY, -1, GETUTCDATE()))
group by
datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)),
GroupingSubject
Order by datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)) asc",
new {Action = action}))
.ToList();
}
public async Task<List<(DateTime, string, string)>> FullTarReport(string key)
{
await using var conn = await Open();
return (await conn.QueryAsync<(DateTime, string, string)>(@"
SELECT u.Timestamp, u.Path, u.MetricsKey FROM
(SELECT al.Timestamp, JSON_VALUE(al.Action, '$.Path') as Path, al.MetricsKey FROM dbo.AccessLog al
WHERE al.MetricsKey = @MetricsKey
UNION ALL
SELECT m.Timestamp, m.Action + ' ' + m.Subject as Path, m.MetricsKey FROM dbo.Metrics m
WHERE m.MetricsKey = @MetricsKey
AND m.Action != 'TarKey') u
ORDER BY u.Timestamp Desc",
new {MetricsKey = key})).ToList();
}
public async Task<bool> ValidMetricsKey(string metricsKey)
{
await using var conn = await Open();
return await conn.QuerySingleOrDefaultAsync<string>(
"SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey",
new {MetricsKey = metricsKey}) != default;
}
public async Task AddMetricsKey(string metricsKey)
{
await using var conn = await Open();
await using var trans = conn.BeginTransaction();
if (await conn.QuerySingleOrDefaultAsync<string>(
"SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey",
new {MetricsKey = metricsKey}, trans) != default)
return;
await conn.ExecuteAsync("INSERT INTO dbo.MetricsKeys (MetricsKey) VALUES (@MetricsKey)",
new {MetricsKey = metricsKey}, trans);
}
public async Task<string[]> AllKeys()
{
await using var conn = await Open();
return (await conn.QueryAsync<string>("SELECT MetricsKey from dbo.MetricsKeys")).ToArray();
}
public async Task<long> UniqueInstalls(string machineUrl)
{
await using var conn = await Open();
return await conn.QueryFirstAsync<long>(
@"SELECT COUNT(*) FROM (
SELECT DISTINCT MetricsKey from dbo.Metrics where Action = 'finish_install' and GroupingSubject in (
SELECT JSON_VALUE(Metadata, '$.title') FROM dbo.ModLists
WHERE JSON_VALUE(Metadata, '$.links.machineURL') = @MachineURL)) s",
new {MachineURL = machineUrl});
}
public async Task<long> TotalInstalls(string machineUrl)
{
await using var conn = await Open();
return await conn.QueryFirstAsync<long>(
@"SELECT COUNT(*) from dbo.Metrics where Action = 'finish_install' and GroupingSubject in (
SELECT JSON_VALUE(Metadata, '$.title') FROM dbo.ModLists
WHERE JSON_VALUE(Metadata, '$.links.machineURL') = @MachineURL)",
new {MachineURL = machineUrl});
}
public async Task<IEnumerable<(string, long)>> GetTotalInstalls()
{
await using var conn = await Open();
return await conn.QueryAsync<(string, long)>(
@"SELECT GroupingSubject, Count(*) as Count
From dbo.Metrics
WHERE
GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics
WHERE action = 'finish_install'
AND MetricsKey is not null)
group by GroupingSubject
order by Count(*) desc");
}
public async Task<IEnumerable<(string, long)>> GetTotalUniqueInstalls()
{
await using var conn = await Open();
return await conn.QueryAsync<(string, long)>(
@"Select GroupingSubject, Count(*) as Count
FROM
(select DISTINCT MetricsKey, GroupingSubject
From dbo.Metrics
WHERE
GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics
WHERE action = 'finish_install'
AND MetricsKey is not null)) m
GROUP BY GroupingSubject
Order by Count(*) desc
");
}
public async IAsyncEnumerable<MetricRow> MetricsDump()
{
var keys = new Dictionary<string, long>();
await using var conn = await Open();
foreach (var row in await conn.QueryAsync<(long, DateTime, string, string, string, string)>(
@"select Id, Timestamp, Action, Subject, MetricsKey, GroupingSubject from dbo.metrics WHERE MetricsKey is not null"))
{
if (!keys.TryGetValue(row.Item5, out var keyid))
{
keyid = keys.Count;
keys[row.Item5] = keyid;
}
yield return new MetricRow
{
Id = row.Item1,
Timestamp = row.Item2,
Action = row.Item3,
Subject = row.Item4,
MetricsKey = keyid,
GroupingSubject = row.Item6
};
}
}
public class MetricRow
{
public string Action;
public string GroupingSubject;
public long Id;
public long MetricsKey;
public string Subject;
public DateTime Timestamp;
}
}

147
Wabbajack.Server/DataLayer/MirroredFiles.cs
View File

@ -1,147 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.DTOs;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer;
public partial class SqlService
{
public async Task<MirroredFile> GetNextMirroredFile()
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(Hash, DateTime, DateTime, string, string)>(
"SELECT Hash, Created, Uploaded, Rationale, FailMessage from dbo.MirroredArchives WHERE Uploaded IS NULL");
if (result == default) return null;
return new MirroredFile
{
Hash = result.Item1, Created = result.Item2, Uploaded = result.Item3, Rationale = result.Item4,
FailMessage = result.Item5
};
}
public async Task<Dictionary<Hash, bool>> GetAllMirroredHashes()
{
await using var conn = await Open();
return (await conn.QueryAsync<(Hash, DateTime?)>("SELECT Hash, Uploaded FROM dbo.MirroredArchives"))
.GroupBy(d => d.Item1)
.ToDictionary(d => d.Key, d => d.First().Item2.HasValue);
}
public async Task StartMirror((Hash Hash, string Reason) mirror)
{
await using var conn = await Open();
await using var trans = await conn.BeginTransactionAsync();
if (await conn.QueryFirstOrDefaultAsync<Hash>(@"SELECT Hash FROM dbo.MirroredArchives WHERE Hash = @Hash",
new {mirror.Hash}, trans) != default)
return;
await conn.ExecuteAsync(
@"INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) VALUES (@Hash, GETUTCDATE(), @Reason)",
new {mirror.Hash, mirror.Reason}, trans);
await trans.CommitAsync();
}
public async Task<Dictionary<Hash, string>> GetAllowedMirrors()
{
await using var conn = await Open();
return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, Reason FROM dbo.AllowedMirrorsCache"))
.GroupBy(d => d.Item1)
.ToDictionary(d => d.Key, d => d.First().Item2);
}
public async Task UpsertMirroredFile(MirroredFile file)
{
await using var conn = await Open();
await using var trans = await conn.BeginTransactionAsync();
await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash", new {file.Hash}, trans);
await conn.ExecuteAsync(
"INSERT INTO dbo.MirroredArchives (Hash, Created, Uploaded, Rationale, FailMessage) VALUES (@Hash, @Created, @Uploaded, @Rationale, @FailMessage)",
new
{
file.Hash,
file.Created,
file.Uploaded,
file.Rationale,
file.FailMessage
}, trans);
await trans.CommitAsync();
}
public async Task DeleteMirroredFile(Hash hash)
{
await using var conn = await Open();
await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash",
new {Hash = hash});
}
public async Task<bool> HaveMirror(Hash hash)
{
await using var conn = await Open();
return await conn.QueryFirstOrDefaultAsync<Hash>("SELECT Hash FROM dbo.MirroredArchives WHERE Hash = @Hash",
new {Hash = hash}) != default;
}
public async Task QueueMirroredFiles()
{
await using var conn = await Open();
await conn.ExecuteAsync(@"
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT hs.Hash, GETUTCDATE(), 'File has re-upload permissions on the Nexus' FROM
(SELECT DISTINCT ad.Hash FROM dbo.NexusModPermissions p
INNER JOIN GameMetadata md on md.NexusGameId = p.NexusGameID
INNER JOIN dbo.ArchiveDownloads ad on ad.PrimaryKeyString like 'NexusDownloader+State|'+md.WabbajackName+'|'+CAST(p.ModID as nvarchar)+'|%'
WHERE p.Permissions = 1
AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
) hs
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT DISTINCT Hash, GETUTCDATE(), 'File is hosted on GitHub'
FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%github.com/%'
AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT DISTINCT Hash, GETUTCDATE(), 'File license allows uploading to any Non-nexus site'
FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%enbdev.com/%'
AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives)
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT DISTINCT Hash, GETUTCDATE(), 'DynDOLOD file' /*, Name*/
from dbo.ModListArchives mla WHERE Name like '%DynDoLOD%standalone%'
and Hash not in (select Hash from dbo.MirroredArchives)
INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale)
SELECT DISTINCT Hash, GETUTCDATE(), 'Distribution allowed by author' /*, Name*/
from dbo.ModListArchives mla WHERE Name like '%particle%patch%'
and Hash not in (select Hash from dbo.MirroredArchives)
");
}
public async Task AddNexusModWithOpenPerms(Game gameGame, long modId)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"INSERT INTO dbo.NexusModsWithOpenPerms(NexusGameID, NexusModID) VALUES(@game, @mod)",
new {game = gameGame.MetaData().NexusGameId, modId});
}
public async Task SyncActiveMirroredFiles()
{
await using var conn = await Open();
await conn.ExecuteAsync(@"EXEC dbo.QueueMirroredFiles");
}
}

117
Wabbajack.Server/DataLayer/ModLists.cs
View File

@ -1,117 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.Hashing.xxHash64;
namespace Wabbajack.Server.DataLayer;
public partial class SqlService
{
public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist, bool brokenDownload)
{
await using var conn = await Open();
await using var tran = await conn.BeginTransactionAsync();
await conn.ExecuteAsync(@"DELETE FROM dbo.ModLists Where MachineUrl = @MachineUrl",
new {MachineUrl = metadata.Links.MachineURL}, tran);
var archives = modlist.Archives;
var directives = modlist.Directives;
modlist.Archives = Array.Empty<Archive>();
modlist.Directives = Array.Empty<Directive>();
await conn.ExecuteAsync(
@"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList, BrokenDownload) VALUES (@MachineUrl, @Hash, @Metadata, @ModList, @BrokenDownload)",
new
{
MachineUrl = metadata.Links.MachineURL,
Hash = hash,
MetaData = _dtos.Serialize(metadata),
ModList = _dtos.Serialize(modlist),
BrokenDownload = brokenDownload
}, tran);
var entries = archives.Select(a =>
new
{
MachineUrl = metadata.Links.MachineURL,
a.Name,
a.Hash,
a.Size,
State = _dtos.Serialize(a.State),
a.State.PrimaryKeyString
}).ToArray();
await conn.ExecuteAsync(@"DELETE FROM dbo.ModListArchives WHERE MachineURL = @machineURL",
new {MachineUrl = metadata.Links.MachineURL}, tran);
foreach (var entry in entries)
await conn.ExecuteAsync(
"INSERT INTO dbo.ModListArchives (MachineURL, Name, Hash, Size, PrimaryKeyString, State) VALUES (@MachineURL, @Name, @Hash, @Size, @PrimaryKeyString, @State)",
entry, tran);
await tran.CommitAsync();
}
public async Task<bool> HaveIndexedModlist(string machineUrl, Hash hash)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT MachineURL from dbo.Modlists WHERE MachineURL = @MachineUrl AND Hash = @Hash",
new {MachineUrl = machineUrl, Hash = hash});
return result != null;
}
public async Task<bool> HashIsInAModlist(Hash hash)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<bool>(
"SELECT Hash FROM dbo.ModListArchives Where Hash = @Hash",
new {Hash = hash});
return result;
}
public async Task<List<Archive>> ModListArchives(string machineURL)
{
await using var conn = await Open();
var archives = await conn.QueryAsync<(string, Hash, long, IDownloadState)>(
"SELECT Name, Hash, Size, State FROM dbo.ModListArchives WHERE MachineUrl = @MachineUrl",
new {MachineUrl = machineURL});
return archives.Select(t => new Archive
{
State = t.Item4,
Name = string.IsNullOrWhiteSpace(t.Item1) ? t.Item4.PrimaryKeyString : t.Item1,
Size = t.Item3,
Hash = t.Item2
}).ToList();
}
public async Task<List<Archive>> ModListArchives()
{
await using var conn = await Open();
var archives =
await conn.QueryAsync<(string, Hash, long, IDownloadState)>(
"SELECT Name, Hash, Size, State FROM dbo.ModListArchives");
return archives.Select(t => new Archive
{
State = t.Item4,
Name = string.IsNullOrWhiteSpace(t.Item1) ? t.Item4.PrimaryKeyString : t.Item1,
Size = t.Item3,
Hash = t.Item2
}).ToList();
}
public async Task<int> PurgeList(string machineURL)
{
await using var conn = await Open();
var ret1 = await conn.ExecuteAsync(@" delete from dbo.ModListArchives where MachineURL = @machineURL",
new {machineURL});
var ret2 = await conn.ExecuteAsync(@" delete from dbo.ModLists where MachineURL = @machineURL",
new {machineURL});
return ret1 + ret2;
}
}

156
Wabbajack.Server/DataLayer/Nexus.cs
View File

@ -1,156 +0,0 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.DTOs;
using Wabbajack.Networking.NexusApi.DTOs;
namespace Wabbajack.Server.DataLayer;
/// <summary>
/// SQL routines that read/write cached information from the Nexus
/// </summary>
public partial class SqlService
{
public async Task<long> DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date)
{
await using var conn = await Open();
var deleted = await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
return deleted;
}
public async Task<long> DeleteNexusModFilesUpdatedBeforeDate(Game game, long modId, DateTime date)
{
await using var conn = await Open();
var deleted = await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
return deleted;
}
public async Task<ModInfo?> GetNexusModInfoString(Game game, long modId)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT Data FROM dbo.NexusModInfos WHERE Game = @Game AND @ModId = ModId",
new {Game = game.MetaData().NexusGameId, ModId = modId});
return result == null ? null : _dtos.Deserialize<ModInfo>(result);
}
public async Task AddNexusModInfo(Game game, long modId, DateTime lastCheckedUtc, ModInfo data)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"MERGE dbo.NexusModInfos AS Target
USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
ON Target.Game = Source.Game AND Target.ModId = Source.ModId
WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
new
{
Game = game.MetaData().NexusGameId,
ModId = modId,
LastChecked = lastCheckedUtc,
Data = _dtos.Serialize(data)
});
}
public async Task AddNexusModFiles(Game game, long modId, DateTime lastCheckedUtc, ModFiles data)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"MERGE dbo.NexusModFiles AS Target
USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
ON Target.Game = Source.Game AND Target.ModId = Source.ModId
WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
new
{
Game = game.MetaData().NexusGameId,
ModId = modId,
LastChecked = lastCheckedUtc,
Data = _dtos.Serialize(data)
});
}
public async Task AddNexusModFileSlow(Game game, long modId, long fileId, DateTime lastCheckedUtc)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"MERGE dbo.NexusModFilesSlow AS Target
USING (SELECT @GameId GameId, @ModId ModId, @LastChecked LastChecked, @FileId FileId) AS Source
ON Target.GameId = Source.GameId AND Target.ModId = Source.ModId AND Target.FileId = Source.FileId
WHEN MATCHED THEN UPDATE SET Target.LastChecked = @LastChecked
WHEN NOT MATCHED THEN INSERT (GameId, ModId, LastChecked, FileId) VALUES (@GameId, @ModId, @LastChecked, @FileId);",
new
{
GameId = game.MetaData().NexusGameId,
ModId = modId,
FileId = fileId,
LastChecked = lastCheckedUtc
});
}
public async Task<ModFiles?> GetModFiles(Game game, long modId)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT Data FROM dbo.NexusModFiles WHERE Game = @Game AND @ModId = ModId",
new {Game = game.MetaData().NexusGameId, ModId = modId});
return result == null ? null : _dtos.Deserialize<ModFiles>(result);
}
public async Task PurgeNexusCache(long modId)
{
await using var conn = await Open();
await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModPermissions WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModFile WHERE ModId = @ModID", new {ModId = modId});
}
public async Task UpdateGameMetadata()
{
await using var conn = await Open();
var existing = (await conn.QueryAsync<string>("SELECT WabbajackName FROM dbo.GameMetadata")).ToHashSet();
var missing = GameRegistry.Games.Values.Where(g => !existing.Contains(g.Game.ToString())).ToList();
foreach (var add in missing.Where(g => g.NexusGameId != 0))
await conn.ExecuteAsync(
"INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, @WabbajackName)",
new {add.NexusGameId, WabbajackName = add.Game.ToString()});
}
public async Task<ModFile?> GetModFile(Game game, long modId, long fileId)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT Data FROM dbo.NexusModFile WHERE Game = @Game AND @ModId = ModId AND @FileId = FileId",
new {Game = game.MetaData().NexusGameId, ModId = modId, FileId = fileId});
return result == null ? null : _dtos.Deserialize<ModFile>(result);
}
public async Task AddNexusModFile(Game game, long modId, long fileId, DateTime lastCheckedUtc, ModFile data)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"INSERT INTO dbo.NexusModFile (Game, ModId, FileId, LastChecked, Data)
VALUES (@Game, @ModId, @FileId, @LastChecked, @Data)",
new
{
Game = game.MetaData().NexusGameId,
ModId = modId,
FileId = fileId,
LastChecked = lastCheckedUtc,
Data = _dtos.Serialize(data)
});
}
}

51
Wabbajack.Server/DataLayer/NexusKeys.cs
View File

@ -1,51 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
namespace Wabbajack.Server.DataLayer;
public partial class SqlService
{
public async Task SetNexusAPIKey(string key, long daily, long hourly)
{
await using var conn = await Open();
await using var trans = await conn.BeginTransactionAsync();
await conn.ExecuteAsync(@"DELETE FROM NexusKeys WHERE ApiKey = @ApiKey", new {ApiKey = key}, trans);
await conn.ExecuteAsync(
@"INSERT INTO NexusKeys (ApiKey, DailyRemain, HourlyRemain) VALUES (@ApiKey, @DailyRemain, @HourlyRemain)",
new {ApiKey = key, DailyRemain = daily, HourlyRemain = hourly}, trans);
await trans.CommitAsync();
}
public async Task DeleteNexusAPIKey(string key)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"DELETE FROM NexusKeys WHERE ApiKey = @ApiKey", new {ApiKey = key});
}
public async Task<List<string>> GetNexusApiKeys(int threshold = 1500)
{
await using var conn = await Open();
return (await conn.QueryAsync<string>(
@"SELECT ApiKey FROM NexusKeys WHERE DailyRemain >= @Threshold ORDER BY DailyRemain DESC",
new {Threshold = threshold})).ToList();
}
public async Task<List<(string Key, int Daily, int Hourly)>> GetNexusApiKeysWithCounts(int threshold = 1500)
{
await using var conn = await Open();
return (await conn.QueryAsync<(string, int, int)>(
@"SELECT ApiKey, DailyRemain, HourlyRemain FROM NexusKeys WHERE DailyRemain >= @Threshold ORDER BY DailyRemain DESC",
new {Threshold = threshold})).ToList();
}
public async Task<bool> HaveKey(string key)
{
await using var conn = await Open();
return (await conn.QueryAsync<string>(@"SELECT ApiKey FROM NexusKeys WHERE ApiKey = @ApiKey",
new {ApiKey = key})).Any();
}
}

43
Wabbajack.Server/DataLayer/NonNexusModlistArchives.cs
View File

@ -1,43 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates;
namespace Wabbajack.Server.DataLayer;
public partial class SqlService
{
public async Task<List<Archive>> GetNonNexusModlistArchives()
{
await using var conn = await Open();
var results = await conn.QueryAsync<(Hashing.xxHash64.Hash Hash, long Size, string State)>(
@"SELECT Hash, Size, State FROM dbo.ModListArchives WHERE PrimaryKeyString NOT LIKE 'NexusDownloader+State|%'");
return results.Select(r => new Archive
{
State = _dtos.Deserialize<IDownloadState>(r.State)!,
Size = r.Size,
Hash = r.Hash
}).ToList();
}
public async Task UpdateNonNexusModlistArchivesStatus(IEnumerable<(Archive Archive, bool IsValid)> results)
{
await using var conn = await Open();
var trans = await conn.BeginTransactionAsync();
await conn.ExecuteAsync("DELETE FROM dbo.ModlistArchiveStatus;", transaction: trans);
foreach (var itm in results.DistinctBy(itm => (itm.Archive.Hash, itm.Archive.State.PrimaryKeyString)))
await conn.ExecuteAsync(
@"INSERT INTO dbo.ModlistArchiveStatus (PrimaryKeyStringHash, PrimaryKeyString, Hash, IsValid)
VALUES (HASHBYTES('SHA2_256', @PrimaryKeyString), @PrimaryKeyString, @Hash, @IsValid)", new
{
itm.Archive.State.PrimaryKeyString,
itm.Archive.Hash,
itm.IsValid
}, trans);
await trans.CommitAsync();
}
}

259
Wabbajack.Server/DataLayer/Patches.cs
View File

@ -1,259 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer;
public partial class SqlService
{
/// <summary>
/// Adds a patch record
/// </summary>
/// <param name="patch"></param>
/// <returns></returns>
public async Task<bool> AddPatch(Patch patch)
{
await using var conn = await Open();
await using var trans = conn.BeginTransaction();
if (await conn.QuerySingleOrDefaultAsync<(Guid, Guid)>(
"Select SrcID, DestID FROM dbo.Patches where SrcID = @SrcId and DestID = @DestId",
new {SrcId = patch.Src.Id, DestId = patch.Dest.Id}, trans) != default)
return false;
await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)",
new {SrcId = patch.Src.Id, DestId = patch.Dest.Id}, trans);
await trans.CommitAsync();
return true;
}
/// <summary>
/// Adds a patch record
/// </summary>
/// <param name="patch"></param>
/// <returns></returns>
public async Task FinializePatch(Patch patch)
{
await using var conn = await Open();
await conn.ExecuteAsync(
"UPDATE dbo.Patches SET PatchSize = @PatchSize, Finished = @Finished, IsFailed = @IsFailed, FailMessage = @FailMessage WHERE SrcId = @SrcId AND DestID = @DestId",
new
{
SrcId = patch.Src.Id,
DestId = patch.Dest.Id,
patch.PatchSize,
patch.Finished,
patch.IsFailed,
patch.FailMessage
});
}
public async Task<Patch> FindPatch(Guid src, Guid dest)
{
await using var conn = await Open();
var patch = await conn.QueryFirstOrDefaultAsync<(long, DateTime?, bool?, string)>(
@"SELECT p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads src ON p.SrcId = src.Id
LEFT JOIN dbo.ArchiveDownloads dest ON p.SrcId = dest.Id
WHERE SrcId = @SrcId
AND DestId = @DestId
AND src.DownloadFinished IS NOT NULL
AND dest.DownloadFinished IS NOT NULL",
new
{
SrcId = src,
DestId = dest
});
if (patch == default)
return default;
return new Patch
{
Src = await GetArchiveDownload(src),
Dest = await GetArchiveDownload(dest),
PatchSize = patch.Item1,
Finished = patch.Item2,
IsFailed = patch.Item3,
FailMessage = patch.Item4
};
}
public async Task<Patch> FindOrEnqueuePatch(Guid src, Guid dest)
{
await using var conn = await Open();
var trans = await conn.BeginTransactionAsync();
var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
"SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestId",
new
{
SrcId = src,
DestId = dest
}, trans);
if (patch == default)
{
await conn.ExecuteAsync("INSERT INTO dbo.Patches (SrcId, DestId) VALUES (@SrcId, @DestId)",
new {SrcId = src, DestId = dest}, trans);
await trans.CommitAsync();
return new Patch {Src = await GetArchiveDownload(src), Dest = await GetArchiveDownload(dest)};
}
return new Patch
{
Src = await GetArchiveDownload(src),
Dest = await GetArchiveDownload(dest),
PatchSize = patch.Item3,
Finished = patch.Item4,
IsFailed = patch.Item5,
FailMessage = patch.Item6
};
}
public async Task<Patch> GetPendingPatch()
{
await using var conn = await Open();
var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
@"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads src ON src.Id = p.SrcId
LEFT JOIN dbo.ArchiveDownloads dest ON dest.Id = p.DestId
WHERE p.Finished is NULL AND src.IsFailed = 0 AND dest.IsFailed = 0 ");
if (patch == default)
return default;
return new Patch
{
Src = await GetArchiveDownload(patch.Item1),
Dest = await GetArchiveDownload(patch.Item2),
PatchSize = patch.Item3,
Finished = patch.Item4,
IsFailed = patch.Item5,
FailMessage = patch.Item6
};
}
public async Task<List<Patch>> PatchesForSource(Guid sourceDownload)
{
await using var conn = await Open();
var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
"SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId",
new {SrcId = sourceDownload});
return await AsPatches(patches);
}
public async Task<List<Patch>> PatchesForSource(Hash sourceHash)
{
await using var conn = await Open();
var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
@"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a ON p.SrcId = a.Id
WHERE a.Hash = @Hash AND p.Finished IS NOT NULL AND p.IsFailed = 0", new {Hash = sourceHash});
return await AsPatches(patches);
}
public async Task MarkPatchUsage(Guid srcId, Guid destId)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"UPDATE dbo.Patches SET Downloads = Downloads + 1, LastUsed = GETUTCDATE() WHERE SrcId = @srcId AND DestID = @destId",
new {SrcId = srcId, DestId = destId});
}
public async Task<List<Patch>> GetOldPatches()
{
await using var conn = await Open();
var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
@"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a ON p.SrcId = a.Id
WHERE a.Hash not in (SELECT Hash FROM dbo.ModListArchives)");
return await AsPatches(patches);
}
private async Task<List<Patch>> AsPatches(IEnumerable<(Guid, Guid, long, DateTime?, bool?, string)> patches)
{
var results = new List<Patch>();
foreach (var (srcId, destId, patchSize, finished, isFailed, failMessage) in patches)
results.Add(new Patch
{
Src = await GetArchiveDownload(srcId),
Dest = await GetArchiveDownload(destId),
PatchSize = patchSize,
Finished = finished,
IsFailed = isFailed,
FailMessage = failMessage
});
return results;
}
public async Task DeletePatch(Patch patch)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"DELETE FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestID",
new
{
SrcId = patch.Src.Id,
DestId = patch.Dest.Id
});
}
public async Task<HashSet<(Hash, Hash)>> AllPatchHashes()
{
await using var conn = await Open();
return (await conn.QueryAsync<(Hash, Hash)>(@"SELECT a1.Hash, a2.Hash
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a1 ON a1.Id = p.SrcId
LEFT JOIN dbo.ArchiveDownloads a2 on a2.Id = p.DestId
WHERE p.Finished IS NOT NULL")).ToHashSet();
}
public async Task DeletePatchesForHashPair((Hash, Hash) sqlFile)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"DELETE p
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a1 ON a1.Id = p.SrcId
LEFT JOIN dbo.ArchiveDownloads a2 on a2.Id = p.DestId
WHERE a1.Hash = @SrcHash
AND a2.Hash = @DestHash", new
{
SrcHash = sqlFile.Item1,
DestHash = sqlFile.Item2
});
}
public async Task PurgePatch(Hash hash, string rationale)
{
await using var conn = await Open();
await using var tx = await conn.BeginTransactionAsync();
await conn.ExecuteAsync(
"DELETE p FROM dbo.Patches p LEFT JOIN dbo.ArchiveDownloads ad ON ad.Id = p.SrcId WHERE ad.Hash = @Hash ",
new {Hash = hash}, tx);
await conn.ExecuteAsync(
"INSERT INTO dbo.NoPatch (Hash, Created, Rationale) VALUES (@Hash, GETUTCDATE(), @Rationale)",
new
{
Hash = hash,
Rationale = rationale
}, tx);
await tx.CommitAsync();
}
public async Task<bool> IsNoPatch(Hash hash)
{
await using var conn = await Open();
return await conn.QueryFirstOrDefaultAsync<Hash>("SELECT Hash FROM NoPatch WHERE Hash = @Hash",
new {Hash = hash}) != default;
}
}

30
Wabbajack.Server/DataLayer/SqlService.cs
View File

@ -1,30 +0,0 @@
using System.Data.SqlClient;
using System.Threading.Tasks;
using Wabbajack.BuildServer;
using Wabbajack.Downloaders;
using Wabbajack.DTOs.JsonConverters;
namespace Wabbajack.Server.DataLayer;
public partial class SqlService
{
private readonly DownloadDispatcher _dispatcher;
private readonly DTOSerializer _dtos;
private readonly AppSettings _settings;
public SqlService(AppSettings settings, DTOSerializer dtos, DownloadDispatcher dispatcher)
{
_settings = settings;
_dtos = dtos;
_dispatcher = dispatcher;
// Ugly hack, but the SQL mappers need it
_dtoStatic = dtos;
}
public async Task<SqlConnection> Open()
{
var conn = new SqlConnection(_settings.SqlConnection);
await conn.OpenAsync();
return conn;
}
}

56
Wabbajack.Server/DataLayer/ValidationData.cs
View File

@ -1,56 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.DTOs;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer;
public partial class SqlService
{
public async Task<ValidationData> GetValidationData()
{
var archiveStatus = AllModListArchivesStatus();
var modLists = AllModLists();
var mirrors = GetAllMirroredHashes();
var authoredFiles = AllAuthoredFiles();
var nexusFiles = await AllNexusFiles();
return new ValidationData
{
NexusFiles = nexusFiles.ToDictionary(nf => (nf.NexusGameId, nf.ModId, nf.FileId), nf => nf.category),
ArchiveStatus = await archiveStatus,
ModLists = await modLists,
Mirrors = await mirrors,
AllowedMirrors = new Lazy<Task<Dictionary<Hash, string>>>(async () => await GetAllowedMirrors()),
AllAuthoredFiles = await authoredFiles
};
}
public async Task<Dictionary<(string PrimaryKeyString, Hash Hash), bool>> AllModListArchivesStatus()
{
await using var conn = await Open();
var results =
await conn.QueryAsync<(string, Hash, bool)>(
@"SELECT PrimaryKeyString, Hash, IsValid FROM dbo.ModListArchiveStatus");
return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3);
}
public async Task<HashSet<(long NexusGameId, long ModId, long FileId, string category)>> AllNexusFiles()
{
await using var conn = await Open();
var results =
await conn.QueryAsync<(long, long, long, string)>(
@"SELECT Game, ModId, FileId, JSON_VALUE(Data, '$.category_name') FROM dbo.NexusModFile");
return results.ToHashSet();
}
public async Task<List<ModlistMetadata>> AllModLists()
{
await using var conn = await Open();
var results = await conn.QueryAsync<ModlistMetadata>(@"SELECT Metadata FROM dbo.ModLists");
return results.ToList();
}
}

110
Wabbajack.Server/DataModels/AuthorFiles.cs Normal file
View File

@ -0,0 +1,110 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.DTOs.CDN;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
namespace Wabbajack.Server.DataModels;
public class AuthorFiles
{
private readonly ILogger<AuthorFiles> _logger;
private readonly AppSettings _settings;
private readonly DTOSerializer _dtos;
private Dictionary<string, FileDefinition> _byServerId = new();
public AbsolutePath AuthorFilesLocation => _settings.AuthoredFilesFolder.ToAbsolutePath();
public AuthorFiles(ILogger<AuthorFiles> logger, AppSettings settings, DTOSerializer dtos)
{
_logger = logger;
_settings = settings;
_dtos = dtos;
}
public IEnumerable<AbsolutePath> AllDefinitions => AuthorFilesLocation.EnumerateFiles("definition.json.gz");
public async Task<FileDefinitionMetadata[]> AllAuthoredFiles()
{
var defs = new List<FileDefinitionMetadata>();
foreach (var file in AllDefinitions)
{
defs.Add(new FileDefinitionMetadata
{
Definition = await ReadDefinition(file),
Updated = file.LastModifiedUtc()
});
}
_byServerId = defs.ToDictionary(f => f.Definition.ServerAssignedUniqueId!, f => f.Definition);
return defs.ToArray();
}
public async Task<Stream> StreamForPart(string mungedName, int part)
{
return AuthorFilesLocation.Combine(mungedName, "parts", part.ToString()).Open(FileMode.Open);
}
public async Task<Stream> CreatePart(string mungedName, int part)
{
return AuthorFilesLocation.Combine(mungedName, "parts", part.ToString()).Open(FileMode.Create, FileAccess.Write, FileShare.None);
}
public async Task WriteDefinition(FileDefinition definition)
{
var path = AuthorFilesLocation.Combine(definition.MungedName, "definition.json.gz");
path.Parent.CreateDirectory();
path.Parent.Combine("parts").CreateDirectory();
await using var ms = new MemoryStream();
await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
{
await _dtos.Serialize(definition, gz);
}
await path.WriteAllBytesAsync(ms.ToArray());
}
public async Task<FileDefinition> ReadDefinition(string mungedName)
{
return await ReadDefinition(AuthorFilesLocation.Combine(mungedName, "definition.json.gz"));
}
private async Task<FileDefinition> ReadDefinition(AbsolutePath file)
{
var gz = new GZipStream(new MemoryStream(await file.ReadAllBytesAsync()), CompressionMode.Decompress);
var definition = (await _dtos.DeserializeAsync<FileDefinition>(gz))!;
return definition;
}
public class FileDefinitionMetadata
{
public FileDefinition Definition { get; set; }
public DateTime Updated { get; set; }
public string HumanSize => Definition.Size.ToFileSizeString();
}
public async Task DeleteFile(FileDefinition definition)
{
var folder = AuthorFilesLocation.Combine(definition.MungedName);
folder.DeleteDirectory();
}
public async Task<FileDefinition> ReadDefinitionForServerId(string serverAssignedUniqueId)
{
if (_byServerId.TryGetValue(serverAssignedUniqueId, out var found))
return found;
await AllAuthoredFiles();
return _byServerId[serverAssignedUniqueId];
}
}

28
Wabbajack.Server/DataModels/AuthorKeys.cs Normal file
View File

@ -0,0 +1,28 @@
using System.Threading.Tasks;
using Wabbajack.BuildServer;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
namespace Wabbajack.Server.DataModels;
public class AuthorKeys
{
private readonly AppSettings _settings;
private AbsolutePath AuthorKeysPath => _settings.AuthorAPIKeyFile.ToAbsolutePath();
public AuthorKeys(AppSettings settings)
{
_settings = settings;
}
public async Task<string?> AuthorForKey(string key)
{
await foreach (var line in AuthorKeysPath.ReadAllLinesAsync())
{
var parts = line.Split("\t");
if (parts[0].Trim() == key)
return parts[1].Trim();
}
return null;
}
}

39
Wabbajack.Server/DataModels/Metrics.cs Normal file
View File

@ -0,0 +1,39 @@
using System;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Toolkit.HighPerformance;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataModels;
public class Metrics
{
private readonly AppSettings _settings;
public SemaphoreSlim _lock = new(1);
private readonly DTOSerializer _dtos;
public Metrics(AppSettings settings, DTOSerializer dtos)
{
_settings = settings;
_dtos = dtos;
}
public async Task Ingest(Metric metric)
{
using var _ = await _lock.Lock();
var data = Encoding.UTF8.GetBytes(_dtos.Serialize(metric));
var metricsFile = _settings.MetricsFolder.ToAbsolutePath().Combine(DateTime.Now.ToString("yyyy_MM_dd") + ".json");
await using var fs = metricsFile.Open(FileMode.Append, FileAccess.Write, FileShare.Read);
fs.Write(data);
fs.Write(Encoding.UTF8.GetBytes("\n"));
}
}

11
Wabbajack.Server/Program.cs
View File

@ -22,16 +22,7 @@ public class Program
webBuilder.UseStartup<Startup>()
.UseKestrel(options =>
{
options.Listen(IPAddress.Any, testMode ? 8080 : 80);
if (!testMode)
options.Listen(IPAddress.Any, 443, listenOptions =>
{
using var store = new X509Store(StoreName.My);
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Find(X509FindType.FindBySubjectName,
"build.wabbajack.org", true)[0];
listenOptions.UseHttps(cert);
});
options.Listen(IPAddress.Any, 5000);
options.Limits.MaxRequestBodySize = null;
});
});

150
Wabbajack.Server/Services/ArchiveDownloader.cs
View File

@ -1,150 +0,0 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Downloaders;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.Networking.NexusApi;
using Wabbajack.Paths.IO;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services;
public class ArchiveDownloader : AbstractService<ArchiveDownloader, int>
{
private readonly DownloadDispatcher _dispatcher;
private readonly TemporaryFileManager _manager;
private readonly ArchiveMaintainer _archiveMaintainer;
private readonly DiscordWebHook _discord;
private NexusApi _nexusClient;
private readonly SqlService _sql;
public ArchiveDownloader(ILogger<ArchiveDownloader> logger, AppSettings settings, SqlService sql,
ArchiveMaintainer archiveMaintainer,
DiscordWebHook discord, QuickSync quickSync, DownloadDispatcher dispatcher, TemporaryFileManager manager)
: base(logger, settings, quickSync, TimeSpan.FromMinutes(10))
{
_sql = sql;
_archiveMaintainer = archiveMaintainer;
_discord = discord;
_dispatcher = dispatcher;
_manager = manager;
}
public override async Task<int> Execute()
{
var count = 0;
while (true)
{
var (_, header) = await _nexusClient.Validate();
var ignoreNexus = header.DailyRemaining < 100 && header.HourlyRemaining < 10;
//var ignoreNexus = true;
if (ignoreNexus)
_logger.LogWarning(
$"Ignoring Nexus Downloads due to low hourly api limit (Daily: {header.DailyRemaining}, Hourly:{header.HourlyRemaining})");
else
_logger.LogInformation(
$"Looking for any download (Daily: {header.DailyRemaining}, Hourly:{header.HourlyRemaining})");
var nextDownload = await _sql.GetNextPendingDownload(ignoreNexus);
if (nextDownload == default)
break;
_logger.LogInformation($"Checking for previously archived {nextDownload.Archive.Hash}");
if (nextDownload.Archive.Hash != default && _archiveMaintainer.HaveArchive(nextDownload.Archive.Hash))
{
await nextDownload.Finish(_sql);
continue;
}
if (nextDownload.Archive.State is Manual or GameFileSource)
{
await nextDownload.Finish(_sql);
continue;
}
try
{
_logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}");
ReportStarting(nextDownload.Archive.State.PrimaryKeyString);
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"
});
await _dispatcher.PrepareAll(new[] {nextDownload.Archive.State});
await using var tempPath = _manager.CreateFile();
if (await _dispatcher.Download(nextDownload.Archive, tempPath.Path, CancellationToken.None) == default)
{
_logger.LogError(
$"Downloader returned false for {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, "Downloader returned false");
continue;
}
var hash = await tempPath.Path.Hash();
if (hash == default || nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)
{
_logger.Log(LogLevel.Warning,
$"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size()}");
await nextDownload.Fail(_sql, "Invalid Hash");
continue;
}
if (nextDownload.Archive.Size != default &&
tempPath.Path.Size() != nextDownload.Archive.Size)
{
await nextDownload.Fail(_sql, "Invalid Size");
continue;
}
nextDownload.Archive.Hash = hash;
nextDownload.Archive.Size = tempPath.Path.Size();
_logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await _archiveMaintainer.Ingest(tempPath.Path);
_logger.Log(LogLevel.Information,
$"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Finish(_sql);
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}"
});
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, ex.ToString());
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}"
});
}
finally
{
ReportEnding(nextDownload.Archive.State.PrimaryKeyString);
}
count++;
}
if (count > 0)
// Wake the Patch builder up in case it needs to build a patch now
await _quickSync.Notify<PatchBuilder>();
return count;
}
}

66
Wabbajack.Server/Services/ArchiveMaintainer.cs
View File

@ -1,66 +0,0 @@
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.VFS;
namespace Wabbajack.Server.Services;
/// <summary>
/// Maintains a concurrent cache of all the files we've downloaded, indexed by Hash.
/// </summary>
public class ArchiveMaintainer
{
private readonly ILogger<ArchiveMaintainer> _logger;
private readonly AppSettings _settings;
public ArchiveMaintainer(ILogger<ArchiveMaintainer> logger, AppSettings settings)
{
_settings = settings;
_logger = logger;
_logger.Log(LogLevel.Information, "Creating Archive Maintainer");
}
private AbsolutePath ArchivePath(Hash hash)
{
return _settings.ArchivePath.Combine(hash.ToHex());
}
public async Task Ingest(AbsolutePath file)
{
var hash = await file.Hash(CancellationToken.None);
if (hash == default) return;
var newPath = ArchivePath(hash);
if (HaveArchive(hash))
{
file.Delete();
return;
}
await file.MoveToAsync(newPath, true, CancellationToken.None);
}
public bool HaveArchive(Hash hash)
{
return ArchivePath(hash).FileExists();
}
public bool TryGetPath(Hash hash, out AbsolutePath path)
{
path = ArchivePath(hash);
return path.FileExists();
}
}
public static class ArchiveMaintainerExtensions
{
public static IServiceCollection UseArchiveMaintainer(this IServiceCollection b)
{
return b.AddSingleton<ArchiveMaintainer>();
}
}

189
Wabbajack.Server/Services/DiscordFrontend.cs
View File

@ -1,189 +0,0 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Discord;
using Discord.WebSocket;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.DTOs;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.TokenProviders;
namespace Wabbajack.Server.Services;
public class DiscordFrontend : IStartable
{
private readonly IDiscordToken _token;
private readonly DiscordSocketClient _client;
private readonly MetricsKeyCache _keyCache;
private readonly ILogger<DiscordFrontend> _logger;
private readonly QuickSync _quickSync;
private AppSettings _settings;
private readonly SqlService _sql;
public DiscordFrontend(ILogger<DiscordFrontend> logger, AppSettings settings, QuickSync quickSync, SqlService sql,
MetricsKeyCache keyCache, IDiscordToken token)
{
_logger = logger;
_settings = settings;
_quickSync = quickSync;
_client = new DiscordSocketClient();
_client.Log += LogAsync;
_client.Ready += ReadyAsync;
_client.MessageReceived += MessageReceivedAsync;
_sql = sql;
_keyCache = keyCache;
_token = token;
}
public async Task Start()
{
await _client.LoginAsync(TokenType.Bot, await _token.Get());
await _client.StartAsync();
}
private async Task MessageReceivedAsync(SocketMessage arg)
{
_logger.LogInformation(arg.Content);
if (arg.Content.StartsWith("!dervenin"))
{
var parts = arg.Content.Split(" ", StringSplitOptions.RemoveEmptyEntries);
if (parts[0] != "!dervenin")
return;
if (parts.Length == 1) await ReplyTo(arg, "Wat?");
if (parts[1] == "purge-nexus-cache")
{
if (parts.Length != 3)
{
await ReplyTo(arg, "Welp you did that wrong, gotta give me a mod-id or url");
return;
}
await PurgeNexusCache(arg, parts[2]);
}
else if (parts[1] == "quick-sync")
{
var options = await _quickSync.Report();
if (parts.Length != 3)
{
var optionsStr = string.Join(", ", options.Select(o => o.Key.Name));
await ReplyTo(arg, $"Can't expect me to quicksync the whole damn world! Try: {optionsStr}");
}
else
{
foreach (var pair in options.Where(o => o.Key.Name == parts[2]))
{
await _quickSync.Notify(pair.Key);
await ReplyTo(arg, $"Notified {pair.Key}");
}
}
}
else if (parts[1] == "purge-list")
{
if (parts.Length != 3)
{
await ReplyTo(arg, "Yeah, I'm not gonna purge the whole server...");
}
else
{
var deleted = await _sql.PurgeList(parts[2]);
await _quickSync.Notify<ModListDownloader>();
await ReplyTo(arg,
$"Purged all traces of #{parts[2]} from the server, triggered list downloading. {deleted} records removed");
}
}
else if (parts[1] == "mirror-mod")
{
await MirrorModCommand(arg, parts);
}
else if (parts[1] == "users")
{
await ReplyTo(arg, $"Wabbajack has {await _keyCache.KeyCount()} known unique users");
}
}
}
private async Task MirrorModCommand(SocketMessage msg, string[] parts)
{
if (parts.Length != 2)
{
await ReplyTo(msg, "Command is: mirror-mod <game-name> <mod-id>");
return;
}
if (long.TryParse(parts[2], out var modId))
{
await ReplyTo(msg, $"Got {modId} for a mod-id, expected a integer");
return;
}
if (GameRegistry.TryGetByFuzzyName(parts[1], out var game))
{
var gameNames = GameRegistry.Games.Select(g => g.Value.NexusName)
.Where(g => !string.IsNullOrWhiteSpace(g))
.Select(g => (string) g)
.ToHashSet();
var joined = string.Join(", ", gameNames.OrderBy(g => g));
await ReplyTo(msg, $"Got {parts[1]} for a game name, expected something like: {joined}");
}
if (game!.NexusGameId == default) await ReplyTo(msg, $"No NexusGameID found for {game}");
await _sql.AddNexusModWithOpenPerms(game.Game, modId);
await _quickSync.Notify<MirrorUploader>();
await ReplyTo(msg, "Done, and I notified the uploader");
}
private async Task PurgeNexusCache(SocketMessage arg, string mod)
{
if (Uri.TryCreate(mod, UriKind.Absolute, out var url))
mod = url.AbsolutePath.Split("/", StringSplitOptions.RemoveEmptyEntries).Last();
if (int.TryParse(mod, out var mod_id))
{
await _sql.PurgeNexusCache(mod_id);
await ReplyTo(arg, $"It is done, {mod_id} has been purged, list validation has been triggered");
}
}
private async Task ReplyTo(SocketMessage socketMessage, string message)
{
await socketMessage.Channel.SendMessageAsync(message);
}
private async Task ReadyAsync()
{
}
private async Task LogAsync(LogMessage arg)
{
switch (arg.Severity)
{
case LogSeverity.Info:
_logger.LogInformation(arg.Message);
break;
case LogSeverity.Warning:
_logger.LogWarning(arg.Message);
break;
case LogSeverity.Critical:
_logger.LogCritical(arg.Message);
break;
case LogSeverity.Error:
_logger.LogError(arg.Exception, arg.Message);
break;
case LogSeverity.Verbose:
_logger.LogTrace(arg.Message);
break;
case LogSeverity.Debug:
_logger.LogDebug(arg.Message);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
}

61
Wabbajack.Server/Services/MetricsKeyCache.cs
View File

@ -1,61 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.Services;
public class MetricsKeyCache : IStartable
{
private HashSet<string> _knownKeys = new();
private readonly AsyncLock _lock = new();
private ILogger<MetricsKeyCache> _logger;
private readonly SqlService _sql;
public MetricsKeyCache(ILogger<MetricsKeyCache> logger, SqlService sql)
{
_logger = logger;
_sql = sql;
}
public async Task Start()
{
_knownKeys = _sql.AllKeys().Result.ToHashSet();
}
public async Task<bool> IsValidKey(string key)
{
using (var _ = await _lock.WaitAsync())
{
if (_knownKeys.Contains(key)) return true;
}
if (await _sql.ValidMetricsKey(key))
{
using var _ = await _lock.WaitAsync();
_knownKeys.Add(key);
return true;
}
return false;
}
public async Task AddKey(string key)
{
using (var _ = await _lock.WaitAsync())
{
if (_knownKeys.Contains(key)) return;
_knownKeys.Add(key);
}
await _sql.AddMetricsKey(key);
}
public async Task<long> KeyCount()
{
using var _ = await _lock.WaitAsync();
return _knownKeys.Count;
}
}

27
Wabbajack.Server/Services/MirrorQueueService.cs
View File

@ -1,27 +0,0 @@
using System;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.Services;
public class MirrorQueueService : AbstractService<MirrorQueueService, int>
{
private DiscordWebHook _discord;
private readonly SqlService _sql;
public MirrorQueueService(ILogger<MirrorQueueService> logger, AppSettings settings, QuickSync quickSync,
DiscordWebHook discordWebHook, SqlService sqlService) :
base(logger, settings, quickSync, TimeSpan.FromMinutes(5))
{
_discord = discordWebHook;
_sql = sqlService;
}
public override async Task<int> Execute()
{
await _sql.QueueMirroredFiles();
return 1;
}
}

227
Wabbajack.Server/Services/MirrorUploader.cs
View File

@ -1,227 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
using FluentFTP.Helpers;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Networking.WabbajackClientApi;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.TokenProviders;
namespace Wabbajack.Server.Services;
public class MirrorUploader : AbstractService<MirrorUploader, int>
{
private readonly IFtpSiteCredentials _credentials;
private readonly DTOSerializer _dtos;
private readonly IFtpSiteCredentials _ftpCreds;
private readonly ParallelOptions _parallelOptions;
private readonly Client _wjClient;
private readonly ArchiveMaintainer _archives;
private readonly DiscordWebHook _discord;
private readonly SqlService _sql;
public MirrorUploader(ILogger<MirrorUploader> logger, AppSettings settings, SqlService sql, QuickSync quickSync,
ArchiveMaintainer archives,
DiscordWebHook discord, IFtpSiteCredentials credentials, Client wjClient, ParallelOptions parallelOptions,
DTOSerializer dtos,
IFtpSiteCredentials ftpCreds)
: base(logger, settings, quickSync, TimeSpan.FromHours(1))
{
_sql = sql;
_archives = archives;
_discord = discord;
_credentials = credentials;
_wjClient = wjClient;
_parallelOptions = parallelOptions;
_dtos = dtos;
_ftpCreds = ftpCreds;
}
public bool ActiveFileSyncEnabled { get; set; } = true;
public override async Task<int> Execute()
{
var uploaded = 0;
if (ActiveFileSyncEnabled)
await _sql.SyncActiveMirroredFiles();
TOP:
var toUpload = await _sql.GetNextMirroredFile();
if (toUpload == default)
{
await DeleteOldMirrorFiles();
return uploaded;
}
uploaded += 1;
try
{
var creds = (await _credentials.Get())[StorageSpace.Mirrors];
if (_archives.TryGetPath(toUpload.Hash, out var path))
{
_logger.LogInformation($"Uploading mirror file {toUpload.Hash} {path.Size().FileSizeToString()}");
var exists = false;
using (var client = await GetClient(creds))
{
exists = await client.FileExistsAsync($"{toUpload.Hash.ToHex()}/definition.json.gz");
}
if (exists)
{
_logger.LogInformation($"Skipping {toUpload.Hash} it's already on the server");
await toUpload.Finish(_sql);
goto TOP;
}
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Uploading {toUpload.Hash} - {toUpload.Created} because {toUpload.Rationale}"
});
var definition = await _wjClient.GenerateFileDefinition(path);
using (var client = await GetClient(creds))
{
await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}");
await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}/parts");
}
string MakePath(long idx)
{
return $"{definition.Hash.ToHex()}/parts/{idx}";
}
await definition.Parts.PDoAll(new Resource<MirrorUploader>(), async part =>
{
_logger.LogInformation("Uploading mirror part ({index}/{length})", part.Index,
definition.Parts.Length);
var buffer = new byte[part.Size];
await using (var fs = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read))
{
fs.Position = part.Offset;
await fs.ReadAsync(buffer);
}
await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () =>
{
using var client = await GetClient(creds);
var name = MakePath(part.Index);
await client.UploadAsync(new MemoryStream(buffer), name);
});
});
await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () =>
{
using var client = await GetClient(creds);
_logger.LogInformation("Finishing mirror upload");
await using var ms = new MemoryStream();
await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
{
await _dtos.Serialize(definition, gz);
}
ms.Position = 0;
var remoteName = $"{definition.Hash.ToHex()}/definition.json.gz";
await client.UploadAsync(ms, remoteName);
});
await toUpload.Finish(_sql);
}
else
{
await toUpload.Fail(_sql, "Archive not found");
}
}
catch (Exception ex)
{
_logger.LogInformation($"{toUpload.Created} {toUpload.Uploaded}");
_logger.LogError(ex, "Error uploading");
await toUpload.Fail(_sql, ex.ToString());
}
goto TOP;
}
private async Task<FtpClient> GetClient(FtpSite? creds = null)
{
return await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () =>
{
creds ??= (await _ftpCreds.Get())[StorageSpace.Mirrors];
var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password));
ftpClient.DataConnectionType = FtpDataConnectionType.EPSV;
await ftpClient.ConnectAsync();
return ftpClient;
});
}
/// <summary>
/// Gets a list of all the Mirrored file hashes that physically exist on the CDN (via FTP lookup)
/// </summary>
/// <returns></returns>
public async Task<HashSet<Hash>> GetHashesOnCDN()
{
using var ftpClient = await GetClient();
var serverFiles = await ftpClient.GetNameListingAsync("\\");
return serverFiles
.Select(f => ((RelativePath) f).FileName)
.Select(l =>
{
try
{
return Hash.FromHex((string) l);
}
catch (Exception)
{
return default;
}
})
.Where(h => h != default)
.ToHashSet();
}
public async Task DeleteOldMirrorFiles()
{
var existingHashes = await GetHashesOnCDN();
var fromSql = await _sql.GetAllMirroredHashes();
foreach (var (hash, _) in fromSql.Where(s => s.Value))
{
_logger.LogInformation("Removing {hash} from SQL it's no longer in the CDN", hash);
if (!existingHashes.Contains(hash))
await _sql.DeleteMirroredFile(hash);
}
var toDelete = existingHashes.Where(h => !fromSql.ContainsKey(h)).ToArray();
using var client = await GetClient();
foreach (var hash in toDelete)
{
await _discord.Send(Channel.Spam,
new DiscordMessage {Content = $"Removing mirrored file {hash}, as it's no longer in sql"});
_logger.LogInformation("Removing {hash} from the CDN it's no longer in SQL", hash);
await client.DeleteDirectoryAsync(hash.ToHex());
}
}
}

140
Wabbajack.Server/Services/ModListDownloader.cs
View File

@ -1,140 +0,0 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Downloaders;
using Wabbajack.DTOs;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Installer;
using Wabbajack.Networking.WabbajackClientApi;
using Wabbajack.Paths.IO;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services;
public class ModListDownloader : AbstractService<ModListDownloader, int>
{
private readonly DownloadDispatcher _dispatcher;
private readonly DTOSerializer _dtos;
private readonly TemporaryFileManager _manager;
private readonly Client _wjClient;
private readonly DiscordWebHook _discord;
private readonly ArchiveMaintainer _maintainer;
private readonly SqlService _sql;
public ModListDownloader(ILogger<ModListDownloader> logger, AppSettings settings, ArchiveMaintainer maintainer,
SqlService sql, DiscordWebHook discord, QuickSync quickSync, Client wjClient, TemporaryFileManager manager,
DownloadDispatcher dispatcher, DTOSerializer dtos)
: base(logger, settings, quickSync, TimeSpan.FromMinutes(1))
{
_logger = logger;
_settings = settings;
_maintainer = maintainer;
_sql = sql;
_discord = discord;
_wjClient = wjClient;
_manager = manager;
_dispatcher = dispatcher;
_dtos = dtos;
}
public override async Task<int> Execute()
{
var downloaded = 0;
var lists = await _wjClient.LoadLists();
foreach (var list in lists)
try
{
ReportStarting(list.Links.MachineURL);
if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
continue;
if (!_maintainer.HaveArchive(list.DownloadMetadata!.Hash))
{
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
var tf = _manager.CreateFile();
var state = _dispatcher.Parse(new Uri(list.Links.Download));
if (state == null)
{
_logger.Log(LogLevel.Error,
$"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}");
continue;
}
downloaded += 1;
await _dispatcher.Download(new Archive {State = state, Name = $"{list.Links.MachineURL}.wabbajack"},
tf.Path, CancellationToken.None);
var hash = await tf.Path.Hash();
if (hash != list.DownloadMetadata.Hash)
{
_logger.Log(LogLevel.Error,
$"Downloaded modlist {list.Links.MachineURL} {list.DownloadMetadata.Hash} didn't match metadata hash of {hash}");
await _sql.IngestModList(list.DownloadMetadata.Hash, list, new ModList(), true);
continue;
}
await _maintainer.Ingest(tf.Path);
}
_maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath);
ModList modlist;
modlist = await StandardInstaller.LoadFromFile(_dtos, modlistPath);
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content = $"Ingesting {list.Links.MachineURL} version {modlist.Version}"
});
await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist, false);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash} - {ex.Message}"
});
}
finally
{
ReportEnding(list.Links.MachineURL);
}
_logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");
if (downloaded > 0)
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"Downloaded {downloaded} new lists"});
var fc = await _sql.EnqueueModListFilesForIndexing();
_logger.Log(LogLevel.Information, $"Enqueing {fc} files for downloading");
if (fc > 0)
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"Enqueing {fc} files for downloading"});
return downloaded;
}
}
public static class ModListDownloaderExtensions
{
public static void UseModListDownloader(this IApplicationBuilder b)
{
var poll = (ModListDownloader) b.ApplicationServices.GetService(typeof(ModListDownloader));
poll.Start();
}
}

97
Wabbajack.Server/Services/NexusPoll.cs
View File

@ -1,97 +0,0 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.DTOs;
using Wabbajack.Networking.NexusApi;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.Services;
public class NexusPoll
{
private readonly NexusApi _api;
private readonly ParallelOptions _parallelOptions;
private readonly GlobalInformation _globalInformation;
private readonly ILogger<NexusPoll> _logger;
private readonly AppSettings _settings;
private readonly SqlService _sql;
public NexusPoll(ILogger<NexusPoll> logger, AppSettings settings, SqlService service,
GlobalInformation globalInformation, ParallelOptions parallelOptions, NexusApi api)
{
_sql = service;
_settings = settings;
_globalInformation = globalInformation;
_logger = logger;
_parallelOptions = parallelOptions;
_api = api;
}
public async Task UpdateNexusCacheAPI(CancellationToken token)
{
using var _ = _logger.BeginScope("Nexus Update via API");
_logger.Log(LogLevel.Information, "Starting Nexus Update via API");
var purged = await GameRegistry.Games.Values
.Where(game => game.NexusName != null)
.SelectMany(async game =>
{
var (mods, _) = await _api.GetUpdates(game.Game, token);
return mods.Select(mod => new {Game = game, Mod = mod});
})
.Select(async row =>
{
var a = row.Mod.LatestFileUpdate.AsUnixTime();
// Mod activity could hide files
var b = row.Mod.LastestModActivity.AsUnixTime();
var t = a > b ? a : b;
long purgeCount = 0;
purgeCount += await _sql.DeleteNexusModInfosUpdatedBeforeDate(row.Game.Game, row.Mod.ModId, t.Date);
purgeCount += await _sql.DeleteNexusModFilesUpdatedBeforeDate(row.Game.Game, row.Mod.ModId, t.Date);
return purgeCount;
})
.SumAsync(x => x);
_logger.Log(LogLevel.Information, "Purged {count} cache entries", purged);
_globalInformation.LastNexusSyncUTC = DateTime.UtcNow;
}
public void Start()
{
if (!_settings.RunBackEndJobs) return;
Task.Run(async () =>
{
while (true)
{
try
{
await UpdateNexusCacheAPI(CancellationToken.None);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting API feed from Nexus");
}
await Task.Delay(_globalInformation.NexusAPIPollRate);
}
});
}
}
public static class NexusPollExtensions
{
public static void UseNexusPoll(this IApplicationBuilder b)
{
var poll = (NexusPoll) b.ApplicationServices.GetService(typeof(NexusPoll));
poll.Start();
}
}

85
Wabbajack.Server/Services/NonNexusDownloadValidator.cs
View File

@ -1,85 +0,0 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Downloaders;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.Services;
public class NonNexusDownloadValidator : AbstractService<NonNexusDownloadValidator, int>
{
private readonly DownloadDispatcher _dispatcher;
private readonly ParallelOptions _parallelOptions;
private readonly SqlService _sql;
public NonNexusDownloadValidator(ILogger<NonNexusDownloadValidator> logger, AppSettings settings, SqlService sql,
QuickSync quickSync, DownloadDispatcher dispatcher, ParallelOptions parallelOptions)
: base(logger, settings, quickSync, TimeSpan.FromHours(2))
{
_sql = sql;
_dispatcher = dispatcher;
_parallelOptions = parallelOptions;
}
public override async Task<int> Execute()
{
var archives = await _sql.GetNonNexusModlistArchives();
_logger.Log(LogLevel.Information, $"Validating {archives.Count} non-Nexus archives");
await _dispatcher.PrepareAll(archives.Select(a => a.State));
var random = new Random();
/*
var results = await archives.PMap(_parallelOptions, async archive =>
{
try
{
await Task.Delay(random.Next(1000, 5000));
var token = new CancellationTokenSource();
token.CancelAfter(TimeSpan.FromMinutes(10));
ReportStarting(archive.State.PrimaryKeyString);
bool isValid = false;
switch (archive.State)
{
//case WabbajackCDNDownloader.State _:
//case GoogleDriveDownloader.State _: // Let's try validating Google again 2/10/2021
case GameFileSource _:
isValid = true;
break;
case Manual _:
case ModDB _:
case Http h when h.Url.ToString().StartsWith("https://wabbajack"):
isValid = true;
break;
default:
isValid = await _dispatcher.Verify(archive, token.Token);
break;
}
return (Archive: archive, IsValid: isValid);
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Error for {archive.Name} {archive.State.PrimaryKeyString} {ex}");
return (Archive: archive, IsValid: false);
}
finally
{
ReportEnding(archive.State.PrimaryKeyString);
}
}).ToList();
await _sql.UpdateNonNexusModlistArchivesStatus(results);
var failed = results.Count(r => !r.IsValid);
var passed = results.Count() - failed;
foreach(var (archive, _) in results.Where(f => !f.IsValid))
_logger.Log(LogLevel.Warning, $"Validation failed for {archive.Name} from {archive.State.PrimaryKeyString}");
_logger.Log(LogLevel.Information, $"Non-nexus validation completed {failed} out of {passed} failed");
*/
return default;
}
}

237
Wabbajack.Server/Services/PatchBuilder.cs
View File

@ -1,237 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
using FluentFTP.Helpers;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Compiler.PatchCache;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.TokenProviders;
namespace Wabbajack.Server.Services;
public class PatchBuilder : AbstractService<PatchBuilder, int>
{
private readonly IFtpSiteCredentials _ftpCreds;
private readonly TemporaryFileManager _manager;
private readonly DiscordWebHook _discordWebHook;
private readonly ArchiveMaintainer _maintainer;
private readonly SqlService _sql;
public PatchBuilder(ILogger<PatchBuilder> logger, SqlService sql, AppSettings settings,
ArchiveMaintainer maintainer,
DiscordWebHook discordWebHook, QuickSync quickSync, TemporaryFileManager manager, IFtpSiteCredentials ftpCreds)
: base(logger, settings, quickSync, TimeSpan.FromMinutes(1))
{
_discordWebHook = discordWebHook;
_sql = sql;
_maintainer = maintainer;
_manager = manager;
_ftpCreds = ftpCreds;
}
public bool NoCleaning { get; set; }
public override async Task<int> Execute()
{
var count = 0;
while (true)
{
count++;
var patch = await _sql.GetPendingPatch();
if (patch == default) break;
try
{
_logger.LogInformation(
$"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}");
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
if (patch.Src.Archive.Hash == patch.Dest.Archive.Hash && patch.Src.Archive.State.PrimaryKeyString ==
patch.Dest.Archive.State.PrimaryKeyString)
{
await patch.Fail(_sql, "Hashes match");
continue;
}
if (patch.Src.Archive.Size > 2_500_000_000 || patch.Dest.Archive.Size > 2_500_000_000)
{
await patch.Fail(_sql, "Too large to patch");
continue;
}
_maintainer.TryGetPath(patch.Src.Archive.Hash, out var srcPath);
_maintainer.TryGetPath(patch.Dest.Archive.Hash, out var destPath);
await using var sigFile = _manager.CreateFile();
await using var patchFile = _manager.CreateFile();
await using var srcStream = srcPath.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
await using var destStream = destPath.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
await using var sigStream = sigFile.Path.Open(FileMode.Create, FileAccess.ReadWrite);
await using var patchOutput = patchFile.Path.Open(FileMode.Create, FileAccess.ReadWrite);
OctoDiff.Create(destStream, srcStream, sigStream, patchOutput);
await patchOutput.DisposeAsync();
var size = patchFile.Path.Size();
await UploadToCDN(patchFile.Path, PatchName(patch));
await patch.Finish(_sql, size);
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Built {size.ToFileSizeString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
}
catch (Exception ex)
{
_logger.LogError(ex, "Error while building patch");
await patch.Fail(_sql, ex.ToString());
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Failure building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
}
}
if (count > 0)
{
}
if (!NoCleaning)
await CleanupOldPatches();
return count;
}
private static string PatchName(Patch patch)
{
return PatchName(patch.Src.Archive.Hash, patch.Dest.Archive.Hash);
}
private static string PatchName(Hash oldHash, Hash newHash)
{
return $"{oldHash.ToHex()}_{newHash.ToHex()}";
}
private async Task CleanupOldPatches()
{
var patches = await _sql.GetOldPatches();
using var client = await GetBunnyCdnFtpClient();
foreach (var patch in patches)
{
_logger.LogInformation($"Cleaning patch {patch.Src.Archive.Hash} -> {patch.Dest.Archive.Hash}");
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Removing {patch.PatchSize.FileSizeToString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString} due it no longer being required by curated lists"
});
if (!await DeleteFromCDN(client, PatchName(patch)))
_logger.LogWarning($"Patch file didn't exist {PatchName(patch)}");
await _sql.DeletePatch(patch);
var pendingPatch = await _sql.GetPendingPatch();
if (pendingPatch != default) break;
}
var files = await client.GetListingAsync("\\");
_logger.LogInformation($"Found {files.Length} on the CDN");
var sqlFiles = await _sql.AllPatchHashes();
_logger.LogInformation($"Found {sqlFiles.Count} in SQL");
HashSet<(Hash, Hash)> NamesToPairs(IEnumerable<FtpListItem> ftpFiles)
{
return ftpFiles.Select(f => f.Name).Where(f => f.Contains("_")).Select(p =>
{
try
{
var lst = p.Split("_", StringSplitOptions.RemoveEmptyEntries).Select(Hash.FromHex).ToArray();
return (lst[0], lst[1]);
}
catch (ArgumentException)
{
return default;
}
catch (FormatException)
{
return default;
}
}).Where(f => f != default).ToHashSet();
}
var oldHashPairs = NamesToPairs(files.Where(f => DateTime.UtcNow - f.Modified > TimeSpan.FromDays(2)));
foreach (var (oldHash, newHash) in oldHashPairs.Where(o => !sqlFiles.Contains(o)))
{
_logger.LogInformation($"Removing CDN File entry for {oldHash} -> {newHash} it's not SQL");
await client.DeleteFileAsync(PatchName(oldHash, newHash));
}
var hashPairs = NamesToPairs(files);
foreach (var sqlFile in sqlFiles.Where(s => !hashPairs.Contains(s)))
{
_logger.LogInformation("Removing SQL File entry for {from} -> {to} it's not on the CDN", sqlFile.Item1,
sqlFile.Item2);
await _sql.DeletePatchesForHashPair(sqlFile);
}
}
private async Task UploadToCDN(AbsolutePath patchFile, string patchName)
{
for (var times = 0; times < 5; times++)
try
{
_logger.Log(LogLevel.Information,
$"Uploading {patchFile.Size().ToFileSizeString()} patch file to CDN {patchName}");
using var client = await GetBunnyCdnFtpClient();
await client.UploadFileAsync(patchFile.ToString(), patchName);
return;
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error uploading {patchFile} to CDN");
}
_logger.Log(LogLevel.Error, $"Couldn't upload {patchFile} to {patchName}");
}
private async Task<bool> DeleteFromCDN(FtpClient client, string patchName)
{
if (!await client.FileExistsAsync(patchName))
return false;
await client.DeleteFileAsync(patchName);
return true;
}
private async Task<FtpClient> GetBunnyCdnFtpClient()
{
var info = (await _ftpCreds.Get())[StorageSpace.Patches];
var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
await client.ConnectAsync();
return client;
}
}

32
Wabbajack.Server/Startup.cs
View File

@ -1,5 +1,6 @@
using System.Diagnostics;
using System.IO;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
@ -12,7 +13,8 @@ using Microsoft.Extensions.FileProviders;
using Microsoft.Extensions.Hosting;
using Newtonsoft.Json;
using Wabbajack.BuildServer;
using Wabbajack.Server.DataLayer;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Server.DataModels;
using Wabbajack.Server.Services;
namespace Wabbajack.Server;
@ -51,20 +53,15 @@ public class Startup
services.AddSingleton<AppSettings>();
services.AddSingleton<QuickSync>();
services.AddSingleton<SqlService>();
services.AddSingleton<GlobalInformation>();
services.AddSingleton<NexusPoll>();
services.AddSingleton<ArchiveMaintainer>();
services.AddSingleton<ModListDownloader>();
services.AddSingleton<NonNexusDownloadValidator>();
services.AddSingleton<ArchiveDownloader>();
services.AddSingleton<DiscordWebHook>();
services.AddSingleton<PatchBuilder>();
services.AddSingleton<MirrorUploader>();
services.AddSingleton<MirrorQueueService>();
services.AddSingleton<Watchdog>();
services.AddSingleton<DiscordFrontend>();
services.AddSingleton<MetricsKeyCache>();
services.AddSingleton<Metrics>();
services.AddSingleton<HttpClient>();
services.AddSingleton<AuthorFiles>();
services.AddSingleton<AuthorKeys>();
services.AddDTOSerializer();
services.AddDTOConverters();
services.AddResponseCompression(options =>
{
options.Providers.Add<BrotliCompressionProvider>();
@ -82,9 +79,6 @@ public class Startup
{
if (env.IsDevelopment()) app.UseDeveloperExceptionPage();
if (this is not TestStartup)
app.UseHttpsRedirection();
app.UseDeveloperExceptionPage();
var provider = new FileExtensionContentTypeProvider();
@ -98,18 +92,10 @@ public class Startup
app.UseAuthentication();
app.UseAuthorization();
app.UseNexusPoll();
app.UseModListDownloader();
app.UseResponseCompression();
app.UseService<NonNexusDownloadValidator>();
app.UseService<ArchiveDownloader>();
app.UseService<DiscordWebHook>();
app.UseService<PatchBuilder>();
app.UseService<MirrorUploader>();
app.UseService<Watchdog>();
app.UseService<DiscordFrontend>();
app.UseService<MetricsKeyCache>();
app.Use(next =>
{

9
Wabbajack.Server/TokenProviders/IFtpSiteCredentials.cs
View File

@ -1,9 +0,0 @@
using System.Collections.Generic;
using Wabbajack.Networking.Http.Interfaces;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.TokenProviders;
public interface IFtpSiteCredentials : ITokenProvider<Dictionary<StorageSpace, FtpSite>>
{
}

4
Wabbajack.Server/Wabbajack.Server.csproj
View File

@ -54,6 +54,10 @@
<ItemGroup>
<Compile Remove="Controllers\UploadedFiles.cs" />
<Compile Remove="Services\ListValidator.cs" />
<Compile Remove="Controllers\ModFiles.cs" />
<Compile Remove="Controllers\Users.cs" />
<Compile Remove="Controllers\NexusCache.cs" />
<Compile Remove="Controllers\AuthorControls.cs" />
</ItemGroup>
<ItemGroup>

13
Wabbajack.Server/appsettings.json
View File

@ -7,15 +7,10 @@
}
},
"WabbajackSettings": {
"DownloadDir": "c:\\tmp\\downloads",
"ArchiveDir": "w:\\archives",
"TempFolder": "c:\\tmp",
"JobRunner": true,
"JobScheduler": false,
"RunFrontEndJobs": true,
"RunBackEndJobs": false,
"BunnyCDN_StorageZone": "wabbajacktest",
"SQLConnection": "Data Source=.\\SQLEXPRESS;Integrated Security=True;Initial Catalog=wabbajack_prod;MultipleActiveResultSets=true"
"TempFolder": "c:\\tmp\\server_temp",
"MetricsFolder": "c:\\tmp\\server_metrics",
"AuthoredFilesFolder": "c:\\tmp\\server_authored_files",
"AuthorAPIKeyFile": "c:\\tmp\\author_keys.txt"
},
"AllowedHosts": "*"
}