From 6c9f6ab5c0597e3612149819ce67533b11668dc2 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 16 Feb 2021 22:46:05 -0700 Subject: [PATCH 1/6] latest server fixes --- Wabbajack.CLI/OptionsDefinition.cs | 3 +- Wabbajack.CLI/Verbs/StressTestURL.cs | 52 +++++ .../Downloaders/AbstractIPS4Downloader.cs | 4 +- .../Downloaders/GoogleDriveDownloader.cs | 14 +- Wabbajack.Lib/Downloaders/HTTPDownloader.cs | 186 +++++++++--------- Wabbajack.Lib/Http/Client.cs | 4 +- Wabbajack.Lib/Http/ClientFactory.cs | 1 + Wabbajack.Lib/Metrics.cs | 4 +- Wabbajack.Server.Test/MetricsTests.cs | 12 +- .../ApiKeyAuthorizationHandler.cs | 18 +- Wabbajack.Server/Controllers/Heartbeat.cs | 3 + Wabbajack.Server/Controllers/Metrics.cs | 87 +++++++- .../Templates/TotalListTemplate.html | 21 ++ Wabbajack.Server/DataLayer/Metrics.cs | 67 +++++++ .../Services/NexusKeyMaintainance.cs | 23 ++- .../Services/NonNexusDownloadValidator.cs | 4 +- Wabbajack.Server/Startup.cs | 8 + Wabbajack.Server/Wabbajack.Server.csproj | 2 + 18 files changed, 391 insertions(+), 122 deletions(-) create mode 100644 Wabbajack.CLI/Verbs/StressTestURL.cs create mode 100644 Wabbajack.Server/Controllers/Templates/TotalListTemplate.html diff --git a/Wabbajack.CLI/OptionsDefinition.cs b/Wabbajack.CLI/OptionsDefinition.cs index c2d93bec..fdeca176 100644 --- a/Wabbajack.CLI/OptionsDefinition.cs +++ b/Wabbajack.CLI/OptionsDefinition.cs @@ -38,7 +38,8 @@ namespace Wabbajack.CLI typeof(PurgeArchive), typeof(AllKnownDownloadStates), typeof(VerifyAllDownloads), - typeof(HashBenchmark) + typeof(HashBenchmark), + typeof(StressTestURL) }; } } diff --git a/Wabbajack.CLI/Verbs/StressTestURL.cs b/Wabbajack.CLI/Verbs/StressTestURL.cs new file mode 100644 index 00000000..f59bd307 --- /dev/null +++ b/Wabbajack.CLI/Verbs/StressTestURL.cs @@ -0,0 +1,52 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("stress-test-url", HelpText = "Verify a file rapidly, trying to make it fail")] + public class StressTestURL : AVerb + { + [Option('i', "input", Required = true, HelpText = "Input url to stress")] + public string Input { get; set; } = ""; + private Uri _Input => new Uri(Input); + + protected override async Task Run() + { + using var queue = new WorkQueue(); + var state = await DownloadDispatcher.Infer(_Input); + if (state == null) + { + Console.WriteLine("Could not parse URL"); + } + + Console.WriteLine("Performing initial download"); + await using var temp = new TempFile(); + var archive = new Archive(state!); + if (!await state!.Download(archive, temp.Path)) + { + Console.WriteLine("Failed initial download"); + } + + var hash = await temp.Path.FileHashAsync(); + archive.Hash = hash!.Value; + archive.Size = temp.Path.Size; + Console.WriteLine($"Hash: {archive.Hash} Size: {archive.Size.ToFileSizeString()}"); + + await Enumerable.Range(0, 100000) + .PMap(queue, async idx => + { + if (!await state.Verify(archive)) + { + throw new Exception($"{idx} Verification failed"); + } + Console.WriteLine($"{idx} Verification passed"); + }); + return ExitCode.Ok; + } + } +} diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs index ba83827c..364ae686 100644 --- a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs @@ -400,7 +400,7 @@ namespace Wabbajack.Lib.Downloaders public async Task GetStringAsync(Uri uri, CancellationToken? token = null) { if (!Downloader.IsCloudFlareProtected) - return await Downloader.AuthedClient.GetStringAsync(uri); + return await Downloader.AuthedClient.GetStringAsync(uri, token); using var driver = await Downloader.GetAuthedDriver(); @@ -448,7 +448,7 @@ namespace Wabbajack.Lib.Downloaders return await Downloader.AuthedClient.GetAsync(uri); using var driver = await Downloader.GetAuthedDriver(); - TaskCompletionSource promise = new TaskCompletionSource(); + TaskCompletionSource promise = new(); driver.DownloadHandler = uri1 => { promise.SetResult(uri); diff --git a/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs b/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs index 68ae9aec..91b31129 100644 --- a/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs @@ -1,4 +1,5 @@ -using System.Text.RegularExpressions; +using System.Linq; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json; @@ -54,10 +55,12 @@ namespace Wabbajack.Lib.Downloaders public override async Task Download(Archive a, AbsolutePath destination) { var state = await ToHttpState(); + if (state == null) + return false; return await state.Download(a, destination); } - private async Task ToHttpState() + private async Task ToHttpState() { var initialURL = $"https://drive.google.com/uc?id={Id}&export=download"; var client = new Wabbajack.Lib.Http.Client(); @@ -65,7 +68,10 @@ namespace Wabbajack.Lib.Downloaders if (!response.IsSuccessStatusCode) throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown"); var regex = new Regex("(?<=/uc\\?export=download&confirm=).*(?=;id=)"); - var confirm = regex.Match(await response.Content.ReadAsStringAsync()); + using var content = response.Content; + var confirm = regex.Match(await content.ReadAsStringAsync()); + if (!confirm.Success) + return null; var url = $"https://drive.google.com/uc?export=download&confirm={confirm}&id={Id}"; var httpState = new HTTPDownloader.State(url) { Client = client }; return httpState; @@ -74,6 +80,8 @@ namespace Wabbajack.Lib.Downloaders public override async Task Verify(Archive a, CancellationToken? token) { var state = await ToHttpState(); + if (state == null) + return false; return await state.Verify(a, token); } diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index a1625aa8..d40a3465 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -82,111 +82,119 @@ namespace Wabbajack.Lib.Downloaders destination.Parent.CreateDirectory(); } - using (var fs = download ? await destination.Create() : null) + await using var fs = download ? await destination.Create() : null; + var client = Client ?? await ClientAPI.GetClient(); + client.Headers.Add(("User-Agent", Consts.UserAgent)); + + foreach (var header in Headers) { - var client = Client ?? await ClientAPI.GetClient(); - client.Headers.Add(("User-Agent", Consts.UserAgent)); + var idx = header.IndexOf(':'); + var k = header.Substring(0, idx); + var v = header.Substring(idx + 1); + client.Headers.Add((k, v)); + } - foreach (var header in Headers) + long totalRead = 0; + const int bufferSize = 1024 * 32 * 8; + + Utils.Status($"Starting Download {a.Name ?? Url}", Percent.Zero); + var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false, token:token); + TOP: + + if (!response.IsSuccessStatusCode) + { + response.Dispose(); + return false; + } + + Stream stream; + try + { + stream = await response.Content.ReadAsStreamAsync(); + } + catch (Exception ex) + { + Utils.Error(ex, $"While downloading {Url}"); + return false; + } + + + var headerVar = a.Size == 0 ? "1" : a.Size.ToString(); + long headerContentSize = 0; + if (response.Content.Headers.Contains("Content-Length")) + { + headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault(); + if (headerVar != null) + long.TryParse(headerVar, out headerContentSize); + } + + if (!download) + { + await stream.DisposeAsync(); + response.Dispose(); + if (a.Size != 0 && headerContentSize != 0) + return a.Size == headerContentSize; + return true; + } + + var supportsResume = response.Headers.AcceptRanges.FirstOrDefault(f => f == "bytes") != null; + + var contentSize = headerVar != null ? long.Parse(headerVar) : 1; + + await using (var webs = stream) + { + var buffer = new byte[bufferSize]; + int readThisCycle = 0; + + while (!(token ?? CancellationToken.None).IsCancellationRequested) { - var idx = header.IndexOf(':'); - var k = header.Substring(0, idx); - var v = header.Substring(idx + 1); - client.Headers.Add((k, v)); - } - - long totalRead = 0; - var bufferSize = 1024 * 32 * 8; - - Utils.Status($"Starting Download {a.Name ?? Url}", Percent.Zero); - var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false, token:token); -TOP: - - if (!response.IsSuccessStatusCode) - { - return false; - } - - Stream stream; - try - { - stream = await response.Content.ReadAsStreamAsync(); - } - catch (Exception ex) - { - Utils.Error(ex, $"While downloading {Url}"); - return false; - } - - - var headerVar = a.Size == 0 ? "1" : a.Size.ToString(); - long header_content_size = 0; - if (response.Content.Headers.Contains("Content-Length")) - { - headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault(); - if (headerVar != null) - long.TryParse(headerVar, out header_content_size); - } - - if (!download) - { - if (a.Size != 0 && header_content_size != 0) - return a.Size == header_content_size; - return true; - } - - var supportsResume = response.Headers.AcceptRanges.FirstOrDefault(f => f == "bytes") != null; - - var contentSize = headerVar != null ? long.Parse(headerVar) : 1; - - using (var webs = stream) - { - var buffer = new byte[bufferSize]; - int readThisCycle = 0; - - while (!(token ?? CancellationToken.None).IsCancellationRequested) + int read = 0; + try { - int read = 0; - try + read = await webs.ReadAsync(buffer, 0, bufferSize); + } + catch (Exception) + { + if (readThisCycle == 0) { - read = await webs.ReadAsync(buffer, 0, bufferSize); + await stream.DisposeAsync(); + response.Dispose(); + throw; } - catch (Exception) + + if (totalRead < contentSize) { - if (readThisCycle == 0) - throw; - - if (totalRead < contentSize) + if (!supportsResume) { - if (supportsResume) - { - Utils.Log( - $"Abort during download, trying to resume {Url} from {totalRead.ToFileSizeString()}"); - - var msg = new HttpRequestMessage(HttpMethod.Get, Url); - msg.Headers.Range = new RangeHeaderValue(totalRead, null); - response.Dispose(); - response = await client.SendAsync(msg); - goto TOP; - } + await stream.DisposeAsync(); + response.Dispose(); throw; } - break; + Utils.Log( + $"Abort during download, trying to resume {Url} from {totalRead.ToFileSizeString()}"); + + var msg = new HttpRequestMessage(HttpMethod.Get, Url); + msg.Headers.Range = new RangeHeaderValue(totalRead, null); + response.Dispose(); + response = await client.SendAsync(msg); + goto TOP; } - readThisCycle += read; - - if (read == 0) break; - Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(totalRead, contentSize)); - - fs!.Write(buffer, 0, read); - totalRead += read; + break; } + + readThisCycle += read; + + if (read == 0) break; + Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(totalRead, contentSize)); + + fs!.Write(buffer, 0, read); + totalRead += read; } - response.Dispose(); - return true; } + response.Dispose(); + return true; } public override async Task Verify(Archive a, CancellationToken? token) diff --git a/Wabbajack.Lib/Http/Client.cs b/Wabbajack.Lib/Http/Client.cs index 3fdf2613..8cafb129 100644 --- a/Wabbajack.Lib/Http/Client.cs +++ b/Wabbajack.Lib/Http/Client.cs @@ -48,10 +48,10 @@ namespace Wabbajack.Lib.Http return await SendStringAsync(request, token: token); } - public async Task GetStringAsync(Uri url) + public async Task GetStringAsync(Uri url, CancellationToken? token = null) { var request = new HttpRequestMessage(HttpMethod.Get, url); - return await SendStringAsync(request); + return await SendStringAsync(request, token: token); } public async Task DeleteStringAsync(string url) diff --git a/Wabbajack.Lib/Http/ClientFactory.cs b/Wabbajack.Lib/Http/ClientFactory.cs index 71fb5aea..07479532 100644 --- a/Wabbajack.Lib/Http/ClientFactory.cs +++ b/Wabbajack.Lib/Http/ClientFactory.cs @@ -22,6 +22,7 @@ namespace Wabbajack.Lib.Http MaxConnectionsPerServer = 20, PooledConnectionLifetime = TimeSpan.FromMilliseconds(100), PooledConnectionIdleTimeout = TimeSpan.FromMilliseconds(100), + AutomaticDecompression = DecompressionMethods.All }; Utils.Log($"Configuring with SSL {_socketsHandler.SslOptions.EnabledSslProtocols}"); diff --git a/Wabbajack.Lib/Metrics.cs b/Wabbajack.Lib/Metrics.cs index 3faafc59..2e48079f 100644 --- a/Wabbajack.Lib/Metrics.cs +++ b/Wabbajack.Lib/Metrics.cs @@ -69,7 +69,7 @@ namespace Wabbajack.Lib /// /// /// - public static async Task Send(string action, string value) + public static async Task Send(string action, string subject) { if (BuildServerStatus.IsBuildServerDown) return; @@ -78,7 +78,7 @@ namespace Wabbajack.Lib Utils.Log($"File hash check (-42) {key}"); var client = new Http.Client(); client.Headers.Add((Consts.MetricsKeyHeader, key)); - await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{value}"); + await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{subject}"); } public static async Task Error(Type type, Exception exception) diff --git a/Wabbajack.Server.Test/MetricsTests.cs b/Wabbajack.Server.Test/MetricsTests.cs index e5dd2af5..130e7776 100644 --- a/Wabbajack.Server.Test/MetricsTests.cs +++ b/Wabbajack.Server.Test/MetricsTests.cs @@ -34,7 +34,17 @@ namespace Wabbajack.BuildServer.Test using var response = await _client.GetAsync(MakeURL($"metrics/report/{action}")); Assert.Equal(TimeSpan.FromHours(1), response.Headers.CacheControl.MaxAge); // we'll just make sure this doesn't error, with limited data that's about all we can do atm - + + using var totalInstalls = await _client.GetAsync(MakeURL($"metrics/total_installs.html")); + Assert.True(totalInstalls.IsSuccessStatusCode); + + using var totalUniqueInstalls = await _client.GetAsync(MakeURL($"metrics/total_unique_installs.html")); + Assert.True(totalUniqueInstalls.IsSuccessStatusCode); + + using var dumpResponse = await _client.GetAsync(MakeURL("metrics/dump.json")); + Assert.True(dumpResponse.IsSuccessStatusCode); + var data = await dumpResponse.Content.ReadAsStringAsync(); + Assert.NotEmpty(data); } } } diff --git a/Wabbajack.Server/ApiKeyAuthorizationHandler.cs b/Wabbajack.Server/ApiKeyAuthorizationHandler.cs index e55700a5..2f36179d 100644 --- a/Wabbajack.Server/ApiKeyAuthorizationHandler.cs +++ b/Wabbajack.Server/ApiKeyAuthorizationHandler.cs @@ -29,7 +29,6 @@ namespace Wabbajack.BuildServer { private const string ProblemDetailsContentType = "application/problem+json"; private readonly SqlService _sql; - private static ConcurrentHashSet _knownKeys = new(); private const string ApiKeyHeaderName = "X-Api-Key"; public ApiKeyAuthenticationHandler( @@ -51,19 +50,25 @@ namespace Wabbajack.BuildServer { if (await _sql.IsTarKey(metricsKey)) { - await _sql.IngestMetric(new Metric {Action = "TarKey", Subject = "Auth", MetricsKey = metricsKey, Timestamp = DateTime.UtcNow}); + await _sql.IngestMetric(new Metric + { + Action = "TarKey", + Subject = "Auth", + MetricsKey = metricsKey, + Timestamp = DateTime.UtcNow + }); await Task.Delay(TimeSpan.FromSeconds(60)); throw new Exception("Error, lipsum timeout of the cross distant cloud."); } } var authorKey = Request.Headers[ApiKeyHeaderName].FirstOrDefault(); - + if (authorKey == null && metricsKey == null) { return AuthenticateResult.NoResult(); } - + if (authorKey != null) { @@ -84,14 +89,12 @@ namespace Wabbajack.BuildServer return AuthenticateResult.Success(ticket); } - if (!_knownKeys.Contains(metricsKey) && !await _sql.ValidMetricsKey(metricsKey)) + if (!await _sql.ValidMetricsKey(metricsKey)) { return AuthenticateResult.Fail("Invalid Metrics Key"); } else { - _knownKeys.Add(metricsKey); - var claims = new List {new(ClaimTypes.Role, "User")}; @@ -102,7 +105,6 @@ namespace Wabbajack.BuildServer return AuthenticateResult.Success(ticket); } - } [JsonName("RequestLog")] diff --git a/Wabbajack.Server/Controllers/Heartbeat.cs b/Wabbajack.Server/Controllers/Heartbeat.cs index 2ed43793..2f86afa8 100644 --- a/Wabbajack.Server/Controllers/Heartbeat.cs +++ b/Wabbajack.Server/Controllers/Heartbeat.cs @@ -2,10 +2,13 @@ using System.Collections.Generic; using System.Linq; using System.Net; +using System.Reflection; +using System.Text; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Nettle; +using Wabbajack.Common; using Wabbajack.Common.StatusFeed; using Wabbajack.Server; using Wabbajack.Server.DataLayer; diff --git a/Wabbajack.Server/Controllers/Metrics.cs b/Wabbajack.Server/Controllers/Metrics.cs index 62963a97..9f4fcd4b 100644 --- a/Wabbajack.Server/Controllers/Metrics.cs +++ b/Wabbajack.Server/Controllers/Metrics.cs @@ -1,9 +1,13 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.IO; using System.Linq; using System.Net; +using System.Reflection; +using System.Text; using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Nettle; @@ -34,6 +38,11 @@ namespace Wabbajack.BuildServer.Controllers public async Task LogMetricAsync(string subject, string value) { var date = DateTime.UtcNow; + + // Used in tests + if (value == "Default" || value == "untitled" || Guid.TryParse(value, out _)) + return new Result { Timestamp = date}; + await Log(date, subject, value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault()); return new Result { Timestamp = date}; } @@ -74,7 +83,7 @@ namespace Wabbajack.BuildServer.Controllers return Ok(results == 0 ? new Badge($"Modlist {name} not found!", "Error") {color = "red"} - : new Badge("Installations: ", $"{results}") {color = "green"}); + : new Badge("Installations: ", "____") {color = "green"}); } [HttpGet] @@ -87,7 +96,7 @@ namespace Wabbajack.BuildServer.Controllers return Ok(results == 0 ? new Badge($"Modlist {name} not found!", "Error") {color = "red"} - : new Badge("Installations: ", $"{results}"){color = "green"}) ; + : new Badge("Installations: ", "____"){color = "green"}) ; } private static readonly Func ReportTemplate = NettleEngine.GetCompiler().Compile(@" @@ -149,5 +158,79 @@ namespace Wabbajack.BuildServer.Controllers { public DateTime Timestamp { get; set; } } + + class TotalListTemplateData + { + public string Title { get; set; } + public long Total { get; set; } + public Item[] Items { get; set; } + + public class Item + { + public long Count { get; set; } + public string Title { get; set; } + } + } + + private static Func _totalListTemplate; + + private static Func TotalListTemplate + { + get + { + if (_totalListTemplate == null) + { + var resource = Assembly.GetExecutingAssembly() + .GetManifestResourceStream("Wabbajack.Server.Controllers.Templates.TotalListTemplate.html")! + .ReadAll(); + _totalListTemplate = NettleEngine.GetCompiler().Compile(Encoding.UTF8.GetString(resource)); + } + + return _totalListTemplate; + } + } + + [HttpGet("total_installs.html")] + [ResponseCache(Duration = 60 * 60)] + public async Task TotalInstalls() + { + var data = await _sql.GetTotalInstalls(); + var result = TotalListTemplate(new TotalListTemplateData + { + Title = "Total Installs", + Total = data.Sum(d => d.Item2), + Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2}) + .ToArray() + }); + return new ContentResult { + ContentType = "text/html", + StatusCode = (int)HttpStatusCode.OK, + Content = result}; + } + + [HttpGet("total_unique_installs.html")] + [ResponseCache(Duration = 60 * 60)] + public async Task TotalUniqueInstalls() + { + var data = await _sql.GetTotalUniqueInstalls(); + var result = TotalListTemplate(new TotalListTemplateData + { + Title = "Total Unique Installs", + Total = data.Sum(d => d.Item2), + Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2}) + .ToArray() + }); + return new ContentResult { + ContentType = "text/html", + StatusCode = (int)HttpStatusCode.OK, + Content = result}; + } + + [HttpGet("dump.json")] + public async Task DataDump() + { + return Ok(await _sql.MetricsDump().ToArrayAsync()); + } + } } diff --git a/Wabbajack.Server/Controllers/Templates/TotalListTemplate.html b/Wabbajack.Server/Controllers/Templates/TotalListTemplate.html new file mode 100644 index 00000000..395af2ae --- /dev/null +++ b/Wabbajack.Server/Controllers/Templates/TotalListTemplate.html @@ -0,0 +1,21 @@ + + + + + Total Installs + + + +

{{$.Title}} - Total: {{$.Total}}

+ + + {{each $.Items }} + + + + + {{/each}} +
{{$.Count}}{{$.Title}}
+ + + \ No newline at end of file diff --git a/Wabbajack.Server/DataLayer/Metrics.cs b/Wabbajack.Server/DataLayer/Metrics.cs index 968286e3..750eec22 100644 --- a/Wabbajack.Server/DataLayer/Metrics.cs +++ b/Wabbajack.Server/DataLayer/Metrics.cs @@ -95,5 +95,72 @@ namespace Wabbajack.Server.DataLayer WHERE JSON_VALUE(Metadata, '$.links.machineURL') = @MachineURL)", new {MachineURL = machineUrl}); } + + public async Task> GetTotalInstalls() + { + await using var conn = await Open(); + return await conn.QueryAsync<(string, long)>( + @"SELECT GroupingSubject, Count(*) as Count + From dbo.Metrics + WHERE + + GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics + WHERE action = 'finish_install' + AND MetricsKey is not null) + group by GroupingSubject + order by Count(*) desc"); + } + + public async Task> GetTotalUniqueInstalls() + { + await using var conn = await Open(); + return await conn.QueryAsync<(string, long)>( + @"Select GroupingSubject, Count(*) as Count + FROM + (select DISTINCT MetricsKey, GroupingSubject + From dbo.Metrics + WHERE + GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics + WHERE action = 'finish_install' + AND MetricsKey is not null)) m + GROUP BY GroupingSubject + Order by Count(*) desc + "); + } + + public async IAsyncEnumerable MetricsDump() + { + var keys = new Dictionary(); + + await using var conn = await Open(); + foreach (var row in await conn.QueryAsync<(long, DateTime, string, string, string, string)>(@"select Id, Timestamp, Action, Subject, MetricsKey, GroupingSubject from dbo.metrics WHERE MetricsKey is not null")) + { + if (!keys.TryGetValue(row.Item5, out var keyid)) + { + keyid = keys.Count; + keys[row.Item5] = keyid; + } + + yield return new MetricRow + { + Id = row.Item1, + Timestamp = row.Item2, + Action = row.Item3, + Subject = row.Item4, + MetricsKey = keyid, + GroupingSubject = row.Item6 + }; + } + } + + public class MetricRow + { + public long Id; + public DateTime Timestamp; + public string Action; + public string Subject; + public string GroupingSubject; + public long MetricsKey; + } } } diff --git a/Wabbajack.Server/Services/NexusKeyMaintainance.cs b/Wabbajack.Server/Services/NexusKeyMaintainance.cs index 32e9c28e..bb26f4da 100644 --- a/Wabbajack.Server/Services/NexusKeyMaintainance.cs +++ b/Wabbajack.Server/Services/NexusKeyMaintainance.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Wabbajack.BuildServer; using Wabbajack.Common; +using Wabbajack.Common.Exceptions; using Wabbajack.Lib.NexusApi; using Wabbajack.Server.DataLayer; @@ -28,13 +29,12 @@ namespace Wabbajack.Server.Services try { var client = new TrackingClient(_sql, key); - if (!await client.IsPremium()) - { - _logger.LogWarning($"Purging non premium key"); - await _sql.DeleteNexusAPIKey(key.Key); - continue; - } - return client; + if (await client.IsPremium()) + return client; + + _logger.LogWarning($"Purging non premium key"); + await _sql.DeleteNexusAPIKey(key.Key); + continue; } catch (Exception ex) { @@ -69,13 +69,16 @@ namespace Wabbajack.Server.Services var (daily, hourly) = await client.GetRemainingApiCalls(); await _sql.SetNexusAPIKey(key.Key, daily, hourly); } - catch (Exception) + catch (HttpException ex) { - _logger.Log(LogLevel.Warning, "Update error, purging API key"); + _logger.Log(LogLevel.Warning, $"Nexus error, not purging API key : {ex.Message}"); + } + catch (Exception ex) + { + _logger.Log(LogLevel.Warning, $"Update error, purging API key : {ex.Message}"); await _sql.DeleteNexusAPIKey(key.Key); } } - return keys.Count; } } diff --git a/Wabbajack.Server/Services/NonNexusDownloadValidator.cs b/Wabbajack.Server/Services/NonNexusDownloadValidator.cs index 2ff52979..f7c0729c 100644 --- a/Wabbajack.Server/Services/NonNexusDownloadValidator.cs +++ b/Wabbajack.Server/Services/NonNexusDownloadValidator.cs @@ -41,8 +41,8 @@ namespace Wabbajack.Server.Services bool isValid = false; switch (archive.State) { - case WabbajackCDNDownloader.State _: - case GoogleDriveDownloader.State _: + case WabbajackCDNDownloader.State _: + //case GoogleDriveDownloader.State _: // Let's try validating Google again 2/10/2021 case ManualDownloader.State _: case ModDBDownloader.State _: case HTTPDownloader.State h when h.Url.StartsWith("https://wabbajack"): diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 2d1c4e2c..68aa79a8 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.ResponseCompression; using Microsoft.AspNetCore.StaticFiles; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -76,6 +77,12 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddResponseCompression(options => + { + options.Providers.Add(); + options.Providers.Add(); + options.MimeTypes = new[] {"*/*"}; + }); services.AddMvc(); services.AddControllers() @@ -123,6 +130,7 @@ namespace Wabbajack.Server app.UseNexusPoll(); app.UseArchiveMaintainer(); app.UseModListDownloader(); + app.UseResponseCompression(); app.UseService(); app.UseService(); diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index d7ea7280..34cfda38 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -45,6 +45,8 @@ + + From 831318c7eb68610910214763a39ed0670fc4f891 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 17 Feb 2021 22:44:54 -0700 Subject: [PATCH 2/6] Few more fixes --- Wabbajack.Server.Test/MetricsTests.cs | 4 ++ Wabbajack.Server.Test/sql/wabbajack_db.sql | 19 ++++++ .../ApiKeyAuthorizationHandler.cs | 9 ++- Wabbajack.Server/Controllers/Metrics.cs | 17 +++-- Wabbajack.Server/DataLayer/Metrics.cs | 24 ++++++- Wabbajack.Server/Services/DiscordFrontend.cs | 8 ++- Wabbajack.Server/Services/MetricsKeyCache.cs | 62 +++++++++++++++++++ Wabbajack.Server/Startup.cs | 2 + 8 files changed, 137 insertions(+), 8 deletions(-) create mode 100644 Wabbajack.Server/Services/MetricsKeyCache.cs diff --git a/Wabbajack.Server.Test/MetricsTests.cs b/Wabbajack.Server.Test/MetricsTests.cs index 130e7776..bda8cfd3 100644 --- a/Wabbajack.Server.Test/MetricsTests.cs +++ b/Wabbajack.Server.Test/MetricsTests.cs @@ -5,6 +5,7 @@ using Dapper; using Wabbajack.Lib; using Wabbajack.Server.DataLayer; using Wabbajack.Server.DTOs; +using Wabbajack.Server.Services; using Xunit; using Xunit.Abstractions; @@ -45,6 +46,9 @@ namespace Wabbajack.BuildServer.Test Assert.True(dumpResponse.IsSuccessStatusCode); var data = await dumpResponse.Content.ReadAsStringAsync(); Assert.NotEmpty(data); + + var cache = Fixture.GetService(); + Assert.True(await cache.KeyCount() > 0); } } } diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index dfc2b02a..12231485 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -550,6 +550,25 @@ CREATE TABLE [dbo].[ModListArchives]( )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] GO + +/****** Object: Table [dbo].[MetricsKeys] Script Date: 2/17/2021 9:20:27 PM ******/ +SET ANSI_NULLS ON +GO + +SET QUOTED_IDENTIFIER ON +GO + +CREATE TABLE [dbo].[MetricsKeys]( +[MetricsKey] [varchar](64) NOT NULL, +CONSTRAINT [PK_MetricsKeys] PRIMARY KEY CLUSTERED + ( + [MetricsKey] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO + + + /****** Object: Table [dbo].[ModListArchiveStatus] Script Date: 12/29/2020 8:55:04 PM ******/ SET ANSI_NULLS ON GO diff --git a/Wabbajack.Server/ApiKeyAuthorizationHandler.cs b/Wabbajack.Server/ApiKeyAuthorizationHandler.cs index 2f36179d..e66f1a5c 100644 --- a/Wabbajack.Server/ApiKeyAuthorizationHandler.cs +++ b/Wabbajack.Server/ApiKeyAuthorizationHandler.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Security.Claims; @@ -13,6 +14,7 @@ using Wabbajack.Common; using Wabbajack.Common.Serialization.Json; using Wabbajack.Server.DataLayer; using Wabbajack.Server.DTOs; +using Wabbajack.Server.Services; namespace Wabbajack.BuildServer @@ -31,14 +33,18 @@ namespace Wabbajack.BuildServer private readonly SqlService _sql; private const string ApiKeyHeaderName = "X-Api-Key"; + private MetricsKeyCache _keyCache; + public ApiKeyAuthenticationHandler( IOptionsMonitor options, ILoggerFactory logger, UrlEncoder encoder, ISystemClock clock, + MetricsKeyCache keyCache, SqlService db) : base(options, logger, encoder, clock) { _sql = db; + _keyCache = keyCache; } protected override async Task HandleAuthenticateAsync() @@ -88,8 +94,9 @@ namespace Wabbajack.BuildServer return AuthenticateResult.Success(ticket); } + - if (!await _sql.ValidMetricsKey(metricsKey)) + if (!await _keyCache.IsValidKey(metricsKey)) { return AuthenticateResult.Fail("Invalid Metrics Key"); } diff --git a/Wabbajack.Server/Controllers/Metrics.cs b/Wabbajack.Server/Controllers/Metrics.cs index 9f4fcd4b..b9b315a5 100644 --- a/Wabbajack.Server/Controllers/Metrics.cs +++ b/Wabbajack.Server/Controllers/Metrics.cs @@ -15,6 +15,7 @@ using Wabbajack.Common; using Wabbajack.Server; using Wabbajack.Server.DataLayer; using Wabbajack.Server.DTOs; +using Wabbajack.Server.Services; using WebSocketSharp; using LogLevel = Microsoft.Extensions.Logging.LogLevel; @@ -26,11 +27,13 @@ namespace Wabbajack.BuildServer.Controllers { private SqlService _sql; private ILogger _logger; + private MetricsKeyCache _keyCache; - public MetricsController(ILogger logger, SqlService sql) + public MetricsController(ILogger logger, SqlService sql, MetricsKeyCache keyCache) { _sql = sql; _logger = logger; + _keyCache = keyCache; } [HttpGet] @@ -38,12 +41,15 @@ namespace Wabbajack.BuildServer.Controllers public async Task LogMetricAsync(string subject, string value) { var date = DateTime.UtcNow; + var metricsKey = Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault(); + if (metricsKey != null) + await _keyCache.AddKey(metricsKey); // Used in tests - if (value == "Default" || value == "untitled" || Guid.TryParse(value, out _)) + if (value == "Default" || value == "untitled" || subject == "failed_download" || Guid.TryParse(value, out _)) return new Result { Timestamp = date}; - await Log(date, subject, value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault()); + await Log(date, subject, value, metricsKey); return new Result { Timestamp = date}; } @@ -174,6 +180,7 @@ namespace Wabbajack.BuildServer.Controllers private static Func _totalListTemplate; + private static Func TotalListTemplate { get @@ -189,6 +196,8 @@ namespace Wabbajack.BuildServer.Controllers return _totalListTemplate; } } + + [HttpGet("total_installs.html")] [ResponseCache(Duration = 60 * 60)] @@ -231,6 +240,6 @@ namespace Wabbajack.BuildServer.Controllers { return Ok(await _sql.MetricsDump().ToArrayAsync()); } - + } } diff --git a/Wabbajack.Server/DataLayer/Metrics.cs b/Wabbajack.Server/DataLayer/Metrics.cs index 750eec22..7305e276 100644 --- a/Wabbajack.Server/DataLayer/Metrics.cs +++ b/Wabbajack.Server/DataLayer/Metrics.cs @@ -70,8 +70,28 @@ namespace Wabbajack.Server.DataLayer public async Task ValidMetricsKey(string metricsKey) { await using var conn = await Open(); - return (await conn.QueryAsync("SELECT TOP(1) MetricsKey from Metrics Where MetricsKey = @MetricsKey", - new {MetricsKey = metricsKey})).FirstOrDefault() != null; + return (await conn.QuerySingleOrDefaultAsync("SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey", + new {MetricsKey = metricsKey})) != default; + } + + public async Task AddMetricsKey(string metricsKey) + { + await using var conn = await Open(); + await using var trans = conn.BeginTransaction(); + + if ((await conn.QuerySingleOrDefaultAsync( + "SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey", + new {MetricsKey = metricsKey}, trans)) != default) + return; + + await conn.ExecuteAsync("INSERT INTO dbo.MetricsKeys (MetricsKey) VALUES (@MetricsKey)", + new {MetricsKey = metricsKey}, trans); + } + + public async Task AllKeys() + { + await using var conn = await Open(); + return (await conn.QueryAsync("SELECT MetricsKey from dbo.MetricsKeys")).ToArray(); } diff --git a/Wabbajack.Server/Services/DiscordFrontend.cs b/Wabbajack.Server/Services/DiscordFrontend.cs index 6f81afa8..ec25be9b 100644 --- a/Wabbajack.Server/Services/DiscordFrontend.cs +++ b/Wabbajack.Server/Services/DiscordFrontend.cs @@ -19,8 +19,9 @@ namespace Wabbajack.Server.Services private QuickSync _quickSync; private DiscordSocketClient _client; private SqlService _sql; + private MetricsKeyCache _keyCache; - public DiscordFrontend(ILogger logger, AppSettings settings, QuickSync quickSync, SqlService sql) + public DiscordFrontend(ILogger logger, AppSettings settings, QuickSync quickSync, SqlService sql, MetricsKeyCache keyCache) { _logger = logger; _settings = settings; @@ -33,6 +34,7 @@ namespace Wabbajack.Server.Services _client.MessageReceived += MessageReceivedAsync; _sql = sql; + _keyCache = keyCache; } private async Task MessageReceivedAsync(SocketMessage arg) @@ -88,6 +90,10 @@ namespace Wabbajack.Server.Services await ReplyTo(arg, $"Purged all traces of #{parts[2]} from the server, triggered list downloading. {deleted} records removed"); } } + else if (parts[1] == "users") + { + await ReplyTo(arg, $"Wabbajack has {await _keyCache.KeyCount()} known unique users"); + } } } diff --git a/Wabbajack.Server/Services/MetricsKeyCache.cs b/Wabbajack.Server/Services/MetricsKeyCache.cs new file mode 100644 index 00000000..4fc14139 --- /dev/null +++ b/Wabbajack.Server/Services/MetricsKeyCache.cs @@ -0,0 +1,62 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.Common; +using Wabbajack.Server.DataLayer; + +namespace Wabbajack.Server.Services +{ + public class MetricsKeyCache : IStartable + { + private ILogger _logger; + private SqlService _sql; + private HashSet _knownKeys = new(); + private AsyncLock _lock = new(); + + public MetricsKeyCache(ILogger logger, SqlService sql) + { + _logger = logger; + _sql = sql; + } + + public async Task IsValidKey(string key) + { + using (var _ = await _lock.WaitAsync()) + { + if (_knownKeys.Contains(key)) return true; + } + + if (await _sql.ValidMetricsKey(key)) + { + using var _ = await _lock.WaitAsync(); + _knownKeys.Add(key); + return true; + } + + return false; + } + + public async Task AddKey(string key) + { + using (var _ = await _lock.WaitAsync()) + { + if (_knownKeys.Contains(key)) return; + _knownKeys.Add(key); + } + + await _sql.AddMetricsKey(key); + } + + public void Start() + { + _knownKeys = (_sql.AllKeys().Result).ToHashSet(); + } + + public async Task KeyCount() + { + using var _ = await _lock.WaitAsync(); + return _knownKeys.Count; + } + } +} diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 68aa79a8..16f1c865 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -77,6 +77,7 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddResponseCompression(options => { options.Providers.Add(); @@ -145,6 +146,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); + app.UseService(); app.Use(next => { From 401c2d232f2137f6edaad62ba9d8950fea3da82d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 18 Feb 2021 17:09:12 -0700 Subject: [PATCH 3/6] Improve the GoogleDrive Downloader a bit --- .../Downloaders/GoogleDriveDownloader.cs | 17 +++++++++++------ Wabbajack.Test/DownloaderTests.cs | 3 +++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs b/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs index 91b31129..902851c1 100644 --- a/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs @@ -1,4 +1,5 @@ using System.Linq; +using System.Net.Http; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -6,6 +7,7 @@ using Newtonsoft.Json; using Wabbajack.Common; using Wabbajack.Common.Exceptions; using Wabbajack.Common.Serialization.Json; +using Wabbajack.Lib.Http; using Wabbajack.Lib.Validation; namespace Wabbajack.Lib.Downloaders @@ -67,12 +69,15 @@ namespace Wabbajack.Lib.Downloaders using var response = await client.GetAsync(initialURL); if (!response.IsSuccessStatusCode) throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown"); - var regex = new Regex("(?<=/uc\\?export=download&confirm=).*(?=;id=)"); - using var content = response.Content; - var confirm = regex.Match(await content.ReadAsStringAsync()); - if (!confirm.Success) - return null; - var url = $"https://drive.google.com/uc?export=download&confirm={confirm}&id={Id}"; + var cookies = response.GetSetCookies(); + var warning = cookies.FirstOrDefault(c => c.Key.StartsWith("download_warning_")); + response.Dispose(); + if (warning == default) + { + return new HTTPDownloader.State(initialURL) { Client = client }; + } + + var url = $"https://drive.google.com/uc?export=download&confirm={warning.Value}&id={Id}"; var httpState = new HTTPDownloader.State(url) { Client = client }; return httpState; } diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index c1a291da..95077405 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -113,6 +113,9 @@ namespace Wabbajack.Test Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); + + var newState = (AbstractDownloadState)new GoogleDriveDownloader.State("1Q_CdeYJStfoTZFLZ79RRVkxI2c_cG0dg"); + Assert.True(await newState.Verify(new Archive(newState) {Size = 0})); } [Fact] From 0f323219710859522a441dd184df89ce0d26c256 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 18 Feb 2021 17:15:00 -0700 Subject: [PATCH 4/6] Improve the GoogleDrive Downloader a bit --- Wabbajack.Lib/Http/HttpExtensions.cs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 Wabbajack.Lib/Http/HttpExtensions.cs diff --git a/Wabbajack.Lib/Http/HttpExtensions.cs b/Wabbajack.Lib/Http/HttpExtensions.cs new file mode 100644 index 00000000..700bbc25 --- /dev/null +++ b/Wabbajack.Lib/Http/HttpExtensions.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; + +namespace Wabbajack.Lib.Http +{ + public static class HttpExtensions + { + public static IEnumerable<(string Key, string Value)> GetSetCookies(this HttpResponseMessage response) + { + if (!response.Headers.TryGetValues("set-cookie", out var values)) + return Array.Empty<(string, string)>(); + + return values + .SelectMany(h => h.Split(";")) + .Select(h => h.Split("=")) + .Where(h => h.Length == 2) + .Select(h => (h[0], h[1])); + } + } +} From 8664c16d135fee3c36dec07ca33276cb0f2a0335 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 22 Feb 2021 21:38:28 -0700 Subject: [PATCH 5/6] Fix tests --- Wabbajack.Server.Test/ABuildServerSystemTest.cs | 3 +++ Wabbajack.Server/Services/ListValidator.cs | 5 +++-- Wabbajack.Server/Startup.cs | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Wabbajack.Server.Test/ABuildServerSystemTest.cs b/Wabbajack.Server.Test/ABuildServerSystemTest.cs index 62bfb879..e1989af9 100644 --- a/Wabbajack.Server.Test/ABuildServerSystemTest.cs +++ b/Wabbajack.Server.Test/ABuildServerSystemTest.cs @@ -15,6 +15,7 @@ using Wabbajack.Lib.ModListRegistry; using Wabbajack.Server; using Wabbajack.Server.DataLayer; using Wabbajack.Server.DTOs; +using Wabbajack.Server.Services; using Xunit; using Xunit.Abstractions; @@ -166,6 +167,8 @@ namespace Wabbajack.BuildServer.Test _client = new Wabbajack.Lib.Http.Client(); _authedClient = new Wabbajack.Lib.Http.Client(); Fixture = fixture.Deref(); + var cache = Fixture.GetService(); + cache.AddKey(Metrics.GetMetricsKey().Result); _authedClient.Headers.Add(("x-api-key", Fixture.APIKey)); AuthorAPI.ApiKeyOverride = Fixture.APIKey; _queue = new WorkQueue(); diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 6b6197c7..39073fde 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -197,6 +197,7 @@ namespace Wabbajack.Server.Services private AsyncLock _healLock = new AsyncLock(); private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive, ModlistMetadata modList) { + using var _ = await _healLock.WaitAsync(); var srcDownload = await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size); if (srcDownload == null || srcDownload.IsFailed == true) { @@ -204,7 +205,7 @@ namespace Wabbajack.Server.Services return (archive, ArchiveStatus.InValid); } - + var patches = await _sql.PatchesForSource(archive.Hash); foreach (var patch in patches) { @@ -219,7 +220,7 @@ namespace Wabbajack.Server.Services return (archive, ArchiveStatus.Updated); } - using var _ = await _healLock.WaitAsync(); + var upgradeTime = DateTime.UtcNow; _logger.LogInformation($"Validator Finding Upgrade for {archive.Hash} {archive.State.PrimaryKeyString}"); diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 16f1c865..1548d9aa 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -82,7 +82,7 @@ namespace Wabbajack.Server { options.Providers.Add(); options.Providers.Add(); - options.MimeTypes = new[] {"*/*"}; + options.MimeTypes = new[] {"application/json"}; }); services.AddMvc(); From 5d5d1c84a129fc2cafc683a60b547288de859d6c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 23 Feb 2021 06:50:09 -0700 Subject: [PATCH 6/6] Fix tests --- Wabbajack.Test/ContentRightsManagementTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Wabbajack.Test/ContentRightsManagementTests.cs b/Wabbajack.Test/ContentRightsManagementTests.cs index a739813d..9db1a885 100644 --- a/Wabbajack.Test/ContentRightsManagementTests.cs +++ b/Wabbajack.Test/ContentRightsManagementTests.cs @@ -119,7 +119,7 @@ namespace Wabbajack.Test { Assert.Equal(HTMLInterface.PermissionValue.No, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 266)); Assert.Equal(HTMLInterface.PermissionValue.Yes, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 1137)); - Assert.Equal(HTMLInterface.PermissionValue.Hidden, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 34604)); + Assert.Contains(await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 34604), new[]{HTMLInterface.PermissionValue.Hidden, HTMLInterface.PermissionValue.NotFound}); Assert.Equal(HTMLInterface.PermissionValue.NotFound, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 24287)); }