latest server fixes

This commit is contained in:
Timothy Baldridge 2021-02-16 22:46:05 -07:00
parent 2079cbc8cd
commit 6c9f6ab5c0
18 changed files with 391 additions and 122 deletions

View File

@ -38,7 +38,8 @@ namespace Wabbajack.CLI
typeof(PurgeArchive),
typeof(AllKnownDownloadStates),
typeof(VerifyAllDownloads),
typeof(HashBenchmark)
typeof(HashBenchmark),
typeof(StressTestURL)
};
}
}

View File

@ -0,0 +1,52 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.CLI.Verbs
{
[Verb("stress-test-url", HelpText = "Verify a file rapidly, trying to make it fail")]
public class StressTestURL : AVerb
{
[Option('i', "input", Required = true, HelpText = "Input url to stress")]
public string Input { get; set; } = "";
private Uri _Input => new Uri(Input);
protected override async Task<ExitCode> Run()
{
using var queue = new WorkQueue();
var state = await DownloadDispatcher.Infer(_Input);
if (state == null)
{
Console.WriteLine("Could not parse URL");
}
Console.WriteLine("Performing initial download");
await using var temp = new TempFile();
var archive = new Archive(state!);
if (!await state!.Download(archive, temp.Path))
{
Console.WriteLine("Failed initial download");
}
var hash = await temp.Path.FileHashAsync();
archive.Hash = hash!.Value;
archive.Size = temp.Path.Size;
Console.WriteLine($"Hash: {archive.Hash} Size: {archive.Size.ToFileSizeString()}");
await Enumerable.Range(0, 100000)
.PMap(queue, async idx =>
{
if (!await state.Verify(archive))
{
throw new Exception($"{idx} Verification failed");
}
Console.WriteLine($"{idx} Verification passed");
});
return ExitCode.Ok;
}
}
}

View File

@ -400,7 +400,7 @@ namespace Wabbajack.Lib.Downloaders
public async Task<string> GetStringAsync(Uri uri, CancellationToken? token = null)
{
if (!Downloader.IsCloudFlareProtected)
return await Downloader.AuthedClient.GetStringAsync(uri);
return await Downloader.AuthedClient.GetStringAsync(uri, token);
using var driver = await Downloader.GetAuthedDriver();
@ -448,7 +448,7 @@ namespace Wabbajack.Lib.Downloaders
return await Downloader.AuthedClient.GetAsync(uri);
using var driver = await Downloader.GetAuthedDriver();
TaskCompletionSource<Uri?> promise = new TaskCompletionSource<Uri?>();
TaskCompletionSource<Uri?> promise = new();
driver.DownloadHandler = uri1 =>
{
promise.SetResult(uri);

View File

@ -1,4 +1,5 @@
using System.Text.RegularExpressions;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
@ -54,10 +55,12 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Download(Archive a, AbsolutePath destination)
{
var state = await ToHttpState();
if (state == null)
return false;
return await state.Download(a, destination);
}
private async Task<HTTPDownloader.State> ToHttpState()
private async Task<HTTPDownloader.State?> ToHttpState()
{
var initialURL = $"https://drive.google.com/uc?id={Id}&export=download";
var client = new Wabbajack.Lib.Http.Client();
@ -65,7 +68,10 @@ namespace Wabbajack.Lib.Downloaders
if (!response.IsSuccessStatusCode)
throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown");
var regex = new Regex("(?<=/uc\\?export=download&amp;confirm=).*(?=;id=)");
var confirm = regex.Match(await response.Content.ReadAsStringAsync());
using var content = response.Content;
var confirm = regex.Match(await content.ReadAsStringAsync());
if (!confirm.Success)
return null;
var url = $"https://drive.google.com/uc?export=download&confirm={confirm}&id={Id}";
var httpState = new HTTPDownloader.State(url) { Client = client };
return httpState;
@ -74,6 +80,8 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Verify(Archive a, CancellationToken? token)
{
var state = await ToHttpState();
if (state == null)
return false;
return await state.Verify(a, token);
}

View File

@ -82,111 +82,119 @@ namespace Wabbajack.Lib.Downloaders
destination.Parent.CreateDirectory();
}
using (var fs = download ? await destination.Create() : null)
await using var fs = download ? await destination.Create() : null;
var client = Client ?? await ClientAPI.GetClient();
client.Headers.Add(("User-Agent", Consts.UserAgent));
foreach (var header in Headers)
{
var client = Client ?? await ClientAPI.GetClient();
client.Headers.Add(("User-Agent", Consts.UserAgent));
var idx = header.IndexOf(':');
var k = header.Substring(0, idx);
var v = header.Substring(idx + 1);
client.Headers.Add((k, v));
}
foreach (var header in Headers)
long totalRead = 0;
const int bufferSize = 1024 * 32 * 8;
Utils.Status($"Starting Download {a.Name ?? Url}", Percent.Zero);
var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false, token:token);
TOP:
if (!response.IsSuccessStatusCode)
{
response.Dispose();
return false;
}
Stream stream;
try
{
stream = await response.Content.ReadAsStreamAsync();
}
catch (Exception ex)
{
Utils.Error(ex, $"While downloading {Url}");
return false;
}
var headerVar = a.Size == 0 ? "1" : a.Size.ToString();
long headerContentSize = 0;
if (response.Content.Headers.Contains("Content-Length"))
{
headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault();
if (headerVar != null)
long.TryParse(headerVar, out headerContentSize);
}
if (!download)
{
await stream.DisposeAsync();
response.Dispose();
if (a.Size != 0 && headerContentSize != 0)
return a.Size == headerContentSize;
return true;
}
var supportsResume = response.Headers.AcceptRanges.FirstOrDefault(f => f == "bytes") != null;
var contentSize = headerVar != null ? long.Parse(headerVar) : 1;
await using (var webs = stream)
{
var buffer = new byte[bufferSize];
int readThisCycle = 0;
while (!(token ?? CancellationToken.None).IsCancellationRequested)
{
var idx = header.IndexOf(':');
var k = header.Substring(0, idx);
var v = header.Substring(idx + 1);
client.Headers.Add((k, v));
}
long totalRead = 0;
var bufferSize = 1024 * 32 * 8;
Utils.Status($"Starting Download {a.Name ?? Url}", Percent.Zero);
var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false, token:token);
TOP:
if (!response.IsSuccessStatusCode)
{
return false;
}
Stream stream;
try
{
stream = await response.Content.ReadAsStreamAsync();
}
catch (Exception ex)
{
Utils.Error(ex, $"While downloading {Url}");
return false;
}
var headerVar = a.Size == 0 ? "1" : a.Size.ToString();
long header_content_size = 0;
if (response.Content.Headers.Contains("Content-Length"))
{
headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault();
if (headerVar != null)
long.TryParse(headerVar, out header_content_size);
}
if (!download)
{
if (a.Size != 0 && header_content_size != 0)
return a.Size == header_content_size;
return true;
}
var supportsResume = response.Headers.AcceptRanges.FirstOrDefault(f => f == "bytes") != null;
var contentSize = headerVar != null ? long.Parse(headerVar) : 1;
using (var webs = stream)
{
var buffer = new byte[bufferSize];
int readThisCycle = 0;
while (!(token ?? CancellationToken.None).IsCancellationRequested)
int read = 0;
try
{
int read = 0;
try
read = await webs.ReadAsync(buffer, 0, bufferSize);
}
catch (Exception)
{
if (readThisCycle == 0)
{
read = await webs.ReadAsync(buffer, 0, bufferSize);
await stream.DisposeAsync();
response.Dispose();
throw;
}
catch (Exception)
if (totalRead < contentSize)
{
if (readThisCycle == 0)
throw;
if (totalRead < contentSize)
if (!supportsResume)
{
if (supportsResume)
{
Utils.Log(
$"Abort during download, trying to resume {Url} from {totalRead.ToFileSizeString()}");
var msg = new HttpRequestMessage(HttpMethod.Get, Url);
msg.Headers.Range = new RangeHeaderValue(totalRead, null);
response.Dispose();
response = await client.SendAsync(msg);
goto TOP;
}
await stream.DisposeAsync();
response.Dispose();
throw;
}
break;
Utils.Log(
$"Abort during download, trying to resume {Url} from {totalRead.ToFileSizeString()}");
var msg = new HttpRequestMessage(HttpMethod.Get, Url);
msg.Headers.Range = new RangeHeaderValue(totalRead, null);
response.Dispose();
response = await client.SendAsync(msg);
goto TOP;
}
readThisCycle += read;
if (read == 0) break;
Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(totalRead, contentSize));
fs!.Write(buffer, 0, read);
totalRead += read;
break;
}
readThisCycle += read;
if (read == 0) break;
Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(totalRead, contentSize));
fs!.Write(buffer, 0, read);
totalRead += read;
}
response.Dispose();
return true;
}
response.Dispose();
return true;
}
public override async Task<bool> Verify(Archive a, CancellationToken? token)

View File

@ -48,10 +48,10 @@ namespace Wabbajack.Lib.Http
return await SendStringAsync(request, token: token);
}
public async Task<string> GetStringAsync(Uri url)
public async Task<string> GetStringAsync(Uri url, CancellationToken? token = null)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
return await SendStringAsync(request);
return await SendStringAsync(request, token: token);
}
public async Task<string> DeleteStringAsync(string url)

View File

@ -22,6 +22,7 @@ namespace Wabbajack.Lib.Http
MaxConnectionsPerServer = 20,
PooledConnectionLifetime = TimeSpan.FromMilliseconds(100),
PooledConnectionIdleTimeout = TimeSpan.FromMilliseconds(100),
AutomaticDecompression = DecompressionMethods.All
};
Utils.Log($"Configuring with SSL {_socketsHandler.SslOptions.EnabledSslProtocols}");

View File

@ -69,7 +69,7 @@ namespace Wabbajack.Lib
/// </summary>
/// <param name="action"></param>
/// <param name="value"></param>
public static async Task Send(string action, string value)
public static async Task Send(string action, string subject)
{
if (BuildServerStatus.IsBuildServerDown)
return;
@ -78,7 +78,7 @@ namespace Wabbajack.Lib
Utils.Log($"File hash check (-42) {key}");
var client = new Http.Client();
client.Headers.Add((Consts.MetricsKeyHeader, key));
await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{value}");
await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{subject}");
}
public static async Task Error(Type type, Exception exception)

View File

@ -34,7 +34,17 @@ namespace Wabbajack.BuildServer.Test
using var response = await _client.GetAsync(MakeURL($"metrics/report/{action}"));
Assert.Equal(TimeSpan.FromHours(1), response.Headers.CacheControl.MaxAge);
// we'll just make sure this doesn't error, with limited data that's about all we can do atm
using var totalInstalls = await _client.GetAsync(MakeURL($"metrics/total_installs.html"));
Assert.True(totalInstalls.IsSuccessStatusCode);
using var totalUniqueInstalls = await _client.GetAsync(MakeURL($"metrics/total_unique_installs.html"));
Assert.True(totalUniqueInstalls.IsSuccessStatusCode);
using var dumpResponse = await _client.GetAsync(MakeURL("metrics/dump.json"));
Assert.True(dumpResponse.IsSuccessStatusCode);
var data = await dumpResponse.Content.ReadAsStringAsync();
Assert.NotEmpty(data);
}
}
}

View File

@ -29,7 +29,6 @@ namespace Wabbajack.BuildServer
{
private const string ProblemDetailsContentType = "application/problem+json";
private readonly SqlService _sql;
private static ConcurrentHashSet<string> _knownKeys = new();
private const string ApiKeyHeaderName = "X-Api-Key";
public ApiKeyAuthenticationHandler(
@ -51,19 +50,25 @@ namespace Wabbajack.BuildServer
{
if (await _sql.IsTarKey(metricsKey))
{
await _sql.IngestMetric(new Metric {Action = "TarKey", Subject = "Auth", MetricsKey = metricsKey, Timestamp = DateTime.UtcNow});
await _sql.IngestMetric(new Metric
{
Action = "TarKey",
Subject = "Auth",
MetricsKey = metricsKey,
Timestamp = DateTime.UtcNow
});
await Task.Delay(TimeSpan.FromSeconds(60));
throw new Exception("Error, lipsum timeout of the cross distant cloud.");
}
}
var authorKey = Request.Headers[ApiKeyHeaderName].FirstOrDefault();
if (authorKey == null && metricsKey == null)
{
return AuthenticateResult.NoResult();
}
if (authorKey != null)
{
@ -84,14 +89,12 @@ namespace Wabbajack.BuildServer
return AuthenticateResult.Success(ticket);
}
if (!_knownKeys.Contains(metricsKey) && !await _sql.ValidMetricsKey(metricsKey))
if (!await _sql.ValidMetricsKey(metricsKey))
{
return AuthenticateResult.Fail("Invalid Metrics Key");
}
else
{
_knownKeys.Add(metricsKey);
var claims = new List<Claim> {new(ClaimTypes.Role, "User")};
@ -102,7 +105,6 @@ namespace Wabbajack.BuildServer
return AuthenticateResult.Success(ticket);
}
}
[JsonName("RequestLog")]

View File

@ -2,10 +2,13 @@
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
using Wabbajack.Common;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Server;
using Wabbajack.Server.DataLayer;

View File

@ -1,9 +1,13 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
@ -34,6 +38,11 @@ namespace Wabbajack.BuildServer.Controllers
public async Task<Result> LogMetricAsync(string subject, string value)
{
var date = DateTime.UtcNow;
// Used in tests
if (value == "Default" || value == "untitled" || Guid.TryParse(value, out _))
return new Result { Timestamp = date};
await Log(date, subject, value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault());
return new Result { Timestamp = date};
}
@ -74,7 +83,7 @@ namespace Wabbajack.BuildServer.Controllers
return Ok(results == 0
? new Badge($"Modlist {name} not found!", "Error") {color = "red"}
: new Badge("Installations: ", $"{results}") {color = "green"});
: new Badge("Installations: ", "____") {color = "green"});
}
[HttpGet]
@ -87,7 +96,7 @@ namespace Wabbajack.BuildServer.Controllers
return Ok(results == 0
? new Badge($"Modlist {name} not found!", "Error") {color = "red"}
: new Badge("Installations: ", $"{results}"){color = "green"}) ;
: new Badge("Installations: ", "____"){color = "green"}) ;
}
private static readonly Func<object, string> ReportTemplate = NettleEngine.GetCompiler().Compile(@"
@ -149,5 +158,79 @@ namespace Wabbajack.BuildServer.Controllers
{
public DateTime Timestamp { get; set; }
}
class TotalListTemplateData
{
public string Title { get; set; }
public long Total { get; set; }
public Item[] Items { get; set; }
public class Item
{
public long Count { get; set; }
public string Title { get; set; }
}
}
private static Func<object, string> _totalListTemplate;
private static Func<object, string> TotalListTemplate
{
get
{
if (_totalListTemplate == null)
{
var resource = Assembly.GetExecutingAssembly()
.GetManifestResourceStream("Wabbajack.Server.Controllers.Templates.TotalListTemplate.html")!
.ReadAll();
_totalListTemplate = NettleEngine.GetCompiler().Compile(Encoding.UTF8.GetString(resource));
}
return _totalListTemplate;
}
}
[HttpGet("total_installs.html")]
[ResponseCache(Duration = 60 * 60)]
public async Task<ContentResult> TotalInstalls()
{
var data = await _sql.GetTotalInstalls();
var result = TotalListTemplate(new TotalListTemplateData
{
Title = "Total Installs",
Total = data.Sum(d => d.Item2),
Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2})
.ToArray()
});
return new ContentResult {
ContentType = "text/html",
StatusCode = (int)HttpStatusCode.OK,
Content = result};
}
[HttpGet("total_unique_installs.html")]
[ResponseCache(Duration = 60 * 60)]
public async Task<ContentResult> TotalUniqueInstalls()
{
var data = await _sql.GetTotalUniqueInstalls();
var result = TotalListTemplate(new TotalListTemplateData
{
Title = "Total Unique Installs",
Total = data.Sum(d => d.Item2),
Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2})
.ToArray()
});
return new ContentResult {
ContentType = "text/html",
StatusCode = (int)HttpStatusCode.OK,
Content = result};
}
[HttpGet("dump.json")]
public async Task<IActionResult> DataDump()
{
return Ok(await _sql.MetricsDump().ToArrayAsync());
}
}
}

View File

@ -0,0 +1,21 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Total Installs</title>
</head>
<body>
<h2>{{$.Title}} - Total: {{$.Total}}</h2>
<table>
{{each $.Items }}
<tr>
<td>{{$.Count}}</td>
<td>{{$.Title}}</td>
</tr>
{{/each}}
</table>
</body>
</html>

View File

@ -95,5 +95,72 @@ namespace Wabbajack.Server.DataLayer
WHERE JSON_VALUE(Metadata, '$.links.machineURL') = @MachineURL)",
new {MachineURL = machineUrl});
}
public async Task<IEnumerable<(string, long)>> GetTotalInstalls()
{
await using var conn = await Open();
return await conn.QueryAsync<(string, long)>(
@"SELECT GroupingSubject, Count(*) as Count
From dbo.Metrics
WHERE
GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics
WHERE action = 'finish_install'
AND MetricsKey is not null)
group by GroupingSubject
order by Count(*) desc");
}
public async Task<IEnumerable<(string, long)>> GetTotalUniqueInstalls()
{
await using var conn = await Open();
return await conn.QueryAsync<(string, long)>(
@"Select GroupingSubject, Count(*) as Count
FROM
(select DISTINCT MetricsKey, GroupingSubject
From dbo.Metrics
WHERE
GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics
WHERE action = 'finish_install'
AND MetricsKey is not null)) m
GROUP BY GroupingSubject
Order by Count(*) desc
");
}
public async IAsyncEnumerable<MetricRow> MetricsDump()
{
var keys = new Dictionary<string, long>();
await using var conn = await Open();
foreach (var row in await conn.QueryAsync<(long, DateTime, string, string, string, string)>(@"select Id, Timestamp, Action, Subject, MetricsKey, GroupingSubject from dbo.metrics WHERE MetricsKey is not null"))
{
if (!keys.TryGetValue(row.Item5, out var keyid))
{
keyid = keys.Count;
keys[row.Item5] = keyid;
}
yield return new MetricRow
{
Id = row.Item1,
Timestamp = row.Item2,
Action = row.Item3,
Subject = row.Item4,
MetricsKey = keyid,
GroupingSubject = row.Item6
};
}
}
public class MetricRow
{
public long Id;
public DateTime Timestamp;
public string Action;
public string Subject;
public string GroupingSubject;
public long MetricsKey;
}
}
}

View File

@ -5,6 +5,7 @@ using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Common.Exceptions;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
@ -28,13 +29,12 @@ namespace Wabbajack.Server.Services
try
{
var client = new TrackingClient(_sql, key);
if (!await client.IsPremium())
{
_logger.LogWarning($"Purging non premium key");
await _sql.DeleteNexusAPIKey(key.Key);
continue;
}
return client;
if (await client.IsPremium())
return client;
_logger.LogWarning($"Purging non premium key");
await _sql.DeleteNexusAPIKey(key.Key);
continue;
}
catch (Exception ex)
{
@ -69,13 +69,16 @@ namespace Wabbajack.Server.Services
var (daily, hourly) = await client.GetRemainingApiCalls();
await _sql.SetNexusAPIKey(key.Key, daily, hourly);
}
catch (Exception)
catch (HttpException ex)
{
_logger.Log(LogLevel.Warning, "Update error, purging API key");
_logger.Log(LogLevel.Warning, $"Nexus error, not purging API key : {ex.Message}");
}
catch (Exception ex)
{
_logger.Log(LogLevel.Warning, $"Update error, purging API key : {ex.Message}");
await _sql.DeleteNexusAPIKey(key.Key);
}
}
return keys.Count;
}
}

View File

@ -41,8 +41,8 @@ namespace Wabbajack.Server.Services
bool isValid = false;
switch (archive.State)
{
case WabbajackCDNDownloader.State _:
case GoogleDriveDownloader.State _:
case WabbajackCDNDownloader.State _:
//case GoogleDriveDownloader.State _: // Let's try validating Google again 2/10/2021
case ManualDownloader.State _:
case ModDBDownloader.State _:
case HTTPDownloader.State h when h.Url.StartsWith("https://wabbajack"):

View File

@ -5,6 +5,7 @@ using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http.Features;
using Microsoft.AspNetCore.ResponseCompression;
using Microsoft.AspNetCore.StaticFiles;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
@ -76,6 +77,12 @@ namespace Wabbajack.Server
services.AddSingleton<Watchdog>();
services.AddSingleton<DiscordFrontend>();
services.AddSingleton<AuthoredFilesCleanup>();
services.AddResponseCompression(options =>
{
options.Providers.Add<BrotliCompressionProvider>();
options.Providers.Add<GzipCompressionProvider>();
options.MimeTypes = new[] {"*/*"};
});
services.AddMvc();
services.AddControllers()
@ -123,6 +130,7 @@ namespace Wabbajack.Server
app.UseNexusPoll();
app.UseArchiveMaintainer();
app.UseModListDownloader();
app.UseResponseCompression();
app.UseService<NonNexusDownloadValidator>();
app.UseService<ListValidator>();

View File

@ -45,6 +45,8 @@
<ItemGroup>
<None Remove="sheo_quotes.txt" />
<EmbeddedResource Include="sheo_quotes.txt" />
<None Remove="Controllers\Templates\TotalListTemplate.html" />
<EmbeddedResource Include="Controllers\Templates\TotalListTemplate.html" />
</ItemGroup>
<ItemGroup>