mirror of
https://github.com/wabbajack-tools/wabbajack.git
synced 2024-08-30 18:42:17 +00:00
latest server fixes
This commit is contained in:
parent
2079cbc8cd
commit
6c9f6ab5c0
@ -38,7 +38,8 @@ namespace Wabbajack.CLI
|
||||
typeof(PurgeArchive),
|
||||
typeof(AllKnownDownloadStates),
|
||||
typeof(VerifyAllDownloads),
|
||||
typeof(HashBenchmark)
|
||||
typeof(HashBenchmark),
|
||||
typeof(StressTestURL)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
52
Wabbajack.CLI/Verbs/StressTestURL.cs
Normal file
52
Wabbajack.CLI/Verbs/StressTestURL.cs
Normal file
@ -0,0 +1,52 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using CommandLine;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
|
||||
namespace Wabbajack.CLI.Verbs
|
||||
{
|
||||
[Verb("stress-test-url", HelpText = "Verify a file rapidly, trying to make it fail")]
|
||||
public class StressTestURL : AVerb
|
||||
{
|
||||
[Option('i', "input", Required = true, HelpText = "Input url to stress")]
|
||||
public string Input { get; set; } = "";
|
||||
private Uri _Input => new Uri(Input);
|
||||
|
||||
protected override async Task<ExitCode> Run()
|
||||
{
|
||||
using var queue = new WorkQueue();
|
||||
var state = await DownloadDispatcher.Infer(_Input);
|
||||
if (state == null)
|
||||
{
|
||||
Console.WriteLine("Could not parse URL");
|
||||
}
|
||||
|
||||
Console.WriteLine("Performing initial download");
|
||||
await using var temp = new TempFile();
|
||||
var archive = new Archive(state!);
|
||||
if (!await state!.Download(archive, temp.Path))
|
||||
{
|
||||
Console.WriteLine("Failed initial download");
|
||||
}
|
||||
|
||||
var hash = await temp.Path.FileHashAsync();
|
||||
archive.Hash = hash!.Value;
|
||||
archive.Size = temp.Path.Size;
|
||||
Console.WriteLine($"Hash: {archive.Hash} Size: {archive.Size.ToFileSizeString()}");
|
||||
|
||||
await Enumerable.Range(0, 100000)
|
||||
.PMap(queue, async idx =>
|
||||
{
|
||||
if (!await state.Verify(archive))
|
||||
{
|
||||
throw new Exception($"{idx} Verification failed");
|
||||
}
|
||||
Console.WriteLine($"{idx} Verification passed");
|
||||
});
|
||||
return ExitCode.Ok;
|
||||
}
|
||||
}
|
||||
}
|
@ -400,7 +400,7 @@ namespace Wabbajack.Lib.Downloaders
|
||||
public async Task<string> GetStringAsync(Uri uri, CancellationToken? token = null)
|
||||
{
|
||||
if (!Downloader.IsCloudFlareProtected)
|
||||
return await Downloader.AuthedClient.GetStringAsync(uri);
|
||||
return await Downloader.AuthedClient.GetStringAsync(uri, token);
|
||||
|
||||
|
||||
using var driver = await Downloader.GetAuthedDriver();
|
||||
@ -448,7 +448,7 @@ namespace Wabbajack.Lib.Downloaders
|
||||
return await Downloader.AuthedClient.GetAsync(uri);
|
||||
|
||||
using var driver = await Downloader.GetAuthedDriver();
|
||||
TaskCompletionSource<Uri?> promise = new TaskCompletionSource<Uri?>();
|
||||
TaskCompletionSource<Uri?> promise = new();
|
||||
driver.DownloadHandler = uri1 =>
|
||||
{
|
||||
promise.SetResult(uri);
|
||||
|
@ -1,4 +1,5 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Newtonsoft.Json;
|
||||
@ -54,10 +55,12 @@ namespace Wabbajack.Lib.Downloaders
|
||||
public override async Task<bool> Download(Archive a, AbsolutePath destination)
|
||||
{
|
||||
var state = await ToHttpState();
|
||||
if (state == null)
|
||||
return false;
|
||||
return await state.Download(a, destination);
|
||||
}
|
||||
|
||||
private async Task<HTTPDownloader.State> ToHttpState()
|
||||
private async Task<HTTPDownloader.State?> ToHttpState()
|
||||
{
|
||||
var initialURL = $"https://drive.google.com/uc?id={Id}&export=download";
|
||||
var client = new Wabbajack.Lib.Http.Client();
|
||||
@ -65,7 +68,10 @@ namespace Wabbajack.Lib.Downloaders
|
||||
if (!response.IsSuccessStatusCode)
|
||||
throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown");
|
||||
var regex = new Regex("(?<=/uc\\?export=download&confirm=).*(?=;id=)");
|
||||
var confirm = regex.Match(await response.Content.ReadAsStringAsync());
|
||||
using var content = response.Content;
|
||||
var confirm = regex.Match(await content.ReadAsStringAsync());
|
||||
if (!confirm.Success)
|
||||
return null;
|
||||
var url = $"https://drive.google.com/uc?export=download&confirm={confirm}&id={Id}";
|
||||
var httpState = new HTTPDownloader.State(url) { Client = client };
|
||||
return httpState;
|
||||
@ -74,6 +80,8 @@ namespace Wabbajack.Lib.Downloaders
|
||||
public override async Task<bool> Verify(Archive a, CancellationToken? token)
|
||||
{
|
||||
var state = await ToHttpState();
|
||||
if (state == null)
|
||||
return false;
|
||||
return await state.Verify(a, token);
|
||||
}
|
||||
|
||||
|
@ -82,111 +82,119 @@ namespace Wabbajack.Lib.Downloaders
|
||||
destination.Parent.CreateDirectory();
|
||||
}
|
||||
|
||||
using (var fs = download ? await destination.Create() : null)
|
||||
await using var fs = download ? await destination.Create() : null;
|
||||
var client = Client ?? await ClientAPI.GetClient();
|
||||
client.Headers.Add(("User-Agent", Consts.UserAgent));
|
||||
|
||||
foreach (var header in Headers)
|
||||
{
|
||||
var client = Client ?? await ClientAPI.GetClient();
|
||||
client.Headers.Add(("User-Agent", Consts.UserAgent));
|
||||
var idx = header.IndexOf(':');
|
||||
var k = header.Substring(0, idx);
|
||||
var v = header.Substring(idx + 1);
|
||||
client.Headers.Add((k, v));
|
||||
}
|
||||
|
||||
foreach (var header in Headers)
|
||||
long totalRead = 0;
|
||||
const int bufferSize = 1024 * 32 * 8;
|
||||
|
||||
Utils.Status($"Starting Download {a.Name ?? Url}", Percent.Zero);
|
||||
var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false, token:token);
|
||||
TOP:
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
response.Dispose();
|
||||
return false;
|
||||
}
|
||||
|
||||
Stream stream;
|
||||
try
|
||||
{
|
||||
stream = await response.Content.ReadAsStreamAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Utils.Error(ex, $"While downloading {Url}");
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
var headerVar = a.Size == 0 ? "1" : a.Size.ToString();
|
||||
long headerContentSize = 0;
|
||||
if (response.Content.Headers.Contains("Content-Length"))
|
||||
{
|
||||
headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault();
|
||||
if (headerVar != null)
|
||||
long.TryParse(headerVar, out headerContentSize);
|
||||
}
|
||||
|
||||
if (!download)
|
||||
{
|
||||
await stream.DisposeAsync();
|
||||
response.Dispose();
|
||||
if (a.Size != 0 && headerContentSize != 0)
|
||||
return a.Size == headerContentSize;
|
||||
return true;
|
||||
}
|
||||
|
||||
var supportsResume = response.Headers.AcceptRanges.FirstOrDefault(f => f == "bytes") != null;
|
||||
|
||||
var contentSize = headerVar != null ? long.Parse(headerVar) : 1;
|
||||
|
||||
await using (var webs = stream)
|
||||
{
|
||||
var buffer = new byte[bufferSize];
|
||||
int readThisCycle = 0;
|
||||
|
||||
while (!(token ?? CancellationToken.None).IsCancellationRequested)
|
||||
{
|
||||
var idx = header.IndexOf(':');
|
||||
var k = header.Substring(0, idx);
|
||||
var v = header.Substring(idx + 1);
|
||||
client.Headers.Add((k, v));
|
||||
}
|
||||
|
||||
long totalRead = 0;
|
||||
var bufferSize = 1024 * 32 * 8;
|
||||
|
||||
Utils.Status($"Starting Download {a.Name ?? Url}", Percent.Zero);
|
||||
var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false, token:token);
|
||||
TOP:
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
Stream stream;
|
||||
try
|
||||
{
|
||||
stream = await response.Content.ReadAsStreamAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Utils.Error(ex, $"While downloading {Url}");
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
var headerVar = a.Size == 0 ? "1" : a.Size.ToString();
|
||||
long header_content_size = 0;
|
||||
if (response.Content.Headers.Contains("Content-Length"))
|
||||
{
|
||||
headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault();
|
||||
if (headerVar != null)
|
||||
long.TryParse(headerVar, out header_content_size);
|
||||
}
|
||||
|
||||
if (!download)
|
||||
{
|
||||
if (a.Size != 0 && header_content_size != 0)
|
||||
return a.Size == header_content_size;
|
||||
return true;
|
||||
}
|
||||
|
||||
var supportsResume = response.Headers.AcceptRanges.FirstOrDefault(f => f == "bytes") != null;
|
||||
|
||||
var contentSize = headerVar != null ? long.Parse(headerVar) : 1;
|
||||
|
||||
using (var webs = stream)
|
||||
{
|
||||
var buffer = new byte[bufferSize];
|
||||
int readThisCycle = 0;
|
||||
|
||||
while (!(token ?? CancellationToken.None).IsCancellationRequested)
|
||||
int read = 0;
|
||||
try
|
||||
{
|
||||
int read = 0;
|
||||
try
|
||||
read = await webs.ReadAsync(buffer, 0, bufferSize);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
if (readThisCycle == 0)
|
||||
{
|
||||
read = await webs.ReadAsync(buffer, 0, bufferSize);
|
||||
await stream.DisposeAsync();
|
||||
response.Dispose();
|
||||
throw;
|
||||
}
|
||||
catch (Exception)
|
||||
|
||||
if (totalRead < contentSize)
|
||||
{
|
||||
if (readThisCycle == 0)
|
||||
throw;
|
||||
|
||||
if (totalRead < contentSize)
|
||||
if (!supportsResume)
|
||||
{
|
||||
if (supportsResume)
|
||||
{
|
||||
Utils.Log(
|
||||
$"Abort during download, trying to resume {Url} from {totalRead.ToFileSizeString()}");
|
||||
|
||||
var msg = new HttpRequestMessage(HttpMethod.Get, Url);
|
||||
msg.Headers.Range = new RangeHeaderValue(totalRead, null);
|
||||
response.Dispose();
|
||||
response = await client.SendAsync(msg);
|
||||
goto TOP;
|
||||
}
|
||||
await stream.DisposeAsync();
|
||||
response.Dispose();
|
||||
throw;
|
||||
}
|
||||
|
||||
break;
|
||||
Utils.Log(
|
||||
$"Abort during download, trying to resume {Url} from {totalRead.ToFileSizeString()}");
|
||||
|
||||
var msg = new HttpRequestMessage(HttpMethod.Get, Url);
|
||||
msg.Headers.Range = new RangeHeaderValue(totalRead, null);
|
||||
response.Dispose();
|
||||
response = await client.SendAsync(msg);
|
||||
goto TOP;
|
||||
}
|
||||
|
||||
readThisCycle += read;
|
||||
|
||||
if (read == 0) break;
|
||||
Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(totalRead, contentSize));
|
||||
|
||||
fs!.Write(buffer, 0, read);
|
||||
totalRead += read;
|
||||
break;
|
||||
}
|
||||
|
||||
readThisCycle += read;
|
||||
|
||||
if (read == 0) break;
|
||||
Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(totalRead, contentSize));
|
||||
|
||||
fs!.Write(buffer, 0, read);
|
||||
totalRead += read;
|
||||
}
|
||||
response.Dispose();
|
||||
return true;
|
||||
}
|
||||
response.Dispose();
|
||||
return true;
|
||||
}
|
||||
|
||||
public override async Task<bool> Verify(Archive a, CancellationToken? token)
|
||||
|
@ -48,10 +48,10 @@ namespace Wabbajack.Lib.Http
|
||||
return await SendStringAsync(request, token: token);
|
||||
}
|
||||
|
||||
public async Task<string> GetStringAsync(Uri url)
|
||||
public async Task<string> GetStringAsync(Uri url, CancellationToken? token = null)
|
||||
{
|
||||
var request = new HttpRequestMessage(HttpMethod.Get, url);
|
||||
return await SendStringAsync(request);
|
||||
return await SendStringAsync(request, token: token);
|
||||
}
|
||||
|
||||
public async Task<string> DeleteStringAsync(string url)
|
||||
|
@ -22,6 +22,7 @@ namespace Wabbajack.Lib.Http
|
||||
MaxConnectionsPerServer = 20,
|
||||
PooledConnectionLifetime = TimeSpan.FromMilliseconds(100),
|
||||
PooledConnectionIdleTimeout = TimeSpan.FromMilliseconds(100),
|
||||
AutomaticDecompression = DecompressionMethods.All
|
||||
|
||||
};
|
||||
Utils.Log($"Configuring with SSL {_socketsHandler.SslOptions.EnabledSslProtocols}");
|
||||
|
@ -69,7 +69,7 @@ namespace Wabbajack.Lib
|
||||
/// </summary>
|
||||
/// <param name="action"></param>
|
||||
/// <param name="value"></param>
|
||||
public static async Task Send(string action, string value)
|
||||
public static async Task Send(string action, string subject)
|
||||
{
|
||||
if (BuildServerStatus.IsBuildServerDown)
|
||||
return;
|
||||
@ -78,7 +78,7 @@ namespace Wabbajack.Lib
|
||||
Utils.Log($"File hash check (-42) {key}");
|
||||
var client = new Http.Client();
|
||||
client.Headers.Add((Consts.MetricsKeyHeader, key));
|
||||
await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{value}");
|
||||
await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{subject}");
|
||||
}
|
||||
|
||||
public static async Task Error(Type type, Exception exception)
|
||||
|
@ -34,7 +34,17 @@ namespace Wabbajack.BuildServer.Test
|
||||
using var response = await _client.GetAsync(MakeURL($"metrics/report/{action}"));
|
||||
Assert.Equal(TimeSpan.FromHours(1), response.Headers.CacheControl.MaxAge);
|
||||
// we'll just make sure this doesn't error, with limited data that's about all we can do atm
|
||||
|
||||
|
||||
using var totalInstalls = await _client.GetAsync(MakeURL($"metrics/total_installs.html"));
|
||||
Assert.True(totalInstalls.IsSuccessStatusCode);
|
||||
|
||||
using var totalUniqueInstalls = await _client.GetAsync(MakeURL($"metrics/total_unique_installs.html"));
|
||||
Assert.True(totalUniqueInstalls.IsSuccessStatusCode);
|
||||
|
||||
using var dumpResponse = await _client.GetAsync(MakeURL("metrics/dump.json"));
|
||||
Assert.True(dumpResponse.IsSuccessStatusCode);
|
||||
var data = await dumpResponse.Content.ReadAsStringAsync();
|
||||
Assert.NotEmpty(data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -29,7 +29,6 @@ namespace Wabbajack.BuildServer
|
||||
{
|
||||
private const string ProblemDetailsContentType = "application/problem+json";
|
||||
private readonly SqlService _sql;
|
||||
private static ConcurrentHashSet<string> _knownKeys = new();
|
||||
private const string ApiKeyHeaderName = "X-Api-Key";
|
||||
|
||||
public ApiKeyAuthenticationHandler(
|
||||
@ -51,19 +50,25 @@ namespace Wabbajack.BuildServer
|
||||
{
|
||||
if (await _sql.IsTarKey(metricsKey))
|
||||
{
|
||||
await _sql.IngestMetric(new Metric {Action = "TarKey", Subject = "Auth", MetricsKey = metricsKey, Timestamp = DateTime.UtcNow});
|
||||
await _sql.IngestMetric(new Metric
|
||||
{
|
||||
Action = "TarKey",
|
||||
Subject = "Auth",
|
||||
MetricsKey = metricsKey,
|
||||
Timestamp = DateTime.UtcNow
|
||||
});
|
||||
await Task.Delay(TimeSpan.FromSeconds(60));
|
||||
throw new Exception("Error, lipsum timeout of the cross distant cloud.");
|
||||
}
|
||||
}
|
||||
|
||||
var authorKey = Request.Headers[ApiKeyHeaderName].FirstOrDefault();
|
||||
|
||||
|
||||
if (authorKey == null && metricsKey == null)
|
||||
{
|
||||
return AuthenticateResult.NoResult();
|
||||
}
|
||||
|
||||
|
||||
|
||||
if (authorKey != null)
|
||||
{
|
||||
@ -84,14 +89,12 @@ namespace Wabbajack.BuildServer
|
||||
return AuthenticateResult.Success(ticket);
|
||||
}
|
||||
|
||||
if (!_knownKeys.Contains(metricsKey) && !await _sql.ValidMetricsKey(metricsKey))
|
||||
if (!await _sql.ValidMetricsKey(metricsKey))
|
||||
{
|
||||
return AuthenticateResult.Fail("Invalid Metrics Key");
|
||||
}
|
||||
else
|
||||
{
|
||||
_knownKeys.Add(metricsKey);
|
||||
|
||||
var claims = new List<Claim> {new(ClaimTypes.Role, "User")};
|
||||
|
||||
|
||||
@ -102,7 +105,6 @@ namespace Wabbajack.BuildServer
|
||||
|
||||
return AuthenticateResult.Success(ticket);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
[JsonName("RequestLog")]
|
||||
|
@ -2,10 +2,13 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Nettle;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Common.StatusFeed;
|
||||
using Wabbajack.Server;
|
||||
using Wabbajack.Server.DataLayer;
|
||||
|
@ -1,9 +1,13 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Nettle;
|
||||
@ -34,6 +38,11 @@ namespace Wabbajack.BuildServer.Controllers
|
||||
public async Task<Result> LogMetricAsync(string subject, string value)
|
||||
{
|
||||
var date = DateTime.UtcNow;
|
||||
|
||||
// Used in tests
|
||||
if (value == "Default" || value == "untitled" || Guid.TryParse(value, out _))
|
||||
return new Result { Timestamp = date};
|
||||
|
||||
await Log(date, subject, value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault());
|
||||
return new Result { Timestamp = date};
|
||||
}
|
||||
@ -74,7 +83,7 @@ namespace Wabbajack.BuildServer.Controllers
|
||||
|
||||
return Ok(results == 0
|
||||
? new Badge($"Modlist {name} not found!", "Error") {color = "red"}
|
||||
: new Badge("Installations: ", $"{results}") {color = "green"});
|
||||
: new Badge("Installations: ", "____") {color = "green"});
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
@ -87,7 +96,7 @@ namespace Wabbajack.BuildServer.Controllers
|
||||
|
||||
return Ok(results == 0
|
||||
? new Badge($"Modlist {name} not found!", "Error") {color = "red"}
|
||||
: new Badge("Installations: ", $"{results}"){color = "green"}) ;
|
||||
: new Badge("Installations: ", "____"){color = "green"}) ;
|
||||
}
|
||||
|
||||
private static readonly Func<object, string> ReportTemplate = NettleEngine.GetCompiler().Compile(@"
|
||||
@ -149,5 +158,79 @@ namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
public DateTime Timestamp { get; set; }
|
||||
}
|
||||
|
||||
class TotalListTemplateData
|
||||
{
|
||||
public string Title { get; set; }
|
||||
public long Total { get; set; }
|
||||
public Item[] Items { get; set; }
|
||||
|
||||
public class Item
|
||||
{
|
||||
public long Count { get; set; }
|
||||
public string Title { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
private static Func<object, string> _totalListTemplate;
|
||||
|
||||
private static Func<object, string> TotalListTemplate
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_totalListTemplate == null)
|
||||
{
|
||||
var resource = Assembly.GetExecutingAssembly()
|
||||
.GetManifestResourceStream("Wabbajack.Server.Controllers.Templates.TotalListTemplate.html")!
|
||||
.ReadAll();
|
||||
_totalListTemplate = NettleEngine.GetCompiler().Compile(Encoding.UTF8.GetString(resource));
|
||||
}
|
||||
|
||||
return _totalListTemplate;
|
||||
}
|
||||
}
|
||||
|
||||
[HttpGet("total_installs.html")]
|
||||
[ResponseCache(Duration = 60 * 60)]
|
||||
public async Task<ContentResult> TotalInstalls()
|
||||
{
|
||||
var data = await _sql.GetTotalInstalls();
|
||||
var result = TotalListTemplate(new TotalListTemplateData
|
||||
{
|
||||
Title = "Total Installs",
|
||||
Total = data.Sum(d => d.Item2),
|
||||
Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2})
|
||||
.ToArray()
|
||||
});
|
||||
return new ContentResult {
|
||||
ContentType = "text/html",
|
||||
StatusCode = (int)HttpStatusCode.OK,
|
||||
Content = result};
|
||||
}
|
||||
|
||||
[HttpGet("total_unique_installs.html")]
|
||||
[ResponseCache(Duration = 60 * 60)]
|
||||
public async Task<ContentResult> TotalUniqueInstalls()
|
||||
{
|
||||
var data = await _sql.GetTotalUniqueInstalls();
|
||||
var result = TotalListTemplate(new TotalListTemplateData
|
||||
{
|
||||
Title = "Total Unique Installs",
|
||||
Total = data.Sum(d => d.Item2),
|
||||
Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2})
|
||||
.ToArray()
|
||||
});
|
||||
return new ContentResult {
|
||||
ContentType = "text/html",
|
||||
StatusCode = (int)HttpStatusCode.OK,
|
||||
Content = result};
|
||||
}
|
||||
|
||||
[HttpGet("dump.json")]
|
||||
public async Task<IActionResult> DataDump()
|
||||
{
|
||||
return Ok(await _sql.MetricsDump().ToArrayAsync());
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
@ -0,0 +1,21 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Total Installs</title>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<h2>{{$.Title}} - Total: {{$.Total}}</h2>
|
||||
|
||||
<table>
|
||||
{{each $.Items }}
|
||||
<tr>
|
||||
<td>{{$.Count}}</td>
|
||||
<td>{{$.Title}}</td>
|
||||
</tr>
|
||||
{{/each}}
|
||||
</table>
|
||||
|
||||
</body>
|
||||
</html>
|
@ -95,5 +95,72 @@ namespace Wabbajack.Server.DataLayer
|
||||
WHERE JSON_VALUE(Metadata, '$.links.machineURL') = @MachineURL)",
|
||||
new {MachineURL = machineUrl});
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<(string, long)>> GetTotalInstalls()
|
||||
{
|
||||
await using var conn = await Open();
|
||||
return await conn.QueryAsync<(string, long)>(
|
||||
@"SELECT GroupingSubject, Count(*) as Count
|
||||
From dbo.Metrics
|
||||
WHERE
|
||||
|
||||
GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics
|
||||
WHERE action = 'finish_install'
|
||||
AND MetricsKey is not null)
|
||||
group by GroupingSubject
|
||||
order by Count(*) desc");
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<(string, long)>> GetTotalUniqueInstalls()
|
||||
{
|
||||
await using var conn = await Open();
|
||||
return await conn.QueryAsync<(string, long)>(
|
||||
@"Select GroupingSubject, Count(*) as Count
|
||||
FROM
|
||||
(select DISTINCT MetricsKey, GroupingSubject
|
||||
From dbo.Metrics
|
||||
WHERE
|
||||
GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics
|
||||
WHERE action = 'finish_install'
|
||||
AND MetricsKey is not null)) m
|
||||
GROUP BY GroupingSubject
|
||||
Order by Count(*) desc
|
||||
");
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<MetricRow> MetricsDump()
|
||||
{
|
||||
var keys = new Dictionary<string, long>();
|
||||
|
||||
await using var conn = await Open();
|
||||
foreach (var row in await conn.QueryAsync<(long, DateTime, string, string, string, string)>(@"select Id, Timestamp, Action, Subject, MetricsKey, GroupingSubject from dbo.metrics WHERE MetricsKey is not null"))
|
||||
{
|
||||
if (!keys.TryGetValue(row.Item5, out var keyid))
|
||||
{
|
||||
keyid = keys.Count;
|
||||
keys[row.Item5] = keyid;
|
||||
}
|
||||
|
||||
yield return new MetricRow
|
||||
{
|
||||
Id = row.Item1,
|
||||
Timestamp = row.Item2,
|
||||
Action = row.Item3,
|
||||
Subject = row.Item4,
|
||||
MetricsKey = keyid,
|
||||
GroupingSubject = row.Item6
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public class MetricRow
|
||||
{
|
||||
public long Id;
|
||||
public DateTime Timestamp;
|
||||
public string Action;
|
||||
public string Subject;
|
||||
public string GroupingSubject;
|
||||
public long MetricsKey;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -5,6 +5,7 @@ using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.BuildServer;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Common.Exceptions;
|
||||
using Wabbajack.Lib.NexusApi;
|
||||
using Wabbajack.Server.DataLayer;
|
||||
|
||||
@ -28,13 +29,12 @@ namespace Wabbajack.Server.Services
|
||||
try
|
||||
{
|
||||
var client = new TrackingClient(_sql, key);
|
||||
if (!await client.IsPremium())
|
||||
{
|
||||
_logger.LogWarning($"Purging non premium key");
|
||||
await _sql.DeleteNexusAPIKey(key.Key);
|
||||
continue;
|
||||
}
|
||||
return client;
|
||||
if (await client.IsPremium())
|
||||
return client;
|
||||
|
||||
_logger.LogWarning($"Purging non premium key");
|
||||
await _sql.DeleteNexusAPIKey(key.Key);
|
||||
continue;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@ -69,13 +69,16 @@ namespace Wabbajack.Server.Services
|
||||
var (daily, hourly) = await client.GetRemainingApiCalls();
|
||||
await _sql.SetNexusAPIKey(key.Key, daily, hourly);
|
||||
}
|
||||
catch (Exception)
|
||||
catch (HttpException ex)
|
||||
{
|
||||
_logger.Log(LogLevel.Warning, "Update error, purging API key");
|
||||
_logger.Log(LogLevel.Warning, $"Nexus error, not purging API key : {ex.Message}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Log(LogLevel.Warning, $"Update error, purging API key : {ex.Message}");
|
||||
await _sql.DeleteNexusAPIKey(key.Key);
|
||||
}
|
||||
}
|
||||
|
||||
return keys.Count;
|
||||
}
|
||||
}
|
||||
|
@ -41,8 +41,8 @@ namespace Wabbajack.Server.Services
|
||||
bool isValid = false;
|
||||
switch (archive.State)
|
||||
{
|
||||
case WabbajackCDNDownloader.State _:
|
||||
case GoogleDriveDownloader.State _:
|
||||
case WabbajackCDNDownloader.State _:
|
||||
//case GoogleDriveDownloader.State _: // Let's try validating Google again 2/10/2021
|
||||
case ManualDownloader.State _:
|
||||
case ModDBDownloader.State _:
|
||||
case HTTPDownloader.State h when h.Url.StartsWith("https://wabbajack"):
|
||||
|
@ -5,6 +5,7 @@ using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Http.Features;
|
||||
using Microsoft.AspNetCore.ResponseCompression;
|
||||
using Microsoft.AspNetCore.StaticFiles;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
@ -76,6 +77,12 @@ namespace Wabbajack.Server
|
||||
services.AddSingleton<Watchdog>();
|
||||
services.AddSingleton<DiscordFrontend>();
|
||||
services.AddSingleton<AuthoredFilesCleanup>();
|
||||
services.AddResponseCompression(options =>
|
||||
{
|
||||
options.Providers.Add<BrotliCompressionProvider>();
|
||||
options.Providers.Add<GzipCompressionProvider>();
|
||||
options.MimeTypes = new[] {"*/*"};
|
||||
});
|
||||
|
||||
services.AddMvc();
|
||||
services.AddControllers()
|
||||
@ -123,6 +130,7 @@ namespace Wabbajack.Server
|
||||
app.UseNexusPoll();
|
||||
app.UseArchiveMaintainer();
|
||||
app.UseModListDownloader();
|
||||
app.UseResponseCompression();
|
||||
|
||||
app.UseService<NonNexusDownloadValidator>();
|
||||
app.UseService<ListValidator>();
|
||||
|
@ -45,6 +45,8 @@
|
||||
<ItemGroup>
|
||||
<None Remove="sheo_quotes.txt" />
|
||||
<EmbeddedResource Include="sheo_quotes.txt" />
|
||||
<None Remove="Controllers\Templates\TotalListTemplate.html" />
|
||||
<EmbeddedResource Include="Controllers\Templates\TotalListTemplate.html" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
Loading…
Reference in New Issue
Block a user