Merge pull request #1322 from wabbajack-tools/latest-server-fixes

latest server fixes

Wabbajack.CLI/OptionsDefinition.cs

@@ -38,7 +38,8 @@ namespace Wabbajack.CLI
             typeof(PurgeArchive),
             typeof(AllKnownDownloadStates),
             typeof(VerifyAllDownloads),
-            typeof(HashBenchmark)
+            typeof(HashBenchmark),
+            typeof(StressTestURL)
         };
     }
 }

Wabbajack.CLI/Verbs/StressTestURL.cs

@@ -0,0 +1,52 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using CommandLine;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+
+namespace Wabbajack.CLI.Verbs
+{
+    [Verb("stress-test-url", HelpText = "Verify a file rapidly, trying to make it fail")]
+    public class StressTestURL : AVerb
+    {
+        [Option('i', "input", Required = true, HelpText = "Input url to stress")]
+        public string Input { get; set; } = "";
+        private Uri _Input => new Uri(Input);
+        
+        protected override async Task<ExitCode> Run()
+        {
+            using var queue = new WorkQueue();
+            var state = await DownloadDispatcher.Infer(_Input);
+            if (state == null)
+            {
+                Console.WriteLine("Could not parse URL");
+            }
+
+            Console.WriteLine("Performing initial download");
+            await using var temp = new TempFile();
+            var archive = new Archive(state!);
+            if (!await state!.Download(archive, temp.Path))
+            {
+                Console.WriteLine("Failed initial download");
+            }
+
+            var hash = await temp.Path.FileHashAsync();
+            archive.Hash = hash!.Value;
+            archive.Size = temp.Path.Size;
+            Console.WriteLine($"Hash: {archive.Hash} Size: {archive.Size.ToFileSizeString()}");
+
+            await Enumerable.Range(0, 100000)
+                .PMap(queue, async idx =>
+                {
+                    if (!await state.Verify(archive))
+                    {
+                        throw new Exception($"{idx} Verification failed");
+                    }
+                    Console.WriteLine($"{idx} Verification passed");
+                });
+            return ExitCode.Ok;
+        }
+    }
+}

Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs

@@ -400,7 +400,7 @@ namespace Wabbajack.Lib.Downloaders
             public async Task<string> GetStringAsync(Uri uri, CancellationToken? token = null)
             {
                 if (!Downloader.IsCloudFlareProtected)
-                    return await Downloader.AuthedClient.GetStringAsync(uri);
+                    return await Downloader.AuthedClient.GetStringAsync(uri, token);
 
                 
                 using var driver = await Downloader.GetAuthedDriver();
@@ -448,7 +448,7 @@ namespace Wabbajack.Lib.Downloaders
                     return await Downloader.AuthedClient.GetAsync(uri);
 
                 using var driver = await Downloader.GetAuthedDriver();
-                TaskCompletionSource<Uri?> promise = new TaskCompletionSource<Uri?>();
+                TaskCompletionSource<Uri?> promise = new();
                 driver.DownloadHandler = uri1 =>
                 {
                     promise.SetResult(uri);

Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs

@@ -1,10 +1,13 @@
-using System.Text.RegularExpressions;
+using System.Linq;
+using System.Net.Http;
+using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
 using Newtonsoft.Json;
 using Wabbajack.Common;
 using Wabbajack.Common.Exceptions;
 using Wabbajack.Common.Serialization.Json;
+using Wabbajack.Lib.Http;
 using Wabbajack.Lib.Validation;
 
 namespace Wabbajack.Lib.Downloaders
@@ -54,19 +57,27 @@ namespace Wabbajack.Lib.Downloaders
             public override async Task<bool> Download(Archive a, AbsolutePath destination)
             {
                 var state = await ToHttpState();
+                if (state == null)
+                    return false;
                 return await state.Download(a, destination);
             }
 
-            private async Task<HTTPDownloader.State> ToHttpState()
+            private async Task<HTTPDownloader.State?> ToHttpState()
             {
                 var initialURL = $"https://drive.google.com/uc?id={Id}&export=download";
                 var client = new Wabbajack.Lib.Http.Client();
                 using var response = await client.GetAsync(initialURL);
                 if (!response.IsSuccessStatusCode)
                     throw new HttpException((int)response.StatusCode, response.ReasonPhrase ?? "Unknown");
-                var regex = new Regex("(?<=/uc\\?export=download&amp;confirm=).*(?=;id=)");
+                var cookies = response.GetSetCookies();
-                var confirm = regex.Match(await response.Content.ReadAsStringAsync());
+                var warning = cookies.FirstOrDefault(c => c.Key.StartsWith("download_warning_"));
-                var url = $"https://drive.google.com/uc?export=download&confirm={confirm}&id={Id}";
+                response.Dispose();
+                if (warning == default)
+                {
+                    return new HTTPDownloader.State(initialURL) { Client = client };
+                }
+
+                var url = $"https://drive.google.com/uc?export=download&confirm={warning.Value}&id={Id}";
                 var httpState = new HTTPDownloader.State(url) { Client = client };
                 return httpState;
             }
@@ -74,6 +85,8 @@ namespace Wabbajack.Lib.Downloaders
             public override async Task<bool> Verify(Archive a, CancellationToken? token)
             {
                 var state = await ToHttpState();
+                if (state == null)
+                    return false;
                 return await state.Verify(a, token);
             }
 

Wabbajack.Lib/Downloaders/HTTPDownloader.cs

@@ -82,111 +82,119 @@ namespace Wabbajack.Lib.Downloaders
                     destination.Parent.CreateDirectory();
                 }
 
-                using (var fs = download ? await destination.Create() : null)
+                await using var fs = download ? await destination.Create() : null;
+                var client = Client ?? await ClientAPI.GetClient();
+                client.Headers.Add(("User-Agent", Consts.UserAgent));
+
+                foreach (var header in Headers)
                 {
-                    var client = Client ?? await ClientAPI.GetClient();
+                    var idx = header.IndexOf(':');
-                    client.Headers.Add(("User-Agent", Consts.UserAgent));
+                    var k = header.Substring(0, idx);
+                    var v = header.Substring(idx + 1);
+                    client.Headers.Add((k, v));
+                }
 
-                    foreach (var header in Headers)
+                long totalRead = 0;
+                const int bufferSize = 1024 * 32 * 8;
+
+                Utils.Status($"Starting Download {a.Name ?? Url}", Percent.Zero);
+                var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false, token:token);
+                TOP:
+
+                if (!response.IsSuccessStatusCode) 
+                {
+                    response.Dispose();
+                    return false;
+                }
+
+                Stream stream;
+                try
+                {
+                    stream = await response.Content.ReadAsStreamAsync();
+                }
+                catch (Exception ex)
+                {
+                    Utils.Error(ex, $"While downloading {Url}");
+                    return false;
+                }
+
+
+                var headerVar = a.Size == 0 ? "1" : a.Size.ToString();
+                long headerContentSize = 0;
+                if (response.Content.Headers.Contains("Content-Length"))
+                {
+                    headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault();
+                    if (headerVar != null)
+                        long.TryParse(headerVar, out headerContentSize);
+                }
+
+                if (!download)
+                {
+                    await stream.DisposeAsync();
+                    response.Dispose();
+                    if (a.Size != 0 && headerContentSize != 0)
+                        return a.Size == headerContentSize;
+                    return true;
+                }
+
+                var supportsResume = response.Headers.AcceptRanges.FirstOrDefault(f => f == "bytes") != null;
+
+                var contentSize = headerVar != null ? long.Parse(headerVar) : 1;
+
+                await using (var webs = stream)
+                {
+                    var buffer = new byte[bufferSize];
+                    int readThisCycle = 0;
+
+                    while (!(token ?? CancellationToken.None).IsCancellationRequested)
                     {
-                        var idx = header.IndexOf(':');
+                        int read = 0;
-                        var k = header.Substring(0, idx);
+                        try
-                        var v = header.Substring(idx + 1);
-                        client.Headers.Add((k, v));
-                    }
-
-                    long totalRead = 0;
-                    var bufferSize = 1024 * 32 * 8;
-
-                    Utils.Status($"Starting Download {a.Name ?? Url}", Percent.Zero);
-                    var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false, token:token);
-TOP:
-
-                    if (!response.IsSuccessStatusCode) 
-                    {
-                        return false;
-                    }
-
-                    Stream stream;
-                    try
-                    {
-                        stream = await response.Content.ReadAsStreamAsync();
-                    }
-                    catch (Exception ex)
-                    {
-                        Utils.Error(ex, $"While downloading {Url}");
-                        return false;
-                    }
-
-
-                    var headerVar = a.Size == 0 ? "1" : a.Size.ToString();
-                    long header_content_size = 0;
-                    if (response.Content.Headers.Contains("Content-Length"))
-                    {
-                        headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault();
-                        if (headerVar != null)
-                            long.TryParse(headerVar, out header_content_size);
-                    }
-
-                    if (!download)
-                    {
-                        if (a.Size != 0 && header_content_size != 0)
-                            return a.Size == header_content_size;
-                        return true;
-                    }
-
-                    var supportsResume = response.Headers.AcceptRanges.FirstOrDefault(f => f == "bytes") != null;
-
-                    var contentSize = headerVar != null ? long.Parse(headerVar) : 1;
-
-                    using (var webs = stream)
-                    {
-                        var buffer = new byte[bufferSize];
-                        int readThisCycle = 0;
-
-                        while (!(token ?? CancellationToken.None).IsCancellationRequested)
                         {
-                            int read = 0;
+                            read = await webs.ReadAsync(buffer, 0, bufferSize);
-                            try
+                        }
+                        catch (Exception)
+                        {
+                            if (readThisCycle == 0)
                             {
-                                read = await webs.ReadAsync(buffer, 0, bufferSize);
+                                await stream.DisposeAsync();
+                                response.Dispose();
+                                throw;
                             }
-                            catch (Exception)
+
+                            if (totalRead < contentSize)
                             {
-                                if (readThisCycle == 0)
+                                if (!supportsResume)
-                                    throw;
-
-                                if (totalRead < contentSize)
                                 {
-                                    if (supportsResume)
+                                    await stream.DisposeAsync();
-                                    {
+                                    response.Dispose();
-                                        Utils.Log(
-                                            $"Abort during download, trying to resume {Url} from {totalRead.ToFileSizeString()}");
-
-                                        var msg = new HttpRequestMessage(HttpMethod.Get, Url);
-                                        msg.Headers.Range = new RangeHeaderValue(totalRead, null);
-                                        response.Dispose();
-                                        response = await client.SendAsync(msg);
-                                        goto TOP;
-                                    }
                                     throw;
                                 }
 
-                                break;
+                                Utils.Log(
+                                    $"Abort during download, trying to resume {Url} from {totalRead.ToFileSizeString()}");
+
+                                var msg = new HttpRequestMessage(HttpMethod.Get, Url);
+                                msg.Headers.Range = new RangeHeaderValue(totalRead, null);
+                                response.Dispose();
+                                response = await client.SendAsync(msg);
+                                goto TOP;
                             }
 
-                            readThisCycle += read;
+                            break;
-
-                            if (read == 0) break;
-                            Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(totalRead, contentSize));
-
-                            fs!.Write(buffer, 0, read);
-                            totalRead += read;
                         }
+
+                        readThisCycle += read;
+
+                        if (read == 0) break;
+                        Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(totalRead, contentSize));
+
+                        fs!.Write(buffer, 0, read);
+                        totalRead += read;
                     }
-                    response.Dispose();
-                    return true;
                 }
+                response.Dispose();
+                return true;
             }
 
             public override async Task<bool> Verify(Archive a, CancellationToken? token)

Wabbajack.Lib/Http/Client.cs

@@ -48,10 +48,10 @@ namespace Wabbajack.Lib.Http
             return await SendStringAsync(request, token: token);
         }
         
-        public async Task<string> GetStringAsync(Uri url)
+        public async Task<string> GetStringAsync(Uri url, CancellationToken? token = null)
         {
             var request = new HttpRequestMessage(HttpMethod.Get, url);
-            return await SendStringAsync(request);
+            return await SendStringAsync(request, token: token);
         }
         
         public async Task<string> DeleteStringAsync(string url)

Wabbajack.Lib/Http/ClientFactory.cs

@@ -22,6 +22,7 @@ namespace Wabbajack.Lib.Http
                 MaxConnectionsPerServer = 20,
                 PooledConnectionLifetime = TimeSpan.FromMilliseconds(100),
                 PooledConnectionIdleTimeout = TimeSpan.FromMilliseconds(100),
+                AutomaticDecompression = DecompressionMethods.All
                
             };
             Utils.Log($"Configuring with SSL {_socketsHandler.SslOptions.EnabledSslProtocols}");

Wabbajack.Lib/Http/HttpExtensions.cs

@@ -0,0 +1,22 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+
+namespace Wabbajack.Lib.Http
+{
+    public static class HttpExtensions
+    {
+        public static IEnumerable<(string Key, string Value)> GetSetCookies(this HttpResponseMessage response)
+        {
+            if (!response.Headers.TryGetValues("set-cookie", out var values))
+                return Array.Empty<(string, string)>();
+            
+            return values
+                .SelectMany(h => h.Split(";"))
+                .Select(h => h.Split("="))
+                .Where(h => h.Length == 2)
+                .Select(h => (h[0], h[1]));
+        }
+    }
+}

Wabbajack.Lib/Metrics.cs

@@ -69,7 +69,7 @@ namespace Wabbajack.Lib
         /// </summary>
         /// <param name="action"></param>
         /// <param name="value"></param>
-        public static async Task Send(string action, string value)
+        public static async Task Send(string action, string subject)
         {
             if (BuildServerStatus.IsBuildServerDown)
                 return;
@@ -78,7 +78,7 @@ namespace Wabbajack.Lib
             Utils.Log($"File hash check (-42) {key}");
             var client = new Http.Client();
             client.Headers.Add((Consts.MetricsKeyHeader, key));
-            await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{value}");
+            await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{subject}");
         }
 
         public static async Task Error(Type type, Exception exception)

Wabbajack.Server.Test/ABuildServerSystemTest.cs

@@ -15,6 +15,7 @@ using Wabbajack.Lib.ModListRegistry;
 using Wabbajack.Server;
 using Wabbajack.Server.DataLayer;
 using Wabbajack.Server.DTOs;
+using Wabbajack.Server.Services;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -166,6 +167,8 @@ namespace Wabbajack.BuildServer.Test
             _client = new Wabbajack.Lib.Http.Client();
             _authedClient = new Wabbajack.Lib.Http.Client();
             Fixture = fixture.Deref();
+            var cache = Fixture.GetService<MetricsKeyCache>();
+            cache.AddKey(Metrics.GetMetricsKey().Result);
             _authedClient.Headers.Add(("x-api-key", Fixture.APIKey));
             AuthorAPI.ApiKeyOverride = Fixture.APIKey;
             _queue = new WorkQueue();

Wabbajack.Server.Test/MetricsTests.cs

@@ -5,6 +5,7 @@ using Dapper;
 using Wabbajack.Lib;
 using Wabbajack.Server.DataLayer;
 using Wabbajack.Server.DTOs;
+using Wabbajack.Server.Services;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -34,7 +35,20 @@ namespace Wabbajack.BuildServer.Test
             using var response = await _client.GetAsync(MakeURL($"metrics/report/{action}"));
             Assert.Equal(TimeSpan.FromHours(1), response.Headers.CacheControl.MaxAge);
             // we'll just make sure this doesn't error, with limited data that's about all we can do atm
-           
+            
+            using var totalInstalls = await _client.GetAsync(MakeURL($"metrics/total_installs.html"));
+            Assert.True(totalInstalls.IsSuccessStatusCode);
+            
+            using var totalUniqueInstalls = await _client.GetAsync(MakeURL($"metrics/total_unique_installs.html"));
+            Assert.True(totalUniqueInstalls.IsSuccessStatusCode);
+
+            using var dumpResponse = await _client.GetAsync(MakeURL("metrics/dump.json"));
+            Assert.True(dumpResponse.IsSuccessStatusCode);
+            var data = await dumpResponse.Content.ReadAsStringAsync();
+            Assert.NotEmpty(data);
+
+            var cache = Fixture.GetService<MetricsKeyCache>();
+            Assert.True(await cache.KeyCount() > 0);
         }
     }
 }

Wabbajack.Server.Test/sql/wabbajack_db.sql

@@ -550,6 +550,25 @@ CREATE TABLE [dbo].[ModListArchives](
 )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
     ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
     GO
+    
+/****** Object:  Table [dbo].[MetricsKeys]    Script Date: 2/17/2021 9:20:27 PM ******/
+SET ANSI_NULLS ON
+GO
+
+SET QUOTED_IDENTIFIER ON
+GO
+
+CREATE TABLE [dbo].[MetricsKeys](
+[MetricsKey] [varchar](64) NOT NULL,
+CONSTRAINT [PK_MetricsKeys] PRIMARY KEY CLUSTERED
+    (
+    [MetricsKey] ASC
+    )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+) ON [PRIMARY]
+GO
+    
+    
+    
 /****** Object:  Table [dbo].[ModListArchiveStatus]    Script Date: 12/29/2020 8:55:04 PM ******/
     SET ANSI_NULLS ON
     GO

Wabbajack.Server/ApiKeyAuthorizationHandler.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Security.Claims;
@@ -13,6 +14,7 @@ using Wabbajack.Common;
 using Wabbajack.Common.Serialization.Json;
 using Wabbajack.Server.DataLayer;
 using Wabbajack.Server.DTOs;
+using Wabbajack.Server.Services;
 
 
 namespace Wabbajack.BuildServer
@@ -29,17 +31,20 @@ namespace Wabbajack.BuildServer
     {
         private const string ProblemDetailsContentType = "application/problem+json";
         private readonly SqlService _sql;
-        private static ConcurrentHashSet<string> _knownKeys = new();
         private const string ApiKeyHeaderName = "X-Api-Key";
 
+        private MetricsKeyCache _keyCache;
+
         public ApiKeyAuthenticationHandler(
             IOptionsMonitor<ApiKeyAuthenticationOptions> options,
             ILoggerFactory logger,
             UrlEncoder encoder,
             ISystemClock clock,
+            MetricsKeyCache keyCache,
             SqlService db) : base(options, logger, encoder, clock)
         {
             _sql = db;
+            _keyCache = keyCache;
         }
 
         protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
@@ -51,19 +56,25 @@ namespace Wabbajack.BuildServer
             {
                 if (await _sql.IsTarKey(metricsKey))
                 {
-                    await _sql.IngestMetric(new Metric {Action = "TarKey", Subject = "Auth", MetricsKey = metricsKey, Timestamp = DateTime.UtcNow});
+                    await _sql.IngestMetric(new Metric
+                    {
+                        Action = "TarKey",
+                        Subject = "Auth",
+                        MetricsKey = metricsKey,
+                        Timestamp = DateTime.UtcNow
+                    });
                     await Task.Delay(TimeSpan.FromSeconds(60));
                     throw new Exception("Error, lipsum timeout of the cross distant cloud.");
                 }
             }
 
             var authorKey = Request.Headers[ApiKeyHeaderName].FirstOrDefault();
-            
+
             if (authorKey == null && metricsKey == null)
             {
                 return AuthenticateResult.NoResult();
             }
-            
+
 
             if (authorKey != null)
             {
@@ -83,15 +94,14 @@ namespace Wabbajack.BuildServer
 
                 return AuthenticateResult.Success(ticket);
             }
+            
 
-            if (!_knownKeys.Contains(metricsKey) && !await _sql.ValidMetricsKey(metricsKey))
+            if (!await _keyCache.IsValidKey(metricsKey))
             {
                 return AuthenticateResult.Fail("Invalid Metrics Key");
             }
             else
             {
-                _knownKeys.Add(metricsKey);
-                
                 var claims = new List<Claim> {new(ClaimTypes.Role, "User")};
 
 
@@ -102,7 +112,6 @@ namespace Wabbajack.BuildServer
 
                 return AuthenticateResult.Success(ticket);
             }
-
         }
 
         [JsonName("RequestLog")]

Wabbajack.Server/Controllers/Heartbeat.cs

@@ -2,10 +2,13 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
+using System.Reflection;
+using System.Text;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using Nettle;
+using Wabbajack.Common;
 using Wabbajack.Common.StatusFeed;
 using Wabbajack.Server;
 using Wabbajack.Server.DataLayer;

Wabbajack.Server/Controllers/Metrics.cs

@@ -1,9 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
+using System.IO;
 using System.Linq;
 using System.Net;
+using System.Reflection;
+using System.Text;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using Nettle;
@@ -11,6 +15,7 @@ using Wabbajack.Common;
 using Wabbajack.Server;
 using Wabbajack.Server.DataLayer;
 using Wabbajack.Server.DTOs;
+using Wabbajack.Server.Services;
 using WebSocketSharp;
 using LogLevel = Microsoft.Extensions.Logging.LogLevel;
 
@@ -22,11 +27,13 @@ namespace Wabbajack.BuildServer.Controllers
     {
         private SqlService _sql;
         private ILogger<MetricsController> _logger;
+        private MetricsKeyCache _keyCache;
 
-        public MetricsController(ILogger<MetricsController> logger, SqlService sql)
+        public MetricsController(ILogger<MetricsController> logger, SqlService sql, MetricsKeyCache keyCache)
         {
             _sql = sql;
             _logger = logger;
+            _keyCache = keyCache;
         }
 
         [HttpGet]
@@ -34,7 +41,15 @@ namespace Wabbajack.BuildServer.Controllers
         public async Task<Result> LogMetricAsync(string subject, string value)
         {
             var date = DateTime.UtcNow;
-            await Log(date, subject, value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault());
+            var metricsKey = Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault();
+            if (metricsKey != null)
+                await _keyCache.AddKey(metricsKey);
+            
+            // Used in tests
+            if (value == "Default" || value == "untitled" || subject == "failed_download" || Guid.TryParse(value, out _))
+                return new Result { Timestamp = date};
+            
+            await Log(date, subject, value, metricsKey);
             return new Result { Timestamp = date};
         }
 
@@ -74,7 +89,7 @@ namespace Wabbajack.BuildServer.Controllers
            
             return Ok(results == 0 
                 ? new Badge($"Modlist {name} not found!", "Error") {color = "red"} 
-                : new Badge("Installations: ", $"{results}") {color = "green"});
+                : new Badge("Installations: ", "____") {color = "green"});
         }
 
         [HttpGet]
@@ -87,7 +102,7 @@ namespace Wabbajack.BuildServer.Controllers
 
             return Ok(results == 0
                 ? new Badge($"Modlist {name} not found!", "Error") {color = "red"}
-                : new Badge("Installations: ", $"{results}"){color = "green"}) ;
+                : new Badge("Installations: ", "____"){color = "green"}) ;
     }
 
         private static readonly Func<object, string> ReportTemplate = NettleEngine.GetCompiler().Compile(@"
@@ -149,5 +164,82 @@ namespace Wabbajack.BuildServer.Controllers
         {
             public DateTime Timestamp { get; set; }
         }
+        
+                class TotalListTemplateData
+        {
+            public string Title { get; set; }
+            public long Total { get; set; }
+            public Item[] Items { get; set; }
+
+            public class Item
+            {
+                public long Count { get; set; }
+                public string Title { get; set; }
+            }
+        }
+
+        private static Func<object, string> _totalListTemplate;
+
+
+        private static Func<object, string> TotalListTemplate
+        {
+            get
+            {
+                if (_totalListTemplate == null)
+                {
+                    var resource = Assembly.GetExecutingAssembly()
+                        .GetManifestResourceStream("Wabbajack.Server.Controllers.Templates.TotalListTemplate.html")!
+                        .ReadAll();
+                    _totalListTemplate = NettleEngine.GetCompiler().Compile(Encoding.UTF8.GetString(resource));
+                }
+
+                return _totalListTemplate;
+            }
+        }
+        
+        
+
+        [HttpGet("total_installs.html")]
+        [ResponseCache(Duration = 60 * 60)]
+        public async Task<ContentResult> TotalInstalls()
+        {
+            var data = await _sql.GetTotalInstalls();
+            var result = TotalListTemplate(new TotalListTemplateData
+            {
+                Title = "Total Installs",
+                Total = data.Sum(d => d.Item2),
+                Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2})
+                    .ToArray()
+            });
+            return new ContentResult {
+                ContentType = "text/html", 
+                StatusCode = (int)HttpStatusCode.OK, 
+                Content = result};
+        }
+        
+        [HttpGet("total_unique_installs.html")]
+        [ResponseCache(Duration = 60 * 60)]
+        public async Task<ContentResult> TotalUniqueInstalls()
+        {
+            var data = await _sql.GetTotalUniqueInstalls();
+            var result = TotalListTemplate(new TotalListTemplateData
+            {
+                Title = "Total Unique Installs",
+                Total = data.Sum(d => d.Item2),
+                Items = data.Select(d => new TotalListTemplateData.Item {Title = d.Item1, Count = d.Item2})
+                    .ToArray()
+            });
+            return new ContentResult {
+                ContentType = "text/html", 
+                StatusCode = (int)HttpStatusCode.OK, 
+                Content = result};
+        }
+
+        [HttpGet("dump.json")]
+        public async Task<IActionResult> DataDump()
+        {
+            return Ok(await _sql.MetricsDump().ToArrayAsync());
+        }
+
     }
 }

Wabbajack.Server/Controllers/Templates/TotalListTemplate.html

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Total Installs</title>
+</head>
+<body>
+
+<h2>{{$.Title}} - Total: {{$.Total}}</h2>
+
+<table>
+    {{each $.Items }}
+    <tr>
+        <td>{{$.Count}}</td>
+        <td>{{$.Title}}</td>
+    </tr>
+    {{/each}}
+</table>
+
+</body>
+</html>

Wabbajack.Server/DataLayer/Metrics.cs

@@ -70,8 +70,28 @@ namespace Wabbajack.Server.DataLayer
         public async Task<bool> ValidMetricsKey(string metricsKey)
         {
             await using var conn = await Open();
-            return (await conn.QueryAsync<string>("SELECT TOP(1) MetricsKey from Metrics Where MetricsKey = @MetricsKey",
+            return (await conn.QuerySingleOrDefaultAsync<string>("SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey",
-                new {MetricsKey = metricsKey})).FirstOrDefault() != null;
+                new {MetricsKey = metricsKey})) != default;
+        }
+
+        public async Task AddMetricsKey(string metricsKey)
+        {
+            await using var conn = await Open();
+            await using var trans = conn.BeginTransaction();
+
+            if ((await conn.QuerySingleOrDefaultAsync<string>(
+                "SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey",
+                new {MetricsKey = metricsKey}, trans)) != default)
+                return;
+            
+            await conn.ExecuteAsync("INSERT INTO dbo.MetricsKeys (MetricsKey) VALUES (@MetricsKey)",
+                new {MetricsKey = metricsKey}, trans);
+        }
+
+        public async Task<string[]> AllKeys()
+        {
+            await using var conn = await Open();
+            return (await conn.QueryAsync<string>("SELECT MetricsKey from dbo.MetricsKeys")).ToArray();
         }
 
 
@@ -95,5 +115,72 @@ namespace Wabbajack.Server.DataLayer
                         WHERE JSON_VALUE(Metadata, '$.links.machineURL') = @MachineURL)",
                 new {MachineURL = machineUrl});
         }
+
+        public async Task<IEnumerable<(string, long)>> GetTotalInstalls()
+        {
+            await using var conn = await Open();
+            return await conn.QueryAsync<(string, long)>(
+                @"SELECT GroupingSubject, Count(*) as Count
+                        From dbo.Metrics
+                        WHERE 
+
+                        GroupingSubject in  (select DISTINCT GroupingSubject from dbo.Metrics
+                            WHERE action = 'finish_install'
+                            AND MetricsKey is not null)
+	                        group by GroupingSubject
+	                        order by Count(*) desc");
+        }
+        
+        public async Task<IEnumerable<(string, long)>> GetTotalUniqueInstalls()
+        {
+            await using var conn = await Open();
+            return await conn.QueryAsync<(string, long)>(
+                @"Select GroupingSubject, Count(*) as Count
+                        FROM
+                        (select DISTINCT MetricsKey, GroupingSubject
+                        From dbo.Metrics
+                        WHERE 
+                        GroupingSubject in  (select DISTINCT GroupingSubject from dbo.Metrics
+                            WHERE action = 'finish_install'
+                            AND MetricsKey is not null)) m
+                        GROUP BY GroupingSubject
+                        Order by Count(*) desc
+                        ");
+        }
+
+        public async IAsyncEnumerable<MetricRow> MetricsDump()
+        {
+            var keys = new Dictionary<string, long>();
+            
+            await using var conn = await Open();
+            foreach (var row in await conn.QueryAsync<(long, DateTime, string, string, string, string)>(@"select Id, Timestamp, Action, Subject, MetricsKey, GroupingSubject from dbo.metrics WHERE MetricsKey is not null"))
+            {
+                if (!keys.TryGetValue(row.Item5, out var keyid))
+                {
+                    keyid = keys.Count;
+                    keys[row.Item5] = keyid;
+                }
+
+                yield return new MetricRow
+                {
+                    Id = row.Item1,
+                    Timestamp = row.Item2,
+                    Action = row.Item3,
+                    Subject = row.Item4,
+                    MetricsKey = keyid,
+                    GroupingSubject = row.Item6
+                };
+            }
+        }
+
+        public class MetricRow
+        {
+            public long Id;
+            public DateTime Timestamp;
+            public string Action;
+            public string Subject;
+            public string GroupingSubject;
+            public long MetricsKey;
+        }
     }
 }

Wabbajack.Server/Services/DiscordFrontend.cs

@@ -19,8 +19,9 @@ namespace Wabbajack.Server.Services
         private QuickSync _quickSync;
         private DiscordSocketClient _client;
         private SqlService _sql;
+        private MetricsKeyCache _keyCache;
 
-        public DiscordFrontend(ILogger<DiscordFrontend> logger, AppSettings settings, QuickSync quickSync, SqlService sql)
+        public DiscordFrontend(ILogger<DiscordFrontend> logger, AppSettings settings, QuickSync quickSync, SqlService sql, MetricsKeyCache keyCache)
         {
             _logger = logger;
             _settings = settings;
@@ -33,6 +34,7 @@ namespace Wabbajack.Server.Services
             _client.MessageReceived += MessageReceivedAsync;
 
             _sql = sql;
+            _keyCache = keyCache;
         }
 
         private async Task MessageReceivedAsync(SocketMessage arg)
@@ -88,6 +90,10 @@ namespace Wabbajack.Server.Services
                         await ReplyTo(arg, $"Purged all traces of #{parts[2]} from the server, triggered list downloading. {deleted} records removed");
                     }
                 }
+                else if (parts[1] == "users")
+                {
+                    await ReplyTo(arg, $"Wabbajack has {await _keyCache.KeyCount()} known unique users");
+                }
             }
         }
 

Wabbajack.Server/Services/ListValidator.cs

@@ -197,6 +197,7 @@ namespace Wabbajack.Server.Services
         private AsyncLock _healLock = new AsyncLock();
         private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive, ModlistMetadata modList)
         {
+            using var _ = await _healLock.WaitAsync();  
             var srcDownload = await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size);
             if (srcDownload == null || srcDownload.IsFailed == true)
             {
@@ -204,7 +205,7 @@ namespace Wabbajack.Server.Services
                 return (archive, ArchiveStatus.InValid);
             }
 
-            
+         
             var patches = await _sql.PatchesForSource(archive.Hash);
             foreach (var patch in patches)
             {
@@ -219,7 +220,7 @@ namespace Wabbajack.Server.Services
                     return (archive, ArchiveStatus.Updated);
             }
 
-            using var _ = await _healLock.WaitAsync();
+
             var upgradeTime = DateTime.UtcNow;
             _logger.LogInformation($"Validator Finding Upgrade for {archive.Hash} {archive.State.PrimaryKeyString}");
 

Wabbajack.Server/Services/MetricsKeyCache.cs

@@ -0,0 +1,62 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.Common;
+using Wabbajack.Server.DataLayer;
+
+namespace Wabbajack.Server.Services
+{
+    public class MetricsKeyCache : IStartable
+    {
+        private ILogger<MetricsKeyCache> _logger;
+        private SqlService _sql;
+        private HashSet<string> _knownKeys = new();
+        private AsyncLock _lock = new();
+
+        public MetricsKeyCache(ILogger<MetricsKeyCache> logger, SqlService sql)
+        {
+            _logger = logger;
+            _sql = sql;
+        }
+
+        public async Task<bool> IsValidKey(string key)
+        {
+            using (var _ = await _lock.WaitAsync())
+            {
+                if (_knownKeys.Contains(key)) return true;
+            }
+
+            if (await _sql.ValidMetricsKey(key))
+            {
+                using var _ = await _lock.WaitAsync();
+                _knownKeys.Add(key);
+                return true;
+            }
+
+            return false;
+        }
+
+        public async Task AddKey(string key)
+        {
+            using (var _ = await _lock.WaitAsync())
+            {
+                if (_knownKeys.Contains(key)) return;
+                _knownKeys.Add(key);
+            }
+
+            await _sql.AddMetricsKey(key);
+        }
+
+        public void Start()
+        {
+            _knownKeys = (_sql.AllKeys().Result).ToHashSet();
+        }
+
+        public async Task<long> KeyCount()
+        {
+            using var _ = await _lock.WaitAsync();
+            return _knownKeys.Count;
+        }
+    }
+}

Wabbajack.Server/Services/NexusKeyMaintainance.cs

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Wabbajack.BuildServer;
 using Wabbajack.Common;
+using Wabbajack.Common.Exceptions;
 using Wabbajack.Lib.NexusApi;
 using Wabbajack.Server.DataLayer;
 
@@ -28,13 +29,12 @@ namespace Wabbajack.Server.Services
                 try
                 {
                     var client = new TrackingClient(_sql, key);
-                    if (!await client.IsPremium())
+                    if (await client.IsPremium())
-                    {
+                        return client;
-                        _logger.LogWarning($"Purging non premium key");
+
-                        await _sql.DeleteNexusAPIKey(key.Key);
+                    _logger.LogWarning($"Purging non premium key");
-                        continue;
+                    await _sql.DeleteNexusAPIKey(key.Key);
-                    }
+                    continue;
-                    return client;
                 }
                 catch (Exception ex)
                 {
@@ -69,13 +69,16 @@ namespace Wabbajack.Server.Services
                     var (daily, hourly) = await client.GetRemainingApiCalls();
                     await _sql.SetNexusAPIKey(key.Key, daily, hourly);
                 }
-                catch (Exception)
+                catch (HttpException ex)
                 {
-                    _logger.Log(LogLevel.Warning, "Update error, purging API key");
+                    _logger.Log(LogLevel.Warning, $"Nexus error, not purging API key : {ex.Message}");
+                }
+                catch (Exception ex)
+                {
+                    _logger.Log(LogLevel.Warning, $"Update error, purging API key : {ex.Message}");
                     await _sql.DeleteNexusAPIKey(key.Key);
                 }
             }
-
             return keys.Count;
         }
     }

Wabbajack.Server/Services/NonNexusDownloadValidator.cs

@@ -41,8 +41,8 @@ namespace Wabbajack.Server.Services
                     bool isValid = false;
                     switch (archive.State)
                     {
-                        case WabbajackCDNDownloader.State _:
+                        case WabbajackCDNDownloader.State _: 
-                        case GoogleDriveDownloader.State _:
+                        //case GoogleDriveDownloader.State _: // Let's try validating Google again 2/10/2021
                         case ManualDownloader.State _:
                         case ModDBDownloader.State _:
                         case HTTPDownloader.State h when h.Url.StartsWith("https://wabbajack"):

Wabbajack.Server/Startup.cs

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Http.Features;
+using Microsoft.AspNetCore.ResponseCompression;
 using Microsoft.AspNetCore.StaticFiles;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -76,6 +77,13 @@ namespace Wabbajack.Server
             services.AddSingleton<Watchdog>();
             services.AddSingleton<DiscordFrontend>();
             services.AddSingleton<AuthoredFilesCleanup>();
+            services.AddSingleton<MetricsKeyCache>();
+            services.AddResponseCompression(options =>
+            {
+                options.Providers.Add<BrotliCompressionProvider>();
+                options.Providers.Add<GzipCompressionProvider>();
+                options.MimeTypes = new[] {"application/json"};
+            });
             
             services.AddMvc();
             services.AddControllers()
@@ -123,6 +131,7 @@ namespace Wabbajack.Server
             app.UseNexusPoll();
             app.UseArchiveMaintainer();
             app.UseModListDownloader();
+            app.UseResponseCompression();
             
             app.UseService<NonNexusDownloadValidator>();
             app.UseService<ListValidator>();
@@ -137,6 +146,7 @@ namespace Wabbajack.Server
             app.UseService<Watchdog>();
             app.UseService<DiscordFrontend>();
             app.UseService<AuthoredFilesCleanup>();
+            app.UseService<MetricsKeyCache>();
 
             app.Use(next =>
             {

Wabbajack.Server/Wabbajack.Server.csproj

@@ -45,6 +45,8 @@
     <ItemGroup>
       <None Remove="sheo_quotes.txt" />
       <EmbeddedResource Include="sheo_quotes.txt" />
+      <None Remove="Controllers\Templates\TotalListTemplate.html" />
+      <EmbeddedResource Include="Controllers\Templates\TotalListTemplate.html" />
     </ItemGroup>
 
     <ItemGroup>

Wabbajack.Test/ContentRightsManagementTests.cs

@@ -119,7 +119,7 @@ namespace Wabbajack.Test
         {
             Assert.Equal(HTMLInterface.PermissionValue.No, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 266));
             Assert.Equal(HTMLInterface.PermissionValue.Yes, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 1137));
-            Assert.Equal(HTMLInterface.PermissionValue.Hidden, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 34604));
+            Assert.Contains(await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 34604), new[]{HTMLInterface.PermissionValue.Hidden, HTMLInterface.PermissionValue.NotFound});
             Assert.Equal(HTMLInterface.PermissionValue.NotFound, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 24287));
             
         }

Wabbajack.Test/DownloaderTests.cs

@@ -113,6 +113,9 @@ namespace Wabbajack.Test
             Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
 
             Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
+
+            var newState = (AbstractDownloadState)new GoogleDriveDownloader.State("1Q_CdeYJStfoTZFLZ79RRVkxI2c_cG0dg");
+            Assert.True(await newState.Verify(new Archive(newState) {Size = 0}));
         }
 
         [Fact]