Few more fixes

Wabbajack.Server.Test/MetricsTests.cs

@@ -5,6 +5,7 @@ using Dapper;
 using Wabbajack.Lib;
 using Wabbajack.Server.DataLayer;
 using Wabbajack.Server.DTOs;
+using Wabbajack.Server.Services;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -45,6 +46,9 @@ namespace Wabbajack.BuildServer.Test
             Assert.True(dumpResponse.IsSuccessStatusCode);
             var data = await dumpResponse.Content.ReadAsStringAsync();
             Assert.NotEmpty(data);
+
+            var cache = Fixture.GetService<MetricsKeyCache>();
+            Assert.True(await cache.KeyCount() > 0);
         }
     }
 }

Wabbajack.Server.Test/sql/wabbajack_db.sql

@@ -550,6 +550,25 @@ CREATE TABLE [dbo].[ModListArchives](
 )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
     ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
     GO
+    
+/****** Object:  Table [dbo].[MetricsKeys]    Script Date: 2/17/2021 9:20:27 PM ******/
+SET ANSI_NULLS ON
+GO
+
+SET QUOTED_IDENTIFIER ON
+GO
+
+CREATE TABLE [dbo].[MetricsKeys](
+[MetricsKey] [varchar](64) NOT NULL,
+CONSTRAINT [PK_MetricsKeys] PRIMARY KEY CLUSTERED
+    (
+    [MetricsKey] ASC
+    )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+) ON [PRIMARY]
+GO
+    
+    
+    
 /****** Object:  Table [dbo].[ModListArchiveStatus]    Script Date: 12/29/2020 8:55:04 PM ******/
     SET ANSI_NULLS ON
     GO

Wabbajack.Server/ApiKeyAuthorizationHandler.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Security.Claims;
@@ -13,6 +14,7 @@ using Wabbajack.Common;
 using Wabbajack.Common.Serialization.Json;
 using Wabbajack.Server.DataLayer;
 using Wabbajack.Server.DTOs;
+using Wabbajack.Server.Services;
 
 
 namespace Wabbajack.BuildServer
@@ -31,14 +33,18 @@ namespace Wabbajack.BuildServer
         private readonly SqlService _sql;
         private const string ApiKeyHeaderName = "X-Api-Key";
 
+        private MetricsKeyCache _keyCache;
+
         public ApiKeyAuthenticationHandler(
             IOptionsMonitor<ApiKeyAuthenticationOptions> options,
             ILoggerFactory logger,
             UrlEncoder encoder,
             ISystemClock clock,
+            MetricsKeyCache keyCache,
             SqlService db) : base(options, logger, encoder, clock)
         {
             _sql = db;
+            _keyCache = keyCache;
         }
 
         protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
@@ -88,8 +94,9 @@ namespace Wabbajack.BuildServer
 
                 return AuthenticateResult.Success(ticket);
             }
+            
 
-            if (!await _sql.ValidMetricsKey(metricsKey))
+            if (!await _keyCache.IsValidKey(metricsKey))
             {
                 return AuthenticateResult.Fail("Invalid Metrics Key");
             }

Wabbajack.Server/Controllers/Metrics.cs

@@ -15,6 +15,7 @@ using Wabbajack.Common;
 using Wabbajack.Server;
 using Wabbajack.Server.DataLayer;
 using Wabbajack.Server.DTOs;
+using Wabbajack.Server.Services;
 using WebSocketSharp;
 using LogLevel = Microsoft.Extensions.Logging.LogLevel;
 
@@ -26,11 +27,13 @@ namespace Wabbajack.BuildServer.Controllers
     {
         private SqlService _sql;
         private ILogger<MetricsController> _logger;
+        private MetricsKeyCache _keyCache;
 
-        public MetricsController(ILogger<MetricsController> logger, SqlService sql)
+        public MetricsController(ILogger<MetricsController> logger, SqlService sql, MetricsKeyCache keyCache)
         {
             _sql = sql;
             _logger = logger;
+            _keyCache = keyCache;
         }
 
         [HttpGet]
@@ -38,12 +41,15 @@ namespace Wabbajack.BuildServer.Controllers
         public async Task<Result> LogMetricAsync(string subject, string value)
         {
             var date = DateTime.UtcNow;
+            var metricsKey = Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault();
+            if (metricsKey != null)
+                await _keyCache.AddKey(metricsKey);
             
             // Used in tests
-            if (value == "Default" || value == "untitled" || Guid.TryParse(value, out _))
+            if (value == "Default" || value == "untitled" || subject == "failed_download" || Guid.TryParse(value, out _))
                 return new Result { Timestamp = date};
             
-            await Log(date, subject, value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault());
+            await Log(date, subject, value, metricsKey);
             return new Result { Timestamp = date};
         }
 
@@ -174,6 +180,7 @@ namespace Wabbajack.BuildServer.Controllers
 
         private static Func<object, string> _totalListTemplate;
 
+
         private static Func<object, string> TotalListTemplate
         {
             get
@@ -189,6 +196,8 @@ namespace Wabbajack.BuildServer.Controllers
                 return _totalListTemplate;
             }
         }
+        
+        
 
         [HttpGet("total_installs.html")]
         [ResponseCache(Duration = 60 * 60)]
@@ -231,6 +240,6 @@ namespace Wabbajack.BuildServer.Controllers
         {
             return Ok(await _sql.MetricsDump().ToArrayAsync());
         }
-        
+
     }
 }

Wabbajack.Server/DataLayer/Metrics.cs

@@ -70,8 +70,28 @@ namespace Wabbajack.Server.DataLayer
         public async Task<bool> ValidMetricsKey(string metricsKey)
         {
             await using var conn = await Open();
-            return (await conn.QueryAsync<string>("SELECT TOP(1) MetricsKey from Metrics Where MetricsKey = @MetricsKey",
+            return (await conn.QuerySingleOrDefaultAsync<string>("SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey",
-                new {MetricsKey = metricsKey})).FirstOrDefault() != null;
+                new {MetricsKey = metricsKey})) != default;
+        }
+
+        public async Task AddMetricsKey(string metricsKey)
+        {
+            await using var conn = await Open();
+            await using var trans = conn.BeginTransaction();
+
+            if ((await conn.QuerySingleOrDefaultAsync<string>(
+                "SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey",
+                new {MetricsKey = metricsKey}, trans)) != default)
+                return;
+            
+            await conn.ExecuteAsync("INSERT INTO dbo.MetricsKeys (MetricsKey) VALUES (@MetricsKey)",
+                new {MetricsKey = metricsKey}, trans);
+        }
+
+        public async Task<string[]> AllKeys()
+        {
+            await using var conn = await Open();
+            return (await conn.QueryAsync<string>("SELECT MetricsKey from dbo.MetricsKeys")).ToArray();
         }
 
 

Wabbajack.Server/Services/DiscordFrontend.cs

@@ -19,8 +19,9 @@ namespace Wabbajack.Server.Services
         private QuickSync _quickSync;
         private DiscordSocketClient _client;
         private SqlService _sql;
+        private MetricsKeyCache _keyCache;
 
-        public DiscordFrontend(ILogger<DiscordFrontend> logger, AppSettings settings, QuickSync quickSync, SqlService sql)
+        public DiscordFrontend(ILogger<DiscordFrontend> logger, AppSettings settings, QuickSync quickSync, SqlService sql, MetricsKeyCache keyCache)
         {
             _logger = logger;
             _settings = settings;
@@ -33,6 +34,7 @@ namespace Wabbajack.Server.Services
             _client.MessageReceived += MessageReceivedAsync;
 
             _sql = sql;
+            _keyCache = keyCache;
         }
 
         private async Task MessageReceivedAsync(SocketMessage arg)
@@ -88,6 +90,10 @@ namespace Wabbajack.Server.Services
                         await ReplyTo(arg, $"Purged all traces of #{parts[2]} from the server, triggered list downloading. {deleted} records removed");
                     }
                 }
+                else if (parts[1] == "users")
+                {
+                    await ReplyTo(arg, $"Wabbajack has {await _keyCache.KeyCount()} known unique users");
+                }
             }
         }
 

Wabbajack.Server/Services/MetricsKeyCache.cs

@@ -0,0 +1,62 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.Common;
+using Wabbajack.Server.DataLayer;
+
+namespace Wabbajack.Server.Services
+{
+    public class MetricsKeyCache : IStartable
+    {
+        private ILogger<MetricsKeyCache> _logger;
+        private SqlService _sql;
+        private HashSet<string> _knownKeys = new();
+        private AsyncLock _lock = new();
+
+        public MetricsKeyCache(ILogger<MetricsKeyCache> logger, SqlService sql)
+        {
+            _logger = logger;
+            _sql = sql;
+        }
+
+        public async Task<bool> IsValidKey(string key)
+        {
+            using (var _ = await _lock.WaitAsync())
+            {
+                if (_knownKeys.Contains(key)) return true;
+            }
+
+            if (await _sql.ValidMetricsKey(key))
+            {
+                using var _ = await _lock.WaitAsync();
+                _knownKeys.Add(key);
+                return true;
+            }
+
+            return false;
+        }
+
+        public async Task AddKey(string key)
+        {
+            using (var _ = await _lock.WaitAsync())
+            {
+                if (_knownKeys.Contains(key)) return;
+                _knownKeys.Add(key);
+            }
+
+            await _sql.AddMetricsKey(key);
+        }
+
+        public void Start()
+        {
+            _knownKeys = (_sql.AllKeys().Result).ToHashSet();
+        }
+
+        public async Task<long> KeyCount()
+        {
+            using var _ = await _lock.WaitAsync();
+            return _knownKeys.Count;
+        }
+    }
+}

Wabbajack.Server/Startup.cs

@@ -77,6 +77,7 @@ namespace Wabbajack.Server
             services.AddSingleton<Watchdog>();
             services.AddSingleton<DiscordFrontend>();
             services.AddSingleton<AuthoredFilesCleanup>();
+            services.AddSingleton<MetricsKeyCache>();
             services.AddResponseCompression(options =>
             {
                 options.Providers.Add<BrotliCompressionProvider>();
@@ -145,6 +146,7 @@ namespace Wabbajack.Server
             app.UseService<Watchdog>();
             app.UseService<DiscordFrontend>();
             app.UseService<AuthoredFilesCleanup>();
+            app.UseService<MetricsKeyCache>();
 
             app.Use(next =>
             {