diff --git a/Wabbajack.Server.Test/MetricsTests.cs b/Wabbajack.Server.Test/MetricsTests.cs index 130e7776..bda8cfd3 100644 --- a/Wabbajack.Server.Test/MetricsTests.cs +++ b/Wabbajack.Server.Test/MetricsTests.cs @@ -5,6 +5,7 @@ using Dapper; using Wabbajack.Lib; using Wabbajack.Server.DataLayer; using Wabbajack.Server.DTOs; +using Wabbajack.Server.Services; using Xunit; using Xunit.Abstractions; @@ -45,6 +46,9 @@ namespace Wabbajack.BuildServer.Test Assert.True(dumpResponse.IsSuccessStatusCode); var data = await dumpResponse.Content.ReadAsStringAsync(); Assert.NotEmpty(data); + + var cache = Fixture.GetService(); + Assert.True(await cache.KeyCount() > 0); } } } diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index dfc2b02a..12231485 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -550,6 +550,25 @@ CREATE TABLE [dbo].[ModListArchives]( )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] GO + +/****** Object: Table [dbo].[MetricsKeys] Script Date: 2/17/2021 9:20:27 PM ******/ +SET ANSI_NULLS ON +GO + +SET QUOTED_IDENTIFIER ON +GO + +CREATE TABLE [dbo].[MetricsKeys]( +[MetricsKey] [varchar](64) NOT NULL, +CONSTRAINT [PK_MetricsKeys] PRIMARY KEY CLUSTERED + ( + [MetricsKey] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO + + + /****** Object: Table [dbo].[ModListArchiveStatus] Script Date: 12/29/2020 8:55:04 PM ******/ SET ANSI_NULLS ON GO diff --git a/Wabbajack.Server/ApiKeyAuthorizationHandler.cs b/Wabbajack.Server/ApiKeyAuthorizationHandler.cs index 2f36179d..e66f1a5c 100644 --- a/Wabbajack.Server/ApiKeyAuthorizationHandler.cs +++ b/Wabbajack.Server/ApiKeyAuthorizationHandler.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Security.Claims; @@ -13,6 +14,7 @@ using Wabbajack.Common; using Wabbajack.Common.Serialization.Json; using Wabbajack.Server.DataLayer; using Wabbajack.Server.DTOs; +using Wabbajack.Server.Services; namespace Wabbajack.BuildServer @@ -31,14 +33,18 @@ namespace Wabbajack.BuildServer private readonly SqlService _sql; private const string ApiKeyHeaderName = "X-Api-Key"; + private MetricsKeyCache _keyCache; + public ApiKeyAuthenticationHandler( IOptionsMonitor options, ILoggerFactory logger, UrlEncoder encoder, ISystemClock clock, + MetricsKeyCache keyCache, SqlService db) : base(options, logger, encoder, clock) { _sql = db; + _keyCache = keyCache; } protected override async Task HandleAuthenticateAsync() @@ -88,8 +94,9 @@ namespace Wabbajack.BuildServer return AuthenticateResult.Success(ticket); } + - if (!await _sql.ValidMetricsKey(metricsKey)) + if (!await _keyCache.IsValidKey(metricsKey)) { return AuthenticateResult.Fail("Invalid Metrics Key"); } diff --git a/Wabbajack.Server/Controllers/Metrics.cs b/Wabbajack.Server/Controllers/Metrics.cs index 9f4fcd4b..b9b315a5 100644 --- a/Wabbajack.Server/Controllers/Metrics.cs +++ b/Wabbajack.Server/Controllers/Metrics.cs @@ -15,6 +15,7 @@ using Wabbajack.Common; using Wabbajack.Server; using Wabbajack.Server.DataLayer; using Wabbajack.Server.DTOs; +using Wabbajack.Server.Services; using WebSocketSharp; using LogLevel = Microsoft.Extensions.Logging.LogLevel; @@ -26,11 +27,13 @@ namespace Wabbajack.BuildServer.Controllers { private SqlService _sql; private ILogger _logger; + private MetricsKeyCache _keyCache; - public MetricsController(ILogger logger, SqlService sql) + public MetricsController(ILogger logger, SqlService sql, MetricsKeyCache keyCache) { _sql = sql; _logger = logger; + _keyCache = keyCache; } [HttpGet] @@ -38,12 +41,15 @@ namespace Wabbajack.BuildServer.Controllers public async Task LogMetricAsync(string subject, string value) { var date = DateTime.UtcNow; + var metricsKey = Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault(); + if (metricsKey != null) + await _keyCache.AddKey(metricsKey); // Used in tests - if (value == "Default" || value == "untitled" || Guid.TryParse(value, out _)) + if (value == "Default" || value == "untitled" || subject == "failed_download" || Guid.TryParse(value, out _)) return new Result { Timestamp = date}; - await Log(date, subject, value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault()); + await Log(date, subject, value, metricsKey); return new Result { Timestamp = date}; } @@ -174,6 +180,7 @@ namespace Wabbajack.BuildServer.Controllers private static Func _totalListTemplate; + private static Func TotalListTemplate { get @@ -189,6 +196,8 @@ namespace Wabbajack.BuildServer.Controllers return _totalListTemplate; } } + + [HttpGet("total_installs.html")] [ResponseCache(Duration = 60 * 60)] @@ -231,6 +240,6 @@ namespace Wabbajack.BuildServer.Controllers { return Ok(await _sql.MetricsDump().ToArrayAsync()); } - + } } diff --git a/Wabbajack.Server/DataLayer/Metrics.cs b/Wabbajack.Server/DataLayer/Metrics.cs index 750eec22..7305e276 100644 --- a/Wabbajack.Server/DataLayer/Metrics.cs +++ b/Wabbajack.Server/DataLayer/Metrics.cs @@ -70,8 +70,28 @@ namespace Wabbajack.Server.DataLayer public async Task ValidMetricsKey(string metricsKey) { await using var conn = await Open(); - return (await conn.QueryAsync("SELECT TOP(1) MetricsKey from Metrics Where MetricsKey = @MetricsKey", - new {MetricsKey = metricsKey})).FirstOrDefault() != null; + return (await conn.QuerySingleOrDefaultAsync("SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey", + new {MetricsKey = metricsKey})) != default; + } + + public async Task AddMetricsKey(string metricsKey) + { + await using var conn = await Open(); + await using var trans = conn.BeginTransaction(); + + if ((await conn.QuerySingleOrDefaultAsync( + "SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey", + new {MetricsKey = metricsKey}, trans)) != default) + return; + + await conn.ExecuteAsync("INSERT INTO dbo.MetricsKeys (MetricsKey) VALUES (@MetricsKey)", + new {MetricsKey = metricsKey}, trans); + } + + public async Task AllKeys() + { + await using var conn = await Open(); + return (await conn.QueryAsync("SELECT MetricsKey from dbo.MetricsKeys")).ToArray(); } diff --git a/Wabbajack.Server/Services/DiscordFrontend.cs b/Wabbajack.Server/Services/DiscordFrontend.cs index 6f81afa8..ec25be9b 100644 --- a/Wabbajack.Server/Services/DiscordFrontend.cs +++ b/Wabbajack.Server/Services/DiscordFrontend.cs @@ -19,8 +19,9 @@ namespace Wabbajack.Server.Services private QuickSync _quickSync; private DiscordSocketClient _client; private SqlService _sql; + private MetricsKeyCache _keyCache; - public DiscordFrontend(ILogger logger, AppSettings settings, QuickSync quickSync, SqlService sql) + public DiscordFrontend(ILogger logger, AppSettings settings, QuickSync quickSync, SqlService sql, MetricsKeyCache keyCache) { _logger = logger; _settings = settings; @@ -33,6 +34,7 @@ namespace Wabbajack.Server.Services _client.MessageReceived += MessageReceivedAsync; _sql = sql; + _keyCache = keyCache; } private async Task MessageReceivedAsync(SocketMessage arg) @@ -88,6 +90,10 @@ namespace Wabbajack.Server.Services await ReplyTo(arg, $"Purged all traces of #{parts[2]} from the server, triggered list downloading. {deleted} records removed"); } } + else if (parts[1] == "users") + { + await ReplyTo(arg, $"Wabbajack has {await _keyCache.KeyCount()} known unique users"); + } } } diff --git a/Wabbajack.Server/Services/MetricsKeyCache.cs b/Wabbajack.Server/Services/MetricsKeyCache.cs new file mode 100644 index 00000000..4fc14139 --- /dev/null +++ b/Wabbajack.Server/Services/MetricsKeyCache.cs @@ -0,0 +1,62 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.Common; +using Wabbajack.Server.DataLayer; + +namespace Wabbajack.Server.Services +{ + public class MetricsKeyCache : IStartable + { + private ILogger _logger; + private SqlService _sql; + private HashSet _knownKeys = new(); + private AsyncLock _lock = new(); + + public MetricsKeyCache(ILogger logger, SqlService sql) + { + _logger = logger; + _sql = sql; + } + + public async Task IsValidKey(string key) + { + using (var _ = await _lock.WaitAsync()) + { + if (_knownKeys.Contains(key)) return true; + } + + if (await _sql.ValidMetricsKey(key)) + { + using var _ = await _lock.WaitAsync(); + _knownKeys.Add(key); + return true; + } + + return false; + } + + public async Task AddKey(string key) + { + using (var _ = await _lock.WaitAsync()) + { + if (_knownKeys.Contains(key)) return; + _knownKeys.Add(key); + } + + await _sql.AddMetricsKey(key); + } + + public void Start() + { + _knownKeys = (_sql.AllKeys().Result).ToHashSet(); + } + + public async Task KeyCount() + { + using var _ = await _lock.WaitAsync(); + return _knownKeys.Count; + } + } +} diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 68aa79a8..16f1c865 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -77,6 +77,7 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddResponseCompression(options => { options.Providers.Add(); @@ -145,6 +146,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); + app.UseService(); app.Use(next => {