Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Few more fixes

Browse Source
This commit is contained in:
Timothy Baldridge 2021-02-17 22:44:54 -07:00
parent 6c9f6ab5c0
commit 831318c7eb
8 changed files with 137 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Wabbajack.Server.Test/MetricsTests.cs
View File

@ -5,6 +5,7 @@ using Dapper;
using Wabbajack.Lib;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
@ -45,6 +46,9 @@ namespace Wabbajack.BuildServer.Test
Assert.True(dumpResponse.IsSuccessStatusCode);
var data = await dumpResponse.Content.ReadAsStringAsync();
Assert.NotEmpty(data);
var cache = Fixture.GetService<MetricsKeyCache>();
Assert.True(await cache.KeyCount() > 0);
}
}
}

19
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

@ -550,6 +550,25 @@ CREATE TABLE [dbo].[ModListArchives](
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
/****** Object: Table [dbo].[MetricsKeys] Script Date: 2/17/2021 9:20:27 PM ******/
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[MetricsKeys](
[MetricsKey] [varchar](64) NOT NULL,
CONSTRAINT [PK_MetricsKeys] PRIMARY KEY CLUSTERED
(
[MetricsKey] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY]
GO
/****** Object: Table [dbo].[ModListArchiveStatus] Script Date: 12/29/2020 8:55:04 PM ******/
SET ANSI_NULLS ON
GO

9
Wabbajack.Server/ApiKeyAuthorizationHandler.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Security.Claims;
@ -13,6 +14,7 @@ using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer
@ -31,14 +33,18 @@ namespace Wabbajack.BuildServer
private readonly SqlService _sql;
private const string ApiKeyHeaderName = "X-Api-Key";
private MetricsKeyCache _keyCache;
public ApiKeyAuthenticationHandler(
IOptionsMonitor<ApiKeyAuthenticationOptions> options,
ILoggerFactory logger,
UrlEncoder encoder,
ISystemClock clock,
MetricsKeyCache keyCache,
SqlService db) : base(options, logger, encoder, clock)
{
_sql = db;
_keyCache = keyCache;
}
protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
@ -88,8 +94,9 @@ namespace Wabbajack.BuildServer
return AuthenticateResult.Success(ticket);
}
if (!await _sql.ValidMetricsKey(metricsKey))
if (!await _keyCache.IsValidKey(metricsKey))
{
return AuthenticateResult.Fail("Invalid Metrics Key");
}

17
Wabbajack.Server/Controllers/Metrics.cs
View File

@ -15,6 +15,7 @@ using Wabbajack.Common;
using Wabbajack.Server;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using WebSocketSharp;
using LogLevel = Microsoft.Extensions.Logging.LogLevel;
@ -26,11 +27,13 @@ namespace Wabbajack.BuildServer.Controllers
{
private SqlService _sql;
private ILogger<MetricsController> _logger;
private MetricsKeyCache _keyCache;
public MetricsController(ILogger<MetricsController> logger, SqlService sql)
public MetricsController(ILogger<MetricsController> logger, SqlService sql, MetricsKeyCache keyCache)
{
_sql = sql;
_logger = logger;
_keyCache = keyCache;
}
[HttpGet]
@ -38,12 +41,15 @@ namespace Wabbajack.BuildServer.Controllers
public async Task<Result> LogMetricAsync(string subject, string value)
{
var date = DateTime.UtcNow;
var metricsKey = Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault();
if (metricsKey != null)
await _keyCache.AddKey(metricsKey);
// Used in tests
if (value == "Default" || value == "untitled" || Guid.TryParse(value, out _))
if (value == "Default" || value == "untitled" || subject == "failed_download" || Guid.TryParse(value, out _))
return new Result { Timestamp = date};
await Log(date, subject, value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault());
await Log(date, subject, value, metricsKey);
return new Result { Timestamp = date};
}
@ -174,6 +180,7 @@ namespace Wabbajack.BuildServer.Controllers
private static Func<object, string> _totalListTemplate;
private static Func<object, string> TotalListTemplate
{
get
@ -189,6 +196,8 @@ namespace Wabbajack.BuildServer.Controllers
return _totalListTemplate;
}
}
[HttpGet("total_installs.html")]
[ResponseCache(Duration = 60 * 60)]
@ -231,6 +240,6 @@ namespace Wabbajack.BuildServer.Controllers
{
return Ok(await _sql.MetricsDump().ToArrayAsync());
}
}
}

24
Wabbajack.Server/DataLayer/Metrics.cs
View File

@ -70,8 +70,28 @@ namespace Wabbajack.Server.DataLayer
public async Task<bool> ValidMetricsKey(string metricsKey)
{
await using var conn = await Open();
return (await conn.QueryAsync<string>("SELECT TOP(1) MetricsKey from Metrics Where MetricsKey = @MetricsKey",
new {MetricsKey = metricsKey})).FirstOrDefault() != null;
return (await conn.QuerySingleOrDefaultAsync<string>("SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey",
new {MetricsKey = metricsKey})) != default;
}
public async Task AddMetricsKey(string metricsKey)
{
await using var conn = await Open();
await using var trans = conn.BeginTransaction();
if ((await conn.QuerySingleOrDefaultAsync<string>(
"SELECT TOP(1) MetricsKey from dbo.MetricsKeys Where MetricsKey = @MetricsKey",
new {MetricsKey = metricsKey}, trans)) != default)
return;
await conn.ExecuteAsync("INSERT INTO dbo.MetricsKeys (MetricsKey) VALUES (@MetricsKey)",
new {MetricsKey = metricsKey}, trans);
}
public async Task<string[]> AllKeys()
{
await using var conn = await Open();
return (await conn.QueryAsync<string>("SELECT MetricsKey from dbo.MetricsKeys")).ToArray();
}

8
Wabbajack.Server/Services/DiscordFrontend.cs
View File

@ -19,8 +19,9 @@ namespace Wabbajack.Server.Services
private QuickSync _quickSync;
private DiscordSocketClient _client;
private SqlService _sql;
private MetricsKeyCache _keyCache;
public DiscordFrontend(ILogger<DiscordFrontend> logger, AppSettings settings, QuickSync quickSync, SqlService sql)
public DiscordFrontend(ILogger<DiscordFrontend> logger, AppSettings settings, QuickSync quickSync, SqlService sql, MetricsKeyCache keyCache)
{
_logger = logger;
_settings = settings;
@ -33,6 +34,7 @@ namespace Wabbajack.Server.Services
_client.MessageReceived += MessageReceivedAsync;
_sql = sql;
_keyCache = keyCache;
}
private async Task MessageReceivedAsync(SocketMessage arg)
@ -88,6 +90,10 @@ namespace Wabbajack.Server.Services
await ReplyTo(arg, $"Purged all traces of #{parts[2]} from the server, triggered list downloading. {deleted} records removed");
}
}
else if (parts[1] == "users")
{
await ReplyTo(arg, $"Wabbajack has {await _keyCache.KeyCount()} known unique users");
}
}
}

62
Wabbajack.Server/Services/MetricsKeyCache.cs Normal file
View File

@ -0,0 +1,62 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.Server.Services
{
public class MetricsKeyCache : IStartable
{
private ILogger<MetricsKeyCache> _logger;
private SqlService _sql;
private HashSet<string> _knownKeys = new();
private AsyncLock _lock = new();
public MetricsKeyCache(ILogger<MetricsKeyCache> logger, SqlService sql)
{
_logger = logger;
_sql = sql;
}
public async Task<bool> IsValidKey(string key)
{
using (var _ = await _lock.WaitAsync())
{
if (_knownKeys.Contains(key)) return true;
}
if (await _sql.ValidMetricsKey(key))
{
using var _ = await _lock.WaitAsync();
_knownKeys.Add(key);
return true;
}
return false;
}
public async Task AddKey(string key)
{
using (var _ = await _lock.WaitAsync())
{
if (_knownKeys.Contains(key)) return;
_knownKeys.Add(key);
}
await _sql.AddMetricsKey(key);
}
public void Start()
{
_knownKeys = (_sql.AllKeys().Result).ToHashSet();
}
public async Task<long> KeyCount()
{
using var _ = await _lock.WaitAsync();
return _knownKeys.Count;
}
}
}

2
Wabbajack.Server/Startup.cs
View File

@ -77,6 +77,7 @@ namespace Wabbajack.Server
services.AddSingleton<Watchdog>();
services.AddSingleton<DiscordFrontend>();
services.AddSingleton<AuthoredFilesCleanup>();
services.AddSingleton<MetricsKeyCache>();
services.AddResponseCompression(options =>
{
options.Providers.Add<BrotliCompressionProvider>();
@ -145,6 +146,7 @@ namespace Wabbajack.Server
app.UseService<Watchdog>();
app.UseService<DiscordFrontend>();
app.UseService<AuthoredFilesCleanup>();
app.UseService<MetricsKeyCache>();
app.Use(next =>
{