From dc530a52b27e294e2929654d9ae6de89b93623ed Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 25 Jun 2022 14:22:06 -0600 Subject: [PATCH] Implement metrics reading from CouchDB --- Wabbajack.Server/AppSettings.cs | 1 + Wabbajack.Server/Controllers/Metrics.cs | 45 ++++++++++++------- .../Services/NexusCacheManager.cs | 8 ++-- Wabbajack.Server/appsettings.json | 1 + 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/Wabbajack.Server/AppSettings.cs b/Wabbajack.Server/AppSettings.cs index a779e614..b9436c50 100644 --- a/Wabbajack.Server/AppSettings.cs +++ b/Wabbajack.Server/AppSettings.cs @@ -10,6 +10,7 @@ public class AppSettings config.Bind("WabbajackSettings", this); } public bool TestMode { get; set; } + public bool RunBackendNexusRoutines { get; set; } = true; public string AuthorAPIKeyFile { get; set; } public string TarKeyFile { get; set; } diff --git a/Wabbajack.Server/Controllers/Metrics.cs b/Wabbajack.Server/Controllers/Metrics.cs index ba6122ca..0e5d558f 100644 --- a/Wabbajack.Server/Controllers/Metrics.cs +++ b/Wabbajack.Server/Controllers/Metrics.cs @@ -2,6 +2,7 @@ using System.Text.Json; using Chronic.Core; using CouchDB.Driver; +using CouchDB.Driver.Views; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Nettle; @@ -98,6 +99,7 @@ public class MetricsController : ControllerBase [Route("dump")] public async Task GetMetrics([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to, [FromQuery] string? subject) { + throw new NotImplementedException(); var parser = new Parser(); to ??= "now"; @@ -121,7 +123,7 @@ public class MetricsController : ControllerBase [HttpGet] [Route("report")] [ResponseCache(Duration = 60 * 60 * 4, VaryByQueryKeys = new [] {"action", "from", "to"})] - public void GetReport([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to) + public async Task GetReport([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to) { var parser = new Parser(); @@ -132,21 +134,15 @@ public class MetricsController : ControllerBase var groupFilterStart = parser.Parse("three days ago").Start!.Value.TruncateToDate(); toDate = new DateTime(toDate.Year, toDate.Month, toDate.Day); - var prefetch = _metricsStore.GetRecordsParallel(groupFilterStart, toDate, action) - .Where(d => d.Action != d.Subject) - .Select(d => d.GroupingSubject) - .ToHashSet();; - + var prefetch = (await GetByAction(action, groupFilterStart, toDate)) + .Select(d => d.Subject) + .ToHashSet(); + var fromDate = parser.Parse(from).Start!.Value.TruncateToDate(); - var counts = _metricsStore.GetRecordsParallel(fromDate, toDate, action) - .Where(r => r.Subject != r.Action) - .Where(r => prefetch.Contains(r.GroupingSubject)) - .Select(r => (r.Timestamp.TruncateToDate(), r.GroupingSubject)) - .ToLookup(r => r, v => 1) - .AsParallel() - .Select(entry => KeyValuePair.Create(entry.Key, entry.Count())) - .ToDictionary(kv => kv.Key, kv => kv.Value); + var counts = (await GetByAction(action, fromDate, toDate)) + .Where(r => prefetch.Contains(r.Subject)) + .ToDictionary(kv => (kv.Date, kv.Subject), kv => kv.Count); Response.Headers.ContentType = "application/json"; var row = new Dictionary(); @@ -162,7 +158,7 @@ public class MetricsController : ControllerBase else row[group] = 0; } - JsonSerializer.Serialize(Response.Body, row); + await JsonSerializer.SerializeAsync(Response.Body, row); Response.Body.Write(EOL); if (d != toDate) Response.Body.Write(COMMA); @@ -173,6 +169,25 @@ public class MetricsController : ControllerBase } + private async Task> GetByAction(string action, DateTime from, DateTime to) + { + var records = await _db.GetViewAsync("Indexes", "ActionDaySubject", + new CouchViewOptions + { + StartKey = new object?[]{action, from.Year, from.Month, from.Day, null}, + EndKey = new object?[]{action, to.Year, to.Month, to.Day, new()}, + Reduce = true, + GroupLevel = 10, + Group = true + }); + + var results = records + .Where(r => r.Key.Length >= 4 && r.Key[4] != null) + .Select(r => + (new DateTime((int)(long)r.Key[1]!, (int)(long)r.Key[2]!, (int)(long)r.Key[3]!), (string)r.Key[4]!, r.Value)); + return results.ToList(); + } + public class Result { diff --git a/Wabbajack.Server/Services/NexusCacheManager.cs b/Wabbajack.Server/Services/NexusCacheManager.cs index 3d3feba2..e38d6eb2 100644 --- a/Wabbajack.Server/Services/NexusCacheManager.cs +++ b/Wabbajack.Server/Services/NexusCacheManager.cs @@ -34,9 +34,11 @@ public class NexusCacheManager _nexusAPI = nexusApi; _discord = discord; - - _timer = new Timer(_ => UpdateNexusCacheAPI().FireAndForget(), null, TimeSpan.FromSeconds(2), - TimeSpan.FromHours(4)); + if (configuration.RunBackendNexusRoutines) + { + _timer = new Timer(_ => UpdateNexusCacheAPI().FireAndForget(), null, TimeSpan.FromSeconds(2), + TimeSpan.FromHours(4)); + } } diff --git a/Wabbajack.Server/appsettings.json b/Wabbajack.Server/appsettings.json index 5547a56c..79381d45 100644 --- a/Wabbajack.Server/appsettings.json +++ b/Wabbajack.Server/appsettings.json @@ -7,6 +7,7 @@ } }, "WabbajackSettings": { + "RunBackendNexusRoutines": false, "TempFolder": "c:\\tmp\\server_temp", "MetricsFolder": "c:\\tmp\\server_metrics", "AuthoredFilesFolder": "c:\\tmp\\server_authored_files",