Perf improvements

Wabbajack.DTOs/ServerResponses/MetricResponse.cs

@@ -8,7 +8,7 @@ public class MetricResult
     public string Action { get; set; }
     public string Subject { get; set; }
     public string GroupingSubject { get; set; }
-    public long MetricKey { get; set; }
+    public ulong MetricKey { get; set; }
     public string UserAgent { get; set; }
     public DateTime Timestamp { get; set; }
 }

Wabbajack.Server/Controllers/Metrics.cs

@@ -112,7 +112,7 @@ public class MetricsController : ControllerBase
     [HttpGet]
     [Route("report")]
     [ResponseCache(Duration = 60 * 60 * 4, VaryByQueryKeys = new [] {"action", "from", "to"})]
-    public async Task GetReport([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to)
+    public void GetReport([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to)
     {
         var parser = new Parser();
         
@@ -123,31 +123,26 @@ public class MetricsController : ControllerBase
         var groupFilterStart = parser.Parse("three days ago").Start!.Value.TruncateToDate();
         toDate = new DateTime(toDate.Year, toDate.Month, toDate.Day);
 
-        var prefetch = await _metricsStore.GetRecords(groupFilterStart, toDate, action)
+        var prefetch = _metricsStore.GetRecordsParallel(groupFilterStart, toDate, action)
-            .Where((Func<MetricResult, bool>) (d => d.Action != d.Subject))
+            .Where(d => d.Action != d.Subject)
-            .Select(async d => d.GroupingSubject)
+            .Select(d => d.GroupingSubject)
             .ToHashSet();;
         
         var fromDate = parser.Parse(from).Start!.Value.TruncateToDate();
 
-        var counts = new Dictionary<(DateTime, string), long>();
+        var counts = _metricsStore.GetRecordsParallel(fromDate, toDate, action)
-
+            .Where(r => r.Subject != r.Action)
-        await foreach (var record in _metricsStore.GetRecords(fromDate, toDate, action))
+            .Where(r => prefetch.Contains(r.GroupingSubject))
-        {
+            .Select(r => (r.Timestamp.TruncateToDate(), r.GroupingSubject))
-            if (record.Subject == record.Action) continue;
+            .ToLookup(r => r, v => 1)
-            if (!prefetch.Contains(record.GroupingSubject)) continue;
+            .AsParallel()
-
+            .Select(entry => KeyValuePair.Create(entry.Key, entry.Count()))
-            var key = (record.Timestamp.TruncateToDate(), record.GroupingSubject);
+            .ToDictionary(kv => kv.Key, kv => kv.Value);
-            if (counts.TryGetValue(key, out var old))
+        
-                counts[key] = old + 1;
-            else
-                counts[key] = 1;
-        }
-
         Response.Headers.ContentType = "application/json";
         var row = new Dictionary<string, object>();
 
-        await Response.Body.WriteAsync(LBRACKET);
+        Response.Body.Write(LBRACKET);
         for (var d = fromDate; d <= toDate; d = d.AddDays(1))
         {
             row["_Timestamp"] = d;
@@ -158,13 +153,13 @@ public class MetricsController : ControllerBase
                 else
                     row[group] = 0;
             }
-            await JsonSerializer.SerializeAsync(Response.Body, row);
+            JsonSerializer.Serialize(Response.Body, row);
-            await Response.Body.WriteAsync(EOL);
+            Response.Body.Write(EOL);
             if (d != toDate)
-                await Response.Body.WriteAsync(COMMA);
+                Response.Body.Write(COMMA);
         }
 
-        await Response.Body.WriteAsync(RBRACKET);
+        Response.Body.Write(RBRACKET);
 
     }
     

Wabbajack.Server/DataModels/Metrics.cs

@@ -1,15 +1,18 @@
 using System;
 using System.Collections;
+using System.Collections.Concurrent;
 using System.IO;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
+using Chronic.Core;
 using Microsoft.Toolkit.HighPerformance;
 using Wabbajack.BuildServer;
 using Wabbajack.Common;
 using Wabbajack.DTOs.JsonConverters;
 using Wabbajack.DTOs.ServerResponses;
+using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Paths;
 using Wabbajack.Paths.IO;
 using Wabbajack.Server.DTOs;
@@ -49,15 +52,12 @@ public class Metrics
 
     public async IAsyncEnumerable<MetricResult> GetRecords(DateTime fromDate, DateTime toDate, string action)
     {
-        var keys = new Dictionary<string, int>();
+        ulong GetMetricKey(string key)
-        int GetMetricKey(string key)
         {
-            if (string.IsNullOrWhiteSpace(key)) return -1;
+            var hash = new xxHashAlgorithm(0);
-            if (keys.TryGetValue(key, out var v))
+            Span<byte> bytes = stackalloc byte[key.Length];
-                return v;
+            Encoding.ASCII.GetBytes(key, bytes);
-            keys.Add(key, keys.Count);
+            return hash.HashBytes(bytes);
-            return keys.Count - 1;
-
         }
         
         foreach (var file in GetFiles(fromDate, toDate))
@@ -82,6 +82,34 @@ public class Metrics
         }
     }
 
+    public ParallelQuery<MetricResult> GetRecordsParallel(DateTime fromDate, DateTime toDate, string action)
+    {
+        ulong GetMetricKey(string key)
+        {
+            if (string.IsNullOrWhiteSpace(key)) return 0;
+            var hash = new xxHashAlgorithm(0);
+            Span<byte> bytes = stackalloc byte[key.Length];
+            Encoding.ASCII.GetBytes(key, bytes);
+            return hash.HashBytes(bytes);
+        }
+
+        var rows = GetFiles(fromDate, toDate).AsParallel()
+            .SelectMany(file => file.ReadAllLines())
+            .Select(row => _dtos.Deserialize<Metric>(row)!)
+            .Where(m => m.Action == action)
+            .Where(m => m.Timestamp >= fromDate && m.Timestamp <= toDate)
+            .Select(m => new MetricResult
+            {
+                Timestamp = m.Timestamp,
+                Subject = m.Subject,
+                Action = m.Action,
+                MetricKey = GetMetricKey(m.MetricsKey),
+                UserAgent = m.UserAgent,
+                GroupingSubject = GetGroupingSubject(m.Subject)
+            });
+        return rows;
+    }
+
     private Regex groupingRegex = new("^[^0-9]*");
     private string GetGroupingSubject(string metricSubject)
     {

Wabbajack.Server/Program.cs

@@ -22,6 +22,7 @@ public class Program
                 webBuilder.UseStartup<Startup>()
                     .UseKestrel(options =>
                     {
+                        options.AllowSynchronousIO = true;
                         options.Listen(IPAddress.Any, 5000);
                         options.Limits.MaxRequestBodySize = null;
                     });