Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Perf improvements

Browse Source
This commit is contained in:
Timothy Baldridge 2021-12-01 17:12:21 -07:00
parent 913b67daf3
commit 37dbd02e18
4 changed files with 56 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.DTOs/ServerResponses/MetricResponse.cs
View File

@ -8,7 +8,7 @@ public class MetricResult
public string Action { get; set; }
public string Subject { get; set; }
public string GroupingSubject { get; set; }
public long MetricKey { get; set; }
public ulong MetricKey { get; set; }
public string UserAgent { get; set; }
public DateTime Timestamp { get; set; }
}

39
Wabbajack.Server/Controllers/Metrics.cs
View File

@ -112,7 +112,7 @@ public class MetricsController : ControllerBase
[HttpGet]
[Route("report")]
[ResponseCache(Duration = 60 * 60 * 4, VaryByQueryKeys = new [] {"action", "from", "to"})]
public async Task GetReport([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to)
public void GetReport([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to)
{
var parser = new Parser();
@ -123,31 +123,26 @@ public class MetricsController : ControllerBase
var groupFilterStart = parser.Parse("three days ago").Start!.Value.TruncateToDate();
toDate = new DateTime(toDate.Year, toDate.Month, toDate.Day);
var prefetch = await _metricsStore.GetRecords(groupFilterStart, toDate, action)
.Where((Func<MetricResult, bool>) (d => d.Action != d.Subject))
.Select(async d => d.GroupingSubject)
var prefetch = _metricsStore.GetRecordsParallel(groupFilterStart, toDate, action)
.Where(d => d.Action != d.Subject)
.Select(d => d.GroupingSubject)
.ToHashSet();;
var fromDate = parser.Parse(from).Start!.Value.TruncateToDate();
var counts = new Dictionary<(DateTime, string), long>();
await foreach (var record in _metricsStore.GetRecords(fromDate, toDate, action))
{
if (record.Subject == record.Action) continue;
if (!prefetch.Contains(record.GroupingSubject)) continue;
var key = (record.Timestamp.TruncateToDate(), record.GroupingSubject);
if (counts.TryGetValue(key, out var old))
counts[key] = old + 1;
else
counts[key] = 1;
}
var counts = _metricsStore.GetRecordsParallel(fromDate, toDate, action)
.Where(r => r.Subject != r.Action)
.Where(r => prefetch.Contains(r.GroupingSubject))
.Select(r => (r.Timestamp.TruncateToDate(), r.GroupingSubject))
.ToLookup(r => r, v => 1)
.AsParallel()
.Select(entry => KeyValuePair.Create(entry.Key, entry.Count()))
.ToDictionary(kv => kv.Key, kv => kv.Value);
Response.Headers.ContentType = "application/json";
var row = new Dictionary<string, object>();
await Response.Body.WriteAsync(LBRACKET);
Response.Body.Write(LBRACKET);
for (var d = fromDate; d <= toDate; d = d.AddDays(1))
{
row["_Timestamp"] = d;
@ -158,13 +153,13 @@ public class MetricsController : ControllerBase
else
row[group] = 0;
}
await JsonSerializer.SerializeAsync(Response.Body, row);
await Response.Body.WriteAsync(EOL);
JsonSerializer.Serialize(Response.Body, row);
Response.Body.Write(EOL);
if (d != toDate)
await Response.Body.WriteAsync(COMMA);
Response.Body.Write(COMMA);
}
await Response.Body.WriteAsync(RBRACKET);
Response.Body.Write(RBRACKET);
}

44
Wabbajack.Server/DataModels/Metrics.cs
View File

@ -1,15 +1,18 @@
using System;
using System.Collections;
using System.Collections.Concurrent;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Chronic.Core;
using Microsoft.Toolkit.HighPerformance;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.DTOs.ServerResponses;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.Server.DTOs;
@ -49,15 +52,12 @@ public class Metrics
public async IAsyncEnumerable<MetricResult> GetRecords(DateTime fromDate, DateTime toDate, string action)
{
var keys = new Dictionary<string, int>();
int GetMetricKey(string key)
ulong GetMetricKey(string key)
{
if (string.IsNullOrWhiteSpace(key)) return -1;
if (keys.TryGetValue(key, out var v))
return v;
keys.Add(key, keys.Count);
return keys.Count - 1;
var hash = new xxHashAlgorithm(0);
Span<byte> bytes = stackalloc byte[key.Length];
Encoding.ASCII.GetBytes(key, bytes);
return hash.HashBytes(bytes);
}
foreach (var file in GetFiles(fromDate, toDate))
@ -82,6 +82,34 @@ public class Metrics
}
}
public ParallelQuery<MetricResult> GetRecordsParallel(DateTime fromDate, DateTime toDate, string action)
{
ulong GetMetricKey(string key)
{
if (string.IsNullOrWhiteSpace(key)) return 0;
var hash = new xxHashAlgorithm(0);
Span<byte> bytes = stackalloc byte[key.Length];
Encoding.ASCII.GetBytes(key, bytes);
return hash.HashBytes(bytes);
}
var rows = GetFiles(fromDate, toDate).AsParallel()
.SelectMany(file => file.ReadAllLines())
.Select(row => _dtos.Deserialize<Metric>(row)!)
.Where(m => m.Action == action)
.Where(m => m.Timestamp >= fromDate && m.Timestamp <= toDate)
.Select(m => new MetricResult
{
Timestamp = m.Timestamp,
Subject = m.Subject,
Action = m.Action,
MetricKey = GetMetricKey(m.MetricsKey),
UserAgent = m.UserAgent,
GroupingSubject = GetGroupingSubject(m.Subject)
});
return rows;
}
private Regex groupingRegex = new("^[^0-9]*");
private string GetGroupingSubject(string metricSubject)
{

1
Wabbajack.Server/Program.cs
View File

@ -22,6 +22,7 @@ public class Program
webBuilder.UseStartup<Startup>()
.UseKestrel(options =>
{
options.AllowSynchronousIO = true;
options.Listen(IPAddress.Any, 5000);
options.Limits.MaxRequestBodySize = null;
});