wabbajack/Wabbajack.Server/Controllers/Metrics.cs

using System.Reflection;
using System.Text.Json;
using Chronic.Core;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
using Wabbajack.Common;
using Wabbajack.DTOs.ServerResponses;
using Wabbajack.Server.DataModels;
using Wabbajack.Server.DTOs;

namespace Wabbajack.BuildServer.Controllers;

[ApiController]
[Route("/metrics")]
public class MetricsController : ControllerBase
{
    private static readonly Func<object, string> ReportTemplate = NettleEngine.GetCompiler().Compile(@"
            <html><body>
                <h2>Tar Report for {{$.key}}</h2>
                <h3>Ban Status: {{$.status}}</h3>
                <table>
                {{each $.log }}
                <tr>
                <td>{{$.Timestamp}}</td>
                <td>{{$.Path}}</td>
                <td>{{$.Key}}</td>
                </tr>
                {{/each}}
                </table>
            </body></html>
        ");

    private static Func<object, string> _totalListTemplate;
    private readonly AppSettings _settings;
    private ILogger<MetricsController> _logger;
    private readonly Metrics _metricsStore;

    public MetricsController(ILogger<MetricsController> logger, Metrics metricsStore,
        AppSettings settings)
    {
        _logger = logger;
        _settings = settings;
        _metricsStore = metricsStore;
    }


    private static Func<object, string> TotalListTemplate
    {
        get
        {
            if (_totalListTemplate == null)
            {
                var resource = Assembly.GetExecutingAssembly()
                    .GetManifestResourceStream("Wabbajack.Server.Controllers.Templates.TotalListTemplate.html")!
                    .ReadAllText();
                _totalListTemplate = NettleEngine.GetCompiler().Compile(resource);
            }

            return _totalListTemplate;
        }
    }

    [HttpGet]
    [Route("{subject}/{value}")]
    public async Task<Result> LogMetricAsync(string subject, string value)
    {
        var date = DateTime.UtcNow;
        var metricsKey = Request.Headers[_settings.MetricsKeyHeader].FirstOrDefault();

        // Used in tests
        if (value is "Default" or "untitled" || subject == "failed_download" || Guid.TryParse(value, out _))
            return new Result {Timestamp = date};

        await _metricsStore.Ingest(new Metric
        {
            Timestamp = DateTime.UtcNow, 
            Action = subject, 
            Subject = value, 
            MetricsKey = metricsKey,
            UserAgent = Request.Headers.UserAgent.FirstOrDefault() ?? "<unknown>",
        });
        return new Result {Timestamp = date};
    }

    private static byte[] EOL = {(byte)'\n'};
    private static byte[] LBRACKET = {(byte)'['};
    private static byte[] RBRACKET = {(byte)']'};
    private static byte[] COMMA = {(byte) ','};
    
    [HttpGet]
    [Route("dump")]
    public async Task GetMetrics([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to, [FromQuery] string? subject)
    {
        var parser = new Parser();
        
        to ??= "now";

        var toDate = parser.Parse(to).Start;
        var fromDate = parser.Parse(from).Start;

        var records = _metricsStore.GetRecords(fromDate!.Value, toDate!.Value, action);

        Response.Headers.ContentType = "application/json";
        await foreach (var record in records)
        {
            if (!string.IsNullOrWhiteSpace(subject) && !record.Subject.Contains(subject))
                continue;
            
            await JsonSerializer.SerializeAsync(Response.Body, record);
            await Response.Body.WriteAsync(EOL);
        }
    }

    [HttpGet]
    [Route("report")]
    [ResponseCache(Duration = 60 * 60 * 4, VaryByQueryKeys = new [] {"action", "from", "to"})]
    public void GetReport([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to)
    {
        var parser = new Parser();
        
        to ??= "now";

        var toDate = parser.Parse(to).Start!.Value.TruncateToDate();
        
        var groupFilterStart = parser.Parse("three days ago").Start!.Value.TruncateToDate();
        toDate = new DateTime(toDate.Year, toDate.Month, toDate.Day);

        var prefetch = _metricsStore.GetRecordsParallel(groupFilterStart, toDate, action)
            .Where(d => d.Action != d.Subject)
            .Select(d => d.GroupingSubject)
            .ToHashSet();;
        
        var fromDate = parser.Parse(from).Start!.Value.TruncateToDate();

        var counts = _metricsStore.GetRecordsParallel(fromDate, toDate, action)
            .Where(r => r.Subject != r.Action)
            .Where(r => prefetch.Contains(r.GroupingSubject))
            .Select(r => (r.Timestamp.TruncateToDate(), r.GroupingSubject))
            .ToLookup(r => r, v => 1)
            .AsParallel()
            .Select(entry => KeyValuePair.Create(entry.Key, entry.Count()))
            .ToDictionary(kv => kv.Key, kv => kv.Value);
        
        Response.Headers.ContentType = "application/json";
        var row = new Dictionary<string, object>();

        Response.Body.Write(LBRACKET);
        for (var d = fromDate; d <= toDate; d = d.AddDays(1))
        {
            row["_Timestamp"] = d;
            foreach (var group in prefetch)
            {
                if (counts.TryGetValue((d, group), out var found))
                    row[group] = found;
                else
                    row[group] = 0;
            }
            JsonSerializer.Serialize(Response.Body, row);
            Response.Body.Write(EOL);
            if (d != toDate)
                Response.Body.Write(COMMA);
        }

        Response.Body.Write(RBRACKET);

    }
    
    
    public class Result
    {
        public DateTime Timestamp { get; set; }
    }
}
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								using System.Reflection;
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
+								using System.Text.Json;
 								using Chronic.Core;
-												Rewriting the build server (mostly from scratch)

											
										
										
											2020-05-09 03:56:06 +00:00
+								using Microsoft.AspNetCore.Mvc;
 								using Microsoft.Extensions.Logging;
-												Fix for install error with DeleteAsync

											
										
										
											2020-06-15 04:05:00 +00:00
+								using Nettle;
-												Rewriting the build server (mostly from scratch)

											
										
										
											2020-05-09 03:56:06 +00:00
+								using Wabbajack.Common;
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								using Wabbajack.DTOs.ServerResponses;
-												Oh yeah, lots off good stuff here

											
										
										
											2021-11-27 18:31:35 +00:00
+								using Wabbajack.Server.DataModels;
-												Rewriting the build server (mostly from scratch)

											
										
										
											2020-05-09 03:56:06 +00:00
+								using Wabbajack.Server.DTOs;
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								namespace Wabbajack.BuildServer.Controllers;
-												Port over metrics.html

											
										
										
											2020-05-09 13:04:38 +00:00
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								[ApiController]
 								[Route("/metrics")]
 								public class MetricsController : ControllerBase
 								{
 								    private static readonly Func<object, string> ReportTemplate = NettleEngine.GetCompiler().Compile(@"
-												Fix for install error with DeleteAsync

											
										
										
											2020-06-15 04:05:00 +00:00
+								            <html><body>
 								                <h2>Tar Report for {{$.key}}</h2>
 								                <h3>Ban Status: {{$.status}}</h3>
 								                <table>
 								                {{each $.log }}
 								                <tr>
 								                <td>{{$.Timestamp}}</td>
 								                <td>{{$.Path}}</td>
 								                <td>{{$.Key}}</td>
 								                </tr>
 								                {{/each}}
 								                </table>
 								            </body></html>
 								        ");
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    private static Func<object, string> _totalListTemplate;
 								    private readonly AppSettings _settings;
 								    private ILogger<MetricsController> _logger;
-												Oh yeah, lots off good stuff here

											
										
										
											2021-11-27 18:31:35 +00:00
+								    private readonly Metrics _metricsStore;
-												Fix for install error with DeleteAsync

											
										
										
											2020-06-15 04:05:00 +00:00
-												Oh yeah, lots off good stuff here

											
										
										
											2021-11-27 18:31:35 +00:00
+								    public MetricsController(ILogger<MetricsController> logger, Metrics metricsStore,
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								        AppSettings settings)
 								    {
 								        _logger = logger;
 								        _settings = settings;
-												Oh yeah, lots off good stuff here

											
										
										
											2021-11-27 18:31:35 +00:00
+								        _metricsStore = metricsStore;
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    }
-												Rewriting the build server (mostly from scratch)

											
										
										
											2020-05-09 03:56:06 +00:00
-												latest server fixes

											
										
										
											2021-02-17 05:46:05 +00:00
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    private static Func<object, string> TotalListTemplate
 								    {
 								        get
 								        {
 								            if (_totalListTemplate == null)
-												latest server fixes

											
										
										
											2021-02-17 05:46:05 +00:00
+								            {
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								                var resource = Assembly.GetExecutingAssembly()
 								                    .GetManifestResourceStream("Wabbajack.Server.Controllers.Templates.TotalListTemplate.html")!
 								                    .ReadAllText();
 								                _totalListTemplate = NettleEngine.GetCompiler().Compile(resource);
-												latest server fixes

											
										
										
											2021-02-17 05:46:05 +00:00
+								            }
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
 								            return _totalListTemplate;
-												latest server fixes

											
										
										
											2021-02-17 05:46:05 +00:00
+								        }
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    }
-												latest server fixes

											
										
										
											2021-02-17 05:46:05 +00:00
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    [HttpGet]
-												fix for duplicated value

											
										
										
											2021-11-30 03:30:24 +00:00
+								    [Route("{subject}/{value}")]
 								    public async Task<Result> LogMetricAsync(string subject, string value)
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    {
 								        var date = DateTime.UtcNow;
 								        var metricsKey = Request.Headers[_settings.MetricsKeyHeader].FirstOrDefault();
-												latest server fixes

											
										
										
											2021-02-17 05:46:05 +00:00
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								        // Used in tests
-												fix for duplicated value

											
										
										
											2021-11-30 03:30:24 +00:00
+								        if (value is "Default" or "untitled" || subject == "failed_download" || Guid.TryParse(value, out _))
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								            return new Result {Timestamp = date};
-												Few more fixes

											
										
										
											2021-02-18 05:44:54 +00:00
-												Oh yeah, lots off good stuff here

											
										
										
											2021-11-27 18:31:35 +00:00
+								        await _metricsStore.Ingest(new Metric
-												latest server fixes

											
										
										
											2021-02-17 05:46:05 +00:00
+								        {
-												Oh yeah, lots off good stuff here

											
										
										
											2021-11-27 18:31:35 +00:00
+								            Timestamp = DateTime.UtcNow,
-												fix for duplicated value

											
										
										
											2021-11-30 03:30:24 +00:00
+								            Action = subject,
 								            Subject = value,
-												Oh yeah, lots off good stuff here

											
										
										
											2021-11-27 18:31:35 +00:00
+								            MetricsKey = metricsKey,
 								            UserAgent = Request.Headers.UserAgent.FirstOrDefault() ?? "<unknown>",
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								        });
-												Oh yeah, lots off good stuff here

											
										
										
											2021-11-27 18:31:35 +00:00
+								        return new Result {Timestamp = date};
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    }
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
+								    private static byte[] EOL = {(byte)'\n'};
-												Restore metrics page behavior

											
										
										
											2021-12-01 22:53:32 +00:00
+								    private static byte[] LBRACKET = {(byte)'['};
 								    private static byte[] RBRACKET = {(byte)']'};
 								    private static byte[] COMMA = {(byte) ','};
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
+								    [HttpGet]
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								    [Route("dump")]
-												Re-implement tar log

											
										
										
											2022-02-09 18:24:31 +00:00
+								    public async Task GetMetrics([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to, [FromQuery] string? subject)
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
+								    {
 								        var parser = new Parser();
 								        to ??= "now";
 								        var toDate = parser.Parse(to).Start;
 								        var fromDate = parser.Parse(from).Start;
 								        var records = _metricsStore.GetRecords(fromDate!.Value, toDate!.Value, action);
-												Re-implement tar log

											
										
										
											2022-02-09 18:24:31 +00:00
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
+								        Response.Headers.ContentType = "application/json";
 								        await foreach (var record in records)
 								        {
-												Re-implement tar log

											
										
										
											2022-02-09 18:24:31 +00:00
+								            if (!string.IsNullOrWhiteSpace(subject) && !record.Subject.Contains(subject))
 								                continue;
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
 								            await JsonSerializer.SerializeAsync(Response.Body, record);
 								            await Response.Body.WriteAsync(EOL);
 								        }
 								    }
-												Re-implement tar log

											
										
										
											2022-02-09 18:24:31 +00:00
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								    [HttpGet]
 								    [Route("report")]
-												Add response caching to metrics

											
										
										
											2021-12-01 06:06:00 +00:00
+								    [ResponseCache(Duration = 60 * 60 * 4, VaryByQueryKeys = new [] {"action", "from", "to"})]
-												Perf improvements

											
										
										
											2021-12-02 00:12:21 +00:00
+								    public void GetReport([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to)
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								    {
 								        var parser = new Parser();
 								        to ??= "now";
 								        var toDate = parser.Parse(to).Start!.Value.TruncateToDate();
 								        var groupFilterStart = parser.Parse("three days ago").Start!.Value.TruncateToDate();
 								        toDate = new DateTime(toDate.Year, toDate.Month, toDate.Day);
-												Perf improvements

											
										
										
											2021-12-02 00:12:21 +00:00
+								        var prefetch = _metricsStore.GetRecordsParallel(groupFilterStart, toDate, action)
 								            .Where(d => d.Action != d.Subject)
 								            .Select(d => d.GroupingSubject)
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								            .ToHashSet();;
 								        var fromDate = parser.Parse(from).Start!.Value.TruncateToDate();
-												Perf improvements

											
										
										
											2021-12-02 00:12:21 +00:00
+								        var counts = _metricsStore.GetRecordsParallel(fromDate, toDate, action)
 								            .Where(r => r.Subject != r.Action)
 								            .Where(r => prefetch.Contains(r.GroupingSubject))
 								            .Select(r => (r.Timestamp.TruncateToDate(), r.GroupingSubject))
 								            .ToLookup(r => r, v => 1)
 								            .AsParallel()
 								            .Select(entry => KeyValuePair.Create(entry.Key, entry.Count()))
 								            .ToDictionary(kv => kv.Key, kv => kv.Value);
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								        Response.Headers.ContentType = "application/json";
 								        var row = new Dictionary<string, object>();
-												Restore metrics page behavior

											
										
										
											2021-12-01 22:53:32 +00:00
-												Perf improvements

											
										
										
											2021-12-02 00:12:21 +00:00
+								        Response.Body.Write(LBRACKET);
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								        for (var d = fromDate; d <= toDate; d = d.AddDays(1))
 								        {
 								            row["_Timestamp"] = d;
 								            foreach (var group in prefetch)
 								            {
 								                if (counts.TryGetValue((d, group), out var found))
 								                    row[group] = found;
 								                else
 								                    row[group] = 0;
 								            }
-												Perf improvements

											
										
										
											2021-12-02 00:12:21 +00:00
+								            JsonSerializer.Serialize(Response.Body, row);
 								            Response.Body.Write(EOL);
-												Restore metrics page behavior

											
										
										
											2021-12-01 22:53:32 +00:00
+								            if (d != toDate)
-												Perf improvements

											
										
										
											2021-12-02 00:12:21 +00:00
+								                Response.Body.Write(COMMA);
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								        }
-												Perf improvements

											
										
										
											2021-12-02 00:12:21 +00:00
+								        Response.Body.Write(RBRACKET);
-												Restore metrics page behavior

											
										
										
											2021-12-01 22:53:32 +00:00
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								    }
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    public class Result
 								    {
 								        public DateTime Timestamp { get; set; }
 								    }
 								}