wabbajack/Wabbajack.Server/Controllers/Metrics.cs

using System.Reflection;
using System.Text.Json;
using Chronic.Core;
using CouchDB.Driver;
using CouchDB.Driver.Views;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
using Wabbajack.Common;
using Wabbajack.DTOs.ServerResponses;
using Wabbajack.Server.DataModels;
using Wabbajack.Server.DTOs;

namespace Wabbajack.BuildServer.Controllers;

[ApiController]
[Route("/metrics")]
public class MetricsController : ControllerBase
{
    private static readonly Func<object, string> ReportTemplate = NettleEngine.GetCompiler().Compile(@"
            <html><body>
                <h2>Tar Report for {{$.key}}</h2>
                <h3>Ban Status: {{$.status}}</h3>
                <table>
                {{each $.log }}
                <tr>
                <td>{{$.Timestamp}}</td>
                <td>{{$.Path}}</td>
                <td>{{$.Key}}</td>
                </tr>
                {{/each}}
                </table>
            </body></html>
        ");

    private static Func<object, string> _totalListTemplate;
    private readonly AppSettings _settings;
    private ILogger<MetricsController> _logger;
    private readonly Metrics _metricsStore;
    private readonly ICouchDatabase<Metric> _db;

    public MetricsController(ILogger<MetricsController> logger, Metrics metricsStore,
        AppSettings settings, ICouchDatabase<Metric> db)
    {
        _logger = logger;
        _settings = settings;
        _metricsStore = metricsStore;
        _db = db;
    }


    private static Func<object, string> TotalListTemplate
    {
        get
        {
            if (_totalListTemplate == null)
            {
                var resource = Assembly.GetExecutingAssembly()
                    .GetManifestResourceStream("Wabbajack.Server.Controllers.Templates.TotalListTemplate.html")!
                    .ReadAllText();
                _totalListTemplate = NettleEngine.GetCompiler().Compile(resource);
            }

            return _totalListTemplate;
        }
    }

    [HttpGet]
    [Route("{subject}/{value}")]
    public async Task<Result> LogMetricAsync(string subject, string value)
    {
        var date = DateTime.UtcNow;
        var metricsKey = Request.Headers[_settings.MetricsKeyHeader].FirstOrDefault();

        // Used in tests
        if (value is "Default" or "untitled" || subject == "failed_download" || Guid.TryParse(value, out _))
            return new Result {Timestamp = date};

        await _db.AddAsync(new Metric
        {
            Timestamp = date,
            Action = subject,
            Subject = value,
            MetricsKey = metricsKey,
            UserAgent = Request.Headers.UserAgent.FirstOrDefault() ?? "<unknown>",
            Ip = Request.Headers["cf-connecting-ip"].FirstOrDefault() ??
                 (Request.HttpContext.Connection.RemoteIpAddress?.ToString() ?? "")
        });

        return new Result {Timestamp = date};
    }

    private static byte[] EOL = {(byte)'\n'};
    private static byte[] LBRACKET = {(byte)'['};
    private static byte[] RBRACKET = {(byte)']'};
    private static byte[] COMMA = {(byte) ','};

    [HttpGet]
    [Route("dump")]
    public async Task GetMetrics([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to, [FromQuery] string? subject)
    {
        throw new NotImplementedException();
        var parser = new Parser();
        
        to ??= "now";

        var toDate = parser.Parse(to).Start;
        var fromDate = parser.Parse(from).Start;

        var records = _metricsStore.GetRecords(fromDate!.Value, toDate!.Value, action);

        Response.Headers.ContentType = "application/json";
        await foreach (var record in records)
        {
            if (!string.IsNullOrWhiteSpace(subject) && !record.Subject.Contains(subject))
                continue;
            
            await JsonSerializer.SerializeAsync(Response.Body, record);
            await Response.Body.WriteAsync(EOL);
        }
    }

    [HttpGet]
    [Route("report")]
    [ResponseCache(Duration = 60 * 10, VaryByQueryKeys = new [] {"action", "from", "to"})]
    public async Task GetReport([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to)
    {
        var parser = new Parser();
        
        to ??= "now";

        var toDate = parser.Parse(to).Start!.Value.TruncateToDate();
        
        var groupFilterStart = parser.Parse("three days ago").Start!.Value.TruncateToDate();
        toDate = new DateTime(toDate.Year, toDate.Month, toDate.Day);

        var prefetch = (await GetByAction(action, groupFilterStart, toDate))
            .Select(d => d.Subject)
            .ToHashSet();

        var fromDate = parser.Parse(from).Start!.Value.TruncateToDate();

        var counts = (await GetByAction(action, fromDate, toDate))
            .Where(r => prefetch.Contains(r.Subject))
            .ToDictionary(kv => (kv.Date, kv.Subject), kv => kv.Count);
        
        Response.Headers.ContentType = "application/json";
        var row = new Dictionary<string, object>();

        Response.Body.Write(LBRACKET);
        for (var d = fromDate; d <= toDate; d = d.AddDays(1))
        {
            row["_Timestamp"] = d;
            foreach (var group in prefetch)
            {
                if (counts.TryGetValue((d, group), out var found))
                    row[group] = found;
                else
                    row[group] = 0;
            }
            await JsonSerializer.SerializeAsync(Response.Body, row);
            Response.Body.Write(EOL);
            if (d != toDate)
                Response.Body.Write(COMMA);
        }

        Response.Body.Write(RBRACKET);

    }
    
    
    private async Task<IReadOnlyList<(DateTime Date, string Subject, long Count)>> GetByAction(string action, DateTime from, DateTime to)
    {
        var records = await _db.GetViewAsync<object?[], long>("Indexes", "ActionDaySubject",
            new CouchViewOptions<object?[]>
            {
                StartKey = new object?[]{action, from.Year, from.Month, from.Day, null},
                EndKey = new object?[]{action, to.Year, to.Month, to.Day, new()},
                Reduce = true,
                GroupLevel = 10,
                Group = true
            });
        
        var results = records
            .Where(r => r.Key.Length >= 4 && r.Key[4] != null)
            .Select(r => 
            (new DateTime((int)(long)r.Key[1]!, (int)(long)r.Key[2]!, (int)(long)r.Key[3]!), (string)r.Key[4]!, r.Value));
        return results.ToList();
    }
    

    public class Result
    {
        public DateTime Timestamp { get; set; }
    }
}
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								using System.Reflection;
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
+								using System.Text.Json;
 								using Chronic.Core;
-												Add Cesi cache

											
										
										
											2022-06-22 01:38:42 +00:00
+								using CouchDB.Driver;
-												Implement metrics reading from CouchDB

											
										
										
											2022-06-25 20:22:06 +00:00
+								using CouchDB.Driver.Views;
-												Rewriting the build server (mostly from scratch)

											
										
										
											2020-05-09 03:56:06 +00:00
+								using Microsoft.AspNetCore.Mvc;
 								using Microsoft.Extensions.Logging;
-												Fix for install error with DeleteAsync

											
										
										
											2020-06-15 04:05:00 +00:00
+								using Nettle;
-												Rewriting the build server (mostly from scratch)

											
										
										
											2020-05-09 03:56:06 +00:00
+								using Wabbajack.Common;
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								using Wabbajack.DTOs.ServerResponses;
-												Oh yeah, lots off good stuff here

											
										
										
											2021-11-27 18:31:35 +00:00
+								using Wabbajack.Server.DataModels;
-												Rewriting the build server (mostly from scratch)

											
										
										
											2020-05-09 03:56:06 +00:00
+								using Wabbajack.Server.DTOs;
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								namespace Wabbajack.BuildServer.Controllers;
-												Port over metrics.html

											
										
										
											2020-05-09 13:04:38 +00:00
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								[ApiController]
 								[Route("/metrics")]
 								public class MetricsController : ControllerBase
 								{
 								    private static readonly Func<object, string> ReportTemplate = NettleEngine.GetCompiler().Compile(@"
-												Fix for install error with DeleteAsync

											
										
										
											2020-06-15 04:05:00 +00:00
+								            <html><body>
 								                <h2>Tar Report for {{$.key}}</h2>
 								                <h3>Ban Status: {{$.status}}</h3>
 								                <table>
 								                {{each $.log }}
 								                <tr>
 								                <td>{{$.Timestamp}}</td>
 								                <td>{{$.Path}}</td>
 								                <td>{{$.Key}}</td>
 								                </tr>
 								                {{/each}}
 								                </table>
 								            </body></html>
 								        ");
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    private static Func<object, string> _totalListTemplate;
 								    private readonly AppSettings _settings;
 								    private ILogger<MetricsController> _logger;
-												Oh yeah, lots off good stuff here

											
										
										
											2021-11-27 18:31:35 +00:00
+								    private readonly Metrics _metricsStore;
-												Add Cesi cache

											
										
										
											2022-06-22 01:38:42 +00:00
+								    private readonly ICouchDatabase<Metric> _db;
-												Fix for install error with DeleteAsync

											
										
										
											2020-06-15 04:05:00 +00:00
-												Oh yeah, lots off good stuff here

											
										
										
											2021-11-27 18:31:35 +00:00
+								    public MetricsController(ILogger<MetricsController> logger, Metrics metricsStore,
-												Add Cesi cache

											
										
										
											2022-06-22 01:38:42 +00:00
+								        AppSettings settings, ICouchDatabase<Metric> db)
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    {
 								        _logger = logger;
 								        _settings = settings;
-												Oh yeah, lots off good stuff here

											
										
										
											2021-11-27 18:31:35 +00:00
+								        _metricsStore = metricsStore;
-												Add Cesi cache

											
										
										
											2022-06-22 01:38:42 +00:00
+								        _db = db;
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    }
-												Rewriting the build server (mostly from scratch)

											
										
										
											2020-05-09 03:56:06 +00:00
-												latest server fixes

											
										
										
											2021-02-17 05:46:05 +00:00
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    private static Func<object, string> TotalListTemplate
 								    {
 								        get
 								        {
 								            if (_totalListTemplate == null)
-												latest server fixes

											
										
										
											2021-02-17 05:46:05 +00:00
+								            {
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								                var resource = Assembly.GetExecutingAssembly()
 								                    .GetManifestResourceStream("Wabbajack.Server.Controllers.Templates.TotalListTemplate.html")!
 								                    .ReadAllText();
 								                _totalListTemplate = NettleEngine.GetCompiler().Compile(resource);
-												latest server fixes

											
										
										
											2021-02-17 05:46:05 +00:00
+								            }
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
 								            return _totalListTemplate;
-												latest server fixes

											
										
										
											2021-02-17 05:46:05 +00:00
+								        }
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    }
-												latest server fixes

											
										
										
											2021-02-17 05:46:05 +00:00
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    [HttpGet]
-												fix for duplicated value

											
										
										
											2021-11-30 03:30:24 +00:00
+								    [Route("{subject}/{value}")]
 								    public async Task<Result> LogMetricAsync(string subject, string value)
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    {
 								        var date = DateTime.UtcNow;
 								        var metricsKey = Request.Headers[_settings.MetricsKeyHeader].FirstOrDefault();
-												latest server fixes

											
										
										
											2021-02-17 05:46:05 +00:00
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								        // Used in tests
-												fix for duplicated value

											
										
										
											2021-11-30 03:30:24 +00:00
+								        if (value is "Default" or "untitled" || subject == "failed_download" || Guid.TryParse(value, out _))
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								            return new Result {Timestamp = date};
-												Few more fixes

											
										
										
											2021-02-18 05:44:54 +00:00
-												Put metrics in CouchDB, fix the proxy, fix list validation

											
										
										
											2022-06-22 21:11:26 +00:00
+								        await _db.AddAsync(new Metric
-												latest server fixes

											
										
										
											2021-02-17 05:46:05 +00:00
+								        {
-												Put metrics in CouchDB, fix the proxy, fix list validation

											
										
										
											2022-06-22 21:11:26 +00:00
+								            Timestamp = date,
 								            Action = subject,
 								            Subject = value,
-												Oh yeah, lots off good stuff here

											
										
										
											2021-11-27 18:31:35 +00:00
+								            MetricsKey = metricsKey,
 								            UserAgent = Request.Headers.UserAgent.FirstOrDefault() ?? "<unknown>",
-												Put metrics in CouchDB, fix the proxy, fix list validation

											
										
										
											2022-06-22 21:11:26 +00:00
+								            Ip = Request.Headers["cf-connecting-ip"].FirstOrDefault() ??
 								                 (Request.HttpContext.Connection.RemoteIpAddress?.ToString() ?? "")
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								        });
-												Put metrics in CouchDB, fix the proxy, fix list validation

											
										
										
											2022-06-22 21:11:26 +00:00
-												Oh yeah, lots off good stuff here

											
										
										
											2021-11-27 18:31:35 +00:00
+								        return new Result {Timestamp = date};
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    }
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
+								    private static byte[] EOL = {(byte)'\n'};
-												Restore metrics page behavior

											
										
										
											2021-12-01 22:53:32 +00:00
+								    private static byte[] LBRACKET = {(byte)'['};
 								    private static byte[] RBRACKET = {(byte)']'};
 								    private static byte[] COMMA = {(byte) ','};
-												Add Cesi cache

											
										
										
											2022-06-22 01:38:42 +00:00
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
+								    [HttpGet]
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								    [Route("dump")]
-												Re-implement tar log

											
										
										
											2022-02-09 18:24:31 +00:00
+								    public async Task GetMetrics([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to, [FromQuery] string? subject)
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
+								    {
-												Implement metrics reading from CouchDB

											
										
										
											2022-06-25 20:22:06 +00:00
+								        throw new NotImplementedException();
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
+								        var parser = new Parser();
 								        to ??= "now";
 								        var toDate = parser.Parse(to).Start;
 								        var fromDate = parser.Parse(from).Start;
 								        var records = _metricsStore.GetRecords(fromDate!.Value, toDate!.Value, action);
-												Re-implement tar log

											
										
										
											2022-02-09 18:24:31 +00:00
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
+								        Response.Headers.ContentType = "application/json";
 								        await foreach (var record in records)
 								        {
-												Re-implement tar log

											
										
										
											2022-02-09 18:24:31 +00:00
+								            if (!string.IsNullOrWhiteSpace(subject) && !record.Subject.Contains(subject))
 								                continue;
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
 								            await JsonSerializer.SerializeAsync(Response.Body, record);
 								            await Response.Body.WriteAsync(EOL);
 								        }
 								    }
-												Re-implement tar log

											
										
										
											2022-02-09 18:24:31 +00:00
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								    [HttpGet]
 								    [Route("report")]
-												Set metric cache time to 10 minutes

											
										
										
											2022-06-25 20:23:08 +00:00
+								    [ResponseCache(Duration = 60 * 10, VaryByQueryKeys = new [] {"action", "from", "to"})]
-												Implement metrics reading from CouchDB

											
										
										
											2022-06-25 20:22:06 +00:00
+								    public async Task GetReport([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to)
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								    {
 								        var parser = new Parser();
 								        to ??= "now";
 								        var toDate = parser.Parse(to).Start!.Value.TruncateToDate();
 								        var groupFilterStart = parser.Parse("three days ago").Start!.Value.TruncateToDate();
 								        toDate = new DateTime(toDate.Year, toDate.Month, toDate.Day);
-												Implement metrics reading from CouchDB

											
										
										
											2022-06-25 20:22:06 +00:00
+								        var prefetch = (await GetByAction(action, groupFilterStart, toDate))
 								            .Select(d => d.Subject)
 								            .ToHashSet();
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								        var fromDate = parser.Parse(from).Start!.Value.TruncateToDate();
-												Implement metrics reading from CouchDB

											
										
										
											2022-06-25 20:22:06 +00:00
+								        var counts = (await GetByAction(action, fromDate, toDate))
 								            .Where(r => prefetch.Contains(r.Subject))
 								            .ToDictionary(kv => (kv.Date, kv.Subject), kv => kv.Count);
-												Perf improvements

											
										
										
											2021-12-02 00:12:21 +00:00
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								        Response.Headers.ContentType = "application/json";
 								        var row = new Dictionary<string, object>();
-												Restore metrics page behavior

											
										
										
											2021-12-01 22:53:32 +00:00
-												Perf improvements

											
										
										
											2021-12-02 00:12:21 +00:00
+								        Response.Body.Write(LBRACKET);
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								        for (var d = fromDate; d <= toDate; d = d.AddDays(1))
 								        {
 								            row["_Timestamp"] = d;
 								            foreach (var group in prefetch)
 								            {
 								                if (counts.TryGetValue((d, group), out var found))
 								                    row[group] = found;
 								                else
 								                    row[group] = 0;
 								            }
-												Implement metrics reading from CouchDB

											
										
										
											2022-06-25 20:22:06 +00:00
+								            await JsonSerializer.SerializeAsync(Response.Body, row);
-												Perf improvements

											
										
										
											2021-12-02 00:12:21 +00:00
+								            Response.Body.Write(EOL);
-												Restore metrics page behavior

											
										
										
											2021-12-01 22:53:32 +00:00
+								            if (d != toDate)
-												Perf improvements

											
										
										
											2021-12-02 00:12:21 +00:00
+								                Response.Body.Write(COMMA);
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								        }
-												Perf improvements

											
										
										
											2021-12-02 00:12:21 +00:00
+								        Response.Body.Write(RBRACKET);
-												Restore metrics page behavior

											
										
										
											2021-12-01 22:53:32 +00:00
-												Add metrics reporting, some work on the app

											
										
										
											2021-12-01 05:14:14 +00:00
+								    }
-												Implement metrics reading from CouchDB

											
										
										
											2022-06-25 20:22:06 +00:00
+								    private async Task<IReadOnlyList<(DateTime Date, string Subject, long Count)>> GetByAction(string action, DateTime from, DateTime to)
 								    {
 								        var records = await _db.GetViewAsync<object?[], long>("Indexes", "ActionDaySubject",
 								            new CouchViewOptions<object?[]>
 								            {
 								                StartKey = new object?[]{action, from.Year, from.Month, from.Day, null},
 								                EndKey = new object?[]{action, to.Year, to.Month, to.Day, new()},
 								                Reduce = true,
 								                GroupLevel = 10,
 								                Group = true
 								            });
 								        var results = records
 								            .Where(r => r.Key.Length >= 4 && r.Key[4] != null)
 								            .Select(r =>
 								            (new DateTime((int)(long)r.Key[1]!, (int)(long)r.Key[2]!, (int)(long)r.Key[3]!), (string)r.Key[4]!, r.Value));
 								        return results.ToList();
 								    }
-												Add metrics getter

											
										
										
											2021-11-29 23:35:23 +00:00
-												Code cleanup and fix test running

											
										
										
											2021-10-23 16:51:17 +00:00
+								    public class Result
 								    {
 								        public DateTime Timestamp { get; set; }
 								    }
 								}