Fix file extractor tests, convert metrics to SQL

Wabbajack.BuildServer/Controllers/GraphQL.cs

@@ -4,6 +4,7 @@ using GraphQL.Types;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using Wabbajack.BuildServer.GraphQL;
+using Wabbajack.BuildServer.Model.Models;
 using Wabbajack.BuildServer.Models;
 
 namespace Wabbajack.BuildServer.Controllers
@@ -12,15 +13,18 @@ namespace Wabbajack.BuildServer.Controllers
     [ApiController]
     public class GraphQL : AControllerBase<GraphQL>
     {
-        public GraphQL(ILogger<GraphQL> logger, DBContext db) : base(logger, db)
+        private SqlService _sql;
+
+        public GraphQL(ILogger<GraphQL> logger, DBContext db, SqlService sql) : base(logger, db)
         {
+            _sql = sql;
         }
 
         [HttpPost]
         public async Task<IActionResult> Post([FromBody] GraphQLQuery query)
         {
             var inputs = query.Variables.ToInputs();
-            var schema = new Schema {Query = new Query(Db), Mutation = new Mutation(Db)};
+            var schema = new Schema {Query = new Query(Db, _sql), Mutation = new Mutation(Db)};
 
             var result = await new DocumentExecuter().ExecuteAsync(_ =>
             {

Wabbajack.BuildServer/Controllers/MetricsController.cs

@@ -1,11 +1,14 @@
 using System;
 using System.Collections.Generic;
+using System.Data.SqlTypes;
 using System.Linq;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using MongoDB.Driver;
 using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.Model.Models;
 using Wabbajack.BuildServer.Models;
 using Wabbajack.Common;
 using Wabbajack.Lib.ModListRegistry;
@@ -16,8 +19,11 @@ namespace Wabbajack.BuildServer.Controllers
     [Route("/metrics")]
     public class MetricsController : AControllerBase<MetricsController>
     {
-        public MetricsController(ILogger<MetricsController> logger, DBContext db) : base(logger, db)
+        private SqlService _sql;
+
+        public MetricsController(ILogger<MetricsController> logger, DBContext db, SqlService sql) : base(logger, db)
         {
+            _sql = sql;
         }
 
         [HttpGet]
@@ -29,10 +35,20 @@ namespace Wabbajack.BuildServer.Controllers
             return new Result { Timestamp = date};
         }
 
+        [Authorize]
+        [HttpGet]
+        [Route("transfer")]
+        public async Task<string> Transfer()
+        {
+            var all_metrics = await Db.Metrics.AsQueryable().ToListAsync();
+            await _sql.IngestAllMetrics(all_metrics);
+            return "done";
+        }
+
         private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
         {
             Logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}");
-            await Db.Metrics.InsertOneAsync(new Metric
+            await _sql.IngestMetric(new Metric
             {
                 Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey
             });

Wabbajack.BuildServer/GraphQL/Query.cs

@@ -1,10 +1,13 @@
 using System;
+using System.Linq;
 using System.Collections.Generic;
+using System.Data.SqlTypes;
 using GraphQL;
 using GraphQL.Types;
 using GraphQLParser.AST;
 using MongoDB.Driver;
 using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.Model.Models;
 using Wabbajack.BuildServer.Models;
 using Wabbajack.Common;
 
@@ -12,7 +15,7 @@ namespace Wabbajack.BuildServer.GraphQL
 {
     public class Query : ObjectGraphType
     {
-        public Query(DBContext db)
+        public Query(DBContext db, SqlService sql)
         {
             Field<ListGraphType<JobType>>("unfinishedJobs", resolve: context =>
             {
@@ -68,7 +71,15 @@ namespace Wabbajack.BuildServer.GraphQL
                 resolve: async context =>
                 {
                     var group = context.GetArgument<string>("metric_type");
-                    return await Metric.Report(db, group);
+                    var data = (await sql.MetricsReport(group))
+                                  .GroupBy(m => m.Subject)
+                                  .Select(g => new MetricResult
+                                  {
+                                      SeriesName = g.Key,
+                                      Labels = g.Select(m => m.Date.ToString()).ToList(),
+                                      Values = g.Select(m => m.Count).ToList()
+                                  });
+                    return data;
                 });
         }
     }

Wabbajack.BuildServer/Models/Metric.cs

@@ -16,11 +16,11 @@ namespace Wabbajack.BuildServer.Models
     public class Metric
     {
         [BsonId]
-        public ObjectId Id;
+        public ObjectId Id { get; set; }
-        public DateTime Timestamp;
+        public DateTime Timestamp { get; set; }
-        public string Action;
+        public string Action { get; set; }
-        public string Subject;
+        public string Subject { get; set; }
-        public string MetricsKey;
+        public string MetricsKey { get; set; }
 
         
         public static async Task<IEnumerable<MetricResult>> Report(DBContext db, string grouping)

Wabbajack.BuildServer/Models/Sql/Results/AggregateMetric.cs

@@ -0,0 +1,11 @@
+using System;
+
+namespace Wabbajack.BuildServer.Model.Models.Results
+{
+    public class AggregateMetric
+    {
+        public DateTime Date { get; set; }
+        public string Subject { get; set; }
+        public int Count { get; set; }
+    }
+}

Wabbajack.BuildServer/Models/Sql/SqlService.cs

@@ -7,6 +7,7 @@ using System.Linq;
 using System.Threading.Tasks;
 using Dapper;
 using Microsoft.Extensions.Configuration;
+using Wabbajack.BuildServer.Model.Models.Results;
 using Wabbajack.BuildServer.Models;
 using Wabbajack.Common;
 using Wabbajack.VirtualFileSystem;
@@ -127,5 +128,40 @@ namespace Wabbajack.BuildServer.Model.Models
             }
             return Build(0).First();
         }
+
+        public async Task IngestAllMetrics(IEnumerable<Metric> allMetrics)
+        {
+            await using var conn = await Open();
+            await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", allMetrics);
+        }
+        public async Task IngestMetric(Metric metric)
+        {
+            await using var conn = await Open();
+            await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", metric);
+        }
+
+        public async Task<IEnumerable<AggregateMetric>> MetricsReport(string action)
+        {
+            await using var conn = await Open();
+            return (await conn.QueryAsync<AggregateMetric>(@"
+                        SELECT d.Date, d.GroupingSubject as Subject, Count(*) as Count FROM 
+                        (select DISTINCT CONVERT(date, Timestamp) as Date, GroupingSubject, Action, MetricsKey from dbo.Metrics) m
+                        RIGHT OUTER JOIN
+                        (SELECT CONVERT(date, DATEADD(DAY, number + 1, dbo.MinMetricDate())) as Date, GroupingSubject, Action
+                        FROM master..spt_values
+                        CROSS JOIN (
+                          SELECT DISTINCT GroupingSubject, Action FROM dbo.Metrics 
+                          WHERE MetricsKey is not null 
+                          AND Subject != 'Default'
+                          AND TRY_CONVERT(uniqueidentifier, Subject) is null) as keys
+                        WHERE type = 'P'
+                        AND DATEADD(DAY, number+1, dbo.MinMetricDate()) < dbo.MaxMetricDate()) as d
+                        ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action
+                        WHERE d.Action = @action
+                        group by d.Date, d.GroupingSubject, d.Action
+                        ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action}))
+                .ToList();
+        }
+
     }
 }

Wabbajack.Common/FileExtractor.cs

@@ -209,7 +209,7 @@ namespace Wabbajack.Common
             {
                 var info = new ProcessStartInfo
                 {
-                    FileName = "innounp.exe",
+                    FileName = @"Extractors\innounp.exe",
                     Arguments = $"-t \"{v}\" ",
                     RedirectStandardError = true,
                     RedirectStandardInput = true,