GraphQL support, endpoints for metrics via GraphQL

Wabbajack.BuildServer/Controllers/GraphQL.cs

@@ -0,0 +1,48 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using GraphQL;
+using GraphQL.Language.AST;
+using GraphQL.Types;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json.Linq;
+using Wabbajack.BuildServer.GraphQL;
+using Wabbajack.BuildServer.Models;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [Route("graphql")]
+    [ApiController]
+    public class GraphQL : AControllerBase<GraphQL>
+    {
+        public GraphQL(ILogger<GraphQL> logger, DBContext db) : base(logger, db)
+        {
+        }
+
+        [HttpPost]
+        public async Task<IActionResult> Post([FromBody] GraphQLQuery query)
+        {
+            var inputs = query.Variables.ToInputs();
+            var schema = new Schema
+            {
+                Query = new Query(Db),
+            };
+            
+            var result = await new DocumentExecuter().ExecuteAsync(_ =>
+            {
+                _.Schema = schema;
+                _.Query = query.Query;
+                _.OperationName = query.OperationName;
+                _.Inputs = inputs;
+            });
+            
+            if(result.Errors?.Count > 0)
+            {
+                return BadRequest();
+            }
+
+            return Ok(result);
+        }
+       
+    }
+}

Wabbajack.BuildServer/Controllers/Metrics.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.Common;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [Route("/metrics")]
+    public class Metrics : AControllerBase<Metrics>
+    {
+        [HttpGet]
+        [Route("{Action}/Value")]
+        public async Task<string> NewMetric(string Action, string Value)
+        {
+            
+            var date = DateTime.UtcNow;
+            await Log(date, Action, Value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault());
+            return date.ToString();
+        }
+
+        public Metrics(ILogger<Metrics> logger, DBContext db) : base(logger, db)
+        {
+        }
+        
+        internal async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
+        {
+            var msg = new[] {string.Join("\t", new[]{timestamp.ToString(), metricsKey, action, subject})};
+            Utils.Log(msg.First());
+            await Db.Metrics.InsertOneAsync(new Metric {Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey});
+        }
+    }
+}

Wabbajack.BuildServer/GraphQL/GraphQLQuery.cs

@@ -0,0 +1,15 @@
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace Wabbajack.BuildServer.GraphQL
+{
+    public class GraphQLQuery
+    {
+
+        public string OperationName { get; set; }
+
+        public string NamedQuery { get; set; }
+        public string Query { get; set; }
+        public JObject Variables { get; set; }
+    }
+}

Wabbajack.BuildServer/GraphQL/JobType.cs

@@ -0,0 +1,17 @@
+using GraphQL.Types;
+using Wabbajack.BuildServer.Models.JobQueue;
+
+namespace Wabbajack.BuildServer.GraphQL
+{
+    public class JobType : ObjectGraphType<Job>
+    {
+        public JobType()
+        {
+            Name = "Job";
+            Field(x => x.Id, type: typeof(IdGraphType)).Description("Unique Id of the Job");
+            Field(x => x.Created, type: typeof(DateTimeGraphType)).Description("Creation time of the Job");
+            Field(x => x.Started, type: typeof(DateTimeGraphType)).Description("Started time of the Job");
+            Field(x => x.Ended, type: typeof(DateTimeGraphType)).Description("Ended time of the Job");
+        }
+    }
+}

Wabbajack.BuildServer/GraphQL/MetricType.cs

@@ -0,0 +1,36 @@
+using System.Collections.Generic;
+using GraphQL.Types;
+
+namespace Wabbajack.BuildServer.GraphQL
+{
+    public class MetricEnum : EnumerationGraphType
+    {
+        public MetricEnum()
+        {
+            Name = "MetricType";
+            Description = "The metric grouping";
+            AddValue("BEGIN_INSTALL", "Installation of a modlist started", "begin_install");
+            AddValue("FINISHED_INSTALL", "Installation of a modlist finished", "finish_install");
+            AddValue("BEGIN_DOWNLOAD", "Downloading of a modlist begain started", "downloading");
+        }
+    }
+
+    public class MetricResultType : ObjectGraphType<MetricResult>
+    {
+        public MetricResultType()
+        {
+            Name = "MetricResult";
+            Description =
+                "A single line of data from a metrics graph. For example, the number of unique downloads each day.";
+            Field(x => x.SeriesName).Description("The name of the data series");
+            Field(x => x.Labels).Description("The name for each plot of data (for example the date for each value");
+            Field(x => x.Values).Description("The value for each plot of data");
+        }
+    }
+    public class MetricResult
+    {
+        public string SeriesName { get; set; }
+        public List<string> Labels { get; set; }
+        public List<int> Values { get; set; }
+    }
+}

Wabbajack.BuildServer/GraphQL/Query.cs

@@ -0,0 +1,54 @@
+using System;
+using System.Collections.Generic;
+using GraphQL;
+using GraphQL.Types;
+using GraphQLParser.AST;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.Common;
+
+namespace Wabbajack.BuildServer.GraphQL
+{
+    public class Query : ObjectGraphType
+    {
+        public Query(DBContext db)
+        {
+            Field<ListGraphType<JobType>>("unfinishedJobs", resolve: context =>
+            {
+                var data =  db.Jobs.AsQueryable().Where(j => j.Ended == null).ToList();
+                return data;
+            });
+            
+            FieldAsync<ListGraphType<JobType>>("job",
+                arguments: new QueryArguments(
+                    new QueryArgument<IdGraphType> {Name = "id", Description = "Id of the Job"}),
+                resolve: async context =>
+                {
+                    var id = Guid.Parse(context.GetArgument<string>("id"));
+                    var data = await db.Jobs.AsQueryable().Where(j => j.Id == id).ToListAsync();
+                    return data;
+                });
+
+            Field<VirtualFileType> ("indexedFileTree",
+                arguments: new QueryArguments(
+                    new QueryArgument<IdGraphType> {Name = "hash", Description = "Hash of the Job"}),
+                resolve: context =>
+                {
+                    var hash = context.GetArgument<string>("hash");
+                    var data = db.IndexedFiles.AsQueryable().Where(j => j.Hash == hash).FirstOrDefault();
+                    return data;
+                });
+
+            FieldAsync<ListGraphType<MetricResultType>>("dailyUniqueMetrics",
+                arguments: new QueryArguments(
+                    new QueryArgument<MetricEnum> {Name = "metric_type", Description = "The grouping of metric data to query"}
+                    ),
+                resolve: async context =>
+                {
+                    var group = context.GetArgument<string>("metric_type");
+                    return await Metric.Report(db, group);
+                });
+        }
+    }
+}

Wabbajack.BuildServer/GraphQL/VirtualFileType.cs

@@ -0,0 +1,32 @@
+using GraphQL.Types;
+using Wabbajack.BuildServer.Models;
+
+namespace Wabbajack.BuildServer.GraphQL
+{
+    public class VirtualFileType : ObjectGraphType<IndexedFileWithChildren>
+    {
+        public VirtualFileType()
+        {
+            Name = "VirtualFile";
+            Field(x => x.Hash, type: typeof(IdGraphType)).Description("xxHash64 of the file, in Base64 encoding");
+            Field(x => x.Size, type: typeof(LongGraphType)).Description("Size of the file");
+            Field(x => x.IsArchive).Description("True if this file is an archive (BSA, zip, 7z, etc.)");
+            Field(x => x.SHA256).Description("SHA256 hash of the file, in hexidecimal encoding");
+            Field(x => x.SHA1).Description("SHA1 hash of the file, in hexidecimal encoding");
+            Field(x => x.MD5).Description("MD5 hash of the file, in hexidecimal encoding");
+            Field(x => x.CRC).Description("CRC32 hash of the file, in hexidecimal encoding");
+            Field(x => x.Children, type: typeof(ChildFileType)).Description("Metadata for the files in this archive (if any)");
+        }
+    }
+
+    public class ChildFileType : ObjectGraphType<ChildFile>
+    {
+        public ChildFileType()
+        {
+            Name = "ChildFile";
+            Field(x => x.Name).Description("The relative path to the file inside the parent archive");
+            Field(x => x.Hash).Description("The hash (xxHash64, Base64 ecoded) of the child file");
+            Field(x => x.Extension).Description("File extension of the child file");
+        }
+    }
+}

Wabbajack.BuildServer/Models/DBContext.cs

@@ -24,6 +24,7 @@ namespace Wabbajack.BuildServer.Models
         
         public IMongoCollection<Job> Jobs => Client.GetCollection<Job>(_settings.Collections["JobQueue"]);
         public IMongoCollection<DownloadState> DownloadStates => Client.GetCollection<DownloadState>(_settings.Collections["DownloadStates"]);
+        public IMongoCollection<Metric> Metrics => Client.GetCollection<Metric>(_settings.Collections["Metrics"]);
         public IMongoCollection<IndexedFile> IndexedFiles => Client.GetCollection<IndexedFile>(_settings.Collections["IndexedFiles"]);
 
         public IMongoCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>> NexusModFiles =>

Wabbajack.BuildServer/Models/IndexedFileWithChildren.cs

@@ -0,0 +1,9 @@
+using System.Collections;
+using System.Collections.Generic;
+
+namespace Wabbajack.BuildServer.Models
+{
+    public class IndexedFileWithChildren : IndexedFile
+    {
+   }
+}

Wabbajack.BuildServer/Models/Metric.cs

@@ -0,0 +1,71 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.GraphQL;
+using Wabbajack.Common;
+
+
+namespace Wabbajack.BuildServer.Models
+{
+    public class Metric
+    {
+        [BsonId]
+        public ObjectId Id;
+        public DateTime Timestamp;
+        public string Action;
+        public string Subject;
+        public string MetricsKey;
+
+        
+        public static async Task<IEnumerable<MetricResult>> Report(DBContext db, string grouping)
+        {
+            var data = await db.Metrics.AsQueryable()
+                .Where(m => m.MetricsKey != null)
+                .Where(m => m.Action == grouping)
+                .Where(m => m.Subject != "Default")
+                .ToListAsync();
+
+            var minDate = DateTime.Parse(data.Min(d => d.Timestamp.ToString("yyyy-MM-dd")));
+            var maxDate = DateTime.Parse(data.Max(d => d.Timestamp.ToString("yyyy-MM-dd")));
+
+            var dateArray = Enumerable.Range(0, (int)(maxDate - minDate).TotalDays + 1)
+                .Select(idx => minDate + TimeSpan.FromDays(idx))
+                .Select(date => date.ToString("yyyy-MM-dd"))
+                .ToList();
+            
+            var results = data
+                .Where(d => !Guid.TryParse(d.Subject, out var _))
+                .GroupBy(d => d.Subject)
+                .Select(by_series =>
+                {
+                    var by_day = by_series.GroupBy(d => d.Timestamp.ToString("yyyy-MM-dd"))
+                        .Select(d => (d.Key, d.DistinctBy(v => v.MetricsKey ?? "").Count()))
+                        .OrderBy(r => r.Key);
+                    
+                    var by_day_idx = by_day.ToDictionary(d => d.Key);
+
+                    (string Key, int) GetEntry(string date)
+                    {
+                        if (by_day_idx.TryGetValue(date, out var result))
+                            return result;
+                        return (date, 0);
+                    } 
+                    
+                    return new MetricResult
+                    {
+                        SeriesName = by_series.Key,
+                        Labels = dateArray.Select(d => GetEntry(d).Key).ToList(),
+                        Values = dateArray.Select(d => GetEntry(d).Item2).ToList()
+                    };
+                })
+                .OrderBy(f => f.SeriesName);
+
+            return results;
+        }
+    }
+}

Wabbajack.BuildServer/Program.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace Wabbajack.BuildServer
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            CreateHostBuilder(args).Build().Run();
+        }
+
+        public static IHostBuilder CreateHostBuilder(string[] args) =>
+            Host.CreateDefaultBuilder(args)
+                .ConfigureWebHostDefaults(webBuilder =>
+                {
+                    webBuilder.UseStartup<Startup>();
+                });
+    }
+}

Wabbajack.BuildServer/SerializerSettings.cs

@@ -10,7 +10,7 @@ using MongoDB.Bson.Serialization;
 using MongoDB.Bson.Serialization.Conventions;
 using Wabbajack.BuildServer.Models.JobQueue;
 using Wabbajack.Lib.Downloaders;
-using Newtonsoft.Json.
+
 
 namespace Wabbajack.BuildServer
 {

Wabbajack.BuildServer/Startup.cs

@@ -0,0 +1,96 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using GraphiQl;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.HttpsPolicy;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Authentication;
+using Microsoft.AspNetCore.Authentication.AzureAD.UI;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.OpenApi.Models;
+using Newtonsoft.Json;
+using Swashbuckle.AspNetCore.Swagger;
+using Wabbajack.BuildServer.Controllers;
+using Wabbajack.BuildServer.Models;
+using Microsoft.AspNetCore.Mvc.NewtonsoftJson;
+using Microsoft.Extensions.FileProviders;
+using Directory = System.IO.Directory;
+
+
+namespace Wabbajack.BuildServer
+{
+    public class Startup
+    {
+        public Startup(IConfiguration configuration)
+        {
+            Configuration = configuration;
+        }
+
+        public IConfiguration Configuration { get; }
+
+        // This method gets called by the runtime. Use this method to add services to the container.
+        public void ConfigureServices(IServiceCollection services)
+        {
+            services.AddAuthentication(AzureADDefaults.BearerAuthenticationScheme)
+                .AddAzureADBearer(options => Configuration.Bind("AzureAd", options));
+            services.AddSwaggerGen(c =>
+            {
+                c.SwaggerDoc("v1", new OpenApiInfo {Title = "Wabbajack Build API", Version = "v1"});
+            });
+           
+            services.AddSingleton<DBContext>();
+            services.AddControllers(o =>
+            {
+                
+            }).AddNewtonsoftJson(o =>
+            {
+                o.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore;
+            });
+        }
+
+        // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
+        public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
+        {
+            SerializerSettings.Init();
+            if (env.IsDevelopment())
+            {
+                app.UseDeveloperExceptionPage();
+            }
+
+            app.UseGraphiQl();
+            app.UseDeveloperExceptionPage();
+            app.UseStaticFiles();
+            app.UseHttpsRedirection();
+
+            app.UseSwagger();
+            app.UseSwaggerUI(c =>
+            {
+                c.SwaggerEndpoint("/swagger/v1/swagger.json", "Wabbajack Build API");
+                c.RoutePrefix = string.Empty;
+            });
+            app.UseRouting();
+
+            app.UseAuthentication();
+            app.UseAuthorization();
+            app.UseFileServer(new FileServerOptions
+            {
+                FileProvider = new PhysicalFileProvider(
+                    Path.Combine(Directory.GetCurrentDirectory(), "public"))
+            });
+
+            app.UseEndpoints(endpoints =>
+            {
+                endpoints.MapControllers();
+            });
+        }
+    }
+}

Wabbajack.BuildServer/Wabbajack.BuildServer.csproj

@@ -7,7 +7,11 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="graphiql" Version="1.2.0" />
+        <PackageReference Include="GraphQL" Version="3.0.0-preview-1352" />
         <PackageReference Include="Microsoft.AspNetCore.Authentication.AzureAD.UI" Version="3.1.0" />
+        <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.0" />
+        <PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
         <PackageReference Include="Microsoft.OpenApi" Version="1.1.4" />
         <PackageReference Include="MongoDB.Driver" Version="2.10.0" />
         <PackageReference Include="MongoDB.Driver.Core" Version="2.10.0" />
@@ -44,6 +48,23 @@
       <None Remove="cef.pak" />
       <None Remove="7z.exe" />
       <None Remove="7z.dll" />
+      <None Remove="swiftshader\**" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <_ContentIncludedByDefault Remove="Views\MetricsDashboard.cshtml" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <Compile Remove="swiftshader\**" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <EmbeddedResource Remove="swiftshader\**" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <Content Remove="swiftshader\**" />
     </ItemGroup>
 
 </Project>

Wabbajack.BuildServer/appsettings.Development.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft": "Warning",
+      "Microsoft.Hosting.Lifetime": "Information"
+    }
+  }
+}

Wabbajack.BuildServer/appsettings.json

@@ -0,0 +1,30 @@
+{
+  "AzureAd": {
+    "Instance": "https://login.microsoftonline.com/",
+    "Domain": "qualified.domain.name",
+    "TenantId": "22222222-2222-2222-2222-222222222222",
+    "ClientId": "11111111-1111-1111-11111111111111111"
+  },
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft": "Warning",
+      "Microsoft.Hosting.Lifetime": "Information"
+    }
+  },
+  "MongoDB": {
+    "Host": "internal.test.mongodb",
+    "Database": "wabbajack",
+    "Collections": {
+      "NexusModInfos": "nexus_mod_infos",
+      "NexusModFiles": "nexus_mod_files",
+      "NexusFileInfos": "nexus_file_infos",
+      "ModListStatus": "mod_lists",
+      "JobQueue": "job_queue",
+      "DownloadStates": "download_states",
+      "IndexedFiles": "indexed_files",
+      "Metrics": "metrics"
+    }
+  },
+  "AllowedHosts": "*"
+}

Wabbajack.BuildServer/public/metrics.html

@@ -0,0 +1,77 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Wabbajack Metrics</title>
+    <script src="//cdn.jsdelivr.net/npm/graphql.js@0.6.6/graphql.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/chart.js@2.8.0"></script>
+    <script src="https://cdn.jsdelivr.net/npm/chartjs-plugin-colorschemes"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/underscore.js/1.9.1/underscore-min.js"></script>
+</head>
+<body>
+
+<h2>Begin Download</h2>
+<canvas id="begin_download_chart" width="800" height="600"></canvas>
+<hr/>
+
+<h2>Begin Install</h2>
+<canvas id="begin_install_chart" width="800" height="600"></canvas>
+<hr/>
+
+<h2>Finished Install</h2>
+<canvas id="finished_install_chart" width="800" height="600"></canvas>
+<hr/>
+
+
+
+<script>
+    var makeChart = function(ele, group) {
+        var graph = graphql("/graphql",
+                {
+                    method: "POST",
+                    asJSON: true,
+                    headers: {
+                        "Content-Type": "application/json"
+                    }
+                });
+        var metrics = graph.query(`($type: MetricType) {
+  dailyUniqueMetrics(metric_type: $type)
+  {
+    seriesName,
+    labels,
+    values
+  }
+}`);
+
+        var result = metrics({type: group})
+                .then(function (data) {
+                    var data = data.dailyUniqueMetrics;
+                    var labels = _.uniq(_.flatten(_.map(data, series => series.labels))).sort();
+                    var datasets = _.map(data, series => {
+                        return {
+                            label: series.seriesName,
+                            fill: false,
+                            data: series.values
+                        }});
+                    var ctx = document.getElementById(ele).getContext('2d');
+                    var chart = new Chart(ctx, {
+                        // The type of chart we want to create
+                        type: 'line',
+
+                        // The data for our dataset
+                        data: {
+                            labels: labels,
+                            datasets: datasets},
+
+                        // Configuration options go here
+                        options: {}
+                    });
+                });
+    };
+    makeChart("begin_download_chart", "BEGIN_DOWNLOAD");
+    makeChart("begin_install_chart", "BEGIN_INSTALL");
+    makeChart("finished_install_chart", "FINISHED_INSTALL");
+
+</script>
+</body>
+</html>

Wabbajack.Common/Utils.cs

@@ -769,6 +769,7 @@ namespace Wabbajack.Common
             {
                 var key = select(v);
                 if (set.Contains(key)) continue;
+                set.Add(key);
                 yield return v;
             }
         }