diff --git a/Wabbajack.BuildServer/Controllers/GraphQL.cs b/Wabbajack.BuildServer/Controllers/GraphQL.cs new file mode 100644 index 00000000..42491d96 --- /dev/null +++ b/Wabbajack.BuildServer/Controllers/GraphQL.cs @@ -0,0 +1,48 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using GraphQL; +using GraphQL.Language.AST; +using GraphQL.Types; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json.Linq; +using Wabbajack.BuildServer.GraphQL; +using Wabbajack.BuildServer.Models; + +namespace Wabbajack.BuildServer.Controllers +{ + [Route("graphql")] + [ApiController] + public class GraphQL : AControllerBase + { + public GraphQL(ILogger logger, DBContext db) : base(logger, db) + { + } + + [HttpPost] + public async Task Post([FromBody] GraphQLQuery query) + { + var inputs = query.Variables.ToInputs(); + var schema = new Schema + { + Query = new Query(Db), + }; + + var result = await new DocumentExecuter().ExecuteAsync(_ => + { + _.Schema = schema; + _.Query = query.Query; + _.OperationName = query.OperationName; + _.Inputs = inputs; + }); + + if(result.Errors?.Count > 0) + { + return BadRequest(); + } + + return Ok(result); + } + + } +} diff --git a/Wabbajack.BuildServer/Controllers/Metrics.cs b/Wabbajack.BuildServer/Controllers/Metrics.cs new file mode 100644 index 00000000..95b4e70c --- /dev/null +++ b/Wabbajack.BuildServer/Controllers/Metrics.cs @@ -0,0 +1,35 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Wabbajack.BuildServer.Models; +using Wabbajack.Common; + +namespace Wabbajack.BuildServer.Controllers +{ + [Route("/metrics")] + public class Metrics : AControllerBase + { + [HttpGet] + [Route("{Action}/Value")] + public async Task NewMetric(string Action, string Value) + { + + var date = DateTime.UtcNow; + await Log(date, Action, Value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault()); + return date.ToString(); + } + + public Metrics(ILogger logger, DBContext db) : base(logger, db) + { + } + + internal async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null) + { + var msg = new[] {string.Join("\t", new[]{timestamp.ToString(), metricsKey, action, subject})}; + Utils.Log(msg.First()); + await Db.Metrics.InsertOneAsync(new Metric {Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey}); + } + } +} diff --git a/Wabbajack.BuildServer/GraphQL/GraphQLQuery.cs b/Wabbajack.BuildServer/GraphQL/GraphQLQuery.cs new file mode 100644 index 00000000..2b655477 --- /dev/null +++ b/Wabbajack.BuildServer/GraphQL/GraphQLQuery.cs @@ -0,0 +1,15 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Wabbajack.BuildServer.GraphQL +{ + public class GraphQLQuery + { + + public string OperationName { get; set; } + + public string NamedQuery { get; set; } + public string Query { get; set; } + public JObject Variables { get; set; } + } +} diff --git a/Wabbajack.BuildServer/GraphQL/JobType.cs b/Wabbajack.BuildServer/GraphQL/JobType.cs new file mode 100644 index 00000000..da176fad --- /dev/null +++ b/Wabbajack.BuildServer/GraphQL/JobType.cs @@ -0,0 +1,17 @@ +using GraphQL.Types; +using Wabbajack.BuildServer.Models.JobQueue; + +namespace Wabbajack.BuildServer.GraphQL +{ + public class JobType : ObjectGraphType + { + public JobType() + { + Name = "Job"; + Field(x => x.Id, type: typeof(IdGraphType)).Description("Unique Id of the Job"); + Field(x => x.Created, type: typeof(DateTimeGraphType)).Description("Creation time of the Job"); + Field(x => x.Started, type: typeof(DateTimeGraphType)).Description("Started time of the Job"); + Field(x => x.Ended, type: typeof(DateTimeGraphType)).Description("Ended time of the Job"); + } + } +} diff --git a/Wabbajack.BuildServer/GraphQL/MetricType.cs b/Wabbajack.BuildServer/GraphQL/MetricType.cs new file mode 100644 index 00000000..08687586 --- /dev/null +++ b/Wabbajack.BuildServer/GraphQL/MetricType.cs @@ -0,0 +1,36 @@ +using System.Collections.Generic; +using GraphQL.Types; + +namespace Wabbajack.BuildServer.GraphQL +{ + public class MetricEnum : EnumerationGraphType + { + public MetricEnum() + { + Name = "MetricType"; + Description = "The metric grouping"; + AddValue("BEGIN_INSTALL", "Installation of a modlist started", "begin_install"); + AddValue("FINISHED_INSTALL", "Installation of a modlist finished", "finish_install"); + AddValue("BEGIN_DOWNLOAD", "Downloading of a modlist begain started", "downloading"); + } + } + + public class MetricResultType : ObjectGraphType + { + public MetricResultType() + { + Name = "MetricResult"; + Description = + "A single line of data from a metrics graph. For example, the number of unique downloads each day."; + Field(x => x.SeriesName).Description("The name of the data series"); + Field(x => x.Labels).Description("The name for each plot of data (for example the date for each value"); + Field(x => x.Values).Description("The value for each plot of data"); + } + } + public class MetricResult + { + public string SeriesName { get; set; } + public List Labels { get; set; } + public List Values { get; set; } + } +} diff --git a/Wabbajack.BuildServer/GraphQL/Query.cs b/Wabbajack.BuildServer/GraphQL/Query.cs new file mode 100644 index 00000000..5beecc2e --- /dev/null +++ b/Wabbajack.BuildServer/GraphQL/Query.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using GraphQL; +using GraphQL.Types; +using GraphQLParser.AST; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Wabbajack.BuildServer.Models; +using Wabbajack.Common; + +namespace Wabbajack.BuildServer.GraphQL +{ + public class Query : ObjectGraphType + { + public Query(DBContext db) + { + Field>("unfinishedJobs", resolve: context => + { + var data = db.Jobs.AsQueryable().Where(j => j.Ended == null).ToList(); + return data; + }); + + FieldAsync>("job", + arguments: new QueryArguments( + new QueryArgument {Name = "id", Description = "Id of the Job"}), + resolve: async context => + { + var id = Guid.Parse(context.GetArgument("id")); + var data = await db.Jobs.AsQueryable().Where(j => j.Id == id).ToListAsync(); + return data; + }); + + Field ("indexedFileTree", + arguments: new QueryArguments( + new QueryArgument {Name = "hash", Description = "Hash of the Job"}), + resolve: context => + { + var hash = context.GetArgument("hash"); + var data = db.IndexedFiles.AsQueryable().Where(j => j.Hash == hash).FirstOrDefault(); + return data; + }); + + FieldAsync>("dailyUniqueMetrics", + arguments: new QueryArguments( + new QueryArgument {Name = "metric_type", Description = "The grouping of metric data to query"} + ), + resolve: async context => + { + var group = context.GetArgument("metric_type"); + return await Metric.Report(db, group); + }); + } + } +} diff --git a/Wabbajack.BuildServer/GraphQL/VirtualFileType.cs b/Wabbajack.BuildServer/GraphQL/VirtualFileType.cs new file mode 100644 index 00000000..92e95b2e --- /dev/null +++ b/Wabbajack.BuildServer/GraphQL/VirtualFileType.cs @@ -0,0 +1,32 @@ +using GraphQL.Types; +using Wabbajack.BuildServer.Models; + +namespace Wabbajack.BuildServer.GraphQL +{ + public class VirtualFileType : ObjectGraphType + { + public VirtualFileType() + { + Name = "VirtualFile"; + Field(x => x.Hash, type: typeof(IdGraphType)).Description("xxHash64 of the file, in Base64 encoding"); + Field(x => x.Size, type: typeof(LongGraphType)).Description("Size of the file"); + Field(x => x.IsArchive).Description("True if this file is an archive (BSA, zip, 7z, etc.)"); + Field(x => x.SHA256).Description("SHA256 hash of the file, in hexidecimal encoding"); + Field(x => x.SHA1).Description("SHA1 hash of the file, in hexidecimal encoding"); + Field(x => x.MD5).Description("MD5 hash of the file, in hexidecimal encoding"); + Field(x => x.CRC).Description("CRC32 hash of the file, in hexidecimal encoding"); + Field(x => x.Children, type: typeof(ChildFileType)).Description("Metadata for the files in this archive (if any)"); + } + } + + public class ChildFileType : ObjectGraphType + { + public ChildFileType() + { + Name = "ChildFile"; + Field(x => x.Name).Description("The relative path to the file inside the parent archive"); + Field(x => x.Hash).Description("The hash (xxHash64, Base64 ecoded) of the child file"); + Field(x => x.Extension).Description("File extension of the child file"); + } + } +} diff --git a/Wabbajack.BuildServer/Models/DBContext.cs b/Wabbajack.BuildServer/Models/DBContext.cs index e7e8f6f6..98c19356 100644 --- a/Wabbajack.BuildServer/Models/DBContext.cs +++ b/Wabbajack.BuildServer/Models/DBContext.cs @@ -24,6 +24,7 @@ namespace Wabbajack.BuildServer.Models public IMongoCollection Jobs => Client.GetCollection(_settings.Collections["JobQueue"]); public IMongoCollection DownloadStates => Client.GetCollection(_settings.Collections["DownloadStates"]); + public IMongoCollection Metrics => Client.GetCollection(_settings.Collections["Metrics"]); public IMongoCollection IndexedFiles => Client.GetCollection(_settings.Collections["IndexedFiles"]); public IMongoCollection> NexusModFiles => diff --git a/Wabbajack.BuildServer/Models/IndexedFileWithChildren.cs b/Wabbajack.BuildServer/Models/IndexedFileWithChildren.cs new file mode 100644 index 00000000..e8032d7d --- /dev/null +++ b/Wabbajack.BuildServer/Models/IndexedFileWithChildren.cs @@ -0,0 +1,9 @@ +using System.Collections; +using System.Collections.Generic; + +namespace Wabbajack.BuildServer.Models +{ + public class IndexedFileWithChildren : IndexedFile + { + } +} diff --git a/Wabbajack.BuildServer/Models/Metric.cs b/Wabbajack.BuildServer/Models/Metric.cs new file mode 100644 index 00000000..61355831 --- /dev/null +++ b/Wabbajack.BuildServer/Models/Metric.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Wabbajack.BuildServer.GraphQL; +using Wabbajack.Common; + + +namespace Wabbajack.BuildServer.Models +{ + public class Metric + { + [BsonId] + public ObjectId Id; + public DateTime Timestamp; + public string Action; + public string Subject; + public string MetricsKey; + + + public static async Task> Report(DBContext db, string grouping) + { + var data = await db.Metrics.AsQueryable() + .Where(m => m.MetricsKey != null) + .Where(m => m.Action == grouping) + .Where(m => m.Subject != "Default") + .ToListAsync(); + + var minDate = DateTime.Parse(data.Min(d => d.Timestamp.ToString("yyyy-MM-dd"))); + var maxDate = DateTime.Parse(data.Max(d => d.Timestamp.ToString("yyyy-MM-dd"))); + + var dateArray = Enumerable.Range(0, (int)(maxDate - minDate).TotalDays + 1) + .Select(idx => minDate + TimeSpan.FromDays(idx)) + .Select(date => date.ToString("yyyy-MM-dd")) + .ToList(); + + var results = data + .Where(d => !Guid.TryParse(d.Subject, out var _)) + .GroupBy(d => d.Subject) + .Select(by_series => + { + var by_day = by_series.GroupBy(d => d.Timestamp.ToString("yyyy-MM-dd")) + .Select(d => (d.Key, d.DistinctBy(v => v.MetricsKey ?? "").Count())) + .OrderBy(r => r.Key); + + var by_day_idx = by_day.ToDictionary(d => d.Key); + + (string Key, int) GetEntry(string date) + { + if (by_day_idx.TryGetValue(date, out var result)) + return result; + return (date, 0); + } + + return new MetricResult + { + SeriesName = by_series.Key, + Labels = dateArray.Select(d => GetEntry(d).Key).ToList(), + Values = dateArray.Select(d => GetEntry(d).Item2).ToList() + }; + }) + .OrderBy(f => f.SeriesName); + + return results; + } + } +} diff --git a/Wabbajack.BuildServer/Program.cs b/Wabbajack.BuildServer/Program.cs new file mode 100644 index 00000000..74504fca --- /dev/null +++ b/Wabbajack.BuildServer/Program.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Wabbajack.BuildServer +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder.UseStartup(); + }); + } +} diff --git a/Wabbajack.BuildServer/SerializerSettings.cs b/Wabbajack.BuildServer/SerializerSettings.cs index 413f0be9..213e2b3f 100644 --- a/Wabbajack.BuildServer/SerializerSettings.cs +++ b/Wabbajack.BuildServer/SerializerSettings.cs @@ -10,7 +10,7 @@ using MongoDB.Bson.Serialization; using MongoDB.Bson.Serialization.Conventions; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Lib.Downloaders; -using Newtonsoft.Json. + namespace Wabbajack.BuildServer { diff --git a/Wabbajack.BuildServer/Startup.cs b/Wabbajack.BuildServer/Startup.cs new file mode 100644 index 00000000..8853f069 --- /dev/null +++ b/Wabbajack.BuildServer/Startup.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using GraphiQl; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.HttpsPolicy; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Authentication.AzureAD.UI; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.OpenApi.Models; +using Newtonsoft.Json; +using Swashbuckle.AspNetCore.Swagger; +using Wabbajack.BuildServer.Controllers; +using Wabbajack.BuildServer.Models; +using Microsoft.AspNetCore.Mvc.NewtonsoftJson; +using Microsoft.Extensions.FileProviders; +using Directory = System.IO.Directory; + + +namespace Wabbajack.BuildServer +{ + public class Startup + { + public Startup(IConfiguration configuration) + { + Configuration = configuration; + } + + public IConfiguration Configuration { get; } + + // This method gets called by the runtime. Use this method to add services to the container. + public void ConfigureServices(IServiceCollection services) + { + services.AddAuthentication(AzureADDefaults.BearerAuthenticationScheme) + .AddAzureADBearer(options => Configuration.Bind("AzureAd", options)); + services.AddSwaggerGen(c => + { + c.SwaggerDoc("v1", new OpenApiInfo {Title = "Wabbajack Build API", Version = "v1"}); + }); + + services.AddSingleton(); + services.AddControllers(o => + { + + }).AddNewtonsoftJson(o => + { + o.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; + }); + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + SerializerSettings.Init(); + if (env.IsDevelopment()) + { + app.UseDeveloperExceptionPage(); + } + + app.UseGraphiQl(); + app.UseDeveloperExceptionPage(); + app.UseStaticFiles(); + app.UseHttpsRedirection(); + + app.UseSwagger(); + app.UseSwaggerUI(c => + { + c.SwaggerEndpoint("/swagger/v1/swagger.json", "Wabbajack Build API"); + c.RoutePrefix = string.Empty; + }); + app.UseRouting(); + + app.UseAuthentication(); + app.UseAuthorization(); + app.UseFileServer(new FileServerOptions + { + FileProvider = new PhysicalFileProvider( + Path.Combine(Directory.GetCurrentDirectory(), "public")) + }); + + app.UseEndpoints(endpoints => + { + endpoints.MapControllers(); + }); + } + } +} diff --git a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj index 5411cfed..590d62ca 100644 --- a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj +++ b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj @@ -7,7 +7,11 @@ + + + + @@ -44,6 +48,23 @@ + + + + + <_ContentIncludedByDefault Remove="Views\MetricsDashboard.cshtml" /> + + + + + + + + + + + + diff --git a/Wabbajack.BuildServer/appsettings.Development.json b/Wabbajack.BuildServer/appsettings.Development.json new file mode 100644 index 00000000..8983e0fc --- /dev/null +++ b/Wabbajack.BuildServer/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/Wabbajack.BuildServer/appsettings.json b/Wabbajack.BuildServer/appsettings.json new file mode 100644 index 00000000..f93c14d6 --- /dev/null +++ b/Wabbajack.BuildServer/appsettings.json @@ -0,0 +1,30 @@ +{ + "AzureAd": { + "Instance": "https://login.microsoftonline.com/", + "Domain": "qualified.domain.name", + "TenantId": "22222222-2222-2222-2222-222222222222", + "ClientId": "11111111-1111-1111-11111111111111111" + }, + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "MongoDB": { + "Host": "internal.test.mongodb", + "Database": "wabbajack", + "Collections": { + "NexusModInfos": "nexus_mod_infos", + "NexusModFiles": "nexus_mod_files", + "NexusFileInfos": "nexus_file_infos", + "ModListStatus": "mod_lists", + "JobQueue": "job_queue", + "DownloadStates": "download_states", + "IndexedFiles": "indexed_files", + "Metrics": "metrics" + } + }, + "AllowedHosts": "*" +} diff --git a/Wabbajack.BuildServer/public/metrics.html b/Wabbajack.BuildServer/public/metrics.html new file mode 100644 index 00000000..27e5b11c --- /dev/null +++ b/Wabbajack.BuildServer/public/metrics.html @@ -0,0 +1,77 @@ + + + + + Wabbajack Metrics + + + + + + + +

Begin Download

+ +
+ +

Begin Install

+ +
+ +

Finished Install

+ +
+ + + + + + \ No newline at end of file diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 377bf7e0..847e1f31 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -769,6 +769,7 @@ namespace Wabbajack.Common { var key = select(v); if (set.Contains(key)) continue; + set.Add(key); yield return v; } }