From 82be6f304b5d9b3332097e48a4925dbef6b562b9 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge <tbaldridge@gmail.com> Date: Sun, 29 Dec 2019 21:35:54 -0700 Subject: [PATCH] Implemented Job queue, status page, and HTML templating --- .../DTOs/JobQueue/AJobPayload.cs | 17 +++++ Wabbajack.CacheServer/DTOs/JobQueue/Job.cs | 65 +++++++++++++++++++ Wabbajack.CacheServer/DTOs/Metric.cs | 6 +- Wabbajack.CacheServer/DTOs/ModListStatus.cs | 10 ++- Wabbajack.CacheServer/Extensions.cs | 12 ++++ Wabbajack.CacheServer/JobQueueEndpoints.cs | 51 +++++++++++++++ .../ListValidationService.cs | 59 +++++++++-------- Wabbajack.CacheServer/Metrics.cs | 6 +- Wabbajack.CacheServer/Program.cs | 2 +- Wabbajack.CacheServer/Server.cs | 13 ++-- .../ServerConfig/BuildServerConfig.cs | 3 + .../Wabbajack.CacheServer.csproj | 10 ++- Wabbajack.CacheServer/config.yaml | 5 +- Wabbajack.Lib/Data.cs | 8 +-- 14 files changed, 205 insertions(+), 62 deletions(-) create mode 100644 Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs create mode 100644 Wabbajack.CacheServer/DTOs/JobQueue/Job.cs create mode 100644 Wabbajack.CacheServer/Extensions.cs create mode 100644 Wabbajack.CacheServer/JobQueueEndpoints.cs diff --git a/Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs b/Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs new file mode 100644 index 00000000..b5fcc319 --- /dev/null +++ b/Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson.Serialization.Attributes; + +namespace Wabbajack.CacheServer.DTOs.JobQueue +{ + public abstract class AJobPayload + { + public static List<Type> KnowSubtypes = new List<Type>(); + + [BsonIgnore] + public abstract string Description { get; } + } +} diff --git a/Wabbajack.CacheServer/DTOs/JobQueue/Job.cs b/Wabbajack.CacheServer/DTOs/JobQueue/Job.cs new file mode 100644 index 00000000..d14998e0 --- /dev/null +++ b/Wabbajack.CacheServer/DTOs/JobQueue/Job.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace Wabbajack.CacheServer.DTOs.JobQueue +{ + public class Job + { + public enum JobPriority : int + { + Low, + Normal, + High, + } + + [BsonId] + public Guid Id { get; set; } + public DateTime Started { get; set; } + public DateTime Ended { get; set; } + public DateTime Created { get; set; } = DateTime.Now; + public JobPriority Priority { get; set; } = JobPriority.Normal; + public bool RequiresNexus { get; set; } = true; + public AJobPayload Payload { get; set; } + + public static async Task<Guid> Enqueue(Job job) + { + await Server.Config.JobQueue.Connect().InsertOneAsync(job); + return job.Id; + } + + public static async Task<Job> GetNext() + { + var filter = new BsonDocument + { + {"query", new BsonDocument {{"Started", null}}}, + {"sort", new BsonDocument{{"Priority", -1}, {"Created", 1}}}, + + }; + var update = new BsonDocument + { + {"update", new BsonDocument {{"$set", new BsonDocument {{"Started", DateTime.Now}}}}} + }; + var job = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update); + return job; + } + + public static async Task<Job> Finish(Job job) + { + var filter = new BsonDocument + { + {"query", new BsonDocument {{"Id", job.Id}}}, + }; + var update = new BsonDocument + { + {"update", new BsonDocument {{"$set", new BsonDocument {{"Ended", DateTime.Now}}}}} + }; + var result = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update); + return result; + } + } +} diff --git a/Wabbajack.CacheServer/DTOs/Metric.cs b/Wabbajack.CacheServer/DTOs/Metric.cs index e3c671d3..cc87f93c 100644 --- a/Wabbajack.CacheServer/DTOs/Metric.cs +++ b/Wabbajack.CacheServer/DTOs/Metric.cs @@ -1,9 +1,5 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using CouchDB.Driver.Types; + namespace Wabbajack.CacheServer.DTOs { diff --git a/Wabbajack.CacheServer/DTOs/ModListStatus.cs b/Wabbajack.CacheServer/DTOs/ModListStatus.cs index 5c64599d..f4058cad 100644 --- a/Wabbajack.CacheServer/DTOs/ModListStatus.cs +++ b/Wabbajack.CacheServer/DTOs/ModListStatus.cs @@ -2,11 +2,9 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using System.Windows.Media.Animation; -using CouchDB.Driver.Extensions; -using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; using MongoDB.Driver; +using MongoDB.Driver.Linq; using Wabbajack.Lib; using Wabbajack.Lib.ModListRegistry; @@ -49,7 +47,7 @@ namespace Wabbajack.CacheServer.DTOs return result.First(); } - public static IQueryable<ModListStatus> All + public static IMongoQueryable<ModListStatus> All { get { @@ -60,8 +58,8 @@ namespace Wabbajack.CacheServer.DTOs public class DetailedStatus { - public string Name; - public DateTime Checked = DateTime.Now; + public string Name { get; set; } + public DateTime Checked { get; set; } = DateTime.Now; public List<DetailedStatusItem> Archives { get; set; } public DownloadMetadata DownloadMetaData { get; set; } public bool HasFailures { get; set; } diff --git a/Wabbajack.CacheServer/Extensions.cs b/Wabbajack.CacheServer/Extensions.cs new file mode 100644 index 00000000..de6bf472 --- /dev/null +++ b/Wabbajack.CacheServer/Extensions.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Wabbajack.CacheServer +{ + public static class Extensions + { + } +} diff --git a/Wabbajack.CacheServer/JobQueueEndpoints.cs b/Wabbajack.CacheServer/JobQueueEndpoints.cs new file mode 100644 index 00000000..679b0e3e --- /dev/null +++ b/Wabbajack.CacheServer/JobQueueEndpoints.cs @@ -0,0 +1,51 @@ +using System; +using System.Linq; +using System.Security.Policy; +using System.Threading.Tasks; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Nancy; +using Nettle; +using Wabbajack.CacheServer.DTOs.JobQueue; + +namespace Wabbajack.CacheServer +{ + public class JobQueueEndpoints : NancyModule + { + public JobQueueEndpoints() : base ("/jobs") + { + Get("/", HandleListJobs); + } + + private readonly Func<object, string> HandleListJobsTemplate = NettleEngine.GetCompiler().Compile(@" + <html><head/><body> + + <h2>Jobs - {{$.jobs.Count}} Pending</h2> + <h3>{{$.time}}</h3> + <ol> + {{each $.jobs}} + <li>{{$.Description}}</li> + {{/each}} + </ol> + + <script> + setTimeout(function() { location.reload();}, 10000); + </script> + + </body></html>"); + + private async Task<Response> HandleListJobs(object arg) + { + var jobs = await Server.Config.JobQueue.Connect() + .AsQueryable<Job>() + .Where(j => j.Ended == null) + .OrderByDescending(j => j.Priority) + .ThenBy(j => j.Created) + .ToListAsync(); + + var response = (Response)HandleListJobsTemplate(new {jobs, time = DateTime.Now}); + response.ContentType = "text/html"; + return response; + } + } +} diff --git a/Wabbajack.CacheServer/ListValidationService.cs b/Wabbajack.CacheServer/ListValidationService.cs index fd21fea0..e3b3cdd6 100644 --- a/Wabbajack.CacheServer/ListValidationService.cs +++ b/Wabbajack.CacheServer/ListValidationService.cs @@ -1,21 +1,18 @@ using System; -using System.Collections; using System.Collections.Generic; -using System.Collections.Immutable; using System.Linq; using System.Net.Http; -using System.Text; -using System.Threading; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; -using CouchDB.Driver.Extensions; +using MongoDB.Driver; using Nancy; -using Nancy.Responses; using Wabbajack.CacheServer.DTOs; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.ModListRegistry; +using MongoDB.Driver.Linq; +using Nettle; namespace Wabbajack.CacheServer { @@ -26,6 +23,7 @@ namespace Wabbajack.CacheServer Get("/status", HandleGetLists); Get("/status/{Name}.json", HandleGetListJson); Get("/status/{Name}.html", HandleGetListHtml); + } private async Task<string> HandleGetLists(object arg) @@ -54,34 +52,35 @@ namespace Wabbajack.CacheServer return lst.ToJSON(); } + + private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@" + <html><body> + <h2>{{lst.Name}} - {{lst.Checked}}</h2> + <h3>Failed ({{failed.Count}}):</h3> + <ul> + {{each $.failed }} + <li>{{$.Archive.Name}}</li> + {{/each}} + </ul> + <h3>Passed ({{passed.Count}}):</h3> + <ul> + {{each $.passed }} + <li>{{$.Archive.Name}}</li> + {{/each}} + </ul> + </body></html> + "); + private async Task<Response> HandleGetListHtml(dynamic arg) { + var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus; - var sb = new StringBuilder(); - - sb.Append("<html><body>"); - sb.Append($"<h2>{lst.Name} - {lst.Checked}</h2>"); - - var failed_list = lst.Archives.Where(a => a.IsFailing).ToList(); - sb.Append($"<h3>Failed ({failed_list.Count}):</h3>"); - sb.Append("<ul>"); - foreach (var archive in failed_list) + var response = (Response)HandleGetListTemplate(new { - sb.Append($"<li>{archive.Archive.Name}</li>"); - } - sb.Append("</ul>"); - - var pased_list = lst.Archives.Where(a => !a.IsFailing).ToList(); - sb.Append($"<h3>Passed ({pased_list.Count}):</h3>"); - sb.Append("<ul>"); - foreach (var archive in pased_list.OrderBy(f => f.Archive.Name)) - { - sb.Append($"<li>{archive.Archive.Name}</li>"); - } - sb.Append("</ul>"); - - sb.Append("</body></html>"); - var response = (Response)sb.ToString(); + lst, + failed = lst.Archives.Where(a => a.IsFailing).ToList(), + passed = lst.Archives.Where(a => !a.IsFailing).ToList() + }); response.ContentType = "text/html"; return response; } diff --git a/Wabbajack.CacheServer/Metrics.cs b/Wabbajack.CacheServer/Metrics.cs index 72490852..12b4a799 100644 --- a/Wabbajack.CacheServer/Metrics.cs +++ b/Wabbajack.CacheServer/Metrics.cs @@ -1,13 +1,11 @@ using System; -using System.Collections.Generic; using System.Linq; -using System.Net; using System.Text; using System.Threading; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; -using CouchDB.Driver.Extensions; using MongoDB.Driver; +using MongoDB.Driver.Linq; using Nancy; using Wabbajack.CacheServer.DTOs; using Wabbajack.Common; @@ -75,7 +73,7 @@ namespace Wabbajack.CacheServer .Where(line => line.Length == 3) .Select(line => new {date = DateTime.Parse(line[0]), Action = line[1], Value = line[2]});*/ - var q = (IQueryable<Metric>)Server.Config.Metrics.Connect().AsQueryable(); + var q = Server.Config.Metrics.Connect().AsQueryable(); // Remove guids / Default, which come from testing diff --git a/Wabbajack.CacheServer/Program.cs b/Wabbajack.CacheServer/Program.cs index 5e0b4930..7b281e04 100644 --- a/Wabbajack.CacheServer/Program.cs +++ b/Wabbajack.CacheServer/Program.cs @@ -15,7 +15,7 @@ namespace Wabbajack.CacheServer Utils.LogMessages.Subscribe(Console.WriteLine); using (var server = new Server("http://localhost:8080")) { - ListValidationService.Start(); + //ListValidationService.Start(); server.Start(); Console.ReadLine(); } diff --git a/Wabbajack.CacheServer/Server.cs b/Wabbajack.CacheServer/Server.cs index 22456cfb..0ad9408b 100644 --- a/Wabbajack.CacheServer/Server.cs +++ b/Wabbajack.CacheServer/Server.cs @@ -1,16 +1,8 @@ using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Net; -using System.Net.Sockets; -using System.Text; -using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; using Nancy; using Nancy.Bootstrapper; using Nancy.Configuration; -using Nancy.Diagnostics; using Nancy.Hosting.Self; using Nancy.TinyIoc; using Wabbajack.CacheServer.ServerConfig; @@ -24,6 +16,11 @@ namespace Wabbajack.CacheServer private HostConfiguration _config; public static BuildServerConfig Config; + static Server() + { + } + + public Server(string address) { Address = address; diff --git a/Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs b/Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs index f26d7b0d..d0b9cb85 100644 --- a/Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs +++ b/Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; using Wabbajack.CacheServer.DTOs; +using Wabbajack.CacheServer.DTOs.JobQueue; namespace Wabbajack.CacheServer.ServerConfig { @@ -11,5 +12,7 @@ namespace Wabbajack.CacheServer.ServerConfig { public MongoConfig<Metric> Metrics { get; set; } public MongoConfig<ModListStatus> ListValidation { get; set; } + + public MongoConfig<Job> JobQueue { get; set; } } } diff --git a/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj b/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj index c9a879b1..62cce843 100644 --- a/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj +++ b/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj @@ -73,10 +73,14 @@ <Reference Include="WindowsBase" /> </ItemGroup> <ItemGroup> + <Compile Include="DTOs\JobQueue\AJobPayload.cs" /> + <Compile Include="DTOs\JobQueue\Job.cs" /> <Compile Include="DTOs\Metric.cs" /> <Compile Include="DTOs\ModListStatus.cs" /> <Compile Include="DTOs\MongoDoc.cs" /> <Compile Include="DTOs\SerializerSettings.cs" /> + <Compile Include="Extensions.cs" /> + <Compile Include="JobQueueEndpoints.cs" /> <Compile Include="ListValidationService.cs" /> <Compile Include="Metrics.cs" /> <Compile Include="NexusCacheModule.cs" /> @@ -105,15 +109,15 @@ </ProjectReference> </ItemGroup> <ItemGroup> - <PackageReference Include="CouchDB.NET"> - <Version>1.1.5</Version> - </PackageReference> <PackageReference Include="MongoDB.Driver"> <Version>2.10.0</Version> </PackageReference> <PackageReference Include="Nancy.Hosting.Self"> <Version>2.0.0</Version> </PackageReference> + <PackageReference Include="Nettle"> + <Version>1.3.0</Version> + </PackageReference> <PackageReference Include="Newtonsoft.Json"> <Version>12.0.3</Version> </PackageReference> diff --git a/Wabbajack.CacheServer/config.yaml b/Wabbajack.CacheServer/config.yaml index a04acc07..0b08e2bc 100644 --- a/Wabbajack.CacheServer/config.yaml +++ b/Wabbajack.CacheServer/config.yaml @@ -7,6 +7,9 @@ ListValidation: Host: internal.test.mongodb Database: wabbajack Collection: mod_lists - +JobQueue: + Host: internal.test.mongodb + Database: wabbajack + Collection: job_queue diff --git a/Wabbajack.Lib/Data.cs b/Wabbajack.Lib/Data.cs index e4a4d43d..df2ae470 100644 --- a/Wabbajack.Lib/Data.cs +++ b/Wabbajack.Lib/Data.cs @@ -245,19 +245,19 @@ namespace Wabbajack.Lib /// <summary> /// MurMur3 Hash of the archive /// </summary> - public string Hash; + public string Hash { get; set; } /// <summary> /// Meta INI for the downloaded archive /// </summary> - public string Meta; + public string Meta { get; set; } /// <summary> /// Human friendly name of this archive /// </summary> - public string Name; + public string Name { get; set; } - public long Size; + public long Size { get; set; } public AbstractDownloadState State { get; set; } }