Implemented Job queue, status page, and HTML templating

Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Wabbajack.CacheServer.DTOs.JobQueue
+{
+    public abstract class AJobPayload
+    {
+        public static List<Type> KnowSubtypes = new List<Type>();
+
+        [BsonIgnore]
+        public abstract string Description { get; }
+    }
+}

Wabbajack.CacheServer/DTOs/JobQueue/Job.cs

@@ -0,0 +1,65 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Wabbajack.CacheServer.DTOs.JobQueue
+{ 
+    public class Job
+    {
+        public enum JobPriority : int
+        {
+            Low,
+            Normal,
+            High,
+        }
+
+        [BsonId]
+        public Guid Id { get; set; }
+        public DateTime Started { get; set; }
+        public DateTime Ended { get; set; }
+        public DateTime Created { get; set; } = DateTime.Now;
+        public JobPriority Priority { get; set; } = JobPriority.Normal;
+        public bool RequiresNexus { get; set; } = true;
+        public AJobPayload Payload { get; set; }
+
+        public static async Task<Guid> Enqueue(Job job)
+        {
+            await Server.Config.JobQueue.Connect().InsertOneAsync(job);
+            return job.Id;
+        }
+
+        public static async Task<Job> GetNext()
+        {
+            var filter = new BsonDocument
+            {
+                {"query", new BsonDocument {{"Started", null}}},
+                {"sort", new BsonDocument{{"Priority", -1}, {"Created", 1}}},
+
+            };
+            var update = new BsonDocument
+            {
+                {"update", new BsonDocument {{"$set", new BsonDocument {{"Started", DateTime.Now}}}}}
+            };
+            var job = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update);
+            return job;
+        }
+
+        public static async Task<Job> Finish(Job job)
+        {
+            var filter = new BsonDocument
+            {
+                {"query", new BsonDocument {{"Id", job.Id}}},
+            };
+            var update = new BsonDocument
+            {
+                {"update", new BsonDocument {{"$set", new BsonDocument {{"Ended", DateTime.Now}}}}}
+            };
+            var result = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update);
+            return result;
+        }
+    }
+}

Wabbajack.CacheServer/DTOs/Metric.cs

@@ -1,9 +1,5 @@
 using System;
-using System.Collections.Generic;
+
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using CouchDB.Driver.Types;
 
 namespace Wabbajack.CacheServer.DTOs
 {

Wabbajack.CacheServer/DTOs/ModListStatus.cs

@@ -2,11 +2,9 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
-using System.Windows.Media.Animation;
-using CouchDB.Driver.Extensions;
-using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
 using MongoDB.Driver;
+using MongoDB.Driver.Linq;
 using Wabbajack.Lib;
 using Wabbajack.Lib.ModListRegistry;
 
@@ -49,7 +47,7 @@ namespace Wabbajack.CacheServer.DTOs
             return result.First();
         }
 
-        public static IQueryable<ModListStatus> All
+        public static IMongoQueryable<ModListStatus> All
         {
             get
             {
@@ -60,8 +58,8 @@ namespace Wabbajack.CacheServer.DTOs
 
     public class DetailedStatus
     {
-        public string Name;
+        public string Name { get; set; }
-        public DateTime Checked = DateTime.Now;
+        public DateTime Checked { get; set; } = DateTime.Now;
         public List<DetailedStatusItem> Archives { get; set; }
         public DownloadMetadata DownloadMetaData { get; set; }
         public bool HasFailures { get; set; }

Wabbajack.CacheServer/Extensions.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Wabbajack.CacheServer
+{
+    public static class Extensions
+    {
+    }
+}

Wabbajack.CacheServer/JobQueueEndpoints.cs

@@ -0,0 +1,51 @@
+using System;
+using System.Linq;
+using System.Security.Policy;
+using System.Threading.Tasks;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Nancy;
+using Nettle;
+using Wabbajack.CacheServer.DTOs.JobQueue;
+
+namespace Wabbajack.CacheServer
+{
+    public class JobQueueEndpoints : NancyModule
+    {
+        public JobQueueEndpoints() : base ("/jobs")
+        {
+            Get("/", HandleListJobs);
+        }
+
+        private readonly Func<object, string> HandleListJobsTemplate = NettleEngine.GetCompiler().Compile(@"
+                <html><head/><body>
+
+                <h2>Jobs - {{$.jobs.Count}} Pending</h2>
+                <h3>{{$.time}}</h3>
+                <ol>
+                {{each $.jobs}}
+                    <li>{{$.Description}}</li>
+                {{/each}}
+                </ol>
+
+                <script>
+                setTimeout(function() { location.reload();}, 10000);
+                </script>
+
+                </body></html>");
+
+        private async Task<Response> HandleListJobs(object arg)
+        {
+            var jobs = await Server.Config.JobQueue.Connect()
+                .AsQueryable<Job>()
+                .Where(j => j.Ended == null)
+                .OrderByDescending(j => j.Priority)
+                .ThenBy(j => j.Created)
+                .ToListAsync();
+
+            var response = (Response)HandleListJobsTemplate(new {jobs, time = DateTime.Now});
+            response.ContentType = "text/html";
+            return response;
+        }
+    }
+}

Wabbajack.CacheServer/ListValidationService.cs

@@ -1,21 +1,18 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
-using System.Collections.Immutable;
 using System.Linq;
 using System.Net.Http;
-using System.Text;
-using System.Threading;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
-using CouchDB.Driver.Extensions;
+using MongoDB.Driver;
 using Nancy;
-using Nancy.Responses;
 using Wabbajack.CacheServer.DTOs;
 using Wabbajack.Common;
 using Wabbajack.Lib;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.ModListRegistry;
+using MongoDB.Driver.Linq;
+using Nettle;
 
 namespace Wabbajack.CacheServer
 {
@@ -26,6 +23,7 @@ namespace Wabbajack.CacheServer
             Get("/status", HandleGetLists);
             Get("/status/{Name}.json", HandleGetListJson);
             Get("/status/{Name}.html", HandleGetListHtml);
+
         }
 
         private async Task<string> HandleGetLists(object arg)
@@ -54,34 +52,35 @@ namespace Wabbajack.CacheServer
             return lst.ToJSON();
         }
 
+        
+        private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
+            <html><body>
+                <h2>{{lst.Name}} - {{lst.Checked}}</h2>
+                <h3>Failed ({{failed.Count}}):</h3>
+                <ul>
+                {{each $.failed }}
+                <li>{{$.Archive.Name}}</li>
+                {{/each}}
+                </ul>
+                <h3>Passed ({{passed.Count}}):</h3>
+                <ul>
+                {{each $.passed }}
+                <li>{{$.Archive.Name}}</li>
+                {{/each}}
+                </ul>
+            </body></html>
+        ");
+
         private async Task<Response> HandleGetListHtml(dynamic arg)
         {
+            
             var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
-            var sb = new StringBuilder();
+            var response = (Response)HandleGetListTemplate(new
-
-            sb.Append("<html><body>");
-            sb.Append($"<h2>{lst.Name} - {lst.Checked}</h2>");
-
-            var failed_list = lst.Archives.Where(a => a.IsFailing).ToList();
-            sb.Append($"<h3>Failed ({failed_list.Count}):</h3>");
-            sb.Append("<ul>");
-            foreach (var archive in failed_list)
             {
-                sb.Append($"<li>{archive.Archive.Name}</li>");
+                lst,
-            }
+                failed = lst.Archives.Where(a => a.IsFailing).ToList(),
-            sb.Append("</ul>");
+                passed = lst.Archives.Where(a => !a.IsFailing).ToList()
-
+            });
-            var pased_list = lst.Archives.Where(a => !a.IsFailing).ToList();
-            sb.Append($"<h3>Passed ({pased_list.Count}):</h3>");
-            sb.Append("<ul>");
-            foreach (var archive in pased_list.OrderBy(f => f.Archive.Name))
-            {
-                sb.Append($"<li>{archive.Archive.Name}</li>");
-            }
-            sb.Append("</ul>");
-
-            sb.Append("</body></html>");
-            var response = (Response)sb.ToString();
             response.ContentType = "text/html";
             return response;
         }

Wabbajack.CacheServer/Metrics.cs

@@ -1,13 +1,11 @@
 using System;
-using System.Collections.Generic;
 using System.Linq;
-using System.Net;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
-using CouchDB.Driver.Extensions;
 using MongoDB.Driver;
+using MongoDB.Driver.Linq;
 using Nancy;
 using Wabbajack.CacheServer.DTOs;
 using Wabbajack.Common;
@@ -75,7 +73,7 @@ namespace Wabbajack.CacheServer
                 .Where(line => line.Length == 3)
                 .Select(line => new {date = DateTime.Parse(line[0]), Action = line[1], Value = line[2]});*/
 
-            var q = (IQueryable<Metric>)Server.Config.Metrics.Connect().AsQueryable();
+            var q = Server.Config.Metrics.Connect().AsQueryable();
 
             // Remove guids / Default, which come from testing
 

Wabbajack.CacheServer/Program.cs

@@ -15,7 +15,7 @@ namespace Wabbajack.CacheServer
             Utils.LogMessages.Subscribe(Console.WriteLine);
             using (var server = new Server("http://localhost:8080"))
             {
-                ListValidationService.Start();
+                //ListValidationService.Start();
                 server.Start();
                 Console.ReadLine();
             }

Wabbajack.CacheServer/Server.cs

@@ -1,16 +1,8 @@
 using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Net;
-using System.Net.Sockets;
-using System.Text;
-using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Nancy;
 using Nancy.Bootstrapper;
 using Nancy.Configuration;
-using Nancy.Diagnostics;
 using Nancy.Hosting.Self;
 using Nancy.TinyIoc;
 using Wabbajack.CacheServer.ServerConfig;
@@ -24,6 +16,11 @@ namespace Wabbajack.CacheServer
         private HostConfiguration _config;
         public static BuildServerConfig Config;
 
+        static Server()
+        {
+        }
+
+
         public Server(string address)
         {
             Address = address;

Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using Wabbajack.CacheServer.DTOs;
+using Wabbajack.CacheServer.DTOs.JobQueue;
 
 namespace Wabbajack.CacheServer.ServerConfig
 {
@@ -11,5 +12,7 @@ namespace Wabbajack.CacheServer.ServerConfig
     {
         public MongoConfig<Metric> Metrics { get; set; }
         public MongoConfig<ModListStatus> ListValidation { get; set; }
+
+        public MongoConfig<Job> JobQueue { get; set; }
     }
 }

Wabbajack.CacheServer/Wabbajack.CacheServer.csproj

@@ -73,10 +73,14 @@
     <Reference Include="WindowsBase" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="DTOs\JobQueue\AJobPayload.cs" />
+    <Compile Include="DTOs\JobQueue\Job.cs" />
     <Compile Include="DTOs\Metric.cs" />
     <Compile Include="DTOs\ModListStatus.cs" />
     <Compile Include="DTOs\MongoDoc.cs" />
     <Compile Include="DTOs\SerializerSettings.cs" />
+    <Compile Include="Extensions.cs" />
+    <Compile Include="JobQueueEndpoints.cs" />
     <Compile Include="ListValidationService.cs" />
     <Compile Include="Metrics.cs" />
     <Compile Include="NexusCacheModule.cs" />
@@ -105,15 +109,15 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="CouchDB.NET">
-      <Version>1.1.5</Version>
-    </PackageReference>
     <PackageReference Include="MongoDB.Driver">
       <Version>2.10.0</Version>
     </PackageReference>
     <PackageReference Include="Nancy.Hosting.Self">
       <Version>2.0.0</Version>
     </PackageReference>
+    <PackageReference Include="Nettle">
+      <Version>1.3.0</Version>
+    </PackageReference>
     <PackageReference Include="Newtonsoft.Json">
       <Version>12.0.3</Version>
     </PackageReference>

Wabbajack.CacheServer/config.yaml

@@ -7,6 +7,9 @@ ListValidation:
     Host: internal.test.mongodb
     Database: wabbajack
     Collection: mod_lists
-    
+JobQueue:
+    Host: internal.test.mongodb
+    Database: wabbajack
+    Collection: job_queue    
 
 

Wabbajack.Lib/Data.cs

@@ -245,19 +245,19 @@ namespace Wabbajack.Lib
         /// <summary>
         ///     MurMur3 Hash of the archive
         /// </summary>
-        public string Hash;
+        public string Hash { get; set; }
 
         /// <summary>
         /// Meta INI for the downloaded archive
         /// </summary>
-        public string Meta;
+        public string Meta { get; set; }
 
         /// <summary>
         ///     Human friendly name of this archive
         /// </summary>
-        public string Name;
+        public string Name { get; set; }
 
-        public long Size;
+        public long Size { get; set; }
         public AbstractDownloadState State { get; set; }
     }