From 82be6f304b5d9b3332097e48a4925dbef6b562b9 Mon Sep 17 00:00:00 2001
From: Timothy Baldridge <tbaldridge@gmail.com>
Date: Sun, 29 Dec 2019 21:35:54 -0700
Subject: [PATCH] Implemented Job queue, status page, and HTML templating

---
 .../DTOs/JobQueue/AJobPayload.cs              | 17 +++++
 Wabbajack.CacheServer/DTOs/JobQueue/Job.cs    | 65 +++++++++++++++++++
 Wabbajack.CacheServer/DTOs/Metric.cs          |  6 +-
 Wabbajack.CacheServer/DTOs/ModListStatus.cs   | 10 ++-
 Wabbajack.CacheServer/Extensions.cs           | 12 ++++
 Wabbajack.CacheServer/JobQueueEndpoints.cs    | 51 +++++++++++++++
 .../ListValidationService.cs                  | 59 +++++++++--------
 Wabbajack.CacheServer/Metrics.cs              |  6 +-
 Wabbajack.CacheServer/Program.cs              |  2 +-
 Wabbajack.CacheServer/Server.cs               | 13 ++--
 .../ServerConfig/BuildServerConfig.cs         |  3 +
 .../Wabbajack.CacheServer.csproj              | 10 ++-
 Wabbajack.CacheServer/config.yaml             |  5 +-
 Wabbajack.Lib/Data.cs                         |  8 +--
 14 files changed, 205 insertions(+), 62 deletions(-)
 create mode 100644 Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs
 create mode 100644 Wabbajack.CacheServer/DTOs/JobQueue/Job.cs
 create mode 100644 Wabbajack.CacheServer/Extensions.cs
 create mode 100644 Wabbajack.CacheServer/JobQueueEndpoints.cs

diff --git a/Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs b/Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs
new file mode 100644
index 00000000..b5fcc319
--- /dev/null
+++ b/Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs
@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Wabbajack.CacheServer.DTOs.JobQueue
+{
+    public abstract class AJobPayload
+    {
+        public static List<Type> KnowSubtypes = new List<Type>();
+
+        [BsonIgnore]
+        public abstract string Description { get; }
+    }
+}
diff --git a/Wabbajack.CacheServer/DTOs/JobQueue/Job.cs b/Wabbajack.CacheServer/DTOs/JobQueue/Job.cs
new file mode 100644
index 00000000..d14998e0
--- /dev/null
+++ b/Wabbajack.CacheServer/DTOs/JobQueue/Job.cs
@@ -0,0 +1,65 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Wabbajack.CacheServer.DTOs.JobQueue
+{ 
+    public class Job
+    {
+        public enum JobPriority : int
+        {
+            Low,
+            Normal,
+            High,
+        }
+
+        [BsonId]
+        public Guid Id { get; set; }
+        public DateTime Started { get; set; }
+        public DateTime Ended { get; set; }
+        public DateTime Created { get; set; } = DateTime.Now;
+        public JobPriority Priority { get; set; } = JobPriority.Normal;
+        public bool RequiresNexus { get; set; } = true;
+        public AJobPayload Payload { get; set; }
+
+        public static async Task<Guid> Enqueue(Job job)
+        {
+            await Server.Config.JobQueue.Connect().InsertOneAsync(job);
+            return job.Id;
+        }
+
+        public static async Task<Job> GetNext()
+        {
+            var filter = new BsonDocument
+            {
+                {"query", new BsonDocument {{"Started", null}}},
+                {"sort", new BsonDocument{{"Priority", -1}, {"Created", 1}}},
+
+            };
+            var update = new BsonDocument
+            {
+                {"update", new BsonDocument {{"$set", new BsonDocument {{"Started", DateTime.Now}}}}}
+            };
+            var job = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update);
+            return job;
+        }
+
+        public static async Task<Job> Finish(Job job)
+        {
+            var filter = new BsonDocument
+            {
+                {"query", new BsonDocument {{"Id", job.Id}}},
+            };
+            var update = new BsonDocument
+            {
+                {"update", new BsonDocument {{"$set", new BsonDocument {{"Ended", DateTime.Now}}}}}
+            };
+            var result = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update);
+            return result;
+        }
+    }
+}
diff --git a/Wabbajack.CacheServer/DTOs/Metric.cs b/Wabbajack.CacheServer/DTOs/Metric.cs
index e3c671d3..cc87f93c 100644
--- a/Wabbajack.CacheServer/DTOs/Metric.cs
+++ b/Wabbajack.CacheServer/DTOs/Metric.cs
@@ -1,9 +1,5 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using CouchDB.Driver.Types;
+
 
 namespace Wabbajack.CacheServer.DTOs
 {
diff --git a/Wabbajack.CacheServer/DTOs/ModListStatus.cs b/Wabbajack.CacheServer/DTOs/ModListStatus.cs
index 5c64599d..f4058cad 100644
--- a/Wabbajack.CacheServer/DTOs/ModListStatus.cs
+++ b/Wabbajack.CacheServer/DTOs/ModListStatus.cs
@@ -2,11 +2,9 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
-using System.Windows.Media.Animation;
-using CouchDB.Driver.Extensions;
-using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
 using MongoDB.Driver;
+using MongoDB.Driver.Linq;
 using Wabbajack.Lib;
 using Wabbajack.Lib.ModListRegistry;
 
@@ -49,7 +47,7 @@ namespace Wabbajack.CacheServer.DTOs
             return result.First();
         }
 
-        public static IQueryable<ModListStatus> All
+        public static IMongoQueryable<ModListStatus> All
         {
             get
             {
@@ -60,8 +58,8 @@ namespace Wabbajack.CacheServer.DTOs
 
     public class DetailedStatus
     {
-        public string Name;
-        public DateTime Checked = DateTime.Now;
+        public string Name { get; set; }
+        public DateTime Checked { get; set; } = DateTime.Now;
         public List<DetailedStatusItem> Archives { get; set; }
         public DownloadMetadata DownloadMetaData { get; set; }
         public bool HasFailures { get; set; }
diff --git a/Wabbajack.CacheServer/Extensions.cs b/Wabbajack.CacheServer/Extensions.cs
new file mode 100644
index 00000000..de6bf472
--- /dev/null
+++ b/Wabbajack.CacheServer/Extensions.cs
@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Wabbajack.CacheServer
+{
+    public static class Extensions
+    {
+    }
+}
diff --git a/Wabbajack.CacheServer/JobQueueEndpoints.cs b/Wabbajack.CacheServer/JobQueueEndpoints.cs
new file mode 100644
index 00000000..679b0e3e
--- /dev/null
+++ b/Wabbajack.CacheServer/JobQueueEndpoints.cs
@@ -0,0 +1,51 @@
+using System;
+using System.Linq;
+using System.Security.Policy;
+using System.Threading.Tasks;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Nancy;
+using Nettle;
+using Wabbajack.CacheServer.DTOs.JobQueue;
+
+namespace Wabbajack.CacheServer
+{
+    public class JobQueueEndpoints : NancyModule
+    {
+        public JobQueueEndpoints() : base ("/jobs")
+        {
+            Get("/", HandleListJobs);
+        }
+
+        private readonly Func<object, string> HandleListJobsTemplate = NettleEngine.GetCompiler().Compile(@"
+                <html><head/><body>
+
+                <h2>Jobs - {{$.jobs.Count}} Pending</h2>
+                <h3>{{$.time}}</h3>
+                <ol>
+                {{each $.jobs}}
+                    <li>{{$.Description}}</li>
+                {{/each}}
+                </ol>
+
+                <script>
+                setTimeout(function() { location.reload();}, 10000);
+                </script>
+
+                </body></html>");
+
+        private async Task<Response> HandleListJobs(object arg)
+        {
+            var jobs = await Server.Config.JobQueue.Connect()
+                .AsQueryable<Job>()
+                .Where(j => j.Ended == null)
+                .OrderByDescending(j => j.Priority)
+                .ThenBy(j => j.Created)
+                .ToListAsync();
+
+            var response = (Response)HandleListJobsTemplate(new {jobs, time = DateTime.Now});
+            response.ContentType = "text/html";
+            return response;
+        }
+    }
+}
diff --git a/Wabbajack.CacheServer/ListValidationService.cs b/Wabbajack.CacheServer/ListValidationService.cs
index fd21fea0..e3b3cdd6 100644
--- a/Wabbajack.CacheServer/ListValidationService.cs
+++ b/Wabbajack.CacheServer/ListValidationService.cs
@@ -1,21 +1,18 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
-using System.Collections.Immutable;
 using System.Linq;
 using System.Net.Http;
-using System.Text;
-using System.Threading;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
-using CouchDB.Driver.Extensions;
+using MongoDB.Driver;
 using Nancy;
-using Nancy.Responses;
 using Wabbajack.CacheServer.DTOs;
 using Wabbajack.Common;
 using Wabbajack.Lib;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.ModListRegistry;
+using MongoDB.Driver.Linq;
+using Nettle;
 
 namespace Wabbajack.CacheServer
 {
@@ -26,6 +23,7 @@ namespace Wabbajack.CacheServer
             Get("/status", HandleGetLists);
             Get("/status/{Name}.json", HandleGetListJson);
             Get("/status/{Name}.html", HandleGetListHtml);
+
         }
 
         private async Task<string> HandleGetLists(object arg)
@@ -54,34 +52,35 @@ namespace Wabbajack.CacheServer
             return lst.ToJSON();
         }
 
+        
+        private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
+            <html><body>
+                <h2>{{lst.Name}} - {{lst.Checked}}</h2>
+                <h3>Failed ({{failed.Count}}):</h3>
+                <ul>
+                {{each $.failed }}
+                <li>{{$.Archive.Name}}</li>
+                {{/each}}
+                </ul>
+                <h3>Passed ({{passed.Count}}):</h3>
+                <ul>
+                {{each $.passed }}
+                <li>{{$.Archive.Name}}</li>
+                {{/each}}
+                </ul>
+            </body></html>
+        ");
+
         private async Task<Response> HandleGetListHtml(dynamic arg)
         {
+            
             var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
-            var sb = new StringBuilder();
-
-            sb.Append("<html><body>");
-            sb.Append($"<h2>{lst.Name} - {lst.Checked}</h2>");
-
-            var failed_list = lst.Archives.Where(a => a.IsFailing).ToList();
-            sb.Append($"<h3>Failed ({failed_list.Count}):</h3>");
-            sb.Append("<ul>");
-            foreach (var archive in failed_list)
+            var response = (Response)HandleGetListTemplate(new
             {
-                sb.Append($"<li>{archive.Archive.Name}</li>");
-            }
-            sb.Append("</ul>");
-
-            var pased_list = lst.Archives.Where(a => !a.IsFailing).ToList();
-            sb.Append($"<h3>Passed ({pased_list.Count}):</h3>");
-            sb.Append("<ul>");
-            foreach (var archive in pased_list.OrderBy(f => f.Archive.Name))
-            {
-                sb.Append($"<li>{archive.Archive.Name}</li>");
-            }
-            sb.Append("</ul>");
-
-            sb.Append("</body></html>");
-            var response = (Response)sb.ToString();
+                lst,
+                failed = lst.Archives.Where(a => a.IsFailing).ToList(),
+                passed = lst.Archives.Where(a => !a.IsFailing).ToList()
+            });
             response.ContentType = "text/html";
             return response;
         }
diff --git a/Wabbajack.CacheServer/Metrics.cs b/Wabbajack.CacheServer/Metrics.cs
index 72490852..12b4a799 100644
--- a/Wabbajack.CacheServer/Metrics.cs
+++ b/Wabbajack.CacheServer/Metrics.cs
@@ -1,13 +1,11 @@
 using System;
-using System.Collections.Generic;
 using System.Linq;
-using System.Net;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
-using CouchDB.Driver.Extensions;
 using MongoDB.Driver;
+using MongoDB.Driver.Linq;
 using Nancy;
 using Wabbajack.CacheServer.DTOs;
 using Wabbajack.Common;
@@ -75,7 +73,7 @@ namespace Wabbajack.CacheServer
                 .Where(line => line.Length == 3)
                 .Select(line => new {date = DateTime.Parse(line[0]), Action = line[1], Value = line[2]});*/
 
-            var q = (IQueryable<Metric>)Server.Config.Metrics.Connect().AsQueryable();
+            var q = Server.Config.Metrics.Connect().AsQueryable();
 
             // Remove guids / Default, which come from testing
 
diff --git a/Wabbajack.CacheServer/Program.cs b/Wabbajack.CacheServer/Program.cs
index 5e0b4930..7b281e04 100644
--- a/Wabbajack.CacheServer/Program.cs
+++ b/Wabbajack.CacheServer/Program.cs
@@ -15,7 +15,7 @@ namespace Wabbajack.CacheServer
             Utils.LogMessages.Subscribe(Console.WriteLine);
             using (var server = new Server("http://localhost:8080"))
             {
-                ListValidationService.Start();
+                //ListValidationService.Start();
                 server.Start();
                 Console.ReadLine();
             }
diff --git a/Wabbajack.CacheServer/Server.cs b/Wabbajack.CacheServer/Server.cs
index 22456cfb..0ad9408b 100644
--- a/Wabbajack.CacheServer/Server.cs
+++ b/Wabbajack.CacheServer/Server.cs
@@ -1,16 +1,8 @@
 using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Net;
-using System.Net.Sockets;
-using System.Text;
-using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Nancy;
 using Nancy.Bootstrapper;
 using Nancy.Configuration;
-using Nancy.Diagnostics;
 using Nancy.Hosting.Self;
 using Nancy.TinyIoc;
 using Wabbajack.CacheServer.ServerConfig;
@@ -24,6 +16,11 @@ namespace Wabbajack.CacheServer
         private HostConfiguration _config;
         public static BuildServerConfig Config;
 
+        static Server()
+        {
+        }
+
+
         public Server(string address)
         {
             Address = address;
diff --git a/Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs b/Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs
index f26d7b0d..d0b9cb85 100644
--- a/Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs
+++ b/Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs
@@ -4,6 +4,7 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using Wabbajack.CacheServer.DTOs;
+using Wabbajack.CacheServer.DTOs.JobQueue;
 
 namespace Wabbajack.CacheServer.ServerConfig
 {
@@ -11,5 +12,7 @@ namespace Wabbajack.CacheServer.ServerConfig
     {
         public MongoConfig<Metric> Metrics { get; set; }
         public MongoConfig<ModListStatus> ListValidation { get; set; }
+
+        public MongoConfig<Job> JobQueue { get; set; }
     }
 }
diff --git a/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj b/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj
index c9a879b1..62cce843 100644
--- a/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj
+++ b/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj
@@ -73,10 +73,14 @@
     <Reference Include="WindowsBase" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="DTOs\JobQueue\AJobPayload.cs" />
+    <Compile Include="DTOs\JobQueue\Job.cs" />
     <Compile Include="DTOs\Metric.cs" />
     <Compile Include="DTOs\ModListStatus.cs" />
     <Compile Include="DTOs\MongoDoc.cs" />
     <Compile Include="DTOs\SerializerSettings.cs" />
+    <Compile Include="Extensions.cs" />
+    <Compile Include="JobQueueEndpoints.cs" />
     <Compile Include="ListValidationService.cs" />
     <Compile Include="Metrics.cs" />
     <Compile Include="NexusCacheModule.cs" />
@@ -105,15 +109,15 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="CouchDB.NET">
-      <Version>1.1.5</Version>
-    </PackageReference>
     <PackageReference Include="MongoDB.Driver">
       <Version>2.10.0</Version>
     </PackageReference>
     <PackageReference Include="Nancy.Hosting.Self">
       <Version>2.0.0</Version>
     </PackageReference>
+    <PackageReference Include="Nettle">
+      <Version>1.3.0</Version>
+    </PackageReference>
     <PackageReference Include="Newtonsoft.Json">
       <Version>12.0.3</Version>
     </PackageReference>
diff --git a/Wabbajack.CacheServer/config.yaml b/Wabbajack.CacheServer/config.yaml
index a04acc07..0b08e2bc 100644
--- a/Wabbajack.CacheServer/config.yaml
+++ b/Wabbajack.CacheServer/config.yaml
@@ -7,6 +7,9 @@ ListValidation:
     Host: internal.test.mongodb
     Database: wabbajack
     Collection: mod_lists
-    
+JobQueue:
+    Host: internal.test.mongodb
+    Database: wabbajack
+    Collection: job_queue    
 
 
diff --git a/Wabbajack.Lib/Data.cs b/Wabbajack.Lib/Data.cs
index e4a4d43d..df2ae470 100644
--- a/Wabbajack.Lib/Data.cs
+++ b/Wabbajack.Lib/Data.cs
@@ -245,19 +245,19 @@ namespace Wabbajack.Lib
         /// <summary>
         ///     MurMur3 Hash of the archive
         /// </summary>
-        public string Hash;
+        public string Hash { get; set; }
 
         /// <summary>
         /// Meta INI for the downloaded archive
         /// </summary>
-        public string Meta;
+        public string Meta { get; set; }
 
         /// <summary>
         ///     Human friendly name of this archive
         /// </summary>
-        public string Name;
+        public string Name { get; set; }
 
-        public long Size;
+        public long Size { get; set; }
         public AbstractDownloadState State { get; set; }
     }