Update modlist job

Wabbajack.BuildServer/Controllers/GraphQL.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Reflection.Metadata;
 using System.Threading.Tasks;
 using GraphQL;
 using GraphQL.Language.AST;
@@ -26,6 +27,7 @@ namespace Wabbajack.BuildServer.Controllers
             var schema = new Schema
             {
                 Query = new Query(Db),
+                Mutation = new Mutation(Db)
             };
             
             var result = await new DocumentExecuter().ExecuteAsync(_ =>

Wabbajack.BuildServer/GraphQL/Mutation.cs

@@ -0,0 +1,22 @@
+using GraphQL.Types;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.BuildServer.Models.Jobs;
+
+namespace Wabbajack.BuildServer.GraphQL
+{
+    public class Mutation : ObjectGraphType
+    {
+        public Mutation(DBContext db)
+        {
+            FieldAsync<IdGraphType>("pollNexusForUpdates",
+                resolve: async context =>
+                {
+                    var job = new Job {Payload = new GetNexusUpdatesJob()};
+                    await db.Jobs.InsertOneAsync(job);
+                    return job.Id;
+                });
+        }
+        
+    }
+}

Wabbajack.BuildServer/JobManager.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using MongoDB.Driver;
@@ -70,7 +71,8 @@ namespace Wabbajack.BuildServer
             while (true)
             {
                 await KillOrphanedJobs();
-                await PollNexusMods();
+                await ScheduledJob<GetNexusUpdatesJob>(TimeSpan.FromHours(2));
+                await ScheduledJob<UpdateModLists>(TimeSpan.FromMinutes(30));
                 await Task.Delay(10000);
             }
         }
@@ -93,31 +95,36 @@ namespace Wabbajack.BuildServer
             }
             catch (Exception ex)
             {
-                Logger.Log(LogLevel.Error, ex, "Error in JobScheduler when scheduling GetNexusUpdatesJob");
+                Logger.Log(LogLevel.Error, ex, "Error in JobScheduler when scheduling KillOrphanedJobs");
             }
         }
-
+        
-        private async Task PollNexusMods()
+        private async Task ScheduledJob<T>(TimeSpan span) where T : AJobPayload, new()
         {
             try
             {
-                var updaters = await Db.Jobs.AsQueryable()
+                var jobs = await Db.Jobs.AsQueryable()
-                    .Where(j => j.Payload is GetNexusUpdatesJob)
+                    .Where(j => j.Payload is T)
-                    .Where(j => j.Started == null)
+                    .OrderByDescending(j => j.Created)
-                    .OrderBy(j => j.Created)
+                    .Take(10)
                     .ToListAsync();
-                if (updaters.Count == 0)
+
+                foreach (var job in jobs)
                 {
-                    await Db.Jobs.InsertOneAsync(new Job
+                    if (job.Started == null || job.Ended == null) return;
-                    {
+                    if (DateTime.Now - job.Ended < span) return;
-                        Payload = new GetNexusUpdatesJob()
-                    });
                 }
+                await Db.Jobs.InsertOneAsync(new Job
+                {
+                    Payload = new T()
+                });
             }
             catch (Exception ex)
             {
-                Logger.Log(LogLevel.Error, ex, "Error in JobScheduler when scheduling GetNexusUpdatesJob");
+                
+                Logger.Log(LogLevel.Error, ex, $"Error in JobScheduler when scheduling {typeof(T).Name}");
             }
         }
+
     }
 }

Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs

@@ -13,7 +13,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue
         public static List<Type> KnownSubTypes = new List<Type>
         {
             typeof(IndexJob),
-            typeof(GetNexusUpdatesJob)
+            typeof(GetNexusUpdatesJob),
+            typeof(UpdateModLists)
         };
         public static Dictionary<Type, string> TypeToName { get; set; }
         public static Dictionary<string, Type> NameToType { get; set; }

Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs

@@ -0,0 +1,111 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.ModListRegistry;
+
+namespace Wabbajack.BuildServer.Models.Jobs
+{
+    public class UpdateModLists : AJobPayload
+    {
+        public override string Description => "Validate curated modlists";
+        public override async Task<JobResult> Execute(DBContext db, AppSettings settings)
+        {
+            Utils.Log("Starting Modlist Validation");
+            var modlists = await ModlistMetadata.LoadFromGithub();
+
+            using (var queue = new WorkQueue())
+            {
+                foreach (var list in modlists)
+                {
+                    try
+                    {
+                        await ValidateList(db, list, queue);
+                    }
+                    catch (Exception ex)
+                    {
+                        Utils.Log(ex.ToString());
+                    }
+                }
+            }
+
+            return JobResult.Success();
+        }
+        
+         private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue)
+        {
+            var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
+            
+            var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ExtensionManager.Extension);
+
+            if (list.NeedsDownload(modlist_path))
+            {
+                if (File.Exists(modlist_path))
+                    File.Delete(modlist_path);
+
+                var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
+                Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
+                await state.Download(modlist_path);
+            }
+            else
+            {
+                Utils.Log($"No changes detected from downloaded modlist");
+            }
+
+
+            Utils.Log($"Loading {modlist_path}");
+
+            var installer = AInstaller.LoadFromFile(modlist_path);
+
+            Utils.Log($"{installer.Archives.Count} archives to validate");
+
+            DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
+
+            var validated = (await installer.Archives
+                    .PMap(queue, async archive =>
+                    {
+                        Utils.Log($"Validating: {archive.Name}");
+                        bool is_failed;
+                        try
+                        {
+                            is_failed = !(await archive.State.Verify());
+                        }
+                        catch (Exception)
+                        {
+                            is_failed = false;
+                        }
+
+                        return new DetailedStatusItem {IsFailing = is_failed, Archive = archive};
+                    }))
+                .ToList();
+
+
+            var status = new DetailedStatus
+            {
+                Name = list.Title,
+                Archives = validated.OrderBy(v => v.Archive.Name).ToList(),
+                DownloadMetaData = list.DownloadMetadata,
+                HasFailures = validated.Any(v => v.IsFailing)
+            };
+
+            var dto = new ModListStatus
+            {
+                Id = list.Links.MachineURL,
+                Summary = new ModlistSummary
+                {
+                    Name = status.Name,
+                    Checked = status.Checked,
+                    Failed = status.Archives.Count(a => a.IsFailing),
+                    Passed = status.Archives.Count(a => !a.IsFailing),
+                },
+                DetailedStatus = status,
+                Metadata = list
+            };
+            await ModListStatus.Update(db, dto);
+        }
+    }
+}