From 153dc55d140d5fa8b473a9c0ee7300eac5a7c2a3 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 9 Jan 2020 05:14:48 -0700 Subject: [PATCH] Update modlist job --- Wabbajack.BuildServer/Controllers/GraphQL.cs | 2 + Wabbajack.BuildServer/GraphQL/Mutation.cs | 22 ++++ Wabbajack.BuildServer/JobManager.cs | 35 +++--- .../Models/JobQueue/AJobPayload.cs | 3 +- .../Models/Jobs/UpdateModLists.cs | 111 ++++++++++++++++++ 5 files changed, 158 insertions(+), 15 deletions(-) create mode 100644 Wabbajack.BuildServer/GraphQL/Mutation.cs create mode 100644 Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs diff --git a/Wabbajack.BuildServer/Controllers/GraphQL.cs b/Wabbajack.BuildServer/Controllers/GraphQL.cs index 42491d96..479692d3 100644 --- a/Wabbajack.BuildServer/Controllers/GraphQL.cs +++ b/Wabbajack.BuildServer/Controllers/GraphQL.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Reflection.Metadata; using System.Threading.Tasks; using GraphQL; using GraphQL.Language.AST; @@ -26,6 +27,7 @@ namespace Wabbajack.BuildServer.Controllers var schema = new Schema { Query = new Query(Db), + Mutation = new Mutation(Db) }; var result = await new DocumentExecuter().ExecuteAsync(_ => diff --git a/Wabbajack.BuildServer/GraphQL/Mutation.cs b/Wabbajack.BuildServer/GraphQL/Mutation.cs new file mode 100644 index 00000000..24da19a7 --- /dev/null +++ b/Wabbajack.BuildServer/GraphQL/Mutation.cs @@ -0,0 +1,22 @@ +using GraphQL.Types; +using Wabbajack.BuildServer.Models; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.BuildServer.Models.Jobs; + +namespace Wabbajack.BuildServer.GraphQL +{ + public class Mutation : ObjectGraphType + { + public Mutation(DBContext db) + { + FieldAsync("pollNexusForUpdates", + resolve: async context => + { + var job = new Job {Payload = new GetNexusUpdatesJob()}; + await db.Jobs.InsertOneAsync(job); + return job.Id; + }); + } + + } +} diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index bd2644d0..6eedabb4 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using MongoDB.Driver; @@ -70,7 +71,8 @@ namespace Wabbajack.BuildServer while (true) { await KillOrphanedJobs(); - await PollNexusMods(); + await ScheduledJob(TimeSpan.FromHours(2)); + await ScheduledJob(TimeSpan.FromMinutes(30)); await Task.Delay(10000); } } @@ -93,31 +95,36 @@ namespace Wabbajack.BuildServer } catch (Exception ex) { - Logger.Log(LogLevel.Error, ex, "Error in JobScheduler when scheduling GetNexusUpdatesJob"); + Logger.Log(LogLevel.Error, ex, "Error in JobScheduler when scheduling KillOrphanedJobs"); } } - - private async Task PollNexusMods() + + private async Task ScheduledJob(TimeSpan span) where T : AJobPayload, new() { try { - var updaters = await Db.Jobs.AsQueryable() - .Where(j => j.Payload is GetNexusUpdatesJob) - .Where(j => j.Started == null) - .OrderBy(j => j.Created) + var jobs = await Db.Jobs.AsQueryable() + .Where(j => j.Payload is T) + .OrderByDescending(j => j.Created) + .Take(10) .ToListAsync(); - if (updaters.Count == 0) + + foreach (var job in jobs) { - await Db.Jobs.InsertOneAsync(new Job - { - Payload = new GetNexusUpdatesJob() - }); + if (job.Started == null || job.Ended == null) return; + if (DateTime.Now - job.Ended < span) return; } + await Db.Jobs.InsertOneAsync(new Job + { + Payload = new T() + }); } catch (Exception ex) { - Logger.Log(LogLevel.Error, ex, "Error in JobScheduler when scheduling GetNexusUpdatesJob"); + + Logger.Log(LogLevel.Error, ex, $"Error in JobScheduler when scheduling {typeof(T).Name}"); } } + } } diff --git a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs index d108fba4..a71369d1 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs @@ -13,7 +13,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue public static List KnownSubTypes = new List { typeof(IndexJob), - typeof(GetNexusUpdatesJob) + typeof(GetNexusUpdatesJob), + typeof(UpdateModLists) }; public static Dictionary TypeToName { get; set; } public static Dictionary NameToType { get; set; } diff --git a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs new file mode 100644 index 00000000..668ad4d4 --- /dev/null +++ b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs @@ -0,0 +1,111 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.ModListRegistry; + +namespace Wabbajack.BuildServer.Models.Jobs +{ + public class UpdateModLists : AJobPayload + { + public override string Description => "Validate curated modlists"; + public override async Task Execute(DBContext db, AppSettings settings) + { + Utils.Log("Starting Modlist Validation"); + var modlists = await ModlistMetadata.LoadFromGithub(); + + using (var queue = new WorkQueue()) + { + foreach (var list in modlists) + { + try + { + await ValidateList(db, list, queue); + } + catch (Exception ex) + { + Utils.Log(ex.ToString()); + } + } + } + + return JobResult.Success(); + } + + private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue) + { + var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL); + + var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ExtensionManager.Extension); + + if (list.NeedsDownload(modlist_path)) + { + if (File.Exists(modlist_path)) + File.Delete(modlist_path); + + var state = DownloadDispatcher.ResolveArchive(list.Links.Download); + Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}"); + await state.Download(modlist_path); + } + else + { + Utils.Log($"No changes detected from downloaded modlist"); + } + + + Utils.Log($"Loading {modlist_path}"); + + var installer = AInstaller.LoadFromFile(modlist_path); + + Utils.Log($"{installer.Archives.Count} archives to validate"); + + DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State)); + + var validated = (await installer.Archives + .PMap(queue, async archive => + { + Utils.Log($"Validating: {archive.Name}"); + bool is_failed; + try + { + is_failed = !(await archive.State.Verify()); + } + catch (Exception) + { + is_failed = false; + } + + return new DetailedStatusItem {IsFailing = is_failed, Archive = archive}; + })) + .ToList(); + + + var status = new DetailedStatus + { + Name = list.Title, + Archives = validated.OrderBy(v => v.Archive.Name).ToList(), + DownloadMetaData = list.DownloadMetadata, + HasFailures = validated.Any(v => v.IsFailing) + }; + + var dto = new ModListStatus + { + Id = list.Links.MachineURL, + Summary = new ModlistSummary + { + Name = status.Name, + Checked = status.Checked, + Failed = status.Archives.Count(a => a.IsFailing), + Passed = status.Archives.Count(a => !a.IsFailing), + }, + DetailedStatus = status, + Metadata = list + }; + await ModListStatus.Update(db, dto); + } + } +}