From 527f30f1c09c1cf16cbd50804fdbfc97bd2bf47a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 12 Jan 2020 21:04:46 -0700 Subject: [PATCH 1/6] Index new files on the nexus each day --- Wabbajack.BuildServer/JobManager.cs | 1 + Wabbajack.BuildServer/Models/DBContext.cs | 1 + .../Models/JobQueue/AJobPayload.cs | 3 +- Wabbajack.BuildServer/Models/JobQueue/Job.cs | 1 + .../Models/Jobs/EnqueueRecentFiles.cs | 84 +++++++++++++++++++ .../Models/Jobs/GetNexusUpdatesJob.cs | 36 ++++---- .../Models/Jobs/UpdateModLists.cs | 2 +- .../Models/NexusCacheData.cs | 8 ++ .../Models/NexusUpdateEntry.cs | 16 ++++ 9 files changed, 135 insertions(+), 17 deletions(-) create mode 100644 Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs create mode 100644 Wabbajack.BuildServer/Models/NexusUpdateEntry.cs diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index 4dbe21fa..9e858642 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -75,6 +75,7 @@ namespace Wabbajack.BuildServer await ScheduledJob(TimeSpan.FromMinutes(30), Job.JobPriority.High); await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.Low); await ScheduledJob(TimeSpan.FromHours(24), Job.JobPriority.High); + await ScheduledJob(TimeSpan.FromHours(6), Job.JobPriority.Low); await Task.Delay(10000); } } diff --git a/Wabbajack.BuildServer/Models/DBContext.cs b/Wabbajack.BuildServer/Models/DBContext.cs index 98c19356..128825c9 100644 --- a/Wabbajack.BuildServer/Models/DBContext.cs +++ b/Wabbajack.BuildServer/Models/DBContext.cs @@ -26,6 +26,7 @@ namespace Wabbajack.BuildServer.Models public IMongoCollection DownloadStates => Client.GetCollection(_settings.Collections["DownloadStates"]); public IMongoCollection Metrics => Client.GetCollection(_settings.Collections["Metrics"]); public IMongoCollection IndexedFiles => Client.GetCollection(_settings.Collections["IndexedFiles"]); + public IMongoCollection>> NexusUpdates => Client.GetCollection>>(_settings.Collections["NexusUpdates"]); public IMongoCollection> NexusModFiles => Client.GetCollection>( diff --git a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs index 792b4f71..76c3181c 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs @@ -16,7 +16,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue typeof(GetNexusUpdatesJob), typeof(UpdateModLists), typeof(EnqueueAllArchives), - typeof(EnqueueAllGameFiles) + typeof(EnqueueAllGameFiles), + typeof(EnqueueRecentFiles) }; public static Dictionary TypeToName { get; set; } public static Dictionary NameToType { get; set; } diff --git a/Wabbajack.BuildServer/Models/JobQueue/Job.cs b/Wabbajack.BuildServer/Models/JobQueue/Job.cs index 006da7aa..52f71a47 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/Job.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/Job.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; using MongoDB.Driver; +using Wabbajack.Lib.NexusApi; namespace Wabbajack.BuildServer.Models.JobQueue { diff --git a/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs b/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs new file mode 100644 index 00000000..eaabf2e9 --- /dev/null +++ b/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using MongoDB.Driver; +using MongoDB.Driver.Core.Authentication; +using MongoDB.Driver.Linq; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.NexusApi; + +namespace Wabbajack.BuildServer.Models.Jobs +{ + public class EnqueueRecentFiles : AJobPayload + { + public override string Description => "Enqueue the past days worth of mods for indexing"; + + private static HashSet GamesToScan = new HashSet + { + Game.Fallout3, Game.Fallout4, Game.Skyrim, Game.SkyrimSpecialEdition, Game.SkyrimVR, Game.FalloutNewVegas, Game.Oblivion + }; + public override async Task Execute(DBContext db, AppSettings settings) + { + using (var queue = new WorkQueue()) + { + var updates = await db.NexusUpdates.AsQueryable().ToListAsync(); + var mods = updates + .Where(list => GamesToScan.Contains(GameRegistry.GetByNexusName(list.Game).Game)) + .SelectMany(list => + list.Data.Where(mod => DateTime.UtcNow - mod.LatestFileUpdate.AsUnixTime() < TimeSpan.FromDays(1)) + .Select(mod => (list.Game, mod.ModId))); + var mod_files = (await mods.PMap(queue, async mod => + { + var client = await NexusApiClient.Get(); + try + { + var files = await client.GetModFiles(GameRegistry.GetByNexusName(mod.Game).Game, + (int)mod.ModId); + return (mod.Game, mod.ModId, files.files); + } + catch (Exception) + { + return default; + } + })).Where(t => t.Game != null).ToList(); + + var archives = + mod_files.SelectMany(mod => mod.files.Select(file => (mod.Game, mod.ModId, File:file)).Where(f => !string.IsNullOrEmpty(f.File.category_name) )) + .Select(tuple => + { + var state = new NexusDownloader.State + { + GameName = tuple.Game, ModID = tuple.ModId.ToString(), FileID = tuple.File.file_id.ToString() + }; + return new Archive {State = state, Name = tuple.File.file_name}; + }).ToList(); + + Utils.Log($"Found {archives.Count} archives from recent Nexus updates to index"); + var searching = archives.Select(a => a.State.PrimaryKeyString).Distinct().ToArray(); + + Utils.Log($"Looking for missing states"); + var knownArchives = (await db.DownloadStates.AsQueryable().Where(s => searching.Contains(s.Key)) + .Select(d => d.Key).ToListAsync()).ToDictionary(a => a); + + Utils.Log($"Found {knownArchives.Count} pre-existing archives"); + var missing = archives.Where(a => !knownArchives.ContainsKey(a.State.PrimaryKeyString)) + .DistinctBy(d => d.State.PrimaryKeyString) + .ToList(); + + Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs"); + + var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}, Priority = Job.JobPriority.Low}); + + Utils.Log($"Writing jobs to the DB"); + await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions {IsOrdered = false}); + Utils.Log($"Done adding archives for Nexus Updates"); + + return JobResult.Success(); + } + } + } +} diff --git a/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs b/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs index 54028fab..1a52a28f 100644 --- a/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs @@ -5,6 +5,7 @@ using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; using Wabbajack.Lib.NexusApi; using MongoDB.Driver; +using Newtonsoft.Json; namespace Wabbajack.BuildServer.Models.Jobs @@ -21,9 +22,17 @@ namespace Wabbajack.BuildServer.Models.Jobs .Where(game => game.NexusName != null) .Select(async game => { - return (game, - mods: await api.Get>( - $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m")); + var mods = await api.Get>( + $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m"); + + var entry = new NexusCacheData>(); + entry.Game = game.NexusName; + entry.Path = $"/v1/games/{game.NexusName}/mods/updated.json?period=1m"; + entry.Data = mods; + + await entry.Upsert(db.NexusUpdates); + + return (game, mods); }) .Select(async rTask => { @@ -40,14 +49,16 @@ namespace Wabbajack.BuildServer.Models.Jobs Utils.Log($"Found {purge.Count} updated mods in the last month"); using (var queue = new WorkQueue()) { - var collected = await purge.Select(d => + var collected = purge.Select(d => { - var a = d.mod.latest_file_update.AsUnixTime(); + var a = d.mod.LatestFileUpdate.AsUnixTime(); // Mod activity could hide files - var b = d.mod.latest_mod_activity.AsUnixTime(); + var b = d.mod.LastestModActivity.AsUnixTime(); - return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.mod_id.ToString()}; - }).PMap(queue, async t => + return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.ModId.ToString()}; + }); + + var purged = await collected.PMap(queue, async t => { var resultA = await db.NexusModInfos.DeleteManyAsync(f => f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); @@ -59,17 +70,12 @@ namespace Wabbajack.BuildServer.Models.Jobs return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount; }); - Utils.Log($"Purged {collected.Sum()} cache entries"); + Utils.Log($"Purged {purged.Sum()} cache entries"); } return JobResult.Success(); } - class UpdatedMod - { - public long mod_id; - public long latest_file_update; - public long latest_mod_activity; - } + } } diff --git a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs index 668ad4d4..7660f127 100644 --- a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs +++ b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs @@ -20,7 +20,7 @@ namespace Wabbajack.BuildServer.Models.Jobs using (var queue = new WorkQueue()) { - foreach (var list in modlists) + foreach (var list in modlists.Skip(1).Take(1)) { try { diff --git a/Wabbajack.BuildServer/Models/NexusCacheData.cs b/Wabbajack.BuildServer/Models/NexusCacheData.cs index 4bb0b19a..a28ca5f5 100644 --- a/Wabbajack.BuildServer/Models/NexusCacheData.cs +++ b/Wabbajack.BuildServer/Models/NexusCacheData.cs @@ -1,5 +1,7 @@ using System; +using System.Threading.Tasks; using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver; namespace Wabbajack.BuildServer.Models { @@ -9,6 +11,8 @@ namespace Wabbajack.BuildServer.Models public string Path { get; set; } public T Data { get; set; } public string Game { get; set; } + + [BsonIgnoreIfNull] public string ModId { get; set; } public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow; @@ -16,5 +20,9 @@ namespace Wabbajack.BuildServer.Models [BsonIgnoreIfNull] public string FileId { get; set; } + public async Task Upsert(IMongoCollection> coll) + { + await coll.FindOneAndReplaceAsync>(s => s.Path == Path, this, new FindOneAndReplaceOptions> {IsUpsert = true}); + } } } diff --git a/Wabbajack.BuildServer/Models/NexusUpdateEntry.cs b/Wabbajack.BuildServer/Models/NexusUpdateEntry.cs new file mode 100644 index 00000000..a1a78944 --- /dev/null +++ b/Wabbajack.BuildServer/Models/NexusUpdateEntry.cs @@ -0,0 +1,16 @@ +using System; +using MongoDB.Bson.Serialization.Attributes; +using Newtonsoft.Json; + +namespace Wabbajack.BuildServer.Models +{ + public class NexusUpdateEntry + { + [JsonProperty("mod_id")] + public long ModId { get; set; } + [JsonProperty("latest_file_update")] + public long LatestFileUpdate { get; set; } + [JsonProperty("latest_mod_activity")] + public long LastestModActivity { get; set; } + } +} From 5edcb4c6730705095ae6f0cd95ab24eb567b6729 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 12 Jan 2020 21:05:07 -0700 Subject: [PATCH 2/6] Update settings json file --- Wabbajack.BuildServer/appsettings.json | 1 + 1 file changed, 1 insertion(+) diff --git a/Wabbajack.BuildServer/appsettings.json b/Wabbajack.BuildServer/appsettings.json index 39adbc69..9e7d3582 100644 --- a/Wabbajack.BuildServer/appsettings.json +++ b/Wabbajack.BuildServer/appsettings.json @@ -19,6 +19,7 @@ "NexusModInfos": "nexus_mod_infos", "NexusModFiles": "nexus_mod_files", "NexusFileInfos": "nexus_file_infos", + "NexusUpdates": "nexus_updates", "ModListStatus": "mod_lists", "JobQueue": "job_queue", "DownloadStates": "download_states", From caa2ec8dd422f50fc2d82c4439a5fbccceaed1cf Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 12 Jan 2020 21:31:43 -0700 Subject: [PATCH 3/6] Fix broken RSS feed and status html --- .../Controllers/ListValidation.cs | 87 ++++++++++++++++++- Wabbajack.BuildServer/JobManager.cs | 3 + .../Wabbajack.BuildServer.csproj | 1 + 3 files changed, 89 insertions(+), 2 deletions(-) diff --git a/Wabbajack.BuildServer/Controllers/ListValidation.cs b/Wabbajack.BuildServer/Controllers/ListValidation.cs index d9de495f..198f56da 100644 --- a/Wabbajack.BuildServer/Controllers/ListValidation.cs +++ b/Wabbajack.BuildServer/Controllers/ListValidation.cs @@ -1,9 +1,15 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.ServiceModel.Syndication; using System.Threading.Tasks; +using Microsoft.AspNetCore.Http.Extensions; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using MongoDB.Driver; using MongoDB.Driver.Linq; +using Nettle; using Wabbajack.BuildServer.Models; using Wabbajack.Lib.ModListRegistry; @@ -16,12 +22,89 @@ namespace Wabbajack.BuildServer.Controllers public ListValidation(ILogger logger, DBContext db) : base(logger, db) { } - + [HttpGet] [Route("status.json")] public async Task> HandleGetLists() { return await Db.ModListStatus.AsQueryable().Select(m => m.Summary).ToListAsync(); } + + private static readonly Func HandleGetRssFeedTemplate = NettleEngine.GetCompiler().Compile(@" + + + + {{lst.Name}} - Broken Mods + http://build.wabbajack.org/status/{{lst.Name}}.html + These are mods that are broken and need updating + {{ each $.failed }} + + {{$.Archive.Name}} + {{$.Archive.Name}} + + {{/each}} + + + "); + + [HttpGet] + [Route("status/{Name}/broken.rss")] + public async Task HandleGetRSSFeed(string Name) + { + var lst = (await ModListStatus.ByName(Db, Name)).DetailedStatus; + var response = HandleGetRssFeedTemplate(new + { + lst, + failed = lst.Archives.Where(a => a.IsFailing).ToList(), + passed = lst.Archives.Where(a => !a.IsFailing).ToList() + }); + return new ContentResult + { + ContentType = "application/rss+xml", + StatusCode = (int) HttpStatusCode.OK, + Content = response + }; + } + + private static readonly Func HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@" + +

{{lst.Name}} - {{lst.Checked}} - {{ago}}min ago

+

Failed ({{failed.Count}}):

+
    + {{each $.failed }} +
  • {{$.Archive.Name}}
    • + {{/each}} +
+

Passed ({{passed.Count}}):

+
    + {{each $.passed }} +
  • {{$.Archive.Name}}
    • + {{/each}} +
+ + "); + + [HttpGet] + [Route("status/{Name}.html")] + public async Task HandleGetListHtml(string Name) + { + + var lst = (await ModListStatus.ByName(Db, Name)).DetailedStatus; + var response = HandleGetListTemplate(new + { + lst, + ago = (DateTime.Now - lst.Checked).TotalMinutes, + failed = lst.Archives.Where(a => a.IsFailing).ToList(), + passed = lst.Archives.Where(a => !a.IsFailing).ToList() + }); + return new ContentResult + { + ContentType = "text/html", + StatusCode = (int) HttpStatusCode.OK, + Content = response + }; + } + + } } diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index 9e858642..0a875090 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -26,6 +26,7 @@ namespace Wabbajack.BuildServer public void StartJobRunners() { + return; for (var idx = 0; idx < 2; idx++) { Task.Run(async () => @@ -70,12 +71,14 @@ namespace Wabbajack.BuildServer Utils.LogMessages.Subscribe(msg => Logger.Log(LogLevel.Information, msg.ToString())); while (true) { + /* await KillOrphanedJobs(); await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.High); await ScheduledJob(TimeSpan.FromMinutes(30), Job.JobPriority.High); await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.Low); await ScheduledJob(TimeSpan.FromHours(24), Job.JobPriority.High); await ScheduledJob(TimeSpan.FromHours(6), Job.JobPriority.Low); + */ await Task.Delay(10000); } } diff --git a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj index a79e2feb..4fc8332a 100644 --- a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj +++ b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj @@ -18,6 +18,7 @@ + From 14f7f45a7770e1163da1ed58dd2f4d1b9a751646 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 13 Jan 2020 15:55:55 -0700 Subject: [PATCH 4/6] Add size validation to HTTPDownloader, to catch stealth updates. Check whitelist during list validation. --- .../Controllers/ListValidation.cs | 2 +- Wabbajack.BuildServer/JobManager.cs | 11 ++++------ .../Models/Jobs/UpdateModLists.cs | 21 +++++++++++++----- Wabbajack.BuildServer/Models/ModListStatus.cs | 2 +- Wabbajack.Lib/ACompiler.cs | 2 +- .../Downloaders/AbstractDownloadState.cs | 2 +- .../Downloaders/AbstractIPS4Downloader.cs | 2 +- .../Downloaders/GameFileSourceDownloader.cs | 2 +- .../Downloaders/GoogleDriveDownloader.cs | 4 ++-- Wabbajack.Lib/Downloaders/HTTPDownloader.cs | 19 ++++++++++++---- Wabbajack.Lib/Downloaders/MEGADownloader.cs | 2 +- Wabbajack.Lib/Downloaders/ManualDownloader.cs | 2 +- .../Downloaders/MediaFireDownloader.cs | 2 +- Wabbajack.Lib/Downloaders/ModDBDownloader.cs | 2 +- Wabbajack.Lib/Downloaders/NexusDownloader.cs | 2 +- .../Downloaders/SteamWorkshopDownloader.cs | 2 +- Wabbajack.Test/DownloaderTests.cs | 22 +++++++++---------- Wabbajack.Test/ModlistMetadataTests.cs | 3 ++- 18 files changed, 62 insertions(+), 42 deletions(-) diff --git a/Wabbajack.BuildServer/Controllers/ListValidation.cs b/Wabbajack.BuildServer/Controllers/ListValidation.cs index 198f56da..d7cf774c 100644 --- a/Wabbajack.BuildServer/Controllers/ListValidation.cs +++ b/Wabbajack.BuildServer/Controllers/ListValidation.cs @@ -93,7 +93,7 @@ namespace Wabbajack.BuildServer.Controllers var response = HandleGetListTemplate(new { lst, - ago = (DateTime.Now - lst.Checked).TotalMinutes, + ago = (DateTime.UtcNow - lst.Checked).TotalMinutes, failed = lst.Archives.Where(a => a.IsFailing).ToList(), passed = lst.Archives.Where(a => !a.IsFailing).ToList() }); diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index 0a875090..12380391 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -26,7 +26,6 @@ namespace Wabbajack.BuildServer public void StartJobRunners() { - return; for (var idx = 0; idx < 2; idx++) { Task.Run(async () => @@ -71,14 +70,12 @@ namespace Wabbajack.BuildServer Utils.LogMessages.Subscribe(msg => Logger.Log(LogLevel.Information, msg.ToString())); while (true) { - /* await KillOrphanedJobs(); - await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.High); + //await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.High); await ScheduledJob(TimeSpan.FromMinutes(30), Job.JobPriority.High); - await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.Low); - await ScheduledJob(TimeSpan.FromHours(24), Job.JobPriority.High); - await ScheduledJob(TimeSpan.FromHours(6), Job.JobPriority.Low); - */ + //await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.Low); + //await ScheduledJob(TimeSpan.FromHours(24), Job.JobPriority.High); + //await ScheduledJob(TimeSpan.FromHours(6), Job.JobPriority.Low); await Task.Delay(10000); } } diff --git a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs index 7660f127..43549101 100644 --- a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs +++ b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs @@ -7,6 +7,8 @@ using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.ModListRegistry; +using Wabbajack.Lib.Validation; +using File = Alphaleonis.Win32.Filesystem.File; namespace Wabbajack.BuildServer.Models.Jobs { @@ -20,11 +22,15 @@ namespace Wabbajack.BuildServer.Models.Jobs using (var queue = new WorkQueue()) { - foreach (var list in modlists.Skip(1).Take(1)) + + var whitelists = new ValidateModlist(queue); + await whitelists.LoadListsFromGithub(); + + foreach (var list in modlists) { try { - await ValidateList(db, list, queue); + await ValidateList(db, list, queue, whitelists); } catch (Exception ex) { @@ -36,7 +42,7 @@ namespace Wabbajack.BuildServer.Models.Jobs return JobResult.Success(); } - private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue) + private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists) { var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL); @@ -65,14 +71,14 @@ namespace Wabbajack.BuildServer.Models.Jobs DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State)); + var validated = (await installer.Archives .PMap(queue, async archive => { - Utils.Log($"Validating: {archive.Name}"); bool is_failed; try { - is_failed = !(await archive.State.Verify()); + is_failed = !(await archive.State.Verify(archive)) || !archive.State.IsWhitelisted(whitelists.ServerWhitelist); } catch (Exception) { @@ -105,7 +111,12 @@ namespace Wabbajack.BuildServer.Models.Jobs DetailedStatus = status, Metadata = list }; + Utils.Log( + $"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed"); await ModListStatus.Update(db, dto); + Utils.Log( + $"Done updating {dto.Summary.Name}"); + } } } diff --git a/Wabbajack.BuildServer/Models/ModListStatus.cs b/Wabbajack.BuildServer/Models/ModListStatus.cs index e42b84c7..04b9c8e7 100644 --- a/Wabbajack.BuildServer/Models/ModListStatus.cs +++ b/Wabbajack.BuildServer/Models/ModListStatus.cs @@ -47,7 +47,7 @@ namespace Wabbajack.BuildServer.Models public class DetailedStatus { public string Name { get; set; } - public DateTime Checked { get; set; } = DateTime.Now; + public DateTime Checked { get; set; } = DateTime.UtcNow; public List Archives { get; set; } public DownloadMetadata DownloadMetaData { get; set; } public bool HasFailures { get; set; } diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index 6e44278e..9274ec9b 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -239,7 +239,7 @@ namespace Wabbajack.Lib result.Meta = archive.Meta; result.Size = archive.File.Size; - if (result.State != null && !await result.State.Verify()) + if (result.State != null && !await result.State.Verify(result)) Error( $"Unable to resolve link for {archive.Name}. If this is hosted on the Nexus the file may have been removed."); diff --git a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs index a37f9056..6cf87d97 100644 --- a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs +++ b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs @@ -75,7 +75,7 @@ namespace Wabbajack.Lib.Downloaders /// Returns true if this link is still valid /// /// - public abstract Task Verify(); + public abstract Task Verify(Archive archive); public abstract IDownloader GetDownloader(); diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs index 8fced59f..a52456de 100644 --- a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs @@ -129,7 +129,7 @@ namespace Wabbajack.Lib.Downloaders public int CurrentTime { get; set; } } - public override async Task Verify() + public override async Task Verify(Archive a) { var stream = await ResolveDownloadStream(); if (stream == null) diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs index 8a35ebb2..703ae2dc 100644 --- a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs @@ -72,7 +72,7 @@ namespace Wabbajack.Lib.Downloaders } } - public override async Task Verify() + public override async Task Verify(Archive a) { return File.Exists(SourcePath) && SourcePath.FileHashCached() == Hash; } diff --git a/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs b/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs index d251fc41..7bdf776c 100644 --- a/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs @@ -64,10 +64,10 @@ namespace Wabbajack.Lib.Downloaders return httpState; } - public override async Task Verify() + public override async Task Verify(Archive a) { var state = await ToHttpState(); - return await state.Verify(); + return await state.Verify(a); } public override IDownloader GetDownloader() diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index c3e14a14..0ab39945 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -119,12 +119,23 @@ namespace Wabbajack.Lib.Downloaders return false; } - if (!download) - return true; var headerVar = a.Size == 0 ? "1" : a.Size.ToString(); + long header_content_size = 0; if (response.Content.Headers.Contains("Content-Length")) + { headerVar = response.Content.Headers.GetValues("Content-Length").FirstOrDefault(); + if (headerVar != null) + long.TryParse(headerVar, out header_content_size); + } + + if (!download) + { + if (a.Size != 0 && header_content_size != 0) + return a.Size == header_content_size; + return true; + } + var supportsResume = response.Headers.AcceptRanges.FirstOrDefault(f => f == "bytes") != null; var contentSize = headerVar != null ? long.Parse(headerVar) : 1; @@ -179,9 +190,9 @@ namespace Wabbajack.Lib.Downloaders } } - public override async Task Verify() + public override async Task Verify(Archive a) { - return await DoDownload(new Archive {Name = ""}, "", false); + return await DoDownload(a, "", false); } public override IDownloader GetDownloader() diff --git a/Wabbajack.Lib/Downloaders/MEGADownloader.cs b/Wabbajack.Lib/Downloaders/MEGADownloader.cs index e8f71701..9a3ad814 100644 --- a/Wabbajack.Lib/Downloaders/MEGADownloader.cs +++ b/Wabbajack.Lib/Downloaders/MEGADownloader.cs @@ -37,7 +37,7 @@ namespace Wabbajack.Lib.Downloaders client.DownloadFile(fileLink, destination); } - public override async Task Verify() + public override async Task Verify(Archive a) { var client = new MegaApiClient(); Utils.Status("Logging into MEGA (as anonymous)"); diff --git a/Wabbajack.Lib/Downloaders/ManualDownloader.cs b/Wabbajack.Lib/Downloaders/ManualDownloader.cs index e1810d14..34aeaf45 100644 --- a/Wabbajack.Lib/Downloaders/ManualDownloader.cs +++ b/Wabbajack.Lib/Downloaders/ManualDownloader.cs @@ -111,7 +111,7 @@ namespace Wabbajack.Lib.Downloaders } } - public override async Task Verify() + public override async Task Verify(Archive a) { return true; } diff --git a/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs b/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs index 7e7bce4e..5dee09cf 100644 --- a/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs +++ b/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs @@ -37,7 +37,7 @@ namespace Wabbajack.Lib.Downloaders await result.Download(a, destination); } - public override async Task Verify() + public override async Task Verify(Archive a) { return await Resolve() != null; } diff --git a/Wabbajack.Lib/Downloaders/ModDBDownloader.cs b/Wabbajack.Lib/Downloaders/ModDBDownloader.cs index b1e80e07..349317b5 100644 --- a/Wabbajack.Lib/Downloaders/ModDBDownloader.cs +++ b/Wabbajack.Lib/Downloaders/ModDBDownloader.cs @@ -92,7 +92,7 @@ namespace Wabbajack.Lib.Downloaders return mirrors.Select(d => d.Link).ToArray(); } - public override async Task Verify() + public override async Task Verify(Archive a) { await GetDownloadUrls(); return true; diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index e42e7edb..3b4f08cd 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -158,7 +158,7 @@ namespace Wabbajack.Lib.Downloaders } - public override async Task Verify() + public override async Task Verify(Archive a) { try { diff --git a/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs b/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs index b154124b..c3e7a9f9 100644 --- a/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs +++ b/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs @@ -79,7 +79,7 @@ namespace Wabbajack.Lib.Downloaders } } - public override async Task Verify() + public override async Task Verify(Archive a) { //TODO: find a way to verify steam workshop items throw new NotImplementedException(); diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index c8aabccf..755c5c78 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -64,7 +64,7 @@ namespace Wabbajack.Test var converted = await state.RoundTripState(); - Assert.IsTrue(await converted.Verify()); + Assert.IsTrue(await converted.Verify(new Archive{Size = 20})); var filename = Guid.NewGuid().ToString(); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List{"https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k" } })); @@ -94,7 +94,7 @@ namespace Wabbajack.Test ((HTTPDownloader.State)url_state).Url); var converted = await state.RoundTripState(); - Assert.IsTrue(await converted.Verify()); + Assert.IsTrue(await converted.Verify(new Archive{Size = 20})); var filename = Guid.NewGuid().ToString(); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List { "https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?" } })); @@ -124,7 +124,7 @@ namespace Wabbajack.Test ((GoogleDriveDownloader.State)url_state).Id); var converted = await state.RoundTripState(); - Assert.IsTrue(await converted.Verify()); + Assert.IsTrue(await converted.Verify(new Archive{Size = 20})); var filename = Guid.NewGuid().ToString(); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List { "1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_" } })); @@ -153,7 +153,7 @@ namespace Wabbajack.Test ((HTTPDownloader.State)url_state).Url); var converted = await state.RoundTripState(); - Assert.IsTrue(await converted.Verify()); + Assert.IsTrue(await converted.Verify(new Archive{Size = 20})); var filename = Guid.NewGuid().ToString(); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List { "http://build.wabbajack.org/" } })); @@ -177,7 +177,7 @@ namespace Wabbajack.Test Assert.IsNotNull(state); var converted = await state.RoundTripState(); - Assert.IsTrue(await converted.Verify()); + Assert.IsTrue(await converted.Verify(new Archive{Size = 20})); var filename = Guid.NewGuid().ToString(); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List { "http://build.wabbajack.org/" } })); @@ -238,9 +238,9 @@ namespace Wabbajack.Test var converted = await state.RoundTripState(); - Assert.IsTrue(await converted.Verify()); + Assert.IsTrue(await converted.Verify(new Archive{Size = 20})); // Exercise the cache code - Assert.IsTrue(await converted.Verify()); + Assert.IsTrue(await converted.Verify(new Archive{Size = 20})); var filename = Guid.NewGuid().ToString(); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List () })); @@ -273,7 +273,7 @@ namespace Wabbajack.Test ((ModDBDownloader.State)url_state).Url); var converted = await state.RoundTripState(); - Assert.IsTrue(await converted.Verify()); + Assert.IsTrue(await converted.Verify(new Archive{Size = 20})); var filename = Guid.NewGuid().ToString(); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); @@ -299,7 +299,7 @@ namespace Wabbajack.Test ((HTTPDownloader.State)url_state).Url); */ var converted = await state.RoundTripState(); - Assert.IsTrue(await converted.Verify()); + Assert.IsTrue(await converted.Verify(new Archive{Size = 20})); var filename = Guid.NewGuid().ToString(); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); @@ -327,7 +327,7 @@ namespace Wabbajack.Test ((HTTPDownloader.State)url_state).Url); */ var converted = await state.RoundTripState(); - Assert.IsTrue(await converted.Verify()); + Assert.IsTrue(await converted.Verify(new Archive{Size = 20})); var filename = Guid.NewGuid().ToString(); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); @@ -354,7 +354,7 @@ namespace Wabbajack.Test Assert.IsNotNull(state); var converted = await state.RoundTripState(); - Assert.IsTrue(await converted.Verify()); + Assert.IsTrue(await converted.Verify(new Archive{Size = 20})); var filename = Guid.NewGuid().ToString(); Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); diff --git a/Wabbajack.Test/ModlistMetadataTests.cs b/Wabbajack.Test/ModlistMetadataTests.cs index cb9c4256..4f7cb93b 100644 --- a/Wabbajack.Test/ModlistMetadataTests.cs +++ b/Wabbajack.Test/ModlistMetadataTests.cs @@ -1,6 +1,7 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.ModListRegistry; @@ -25,7 +26,7 @@ namespace Wabbajack.Test { var logo_state = DownloadDispatcher.ResolveArchive(modlist.ImageUri); Assert.IsNotNull(logo_state); - Assert.IsTrue(await logo_state.Verify(), $"{modlist.ImageUri} is not valid"); + Assert.IsTrue(await logo_state.Verify(new Archive{Size = 0}), $"{modlist.ImageUri} is not valid"); } } } From 213df5bc0ef12bf0df46dbcfb1dd4f15fc778e28 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 13 Jan 2020 16:06:09 -0700 Subject: [PATCH 5/6] Gotta stop disabling the job queue --- Wabbajack.BuildServer/JobManager.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index 12380391..9e858642 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -71,11 +71,11 @@ namespace Wabbajack.BuildServer while (true) { await KillOrphanedJobs(); - //await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.High); + await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.High); await ScheduledJob(TimeSpan.FromMinutes(30), Job.JobPriority.High); - //await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.Low); - //await ScheduledJob(TimeSpan.FromHours(24), Job.JobPriority.High); - //await ScheduledJob(TimeSpan.FromHours(6), Job.JobPriority.Low); + await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.Low); + await ScheduledJob(TimeSpan.FromHours(24), Job.JobPriority.High); + await ScheduledJob(TimeSpan.FromHours(6), Job.JobPriority.Low); await Task.Delay(10000); } } From af9f0dbf89583a765fd90978a5c92da0c81c91b8 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 14 Jan 2020 06:35:50 -0700 Subject: [PATCH 6/6] Can manually enqueue jobs --- Wabbajack.BuildServer/Controllers/Jobs.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Wabbajack.BuildServer/Controllers/Jobs.cs b/Wabbajack.BuildServer/Controllers/Jobs.cs index fa5a34bd..4079aada 100644 --- a/Wabbajack.BuildServer/Controllers/Jobs.cs +++ b/Wabbajack.BuildServer/Controllers/Jobs.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; @@ -26,5 +27,15 @@ namespace Wabbajack.BuildServer.Controllers .OrderByDescending(j => j.Priority) .ToListAsync(); } + + [HttpGet] + [Route("enqueue_job/{JobName}")] + public async Task EnqueueJob(string JobName) + { + var jobtype = AJobPayload.NameToType[JobName]; + var job = new Job{Priority = Job.JobPriority.High, Payload = (AJobPayload)jobtype.GetConstructor(new Type[0]).Invoke(new object?[0])}; + await Db.Jobs.InsertOneAsync(job); + return job.Id; + } } }