diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index 4dbe21fa..9e858642 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -75,6 +75,7 @@ namespace Wabbajack.BuildServer await ScheduledJob(TimeSpan.FromMinutes(30), Job.JobPriority.High); await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.Low); await ScheduledJob(TimeSpan.FromHours(24), Job.JobPriority.High); + await ScheduledJob(TimeSpan.FromHours(6), Job.JobPriority.Low); await Task.Delay(10000); } } diff --git a/Wabbajack.BuildServer/Models/DBContext.cs b/Wabbajack.BuildServer/Models/DBContext.cs index 98c19356..128825c9 100644 --- a/Wabbajack.BuildServer/Models/DBContext.cs +++ b/Wabbajack.BuildServer/Models/DBContext.cs @@ -26,6 +26,7 @@ namespace Wabbajack.BuildServer.Models public IMongoCollection DownloadStates => Client.GetCollection(_settings.Collections["DownloadStates"]); public IMongoCollection Metrics => Client.GetCollection(_settings.Collections["Metrics"]); public IMongoCollection IndexedFiles => Client.GetCollection(_settings.Collections["IndexedFiles"]); + public IMongoCollection>> NexusUpdates => Client.GetCollection>>(_settings.Collections["NexusUpdates"]); public IMongoCollection> NexusModFiles => Client.GetCollection>( diff --git a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs index 792b4f71..76c3181c 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs @@ -16,7 +16,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue typeof(GetNexusUpdatesJob), typeof(UpdateModLists), typeof(EnqueueAllArchives), - typeof(EnqueueAllGameFiles) + typeof(EnqueueAllGameFiles), + typeof(EnqueueRecentFiles) }; public static Dictionary TypeToName { get; set; } public static Dictionary NameToType { get; set; } diff --git a/Wabbajack.BuildServer/Models/JobQueue/Job.cs b/Wabbajack.BuildServer/Models/JobQueue/Job.cs index 006da7aa..52f71a47 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/Job.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/Job.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; using MongoDB.Driver; +using Wabbajack.Lib.NexusApi; namespace Wabbajack.BuildServer.Models.JobQueue { diff --git a/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs b/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs new file mode 100644 index 00000000..eaabf2e9 --- /dev/null +++ b/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using MongoDB.Driver; +using MongoDB.Driver.Core.Authentication; +using MongoDB.Driver.Linq; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.NexusApi; + +namespace Wabbajack.BuildServer.Models.Jobs +{ + public class EnqueueRecentFiles : AJobPayload + { + public override string Description => "Enqueue the past days worth of mods for indexing"; + + private static HashSet GamesToScan = new HashSet + { + Game.Fallout3, Game.Fallout4, Game.Skyrim, Game.SkyrimSpecialEdition, Game.SkyrimVR, Game.FalloutNewVegas, Game.Oblivion + }; + public override async Task Execute(DBContext db, AppSettings settings) + { + using (var queue = new WorkQueue()) + { + var updates = await db.NexusUpdates.AsQueryable().ToListAsync(); + var mods = updates + .Where(list => GamesToScan.Contains(GameRegistry.GetByNexusName(list.Game).Game)) + .SelectMany(list => + list.Data.Where(mod => DateTime.UtcNow - mod.LatestFileUpdate.AsUnixTime() < TimeSpan.FromDays(1)) + .Select(mod => (list.Game, mod.ModId))); + var mod_files = (await mods.PMap(queue, async mod => + { + var client = await NexusApiClient.Get(); + try + { + var files = await client.GetModFiles(GameRegistry.GetByNexusName(mod.Game).Game, + (int)mod.ModId); + return (mod.Game, mod.ModId, files.files); + } + catch (Exception) + { + return default; + } + })).Where(t => t.Game != null).ToList(); + + var archives = + mod_files.SelectMany(mod => mod.files.Select(file => (mod.Game, mod.ModId, File:file)).Where(f => !string.IsNullOrEmpty(f.File.category_name) )) + .Select(tuple => + { + var state = new NexusDownloader.State + { + GameName = tuple.Game, ModID = tuple.ModId.ToString(), FileID = tuple.File.file_id.ToString() + }; + return new Archive {State = state, Name = tuple.File.file_name}; + }).ToList(); + + Utils.Log($"Found {archives.Count} archives from recent Nexus updates to index"); + var searching = archives.Select(a => a.State.PrimaryKeyString).Distinct().ToArray(); + + Utils.Log($"Looking for missing states"); + var knownArchives = (await db.DownloadStates.AsQueryable().Where(s => searching.Contains(s.Key)) + .Select(d => d.Key).ToListAsync()).ToDictionary(a => a); + + Utils.Log($"Found {knownArchives.Count} pre-existing archives"); + var missing = archives.Where(a => !knownArchives.ContainsKey(a.State.PrimaryKeyString)) + .DistinctBy(d => d.State.PrimaryKeyString) + .ToList(); + + Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs"); + + var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}, Priority = Job.JobPriority.Low}); + + Utils.Log($"Writing jobs to the DB"); + await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions {IsOrdered = false}); + Utils.Log($"Done adding archives for Nexus Updates"); + + return JobResult.Success(); + } + } + } +} diff --git a/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs b/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs index 54028fab..1a52a28f 100644 --- a/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs @@ -5,6 +5,7 @@ using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; using Wabbajack.Lib.NexusApi; using MongoDB.Driver; +using Newtonsoft.Json; namespace Wabbajack.BuildServer.Models.Jobs @@ -21,9 +22,17 @@ namespace Wabbajack.BuildServer.Models.Jobs .Where(game => game.NexusName != null) .Select(async game => { - return (game, - mods: await api.Get>( - $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m")); + var mods = await api.Get>( + $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m"); + + var entry = new NexusCacheData>(); + entry.Game = game.NexusName; + entry.Path = $"/v1/games/{game.NexusName}/mods/updated.json?period=1m"; + entry.Data = mods; + + await entry.Upsert(db.NexusUpdates); + + return (game, mods); }) .Select(async rTask => { @@ -40,14 +49,16 @@ namespace Wabbajack.BuildServer.Models.Jobs Utils.Log($"Found {purge.Count} updated mods in the last month"); using (var queue = new WorkQueue()) { - var collected = await purge.Select(d => + var collected = purge.Select(d => { - var a = d.mod.latest_file_update.AsUnixTime(); + var a = d.mod.LatestFileUpdate.AsUnixTime(); // Mod activity could hide files - var b = d.mod.latest_mod_activity.AsUnixTime(); + var b = d.mod.LastestModActivity.AsUnixTime(); - return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.mod_id.ToString()}; - }).PMap(queue, async t => + return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.ModId.ToString()}; + }); + + var purged = await collected.PMap(queue, async t => { var resultA = await db.NexusModInfos.DeleteManyAsync(f => f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); @@ -59,17 +70,12 @@ namespace Wabbajack.BuildServer.Models.Jobs return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount; }); - Utils.Log($"Purged {collected.Sum()} cache entries"); + Utils.Log($"Purged {purged.Sum()} cache entries"); } return JobResult.Success(); } - class UpdatedMod - { - public long mod_id; - public long latest_file_update; - public long latest_mod_activity; - } + } } diff --git a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs index 668ad4d4..7660f127 100644 --- a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs +++ b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs @@ -20,7 +20,7 @@ namespace Wabbajack.BuildServer.Models.Jobs using (var queue = new WorkQueue()) { - foreach (var list in modlists) + foreach (var list in modlists.Skip(1).Take(1)) { try { diff --git a/Wabbajack.BuildServer/Models/NexusCacheData.cs b/Wabbajack.BuildServer/Models/NexusCacheData.cs index 4bb0b19a..a28ca5f5 100644 --- a/Wabbajack.BuildServer/Models/NexusCacheData.cs +++ b/Wabbajack.BuildServer/Models/NexusCacheData.cs @@ -1,5 +1,7 @@ using System; +using System.Threading.Tasks; using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver; namespace Wabbajack.BuildServer.Models { @@ -9,6 +11,8 @@ namespace Wabbajack.BuildServer.Models public string Path { get; set; } public T Data { get; set; } public string Game { get; set; } + + [BsonIgnoreIfNull] public string ModId { get; set; } public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow; @@ -16,5 +20,9 @@ namespace Wabbajack.BuildServer.Models [BsonIgnoreIfNull] public string FileId { get; set; } + public async Task Upsert(IMongoCollection> coll) + { + await coll.FindOneAndReplaceAsync>(s => s.Path == Path, this, new FindOneAndReplaceOptions> {IsUpsert = true}); + } } } diff --git a/Wabbajack.BuildServer/Models/NexusUpdateEntry.cs b/Wabbajack.BuildServer/Models/NexusUpdateEntry.cs new file mode 100644 index 00000000..a1a78944 --- /dev/null +++ b/Wabbajack.BuildServer/Models/NexusUpdateEntry.cs @@ -0,0 +1,16 @@ +using System; +using MongoDB.Bson.Serialization.Attributes; +using Newtonsoft.Json; + +namespace Wabbajack.BuildServer.Models +{ + public class NexusUpdateEntry + { + [JsonProperty("mod_id")] + public long ModId { get; set; } + [JsonProperty("latest_file_update")] + public long LatestFileUpdate { get; set; } + [JsonProperty("latest_mod_activity")] + public long LastestModActivity { get; set; } + } +}