using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; using Wabbajack.Lib.NexusApi; using Wabbajack.BuildServer.Model.Models; using Wabbajack.Common.Serialization.Json; namespace Wabbajack.BuildServer.Models.Jobs { [JsonName("GetNexusUpdatesJob")] public class GetNexusUpdatesJob : AJobPayload, IFrontEndJob { public override string Description => "Poll the Nexus for updated mods, and clean any references to those mods"; public override async Task Execute(SqlService sql, AppSettings settings) { var api = await NexusApiClient.Get(); var gameTasks = GameRegistry.Games.Values .Where(game => game.NexusName != null) .Select(async game => { var mods = await api.Get>( $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m"); var entry = new NexusCacheData>(); entry.Game = game.NexusName; entry.Path = $"/v1/games/{game.NexusName}/mods/updated.json?period=1m"; entry.Data = mods; return (game, mods); }) .Select(async rTask => { var (game, mods) = await rTask; return mods.Select(mod => new { game = game, mod = mod }); }).ToList(); Utils.Log($"Getting update list for {gameTasks.Count} games"); var purge = (await Task.WhenAll(gameTasks)) .SelectMany(i => i) .ToList(); Utils.Log($"Found {purge.Count} updated mods in the last month"); using (var queue = new WorkQueue()) { var collected = purge.Select(d => { var a = d.mod.LatestFileUpdate.AsUnixTime(); // Mod activity could hide files var b = d.mod.LastestModActivity.AsUnixTime(); return new {Game = d.game.Game, Date = (a > b ? a : b), ModId = d.mod.ModId}; }); var purged = await collected.PMap(queue, async t => { var resultA = await sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date); var resultB = await sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date); return resultA + resultB; }); Utils.Log($"Purged {purged.Sum()} cache entries"); } return JobResult.Success(); } public static DateTime LastNexusSync { get; set; } = DateTime.Now; public static async Task UpdateNexusCacheFast(SqlService sql) { var results = await NexusUpdatesFeeds.GetUpdates(); NexusApiClient client = null; long updated = 0; foreach (var result in results) { var purgedMods = await sql.DeleteNexusModFilesUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp); var purgedFiles = await sql.DeleteNexusModInfosUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp); var totalPurged = purgedFiles + purgedMods; if (totalPurged > 0) Utils.Log($"Purged {totalPurged} cache items"); if (await sql.GetNexusModInfoString(result.Game, result.ModId) != null) continue; // Lazily create the client client ??= await NexusApiClient.Get(); // Cache the info var files = await client.GetModFiles(result.Game, result.ModId, false); await sql.AddNexusModFiles(result.Game, result.ModId, result.TimeStamp, files); var modInfo = await client.GetModInfo(result.Game, result.ModId); await sql.AddNexusModInfo(result.Game, result.ModId, result.TimeStamp, modInfo); updated++; } if (updated > 0) Utils.Log($"Primed {updated} nexus cache entries"); LastNexusSync = DateTime.Now; return updated; } } }