From c4ef7f3be10dc08516284accb616893f5c51975a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 10 Apr 2020 16:31:06 -0600 Subject: [PATCH 1/4] Can load and dedupe RSS feeds --- Wabbajack.Lib/NexusApi/NexusUpdatesFeeds.cs | 80 +++++++++++++++++++++ Wabbajack.Lib/Wabbajack.Lib.csproj | 3 + Wabbajack.Test/NexusTests.cs | 32 +++++++++ 3 files changed, 115 insertions(+) create mode 100644 Wabbajack.Lib/NexusApi/NexusUpdatesFeeds.cs create mode 100644 Wabbajack.Test/NexusTests.cs diff --git a/Wabbajack.Lib/NexusApi/NexusUpdatesFeeds.cs b/Wabbajack.Lib/NexusApi/NexusUpdatesFeeds.cs new file mode 100644 index 00000000..c33655c2 --- /dev/null +++ b/Wabbajack.Lib/NexusApi/NexusUpdatesFeeds.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.ServiceModel.Syndication; +using System.Threading.Tasks; +using System.Xml; +using Wabbajack.Common; + +namespace Wabbajack.Lib.NexusApi +{ + public class NexusUpdatesFeeds + { + + public static async Task> GetUpdates() + { + var updated = GetFeed(new Uri("https://www.nexusmods.com/rss/updatedtoday")); + var newToday = GetFeed(new Uri("https://www.nexusmods.com/rss/newtoday")); + + var sorted = (await updated).Concat(await newToday).OrderByDescending(f => f.TimeStamp); + var deduped = sorted.GroupBy(g => (g.Game, g.ModId)).Select(g => g.First()); + return deduped; + } + + private static bool TryParseGameUrl(SyndicationLink link, out Game game, out long modId) + { + var parts = link.Uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries); + + var foundGame = GameRegistry.GetByFuzzyName(parts[0]); + if (foundGame == null) + { + game = Game.Oblivion; + modId = 0; + return false; + } + + if (long.TryParse(parts[2], out modId)) + { + game = foundGame.Game; + return true; + } + + game = Game.Oblivion; + modId = 0; + return false; + } + + private static async Task> GetFeed(Uri uri) + { + var client = new Common.Http.Client(); + var data = await client.GetStringAsync(uri); + var reader = XmlReader.Create(new StringReader(data)); + var results = SyndicationFeed.Load(reader); + return results.Items + .Select(itm => + { + if (TryParseGameUrl(itm.Links.First(), out var game, out var modId)) + { + return new UpdateRecord + { + TimeStamp = itm.PublishDate.UtcDateTime, + Game = game, + ModId = modId + }; + } + + return null; + }).Where(v => v != null); + } + + + public class UpdateRecord + { + public Game Game { get; set; } + public long ModId { get; set; } + public DateTime TimeStamp { get; set; } + } + + } +} diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index b79980e0..7e8793e3 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -51,6 +51,9 @@ 4.3.4 + + 4.7.0 + 1.0.1 diff --git a/Wabbajack.Test/NexusTests.cs b/Wabbajack.Test/NexusTests.cs new file mode 100644 index 00000000..49391260 --- /dev/null +++ b/Wabbajack.Test/NexusTests.cs @@ -0,0 +1,32 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Lib.NexusApi; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.Test +{ + public class NexusTests : ATestBase + { + [Fact] + public async Task CanGetNexusRSSUpdates() + { + var results = (await NexusUpdatesFeeds.GetUpdates()).ToArray(); + + Assert.NotEmpty(results); + + Utils.Log($"Loaded {results.Length} updates from the Nexus"); + + foreach (var result in results) + { + Assert.True(DateTime.UtcNow - result.TimeStamp < TimeSpan.FromDays(1)); + } + } + + public NexusTests(ITestOutputHelper output) : base(output) + { + } + } +} From 26a42d3ceb4b45113ab6d571d018dee2fdd0ac89 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 10 Apr 2020 16:48:53 -0600 Subject: [PATCH 2/4] Verified that nexus entries are purged by the updates job --- Wabbajack.BuildServer.Test/JobTests.cs | 53 +++++++++++++++++++++ Wabbajack.Lib/NexusApi/NexusUpdatesFeeds.cs | 4 +- 2 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 Wabbajack.BuildServer.Test/JobTests.cs diff --git a/Wabbajack.BuildServer.Test/JobTests.cs b/Wabbajack.BuildServer.Test/JobTests.cs new file mode 100644 index 00000000..9d588658 --- /dev/null +++ b/Wabbajack.BuildServer.Test/JobTests.cs @@ -0,0 +1,53 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Wabbajack.BuildServer.Model.Models; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.BuildServer.Models.Jobs; +using Wabbajack.Common; +using Wabbajack.Lib.NexusApi; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.BuildServer.Test +{ + + public class JobTests : ABuildServerSystemTest + { + public JobTests(ITestOutputHelper output, SingletonAdaptor fixture) : base(output, fixture) + { + } + + [Fact] + public async Task CanRunNexusUpdateJob() + { + var sql = Fixture.GetService(); + + var oldRecords = await NexusUpdatesFeeds.GetUpdates(); + foreach (var record in oldRecords) + { + await sql.AddNexusModInfo(record.Game, record.ModId, DateTime.UtcNow - TimeSpan.FromDays(1), + new ModInfo()); + await sql.AddNexusModFiles(record.Game, record.ModId, DateTime.UtcNow - TimeSpan.FromDays(1), + new NexusApiClient.GetModFilesResponse()); + + Assert.NotNull(await sql.GetModFiles(record.Game, record.ModId)); + Assert.NotNull(await sql.GetNexusModInfoString(record.Game, record.ModId)); + } + + Utils.Log($"Ingested {oldRecords.Count()} nexus records"); + + // We know this will load the same records as above, but the date will be more recent, so the above records + // should no longer exist in SQL after this job is run + await sql.EnqueueJob(new Job {Payload = new GetNexusUpdatesJob()}); + await RunAllJobs(); + + foreach (var record in oldRecords) + { + Assert.Null(await sql.GetModFiles(record.Game, record.ModId)); + Assert.Null(await sql.GetNexusModInfoString(record.Game, record.ModId)); + } + + } + } +} diff --git a/Wabbajack.Lib/NexusApi/NexusUpdatesFeeds.cs b/Wabbajack.Lib/NexusApi/NexusUpdatesFeeds.cs index c33655c2..164d2250 100644 --- a/Wabbajack.Lib/NexusApi/NexusUpdatesFeeds.cs +++ b/Wabbajack.Lib/NexusApi/NexusUpdatesFeeds.cs @@ -12,13 +12,13 @@ namespace Wabbajack.Lib.NexusApi public class NexusUpdatesFeeds { - public static async Task> GetUpdates() + public static async Task> GetUpdates() { var updated = GetFeed(new Uri("https://www.nexusmods.com/rss/updatedtoday")); var newToday = GetFeed(new Uri("https://www.nexusmods.com/rss/newtoday")); var sorted = (await updated).Concat(await newToday).OrderByDescending(f => f.TimeStamp); - var deduped = sorted.GroupBy(g => (g.Game, g.ModId)).Select(g => g.First()); + var deduped = sorted.GroupBy(g => (g.Game, g.ModId)).Select(g => g.First()).ToList(); return deduped; } From 508eb32230713f87e879e8b736e2b0a3702de271 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 10 Apr 2020 21:16:10 -0600 Subject: [PATCH 3/4] Fixes for nexus cache priming --- Wabbajack.BuildServer.Test/JobTests.cs | 8 ++++ .../Controllers/Heartbeat.cs | 9 ++++- .../Controllers/NexusCache.cs | 12 +++--- Wabbajack.BuildServer/JobManager.cs | 13 ++++++ .../Models/Jobs/GetNexusUpdatesJob.cs | 40 ++++++++++++++++++- .../Models/Sql/SqlService.cs | 6 +-- Wabbajack.Common/Json.cs | 17 ++++---- Wabbajack.Lib/NexusApi/NexusApi.cs | 16 +++++--- 8 files changed, 96 insertions(+), 25 deletions(-) diff --git a/Wabbajack.BuildServer.Test/JobTests.cs b/Wabbajack.BuildServer.Test/JobTests.cs index 9d588658..e2f7e372 100644 --- a/Wabbajack.BuildServer.Test/JobTests.cs +++ b/Wabbajack.BuildServer.Test/JobTests.cs @@ -47,7 +47,15 @@ namespace Wabbajack.BuildServer.Test Assert.Null(await sql.GetModFiles(record.Game, record.ModId)); Assert.Null(await sql.GetNexusModInfoString(record.Game, record.ModId)); } + } + [Fact] + public async Task CanPrimeTheNexusCache() + { + var sql = Fixture.GetService(); + + Assert.True(await GetNexusUpdatesJob.UpdateNexusCacheFast(sql) > 0); + Assert.True(await GetNexusUpdatesJob.UpdateNexusCacheFast(sql) == 0); } } } diff --git a/Wabbajack.BuildServer/Controllers/Heartbeat.cs b/Wabbajack.BuildServer/Controllers/Heartbeat.cs index 5f696e20..87039276 100644 --- a/Wabbajack.BuildServer/Controllers/Heartbeat.cs +++ b/Wabbajack.BuildServer/Controllers/Heartbeat.cs @@ -5,6 +5,7 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Wabbajack.BuildServer.Model.Models; +using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.Common.StatusFeed; namespace Wabbajack.BuildServer.Controllers @@ -36,9 +37,13 @@ namespace Wabbajack.BuildServer.Controllers } [HttpGet] - public async Task GetHeartbeat() + public async Task GetHeartbeat() { - return DateTime.Now - _startTime; + return Ok(new + { + Uptime = DateTime.Now - _startTime, + LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync + }); } [HttpGet("only-authenticated")] diff --git a/Wabbajack.BuildServer/Controllers/NexusCache.cs b/Wabbajack.BuildServer/Controllers/NexusCache.cs index 162e874e..3fc990c2 100644 --- a/Wabbajack.BuildServer/Controllers/NexusCache.cs +++ b/Wabbajack.BuildServer/Controllers/NexusCache.cs @@ -49,10 +49,10 @@ namespace Wabbajack.BuildServer.Controllers if (result == null) { var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - var path = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{ModId}.json"; - var body = await api.Get(path); - await SQL.AddNexusModInfo(game, ModId, DateTime.Now, body); + result = await api.GetModInfo(game, ModId, false); + await SQL.AddNexusModInfo(game, ModId, DateTime.UtcNow, result); + method = "NOT_CACHED"; Interlocked.Increment(ref ForwardCount); } @@ -78,10 +78,8 @@ namespace Wabbajack.BuildServer.Controllers if (result == null) { var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files.json"; - var body = await api.Get(path); - await SQL.AddNexusModFiles(game, ModId, DateTime.Now, body); - + result = await api.GetModFiles(game, ModId, false); + await SQL.AddNexusModFiles(game, ModId, DateTime.UtcNow, result); method = "NOT_CACHED"; Interlocked.Increment(ref ForwardCount); diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index d4a6ddf3..a98ab1a2 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -10,6 +10,7 @@ using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.Common; +using Wabbajack.Lib.NexusApi; namespace Wabbajack.BuildServer { @@ -74,6 +75,9 @@ namespace Wabbajack.BuildServer Utils.LogMessages.Subscribe(Heartbeat.AddToLog); Utils.LogMessages.OfType().Subscribe(u => u.Cancel()); if (!Settings.JobScheduler) return; + + var task = RunNexusCacheLoop(); + while (true) { await KillOrphanedJobs(); @@ -86,6 +90,15 @@ namespace Wabbajack.BuildServer } } + private async Task RunNexusCacheLoop() + { + while (true) + { + await GetNexusUpdatesJob.UpdateNexusCacheFast(Sql); + await Task.Delay(TimeSpan.FromMinutes(1)); + } + } + private async Task KillOrphanedJobs() { try diff --git a/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs b/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs index 6754fe1a..07e893cc 100644 --- a/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Wabbajack.BuildServer.Models.JobQueue; @@ -69,6 +70,43 @@ namespace Wabbajack.BuildServer.Models.Jobs return JobResult.Success(); } + + public static DateTime LastNexusSync { get; set; } = DateTime.Now; + public static async Task UpdateNexusCacheFast(SqlService sql) + { + var results = await NexusUpdatesFeeds.GetUpdates(); + NexusApiClient client = null; + long updated = 0; + foreach (var result in results) + { + var purgedMods = await sql.DeleteNexusModFilesUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp); + var purgedFiles = await sql.DeleteNexusModInfosUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp); + + var totalPurged = purgedFiles + purgedMods; + if (totalPurged > 0) + Utils.Log($"Purged {totalPurged} cache items"); + + if (await sql.GetNexusModInfoString(result.Game, result.ModId) != null) continue; + + // Lazily create the client + client ??= await NexusApiClient.Get(); + + // Cache the info + var files = await client.GetModFiles(result.Game, result.ModId, false); + await sql.AddNexusModFiles(result.Game, result.ModId, result.TimeStamp, files); + + var modInfo = await client.GetModInfo(result.Game, result.ModId); + await sql.AddNexusModInfo(result.Game, result.ModId, result.TimeStamp, modInfo); + updated++; + + } + + if (updated > 0) + Utils.Log($"Primed {updated} nexus cache entries"); + + LastNexusSync = DateTime.Now; + return updated; + } } diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index b1756323..87a6be95 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -575,7 +575,7 @@ namespace Wabbajack.BuildServer.Model.Models { await using var conn = await Open(); var deleted = await conn.ExecuteScalarAsync( - @"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date + @"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date SELECT @@ROWCOUNT AS Deleted", new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date}); return deleted; @@ -585,9 +585,9 @@ namespace Wabbajack.BuildServer.Model.Models { await using var conn = await Open(); var deleted = await conn.ExecuteScalarAsync( - @"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date + @"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date SELECT @@ROWCOUNT AS Deleted", - new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date}); + new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date}); return deleted; } diff --git a/Wabbajack.Common/Json.cs b/Wabbajack.Common/Json.cs index 4d2584c7..2b390e12 100644 --- a/Wabbajack.Common/Json.cs +++ b/Wabbajack.Common/Json.cs @@ -27,11 +27,14 @@ namespace Wabbajack.Common }; public static JsonSerializerSettings JsonSettings => - new JsonSerializerSettings { - TypeNameHandling = TypeNameHandling.Objects, - SerializationBinder = new JsonNameSerializationBinder(), - Converters = Converters}; - + new JsonSerializerSettings { + TypeNameHandling = TypeNameHandling.Objects, + SerializationBinder = new JsonNameSerializationBinder(), + Converters = Converters}; + + public static JsonSerializerSettings GenericJsonSettings => + new JsonSerializerSettings { }; + public static void ToJson(this T obj, string filename) { @@ -73,11 +76,11 @@ namespace Wabbajack.Common return JsonConvert.DeserializeObject(data, JsonSettings)!; } - public static T FromJson(this Stream stream) + public static T FromJson(this Stream stream, bool genericReader = false) { using var tr = new StreamReader(stream, Encoding.UTF8, leaveOpen: true); using var reader = new JsonTextReader(tr); - var ser = JsonSerializer.Create(JsonSettings); + var ser = JsonSerializer.Create(genericReader ? GenericJsonSettings : JsonSettings); return ser.Deserialize(reader); } diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index eb5d486b..5e020b27 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -252,7 +252,7 @@ namespace Wabbajack.Lib.NexusApi await using var stream = await response.Content.ReadAsStreamAsync(); - return stream.FromJson(); + return stream.FromJson(genericReader:true); } catch (TimeoutException) { @@ -321,10 +321,11 @@ namespace Wabbajack.Lib.NexusApi public List files { get; set; } } - public async Task GetModFiles(Game game, long modid) + public async Task GetModFiles(Game game, long modid, bool useCache = true) + { var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modid}/files.json"; - var result = await GetCached(url); + var result = useCache ? await GetCached(url) : await Get(url); if (result.files == null) throw new InvalidOperationException("Got Null data from the Nexus while finding mod files"); return result; @@ -336,10 +337,15 @@ namespace Wabbajack.Lib.NexusApi return await Get>(url); } - public async Task GetModInfo(Game game, long modId) + public async Task GetModInfo(Game game, long modId, bool useCache = true) { var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modId}.json"; - return await GetCached(url); + if (useCache) + { + return await GetCached(url); + } + + return await Get(url); } private class DownloadLink From f59fe643f132553dda2d6ed7d3d8037fc2b236cb Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 10 Apr 2020 22:42:07 -0600 Subject: [PATCH 4/4] Fix heartbeat test --- Wabbajack.BuildServer.Test/BasicServerTests.cs | 5 +++-- Wabbajack.BuildServer/Controllers/Heartbeat.cs | 11 ++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Wabbajack.BuildServer.Test/BasicServerTests.cs b/Wabbajack.BuildServer.Test/BasicServerTests.cs index b1f49eea..9100d8a1 100644 --- a/Wabbajack.BuildServer.Test/BasicServerTests.cs +++ b/Wabbajack.BuildServer.Test/BasicServerTests.cs @@ -1,5 +1,6 @@ using System; using System.Threading.Tasks; +using Wabbajack.BuildServer.Controllers; using Wabbajack.Common; using Xunit; using Xunit.Abstractions; @@ -15,8 +16,8 @@ namespace Wabbajack.BuildServer.Test [Fact] public async Task CanGetHeartbeat() { - var heartbeat = (await _client.GetStringAsync(MakeURL("heartbeat"))).FromJsonString(); - Assert.True(TimeSpan.Parse(heartbeat) > TimeSpan.Zero); + var heartbeat = (await _client.GetStringAsync(MakeURL("heartbeat"))).FromJsonString(); + Assert.True(heartbeat.Uptime > TimeSpan.Zero); } [Fact] diff --git a/Wabbajack.BuildServer/Controllers/Heartbeat.cs b/Wabbajack.BuildServer/Controllers/Heartbeat.cs index 87039276..6e97518a 100644 --- a/Wabbajack.BuildServer/Controllers/Heartbeat.cs +++ b/Wabbajack.BuildServer/Controllers/Heartbeat.cs @@ -4,8 +4,10 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; +using Org.BouncyCastle.Asn1.Cms; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models.Jobs; +using Wabbajack.Common.Serialization.Json; using Wabbajack.Common.StatusFeed; namespace Wabbajack.BuildServer.Controllers @@ -39,13 +41,20 @@ namespace Wabbajack.BuildServer.Controllers [HttpGet] public async Task GetHeartbeat() { - return Ok(new + return Ok(new HeartbeatResult { Uptime = DateTime.Now - _startTime, LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync }); } + [JsonName("HeartbeatResult")] + public class HeartbeatResult + { + public TimeSpan Uptime { get; set; } + public TimeSpan LastNexusUpdate { get; set; } + } + [HttpGet("only-authenticated")] [Authorize] public IActionResult OnlyAuthenticated()