Merge pull request #706 from wabbajack-tools/issue-705

Use RSS feeds to improve cache support

Wabbajack.BuildServer.Test/BasicServerTests.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Threading.Tasks;
+using Wabbajack.BuildServer.Controllers;
 using Wabbajack.Common;
 using Xunit;
 using Xunit.Abstractions;
@@ -15,8 +16,8 @@ namespace Wabbajack.BuildServer.Test
         [Fact]
         public async Task CanGetHeartbeat()
         {
-            var heartbeat = (await _client.GetStringAsync(MakeURL("heartbeat"))).FromJsonString<string>();
-            Assert.True(TimeSpan.Parse(heartbeat) > TimeSpan.Zero);
+            var heartbeat = (await _client.GetStringAsync(MakeURL("heartbeat"))).FromJsonString<Heartbeat.HeartbeatResult>();
+            Assert.True(heartbeat.Uptime > TimeSpan.Zero);
         }
 
         [Fact]

Wabbajack.BuildServer.Test/JobTests.cs

@@ -0,0 +1,61 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Wabbajack.BuildServer.Model.Models;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.BuildServer.Models.Jobs;
+using Wabbajack.Common;
+using Wabbajack.Lib.NexusApi;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Wabbajack.BuildServer.Test
+{
+    
+    public class JobTests : ABuildServerSystemTest
+    {
+        public JobTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
+        {
+        }
+
+        [Fact]
+        public async Task CanRunNexusUpdateJob()
+        {
+            var sql = Fixture.GetService<SqlService>();
+
+            var oldRecords = await NexusUpdatesFeeds.GetUpdates();
+            foreach (var record in oldRecords)
+            {
+                await sql.AddNexusModInfo(record.Game, record.ModId, DateTime.UtcNow - TimeSpan.FromDays(1),
+                    new ModInfo());
+                await sql.AddNexusModFiles(record.Game, record.ModId, DateTime.UtcNow - TimeSpan.FromDays(1),
+                    new NexusApiClient.GetModFilesResponse());
+                
+                Assert.NotNull(await sql.GetModFiles(record.Game, record.ModId));
+                Assert.NotNull(await sql.GetNexusModInfoString(record.Game, record.ModId));
+            }
+
+            Utils.Log($"Ingested {oldRecords.Count()} nexus records");
+            
+            // We know this will load the same records as above, but the date will be more recent, so the above records
+            // should no longer exist in SQL after this job is run
+            await sql.EnqueueJob(new Job {Payload = new GetNexusUpdatesJob()});
+            await RunAllJobs();
+
+            foreach (var record in oldRecords)
+            {
+                Assert.Null(await sql.GetModFiles(record.Game, record.ModId));
+                Assert.Null(await sql.GetNexusModInfoString(record.Game, record.ModId));
+            }
+        }
+
+        [Fact]
+        public async Task CanPrimeTheNexusCache()
+        {
+            var sql = Fixture.GetService<SqlService>();
+
+            Assert.True(await GetNexusUpdatesJob.UpdateNexusCacheFast(sql) > 0);
+            Assert.True(await GetNexusUpdatesJob.UpdateNexusCacheFast(sql) == 0);
+        }
+    }
+}

Wabbajack.BuildServer/Controllers/Heartbeat.cs

@@ -4,7 +4,10 @@ using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
+using Org.BouncyCastle.Asn1.Cms;
 using Wabbajack.BuildServer.Model.Models;
+using Wabbajack.BuildServer.Models.Jobs;
+using Wabbajack.Common.Serialization.Json;
 using Wabbajack.Common.StatusFeed;
 
 namespace Wabbajack.BuildServer.Controllers
@@ -36,9 +39,20 @@ namespace Wabbajack.BuildServer.Controllers
         }
 
         [HttpGet]
-        public async Task<TimeSpan> GetHeartbeat()
+        public async Task<IActionResult> GetHeartbeat()
         {
-            return DateTime.Now - _startTime;
+            return Ok(new HeartbeatResult
+            {
+                Uptime = DateTime.Now - _startTime,
+                LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync
+            });
+        }
+
+        [JsonName("HeartbeatResult")]
+        public class HeartbeatResult
+        {
+            public TimeSpan Uptime { get; set; }
+            public TimeSpan LastNexusUpdate { get; set; }
         }
 
         [HttpGet("only-authenticated")]

Wabbajack.BuildServer/Controllers/NexusCache.cs

@@ -49,10 +49,10 @@ namespace Wabbajack.BuildServer.Controllers
             if (result == null)
             {
                 var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
-                var path = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{ModId}.json";
-                var body = await api.Get<ModInfo>(path);
-                await SQL.AddNexusModInfo(game, ModId, DateTime.Now, body);
+                result = await api.GetModInfo(game, ModId, false);
+                await SQL.AddNexusModInfo(game, ModId, DateTime.UtcNow, result);
 
+                
                 method = "NOT_CACHED";
                 Interlocked.Increment(ref ForwardCount);
             }
@@ -78,10 +78,8 @@ namespace Wabbajack.BuildServer.Controllers
             if (result == null)
             {
                 var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
-                var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files.json";
-                var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
-                await SQL.AddNexusModFiles(game, ModId, DateTime.Now, body);
-
+                result = await api.GetModFiles(game, ModId, false);
+                await SQL.AddNexusModFiles(game, ModId, DateTime.UtcNow, result);
 
                 method = "NOT_CACHED";
                 Interlocked.Increment(ref ForwardCount);

Wabbajack.BuildServer/JobManager.cs

@@ -10,6 +10,7 @@ using Wabbajack.BuildServer.Models;
 using Wabbajack.BuildServer.Models.JobQueue;
 using Wabbajack.BuildServer.Models.Jobs;
 using Wabbajack.Common;
+using Wabbajack.Lib.NexusApi;
 
 namespace Wabbajack.BuildServer
 {
@@ -74,6 +75,9 @@ namespace Wabbajack.BuildServer
             Utils.LogMessages.Subscribe(Heartbeat.AddToLog);
             Utils.LogMessages.OfType<IUserIntervention>().Subscribe(u => u.Cancel());
             if (!Settings.JobScheduler) return;
+
+            var task = RunNexusCacheLoop();
+            
             while (true)
             {
                 await KillOrphanedJobs();
@@ -86,6 +90,15 @@ namespace Wabbajack.BuildServer
             }
         }
 
+        private async Task RunNexusCacheLoop()
+        {
+            while (true)
+            {
+                await GetNexusUpdatesJob.UpdateNexusCacheFast(Sql);
+                await Task.Delay(TimeSpan.FromMinutes(1));
+            }
+        }
+
         private async Task KillOrphanedJobs()
         {
             try

Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Wabbajack.BuildServer.Models.JobQueue;
@@ -69,6 +70,43 @@ namespace Wabbajack.BuildServer.Models.Jobs
 
             return JobResult.Success();
         }
+
+        public static DateTime LastNexusSync { get; set; } = DateTime.Now;
+        public static async Task<long> UpdateNexusCacheFast(SqlService sql)
+        {
+            var results = await NexusUpdatesFeeds.GetUpdates();
+            NexusApiClient client = null;
+            long updated = 0;
+            foreach (var result in results)
+            {
+                var purgedMods = await sql.DeleteNexusModFilesUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
+                var purgedFiles = await sql.DeleteNexusModInfosUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
+
+                var totalPurged = purgedFiles + purgedMods;
+                if (totalPurged > 0)
+                    Utils.Log($"Purged {totalPurged} cache items");
+
+                if (await sql.GetNexusModInfoString(result.Game, result.ModId) != null) continue;
+
+                // Lazily create the client
+                client ??= await NexusApiClient.Get();
+                
+                // Cache the info
+                var files = await client.GetModFiles(result.Game, result.ModId, false);
+                await sql.AddNexusModFiles(result.Game, result.ModId, result.TimeStamp, files);
+                
+                var modInfo = await client.GetModInfo(result.Game, result.ModId);
+                await sql.AddNexusModInfo(result.Game, result.ModId, result.TimeStamp, modInfo);
+                updated++;
+
+            }
+
+            if (updated > 0) 
+                Utils.Log($"Primed {updated} nexus cache entries");
+
+            LastNexusSync = DateTime.Now;
+            return updated;
+        }
         
 
     }

Wabbajack.BuildServer/Models/Sql/SqlService.cs

@@ -575,7 +575,7 @@ namespace Wabbajack.BuildServer.Model.Models
         {
             await using var conn = await Open();
             var deleted = await conn.ExecuteScalarAsync<long>(
-                @"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date
+                @"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
                       SELECT @@ROWCOUNT AS Deleted",
                 new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
             return deleted;
@@ -585,9 +585,9 @@ namespace Wabbajack.BuildServer.Model.Models
         {
             await using var conn = await Open();
             var deleted = await conn.ExecuteScalarAsync<long>(
-                @"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date
+                @"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
                       SELECT @@ROWCOUNT AS Deleted",
-                new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
+                new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
             return deleted;
         }
 

Wabbajack.Common/Json.cs

@@ -27,11 +27,14 @@ namespace Wabbajack.Common
         };
         
         public static JsonSerializerSettings JsonSettings  =>
-                    new JsonSerializerSettings {
-            TypeNameHandling = TypeNameHandling.Objects,
-            SerializationBinder = new JsonNameSerializationBinder(),
-            Converters = Converters};
-        
+            new JsonSerializerSettings {
+                TypeNameHandling = TypeNameHandling.Objects,
+                SerializationBinder = new JsonNameSerializationBinder(),
+                Converters = Converters};
+
+        public static JsonSerializerSettings GenericJsonSettings =>
+            new JsonSerializerSettings { };
+
 
         public static void ToJson<T>(this T obj, string filename)
         {
@@ -73,11 +76,11 @@ namespace Wabbajack.Common
             return JsonConvert.DeserializeObject<T>(data, JsonSettings)!;
         }
 
-        public static T FromJson<T>(this Stream stream)
+        public static T FromJson<T>(this Stream stream, bool genericReader = false)
         {
             using var tr = new StreamReader(stream, Encoding.UTF8, leaveOpen: true);
             using var reader = new JsonTextReader(tr);
-            var ser = JsonSerializer.Create(JsonSettings);
+            var ser = JsonSerializer.Create(genericReader ? GenericJsonSettings : JsonSettings);
             var result = ser.Deserialize<T>(reader);
             if (result == null)
                 throw new JsonException("Type deserialized into null");

Wabbajack.Lib/NexusApi/NexusApi.cs

@@ -252,7 +252,7 @@ namespace Wabbajack.Lib.NexusApi
 
 
                 await using var stream = await response.Content.ReadAsStreamAsync();
-                return stream.FromJson<T>();
+                return stream.FromJson<T>(genericReader:true);
             }
             catch (TimeoutException)
             {
@@ -321,10 +321,11 @@ namespace Wabbajack.Lib.NexusApi
             public List<NexusFileInfo> files { get; set; }
         }
 
-        public async Task<GetModFilesResponse> GetModFiles(Game game, long modid)
+        public async Task<GetModFilesResponse> GetModFiles(Game game, long modid, bool useCache = true)
+        
         {
             var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modid}/files.json";
-            var result = await GetCached<GetModFilesResponse>(url);
+            var result = useCache ? await GetCached<GetModFilesResponse>(url) : await Get<GetModFilesResponse>(url);
             if (result.files == null)
                 throw new InvalidOperationException("Got Null data from the Nexus while finding mod files");
             return result;
@@ -336,10 +337,15 @@ namespace Wabbajack.Lib.NexusApi
             return await Get<List<MD5Response>>(url);
         }
 
-        public async Task<ModInfo> GetModInfo(Game game, long modId)
+        public async Task<ModInfo> GetModInfo(Game game, long modId, bool useCache = true)
         {
             var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modId}.json";
-            return await GetCached<ModInfo>(url);
+            if (useCache)
+            {
+                return await GetCached<ModInfo>(url);
+            }
+
+            return await Get<ModInfo>(url);
         }
 
         private class DownloadLink

Wabbajack.Lib/NexusApi/NexusUpdatesFeeds.cs

@@ -0,0 +1,80 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.ServiceModel.Syndication;
+using System.Threading.Tasks;
+using System.Xml;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.NexusApi
+{
+    public class NexusUpdatesFeeds
+    {
+
+        public static async Task<List<UpdateRecord>> GetUpdates()
+        {
+            var updated = GetFeed(new Uri("https://www.nexusmods.com/rss/updatedtoday"));
+            var newToday = GetFeed(new Uri("https://www.nexusmods.com/rss/newtoday"));
+
+            var sorted = (await updated).Concat(await newToday).OrderByDescending(f => f.TimeStamp);
+            var deduped = sorted.GroupBy(g => (g.Game, g.ModId)).Select(g => g.First()).ToList();
+            return deduped;
+        }
+
+        private static bool TryParseGameUrl(SyndicationLink link, out Game game, out long modId)
+        {
+            var parts = link.Uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries);
+
+            var foundGame = GameRegistry.GetByFuzzyName(parts[0]);
+            if (foundGame == null)
+            {
+                game = Game.Oblivion;
+                modId = 0;
+                return false;
+            }
+
+            if (long.TryParse(parts[2], out modId))
+            {
+                game = foundGame.Game;
+                return true;
+            }
+
+            game = Game.Oblivion;
+            modId = 0;
+            return false;
+        }
+
+        private static async Task<IEnumerable<UpdateRecord>> GetFeed(Uri uri)
+        {
+            var client = new Common.Http.Client();
+            var data = await client.GetStringAsync(uri);
+            var reader = XmlReader.Create(new StringReader(data));
+            var results = SyndicationFeed.Load(reader);
+            return results.Items
+                .Select(itm =>
+                {
+                    if (TryParseGameUrl(itm.Links.First(), out var game, out var modId))
+                    {
+                        return new UpdateRecord
+                        {
+                            TimeStamp = itm.PublishDate.UtcDateTime, 
+                            Game = game, 
+                            ModId = modId
+                        };
+                    }
+
+                    return null;
+                }).Where(v => v != null);
+        }
+
+
+        public class UpdateRecord
+        {
+            public Game Game { get; set; }
+            public long ModId { get; set; }
+            public DateTime TimeStamp { get; set; }
+        }
+            
+    }
+}

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -51,6 +51,9 @@
       <PackageReference Include="System.Net.Http">
         <Version>4.3.4</Version>
       </PackageReference>
+      <PackageReference Include="System.ServiceModel.Syndication">
+        <Version>4.7.0</Version>
+      </PackageReference>
       <PackageReference Include="WebSocketSharp-netstandard">
         <Version>1.0.1</Version>
       </PackageReference>

Wabbajack.Test/NexusTests.cs

@@ -0,0 +1,32 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.Lib.NexusApi;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Wabbajack.Test
+{
+    public class NexusTests : ATestBase
+    {
+        [Fact]
+        public async Task CanGetNexusRSSUpdates()
+        {
+            var results = (await NexusUpdatesFeeds.GetUpdates()).ToArray();
+            
+            Assert.NotEmpty(results);
+
+            Utils.Log($"Loaded {results.Length} updates from the Nexus");
+
+            foreach (var result in results)
+            {
+                Assert.True(DateTime.UtcNow - result.TimeStamp < TimeSpan.FromDays(1));
+            }
+        }
+
+        public NexusTests(ITestOutputHelper output) : base(output)
+        {
+        }
+    }
+}