Fixes for nexus cache priming

Wabbajack.BuildServer.Test/JobTests.cs

@@ -47,7 +47,15 @@ namespace Wabbajack.BuildServer.Test
                 Assert.Null(await sql.GetModFiles(record.Game, record.ModId));
                 Assert.Null(await sql.GetNexusModInfoString(record.Game, record.ModId));
             }
+        }
 
+        [Fact]
+        public async Task CanPrimeTheNexusCache()
+        {
+            var sql = Fixture.GetService<SqlService>();
+
+            Assert.True(await GetNexusUpdatesJob.UpdateNexusCacheFast(sql) > 0);
+            Assert.True(await GetNexusUpdatesJob.UpdateNexusCacheFast(sql) == 0);
         }
     }
 }

Wabbajack.BuildServer/Controllers/Heartbeat.cs

@@ -5,6 +5,7 @@ using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using Wabbajack.BuildServer.Model.Models;
+using Wabbajack.BuildServer.Models.Jobs;
 using Wabbajack.Common.StatusFeed;
 
 namespace Wabbajack.BuildServer.Controllers
@@ -36,9 +37,13 @@ namespace Wabbajack.BuildServer.Controllers
         }
 
         [HttpGet]
-        public async Task<TimeSpan> GetHeartbeat()
+        public async Task<IActionResult> GetHeartbeat()
         {
-            return DateTime.Now - _startTime;
+            return Ok(new
+            {
+                Uptime = DateTime.Now - _startTime,
+                LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync
+            });
         }
 
         [HttpGet("only-authenticated")]

Wabbajack.BuildServer/Controllers/NexusCache.cs

@@ -49,10 +49,10 @@ namespace Wabbajack.BuildServer.Controllers
             if (result == null)
             {
                 var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
-                var path = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{ModId}.json";
+                result = await api.GetModInfo(game, ModId, false);
-                var body = await api.Get<ModInfo>(path);
+                await SQL.AddNexusModInfo(game, ModId, DateTime.UtcNow, result);
-                await SQL.AddNexusModInfo(game, ModId, DateTime.Now, body);
 
+                
                 method = "NOT_CACHED";
                 Interlocked.Increment(ref ForwardCount);
             }
@@ -78,10 +78,8 @@ namespace Wabbajack.BuildServer.Controllers
             if (result == null)
             {
                 var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
-                var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files.json";
+                result = await api.GetModFiles(game, ModId, false);
-                var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
+                await SQL.AddNexusModFiles(game, ModId, DateTime.UtcNow, result);
-                await SQL.AddNexusModFiles(game, ModId, DateTime.Now, body);
-
 
                 method = "NOT_CACHED";
                 Interlocked.Increment(ref ForwardCount);

Wabbajack.BuildServer/JobManager.cs

@@ -10,6 +10,7 @@ using Wabbajack.BuildServer.Models;
 using Wabbajack.BuildServer.Models.JobQueue;
 using Wabbajack.BuildServer.Models.Jobs;
 using Wabbajack.Common;
+using Wabbajack.Lib.NexusApi;
 
 namespace Wabbajack.BuildServer
 {
@@ -74,6 +75,9 @@ namespace Wabbajack.BuildServer
             Utils.LogMessages.Subscribe(Heartbeat.AddToLog);
             Utils.LogMessages.OfType<IUserIntervention>().Subscribe(u => u.Cancel());
             if (!Settings.JobScheduler) return;
+
+            var task = RunNexusCacheLoop();
+            
             while (true)
             {
                 await KillOrphanedJobs();
@@ -86,6 +90,15 @@ namespace Wabbajack.BuildServer
             }
         }
 
+        private async Task RunNexusCacheLoop()
+        {
+            while (true)
+            {
+                await GetNexusUpdatesJob.UpdateNexusCacheFast(Sql);
+                await Task.Delay(TimeSpan.FromMinutes(1));
+            }
+        }
+
         private async Task KillOrphanedJobs()
         {
             try

Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Wabbajack.BuildServer.Models.JobQueue;
@@ -69,6 +70,43 @@ namespace Wabbajack.BuildServer.Models.Jobs
 
             return JobResult.Success();
         }
+
+        public static DateTime LastNexusSync { get; set; } = DateTime.Now;
+        public static async Task<long> UpdateNexusCacheFast(SqlService sql)
+        {
+            var results = await NexusUpdatesFeeds.GetUpdates();
+            NexusApiClient client = null;
+            long updated = 0;
+            foreach (var result in results)
+            {
+                var purgedMods = await sql.DeleteNexusModFilesUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
+                var purgedFiles = await sql.DeleteNexusModInfosUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
+
+                var totalPurged = purgedFiles + purgedMods;
+                if (totalPurged > 0)
+                    Utils.Log($"Purged {totalPurged} cache items");
+
+                if (await sql.GetNexusModInfoString(result.Game, result.ModId) != null) continue;
+
+                // Lazily create the client
+                client ??= await NexusApiClient.Get();
+                
+                // Cache the info
+                var files = await client.GetModFiles(result.Game, result.ModId, false);
+                await sql.AddNexusModFiles(result.Game, result.ModId, result.TimeStamp, files);
+                
+                var modInfo = await client.GetModInfo(result.Game, result.ModId);
+                await sql.AddNexusModInfo(result.Game, result.ModId, result.TimeStamp, modInfo);
+                updated++;
+
+            }
+
+            if (updated > 0) 
+                Utils.Log($"Primed {updated} nexus cache entries");
+
+            LastNexusSync = DateTime.Now;
+            return updated;
+        }
         
 
     }

Wabbajack.BuildServer/Models/Sql/SqlService.cs

@@ -575,7 +575,7 @@ namespace Wabbajack.BuildServer.Model.Models
         {
             await using var conn = await Open();
             var deleted = await conn.ExecuteScalarAsync<long>(
-                @"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date
+                @"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
                       SELECT @@ROWCOUNT AS Deleted",
                 new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
             return deleted;
@@ -585,9 +585,9 @@ namespace Wabbajack.BuildServer.Model.Models
         {
             await using var conn = await Open();
             var deleted = await conn.ExecuteScalarAsync<long>(
-                @"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date
+                @"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
                       SELECT @@ROWCOUNT AS Deleted",
-                new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
+                new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
             return deleted;
         }
 

Wabbajack.Common/Json.cs

@@ -27,11 +27,14 @@ namespace Wabbajack.Common
         };
         
         public static JsonSerializerSettings JsonSettings  =>
-                    new JsonSerializerSettings {
+            new JsonSerializerSettings {
-            TypeNameHandling = TypeNameHandling.Objects,
+                TypeNameHandling = TypeNameHandling.Objects,
-            SerializationBinder = new JsonNameSerializationBinder(),
+                SerializationBinder = new JsonNameSerializationBinder(),
-            Converters = Converters};
+                Converters = Converters};
-        
+
+        public static JsonSerializerSettings GenericJsonSettings =>
+            new JsonSerializerSettings { };
+
 
         public static void ToJson<T>(this T obj, string filename)
         {
@@ -73,11 +76,11 @@ namespace Wabbajack.Common
             return JsonConvert.DeserializeObject<T>(data, JsonSettings)!;
         }
 
-        public static T FromJson<T>(this Stream stream)
+        public static T FromJson<T>(this Stream stream, bool genericReader = false)
         {
             using var tr = new StreamReader(stream, Encoding.UTF8, leaveOpen: true);
             using var reader = new JsonTextReader(tr);
-            var ser = JsonSerializer.Create(JsonSettings);
+            var ser = JsonSerializer.Create(genericReader ? GenericJsonSettings : JsonSettings);
             return ser.Deserialize<T>(reader);
         }
       

Wabbajack.Lib/NexusApi/NexusApi.cs

@@ -252,7 +252,7 @@ namespace Wabbajack.Lib.NexusApi
 
 
                 await using var stream = await response.Content.ReadAsStreamAsync();
-                return stream.FromJson<T>();
+                return stream.FromJson<T>(genericReader:true);
             }
             catch (TimeoutException)
             {
@@ -321,10 +321,11 @@ namespace Wabbajack.Lib.NexusApi
             public List<NexusFileInfo> files { get; set; }
         }
 
-        public async Task<GetModFilesResponse> GetModFiles(Game game, long modid)
+        public async Task<GetModFilesResponse> GetModFiles(Game game, long modid, bool useCache = true)
+        
         {
             var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modid}/files.json";
-            var result = await GetCached<GetModFilesResponse>(url);
+            var result = useCache ? await GetCached<GetModFilesResponse>(url) : await Get<GetModFilesResponse>(url);
             if (result.files == null)
                 throw new InvalidOperationException("Got Null data from the Nexus while finding mod files");
             return result;
@@ -336,10 +337,15 @@ namespace Wabbajack.Lib.NexusApi
             return await Get<List<MD5Response>>(url);
         }
 
-        public async Task<ModInfo> GetModInfo(Game game, long modId)
+        public async Task<ModInfo> GetModInfo(Game game, long modId, bool useCache = true)
         {
             var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modId}.json";
-            return await GetCached<ModInfo>(url);
+            if (useCache)
+            {
+                return await GetCached<ModInfo>(url);
+            }
+
+            return await Get<ModInfo>(url);
         }
 
         private class DownloadLink