Nexus caching re-implemented on the server

Wabbajack.Networking.NexusApi/DTOs/ModInfo.cs

@@ -16,7 +16,7 @@ public class Endorsement
 {
     [JsonPropertyName("endorse_status")] public string EndorseStatus { get; set; } = "";
 
-    [JsonPropertyName("timestamp")] public int Timestamp { get; set; }
+    [JsonPropertyName("timestamp")] public string? Timestamp { get; set; }
 
     [JsonPropertyName("version")] public string Version { get; set; } = "";
 }

Wabbajack.Server/AppSettings.cs

@@ -22,6 +22,7 @@ public class AppSettings
     
     public string PatchesFilesFolder { get; set; }
     public string MirrorFilesFolder { get; set; }
+    public string NexusCacheFolder { get; set; }
     public string MetricsFolder { get; set; } = "";
     public string TarLogPath { get; set; }
     public string GitHubKey { get; set; } = "";

Wabbajack.Server/Controllers/NexusCache.cs

@@ -1,12 +1,18 @@
 using System;
 using System.Net.Http;
+using System.Text.Json;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
+using Wabbajack.Common;
+using Wabbajack.DTOs.JsonConverters;
 using Wabbajack.Networking.NexusApi;
 using Wabbajack.Networking.NexusApi.DTOs;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+using Wabbajack.Server.Services;
 
 namespace Wabbajack.BuildServer.Controllers;
 
@@ -20,12 +26,17 @@ public class NexusCache : ControllerBase
     private readonly ILogger<NexusCache> _logger;
     private AppSettings _settings;
     private readonly HttpClient _client;
+    private readonly AbsolutePath _cacheFolder;
+    private readonly DTOSerializer _dtos;
+    private readonly NexusCacheManager _cache;
 
-    public NexusCache(ILogger<NexusCache> logger,AppSettings settings, HttpClient client)
+    public NexusCache(ILogger<NexusCache> logger,AppSettings settings, HttpClient client, NexusCacheManager cache, DTOSerializer dtos)
     {
         _settings = settings;
         _logger = logger;
         _client = client;
+        _cache = cache;
+        _dtos = dtos;
     }
 
     private async Task ForwardToNexus(HttpRequest src)
@@ -40,6 +51,16 @@ public class NexusCache : ControllerBase
         await response.Content.CopyToAsync(Response.Body);
     }
 
+    private async Task<T> ForwardRequest<T>(HttpRequest src, CancellationToken token)
+    {
+        _logger.LogInformation("Nexus Cache Forwarding: {path}", src.Path);
+        var request = new HttpRequestMessage(HttpMethod.Get, (Uri?)new Uri("https://api.nexusmods.com/" + src.Path));
+        request.Headers.Add("apikey", (string?)src.Headers["apikey"]);
+        request.Headers.Add("User-Agent", (string?)src.Headers.UserAgent);
+        using var response = await _client.SendAsync(request, token);
+        return (await JsonSerializer.DeserializeAsync<T>(await response.Content.ReadAsStreamAsync(token), _dtos.Options, token))!;
+    }
+
     /// <summary>
     ///     Looks up the mod details for a given Gamename/ModId pair. If the entry is not found in the cache it will
     ///     be requested from the server (using the caller's Nexus API key if provided).
@@ -50,22 +71,41 @@ public class NexusCache : ControllerBase
     /// <returns>A Mod Info result</returns>
     [HttpGet]
     [Route("{GameName}/mods/{ModId}.json")]
-    public async Task GetModInfo(string GameName, long ModId)
+    public async Task GetModInfo(string GameName, long ModId, CancellationToken token)
     {
-        await ForwardToNexus(Request);
+        var key = $"modinfo_{GameName}_{ModId}";
+        await ReturnCachedResult<ModInfo>(key, token);
+    }
+
+    private async Task ReturnCachedResult<T>(string key, CancellationToken token)
+    {
+        key = key.ToLowerInvariant();
+        var cached = await _cache.GetCache<T>(key, token);
+        if (cached == null)
+        {
+            var returned = await ForwardRequest<T>(Request, token);
+            await _cache.SaveCache(key, returned, token);
+            Response.StatusCode = 200;
+            Response.ContentType = "application/json";
+            await JsonSerializer.SerializeAsync(Response.Body, returned, _dtos.Options, cancellationToken: token);
+        }
+
+        await JsonSerializer.SerializeAsync(Response.Body, cached, _dtos.Options, cancellationToken: token);
     }
 
     [HttpGet]
     [Route("{GameName}/mods/{ModId}/files.json")]
-    public async Task GetModFiles(string GameName, long ModId)
+    public async Task GetModFiles(string GameName, long ModId, CancellationToken token)
     {
-        await ForwardToNexus(Request);
+        var key = $"modfiles_{GameName}_{ModId}";
+        await ReturnCachedResult<ModFiles>(key, token);
     }
 
     [HttpGet]
     [Route("{GameName}/mods/{ModId}/files/{FileId}.json")]
-    public async Task GetModFile(string GameName, long ModId, long FileId)
+    public async Task GetModFile(string GameName, long ModId, long FileId, CancellationToken token)
     {
-        await ForwardToNexus(Request);
+        var key = $"modfile_{GameName}_{ModId}_{FileId}";
+        await ReturnCachedResult<ModFile>(key, token);
     }
 }

Wabbajack.Server/Services/NexusCacheManager.cs

@@ -0,0 +1,148 @@
+using System.Text.Json;
+using K4os.Compression.LZ4.Internal;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer;
+using Wabbajack.Common;
+using Wabbajack.DTOs;
+using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Networking.NexusApi;
+using Wabbajack.Networking.NexusApi.DTOs;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+using Wabbajack.Server.DTOs;
+
+namespace Wabbajack.Server.Services;
+
+public class NexusCacheManager
+{
+    private readonly ILogger<NexusCacheManager> _loggger;
+    private readonly DTOSerializer _dtos;
+    private readonly AppSettings _configuration;
+    private readonly AbsolutePath _cacheFolder;
+    private readonly SemaphoreSlim _lockObject;
+    private readonly NexusApi _nexusAPI;
+    private readonly Timer _timer;
+    private readonly DiscordWebHook _discord;
+
+    public NexusCacheManager(ILogger<NexusCacheManager> logger, DTOSerializer dtos, AppSettings configuration, NexusApi nexusApi, DiscordWebHook discord)
+    {
+        _loggger = logger;
+        _dtos = dtos;
+        _configuration = configuration;
+        _cacheFolder = configuration.NexusCacheFolder.ToAbsolutePath();
+        _lockObject = new SemaphoreSlim(1);
+        _nexusAPI = nexusApi;
+        _discord = discord;
+
+        _timer = new Timer(_ => UpdateNexusCacheAPI().FireAndForget(), null, TimeSpan.FromSeconds(2),
+            TimeSpan.FromHours(4));
+    }
+    
+    
+    private AbsolutePath CacheFile(string key)
+    {
+        return _cacheFolder.Combine(key).WithExtension(Ext.Json);
+    }
+
+        
+    private bool HaveCache(string key)
+    {
+        return CacheFile(key).FileExists();
+    }
+
+    public async Task SaveCache<T>(string key, T value, CancellationToken token)
+    {
+        var ms = new MemoryStream();
+        await JsonSerializer.SerializeAsync(ms, value, _dtos.Options, token);
+        await ms.FlushAsync(token);
+        var data = ms.ToArray();
+        await _lockObject.WaitAsync(token);
+        try
+        {
+            await CacheFile(key).WriteAllBytesAsync(data, token: token);
+        }
+        finally
+        {
+            _lockObject.Release();
+        }
+    }
+
+    public async Task<T?> GetCache<T>(string key, CancellationToken token)
+    {
+        if (!HaveCache(key)) return default;
+        
+        var file = CacheFile(key);
+        await _lockObject.WaitAsync(token);
+        byte[] data;
+        try
+        {
+            data = await file.ReadAllBytesAsync(token);
+        }
+        catch (FileNotFoundException ex)
+        {
+            return default;
+        }
+        finally
+        {
+            _lockObject.Release();
+        }
+        return await JsonSerializer.DeserializeAsync<T>(new MemoryStream(data), _dtos.Options, token);
+    }
+
+    public async Task UpdateNexusCacheAPI()
+    {
+        var gameTasks = GameRegistry.Games.Values
+            .Where(g => g.NexusName != null)
+            .SelectAsync(async game =>
+            {
+                    var mods = await _nexusAPI.GetUpdates(game.Game, CancellationToken.None);
+                    return (game, mods); 
+            });
+
+
+        var purgeList = new List<(string Key, DateTime Date)>();
+        
+        await foreach (var (game, mods) in gameTasks)
+        {
+            foreach (var mod in mods.Item1)
+            {
+                var date = Math.Max(mod.LastestModActivity, mod.LatestFileUpdate).AsUnixTime();
+                purgeList.Add(($"_{game.Game.MetaData().NexusName!.ToLowerInvariant()}_{mod.ModId}_", date));
+            }
+        }
+
+        // This is O(m * n) where n and m are 15,000 items, we really should improve this
+        var files = (from file in _cacheFolder.EnumerateFiles().AsParallel()
+            from entry in purgeList
+            where file.FileName.ToString().Contains(entry.Key)
+            where file.LastModifiedUtc() < entry.Date
+            select file).ToHashSet();
+
+        foreach (var file in files)
+        {
+            await PurgeCacheEntry(file);
+        }
+        
+        await _discord.Send(Channel.Ham, new DiscordMessage
+        {
+            Content = $"Cleared {files.Count} Nexus cache entries due to updates"
+        });
+    }
+
+    private async Task PurgeCacheEntry(AbsolutePath file)
+    {
+        await _lockObject.WaitAsync();
+        try
+        {
+            if (file.FileExists()) file.Delete();
+        }
+        catch (FileNotFoundException)
+        {
+            return;
+        }
+        finally
+        {
+            _lockObject.Release();
+        }
+    }
+}

Wabbajack.Server/Startup.cs

@@ -1,6 +1,7 @@
 using System.Diagnostics;
 using System.IO;
 using System.Net.Http;
+using System.Runtime.InteropServices;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
@@ -16,11 +17,18 @@ using Nettle.Compiler;
 using Newtonsoft.Json;
 using Octokit;
 using Wabbajack.BuildServer;
+using Wabbajack.DTOs;
 using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.DTOs.Logins;
 using Wabbajack.Networking.GitHub;
+using Wabbajack.Networking.Http.Interfaces;
+using Wabbajack.Networking.NexusApi;
+using Wabbajack.Paths;
+using Wabbajack.RateLimiter;
 using Wabbajack.Server.DataModels;
 using Wabbajack.Server.Extensions;
 using Wabbajack.Server.Services;
+using Wabbajack.Services.OSIntegrated.TokenProviders;
 
 namespace Wabbajack.Server;
 
@@ -66,6 +74,27 @@ public class Startup
         services.AddSingleton<AuthorFiles>();
         services.AddSingleton<AuthorKeys>();
         services.AddSingleton<Client>();
+        services.AddSingleton<NexusCacheManager>();
+        services.AddSingleton<NexusApi>();
+        services.AddAllSingleton<ITokenProvider<NexusApiState>, NexusApiTokenProvider>();
+        services.AddAllSingleton<IResource, IResource<HttpClient>>(s => new Resource<HttpClient>("Web Requests", 12));
+        // Application Info
+        
+        var version =
+            $"{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Patch}{ThisAssembly.Git.SemVer.DashLabel}";
+        services.AddSingleton(s => new ApplicationInfo
+        {
+            ApplicationSlug = "Wabbajack",
+            ApplicationName = Environment.ProcessPath?.ToAbsolutePath().FileName.ToString() ?? "Wabbajack",
+            ApplicationSha = ThisAssembly.Git.Sha,
+            Platform = RuntimeInformation.ProcessArchitecture.ToString(),
+            OperatingSystemDescription = RuntimeInformation.OSDescription,
+            RuntimeIdentifier = RuntimeInformation.RuntimeIdentifier,
+            OSVersion = Environment.OSVersion.VersionString,
+            Version = version
+        });
+
+        
         services.AddResponseCaching();
         services.AddSingleton(s =>
         {
@@ -147,5 +176,8 @@ public class Startup
         });
 
         app.UseEndpoints(endpoints => { endpoints.MapControllers(); });
+
+        // Trigger the internal update code
+        var _ = app.ApplicationServices.GetRequiredService<NexusCacheManager>();
     }
 }

Wabbajack.Server/Wabbajack.Server.csproj

@@ -12,6 +12,7 @@
         <PackageReference Include="Dapper" Version="2.0.123" />
         <PackageReference Include="Discord.Net.WebSocket" Version="2.4.0" />
         <PackageReference Include="FluentFTP" Version="35.0.5" />
+        <PackageReference Include="GitInfo" Version="2.2.0" />
         <PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
         <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.0" />
         <PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
@@ -61,6 +62,7 @@
         <ProjectReference Include="..\Wabbajack.Networking.NexusApi\Wabbajack.Networking.NexusApi.csproj" />
         <ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
         <ProjectReference Include="..\Wabbajack.Paths\Wabbajack.Paths.csproj" />
+        <ProjectReference Include="..\Wabbajack.Services.OSIntegrated\Wabbajack.Services.OSIntegrated.csproj" />
     </ItemGroup>
 
 

Wabbajack.Server/appsettings.json

@@ -13,6 +13,7 @@
     "AuthorAPIKeyFile": "c:\\tmp\\author_keys.txt",
     "PatchesFilesFolder": "c:\\tmp\\patches",
     "MirrorFilesFolder": "c:\\tmp\\mirrors",
+    "NexusCacheFolder": "c:\\tmp\\nexus-cache",
     "GitHubKey": ""
   },
   "AllowedHosts": "*"