Server side fixes for the new Nexus API

Wabbajack.Lib/Downloaders/AbstractDownloadState.cs

@@ -27,6 +27,7 @@ namespace Wabbajack.Lib.Downloaders
         public static List<Type> KnownSubTypes = new List<Type>
         {
             typeof(DeprecatedLoversLabDownloader.State),
+            typeof(DeprecatedVectorPlexusDownloader),
             typeof(HTTPDownloader.State),
             typeof(GameFileSourceDownloader.State),
             typeof(GoogleDriveDownloader.State),

Wabbajack.Lib/Downloaders/NexusDownloader.cs

@@ -74,8 +74,12 @@ namespace Wabbajack.Lib.Downloaders
                 }
                 catch (Exception)
                 {
-                    Utils.Error($"Error getting mod info for Nexus mod with {general.modID}");
+                    return new State
-                    throw;
+                    {
+                        Game = GameRegistry.GetByFuzzyName((string)general.gameName).Game,
+                        ModID = long.Parse(general.modID),
+                        FileID = long.Parse(general.fileID),
+                    };
                 }
 
                 try
@@ -220,15 +224,8 @@ namespace Wabbajack.Lib.Downloaders
                     var nclient = DownloadDispatcher.GetInstance<NexusDownloader>();
                     await nclient.Prepare();
                     var client = nclient.Client!;
-
+                    var file = await client.GetModFile(Game, ModID, FileID);
-                    var modInfo = await client.GetModInfo(Game, ModID);
+                    return file?.category_name != null;
-                    if (!modInfo.available) return false;
-                    var modFiles = await client.GetModFiles(Game, ModID);
-
-                    var found = modFiles.files
-                        .FirstOrDefault(file => file.file_id == FileID && file.category_name != null);
-
-                    return found != null;
                 }
                 catch (Exception ex)
                 {
@@ -245,7 +242,7 @@ namespace Wabbajack.Lib.Downloaders
 
             public override string GetManifestURL(Archive a)
             {
-                return $"http://nexusmods.com/{Game.MetaData().NexusName}/mods/{ModID}";
+                return $"https://www.nexusmods.com/{Game.MetaData().NexusName}/mods/{ModID}/?tab=files&file_id={FileID}";
             }
 
             public override string[] GetMetaIni()

Wabbajack.Lib/NexusApi/INexusApi.cs

@@ -8,6 +8,7 @@ namespace Wabbajack.Lib.NexusApi
     {
         public Task<string> GetNexusDownloadLink(NexusDownloader.State archive);
         public Task<NexusApiClient.GetModFilesResponse> GetModFiles(Game game, long modid, bool useCache = true);
+        public Task<NexusFileInfo> GetModFile(Game game, long modid, long fileId, bool useCache = true);
         public Task<ModInfo> GetModInfo(Game game, long modId, bool useCache = true);
 
         public Task<UserStatus> GetUserStatus();

Wabbajack.Lib/NexusApi/NexusApi.cs

@@ -357,6 +357,13 @@ namespace Wabbajack.Lib.NexusApi
             return result;
         }
 
+        public async Task<NexusFileInfo> GetModFile(Game game, long modId, long fileId, bool useCache = true)
+        {
+            var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modId}/files/{fileId}.json";
+            var result = useCache ? await GetCached<NexusFileInfo>(url) : await Get<NexusFileInfo>(url);
+            return result;
+        }
+
         public async Task<List<MD5Response>> GetModInfoFromMD5(Game game, string md5Hash)
         {
             var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/md5_search/{md5Hash}.json";

Wabbajack.Server.Test/NexusCacheTests.cs

@@ -103,23 +103,13 @@ namespace Wabbajack.BuildServer.Test
                 h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449);
             Assert.True(found != default);
             
-            Assert.True(found.LastChecked > startTime && found.LastChecked < DateTime.UtcNow);
-
             // Delete with exactly the same date, shouldn't clear out the record
-            await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, found.LastChecked);
             var hs2 = await sql.AllNexusFiles();
            
             var found2 = hs2.FirstOrDefault(h =>
                 h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449);
             Assert.True(found != default);
             
-            Assert.True(found2.LastChecked == found.LastChecked);
-            
-            // Delete all the records, it should now be gone
-            await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow);
-            var hs3 = await sql.AllNexusFiles();
-            Assert.DoesNotContain(hs3, f => f.NexusGameId == gameId && f.ModId == 1137);
-
         }
 
 

Wabbajack.Server.Test/sql/wabbajack_db.sql

@@ -806,6 +806,29 @@ CREATE TABLE [dbo].[NexusModFiles](
 )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
 ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
 GO
+    
+/****** Object:  Table [dbo].[NexusModFile]    Script Date: 6/24/2021 2:39:17 PM ******/
+SET ANSI_NULLS ON
+GO
+
+SET QUOTED_IDENTIFIER ON
+GO
+
+CREATE TABLE [dbo].[NexusModFile](
+                                     [Game] [int] NOT NULL,
+                                     [ModId] [bigint] NOT NULL,
+                                     [FileId] [bigint] NOT NULL,
+                                     [Data] [nvarchar](max) NOT NULL,
+	[LastChecked] [datetime] NOT NULL,
+ CONSTRAINT [PK_NexusModFile] PRIMARY KEY CLUSTERED 
+(
+	[Game] ASC,
+	[ModId] ASC,
+	[FileId] ASC
+)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
+GO
+    
 /****** Object:  Table [dbo].[NexusModFilesSlow]    Script Date: 3/9/2021 11:12:53 PM ******/
 SET ANSI_NULLS ON
 GO

Wabbajack.Server/Controllers/Heartbeat.cs

@@ -41,7 +41,7 @@ namespace Wabbajack.BuildServer.Controllers
         }
 
         private const int MAX_LOG_SIZE = 128;
-        private static List<string> Log  = new List<string>();
+        private static List<string> Log  = new();
         private GlobalInformation _globalInformation;
         private SqlService _sql;
         private ILogger<Heartbeat> _logger;

Wabbajack.Server/Controllers/NexusCache.cs

@@ -126,6 +126,43 @@ namespace Wabbajack.BuildServer.Controllers
             Response.Headers.Add("x-cache-result", method);
             return result;
         }
+        
+        [HttpGet]
+        [Route("{GameName}/mods/{ModId}/files/{FileId}.json")]
+        public async Task<ActionResult<NexusFileInfo>> GetModFile(string GameName, long ModId, long FileId)
+        {
+            try
+            {
+                var game = GameRegistry.GetByFuzzyName(GameName).Game;
+                var result = await _sql.GetModFile(game, ModId, FileId);
+
+                string method = "CACHED";
+                if (result == null)
+                {
+                    var api = await GetClient();
+                    result = await api.GetModFile(game, ModId, FileId, false);
+
+                    var date = result.uploaded_time;
+                    date = date == default ? DateTime.UtcNow : date;
+                    await _sql.AddNexusModFile(game, ModId, FileId, date, result);
+
+                    method = "NOT_CACHED";
+                    Interlocked.Increment(ref ForwardCount);
+                }
+                else
+                {
+                    Interlocked.Increment(ref CachedCount);
+                }
+
+                Response.Headers.Add("x-cache-result", method);
+                return result;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogInformation("Unable to find mod file {GameName} {ModId}, {FileId}", GameName, ModId, FileId);
+                return NotFound();
+            }
+        }
 
         [HttpGet]
         [Authorize(Roles ="Author")]

Wabbajack.Server/DTOs/ValidationData.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using Wabbajack.Common;
@@ -9,7 +10,7 @@ namespace Wabbajack.Server.DTOs
 {
     public class ValidationData
     {
-        public ConcurrentHashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; } = new ConcurrentHashSet<(long Game, long ModId, long FileId)>();
+        public Dictionary<(long Game, long ModId, long FileId), string> NexusFiles { get; set; } = new ();
         public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; }
         public List<ModlistMetadata> ModLists { get; set; }
         

Wabbajack.Server/DataLayer/Nexus.cs

@@ -195,5 +195,32 @@ namespace Wabbajack.Server.DataLayer
 
             await tx.CommitAsync();
         }
+
+        public async Task<NexusFileInfo> GetModFile(Game game, long modId, long fileId)
+        {
+            await using var conn = await Open();
+            var result = await conn.QueryFirstOrDefaultAsync<string>(
+                "SELECT Data FROM dbo.NexusModFile WHERE Game = @Game AND @ModId = ModId AND @FileId = FileId",
+                new {Game = game.MetaData().NexusGameId, ModId = modId, FileId = fileId});
+            return result == null ? null : JsonConvert.DeserializeObject<NexusFileInfo>(result);
+        }
+        
+        public async Task AddNexusModFile(Game game, long modId, long fileId, DateTime lastCheckedUtc, NexusFileInfo data)
+        {
+            await using var conn = await Open();
+
+            await conn.ExecuteAsync(                
+                @"INSERT INTO dbo.NexusModFile (Game, ModId, FileId, LastChecked, Data)
+                     VALUES (@Game, @ModId, @FileId, @LastChecked, @Data)",
+                new
+                {
+                    Game = game.MetaData().NexusGameId,
+                    ModId = modId,
+                    FileId = fileId,
+                    LastChecked = lastCheckedUtc,
+                    Data = JsonConvert.SerializeObject(data)
+                });
+        }
+        
     }
 }

Wabbajack.Server/DataLayer/ValidationData.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
@@ -14,14 +15,15 @@ namespace Wabbajack.Server.DataLayer
     {
         public async Task<ValidationData> GetValidationData()
         {
-            var nexusFiles = AllNexusFiles();
+
             var archiveStatus = AllModListArchivesStatus();
             var modLists = AllModLists();
             var mirrors = GetAllMirroredHashes();
             var authoredFiles = AllAuthoredFiles();
+            var nexusFiles = await AllNexusFiles();
             return new ValidationData
             {
-                NexusFiles = new ConcurrentHashSet<(long Game, long ModId, long FileId)>((await nexusFiles).Select(f => (f.NexusGameId, f.ModId, f.FileId))),
+                NexusFiles = nexusFiles.ToDictionary(nf => (nf.NexusGameId, nf.ModId, nf.FileId), nf => nf.category),
                 ArchiveStatus = await archiveStatus,
                 ModLists = await modLists,
                 Mirrors = await mirrors,
@@ -39,17 +41,10 @@ namespace Wabbajack.Server.DataLayer
             return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3);
         }
 
-        public async Task<HashSet<(long NexusGameId, long ModId, long FileId, DateTime LastChecked)>> AllNexusFiles()
+        public async Task<HashSet<(long NexusGameId, long ModId, long FileId, string category)>> AllNexusFiles()
         {
             await using var conn = await Open();
-            var results = await conn.QueryAsync<(long, long, long, DateTime)>(@"SELECT Game, ModId, p.file_id, LastChecked
+            var results = await conn.QueryAsync<(long, long, long, string)>(@"SELECT Game, ModId, FileId, JSON_VALUE(Data, '$.category') FROM dbo.NexusModFile");
-          FROM [NexusModFiles] files
-          CROSS APPLY
-          OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p 
-          WHERE p.category is not null
-          UNION 
-          SELECT GameId, ModId, FileId, LastChecked FROM dbo.NexusModFilesSlow
-          ");
             return results.ToHashSet();
         }
         

Wabbajack.Server/Services/ListValidator.cs

@@ -102,6 +102,9 @@ namespace Wabbajack.Server.Services
                                 await _sql.StartMirror((archive.Hash, reason));
                                 return (archive, ArchiveStatus.Updating);
                             }
+
+                            if (archive.State is NexusDownloader.State)
+                                return (archive, result);
                             return await TryToHeal(data, archive, metadata);
                         }
 
@@ -349,9 +352,9 @@ namespace Wabbajack.Server.Services
                 case GoogleDriveDownloader.State _:
                     // Disabled for now due to GDrive rate-limiting the build server
                     return (archive, ArchiveStatus.Valid);
-                case NexusDownloader.State nexusState when data.NexusFiles.Contains((
+                case NexusDownloader.State nexusState when data.NexusFiles.TryGetValue(
-                    nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)):
+                    (nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID), out var category):
-                    return (archive, ArchiveStatus.Valid);
+                    return (archive, category != null ? ArchiveStatus.Valid : ArchiveStatus.InValid);
                 case NexusDownloader.State ns:
                     return (archive, await FastNexusModStats(ns));
                 case ManualDownloader.State _:
@@ -362,6 +365,10 @@ namespace Wabbajack.Server.Services
                     return (archive, ArchiveStatus.Valid);
                 case MediaFireDownloader.State _:
                     return (archive, ArchiveStatus.Valid);
+                case DeprecatedLoversLabDownloader.State _:
+                    return (archive, ArchiveStatus.Valid);
+                case DeprecatedVectorPlexusDownloader.State _:
+                    return (archive, ArchiveStatus.Valid);
                 default:
                 {
                     if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash),
@@ -374,94 +381,45 @@ namespace Wabbajack.Server.Services
                 }
             }
         }
-        
-        private AsyncLock _lock = new();
 
         public async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns)
         {
             // Check if some other thread has added them
-            var mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID);
+            var file = await _sql.GetModFile(ns.Game, ns.ModID, ns.FileID);
-            var files = await _sql.GetModFiles(ns.Game, ns.ModID);
 
-            if (mod == null || files == null)
+            if (file == null)
             {
-                // Acquire the lock
+                try
-                using var lck = await _lock.WaitAsync();
-                
-                // Check again
-                mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID);
-                files = await _sql.GetModFiles(ns.Game, ns.ModID);
-
-                if (mod == null || files == null)
                 {
+                    NexusApiClient nexusClient = await _nexus.GetClient();
+                    var queryTime = DateTime.UtcNow;
 
+                    _logger.Log(LogLevel.Information, "Found missing Nexus file info {Game} {ModID} {FileID}", ns.Game, ns.ModID, ns.FileID);
+                    try
+                    {
+                        file = await nexusClient.GetModFile(ns.Game, ns.ModID, ns.FileID, false);
+                    }
+                    catch
+                    {
+                        file = new NexusFileInfo() {category_name = null};
+                    }
 
                     try
                     {
-                        NexusApiClient nexusClient = await _nexus.GetClient();
+                        await _sql.AddNexusModFile(ns.Game, ns.ModID, ns.FileID, queryTime, file);
-                        var queryTime = DateTime.UtcNow;
-
-                        if (mod == null)
-                        {
-                            _logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
-                            try
-                            {
-                                mod = await nexusClient.GetModInfo(ns.Game, ns.ModID, false);
-                            }
-                            catch (Exception ex)
-                            {
-                                Utils.Log("Exception in Nexus Validation " + ex);
-                                mod = new ModInfo
-                                {
-                                    mod_id = ns.ModID.ToString(),
-                                    game_id = ns.Game.MetaData().NexusGameId,
-                                    available = false
-                                };
-                            }
-
-                            try
-                            {
-                                await _sql.AddNexusModInfo(ns.Game, ns.ModID, queryTime, mod);
-                            }
-                            catch (Exception)
-                            {
-                                // Could be a PK constraint failure
-                            }
-
-                        }
-
-                        if (files == null)
-                        {
-                            _logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
-                            try
-                            {
-                                files = await nexusClient.GetModFiles(ns.Game, ns.ModID, false);
-                            }
-                            catch
-                            {
-                                files = new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>()};
-                            }
-
-                            try
-                            {
-                                await _sql.AddNexusModFiles(ns.Game, ns.ModID, queryTime, files);
-                            }
-                            catch (Exception)
-                            {
-                                // Could be a PK constraint failure
-                            }
-                        }
                     }
                     catch (Exception)
                     {
-                        return ArchiveStatus.InValid;
+                        // Could be a PK constraint failure
                     }
                 }
+                catch (Exception)
+                {
+                    return ArchiveStatus.InValid;
+                }
             }
 
-            if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID))
+            return file?.category_name != null ? ArchiveStatus.Valid : ArchiveStatus.InValid;
-                return ArchiveStatus.Valid;
-            return ArchiveStatus.InValid;
 
         }
     }

Wabbajack.Test/DownloaderTests.cs

@@ -256,6 +256,11 @@ namespace Wabbajack.Test
             await converted.Download(new Archive(state: null!) { Name = "SkyUI.7z" }, filename.Path);
 
             Assert.Equal(Hash.FromBase64("dF2yafV2Oks="), await filename.Path.FileHashAsync());
+
+            // Verify that we can see a older file
+            var data = await (await NexusApiClient.Get()).GetModFile(Game.SkyrimSpecialEdition, 45221, 185392, useCache:false);
+            Assert.Equal("Smooth Combat - Non Combat Animation System 2.3", data.name);
+            
         }
 
         [Fact]