From ab49b08f19ad27b2c85b9412562c75146af062b3 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 24 Jun 2021 17:01:03 -0600 Subject: [PATCH] Server side fixes for the new Nexus API --- .../Downloaders/AbstractDownloadState.cs | 1 + Wabbajack.Lib/Downloaders/NexusDownloader.cs | 21 ++-- Wabbajack.Lib/NexusApi/INexusApi.cs | 1 + Wabbajack.Lib/NexusApi/NexusApi.cs | 7 ++ Wabbajack.Server.Test/NexusCacheTests.cs | 10 -- Wabbajack.Server.Test/sql/wabbajack_db.sql | 23 ++++ Wabbajack.Server/Controllers/Heartbeat.cs | 2 +- Wabbajack.Server/Controllers/NexusCache.cs | 37 +++++++ Wabbajack.Server/DTOs/ValidationData.cs | 3 +- Wabbajack.Server/DataLayer/Nexus.cs | 27 +++++ Wabbajack.Server/DataLayer/ValidationData.cs | 17 +-- Wabbajack.Server/Services/ListValidator.cs | 104 ++++++------------ Wabbajack.Test/DownloaderTests.cs | 5 + 13 files changed, 150 insertions(+), 108 deletions(-) diff --git a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs index 6974a5c6..6353d678 100644 --- a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs +++ b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs @@ -27,6 +27,7 @@ namespace Wabbajack.Lib.Downloaders public static List KnownSubTypes = new List { typeof(DeprecatedLoversLabDownloader.State), + typeof(DeprecatedVectorPlexusDownloader), typeof(HTTPDownloader.State), typeof(GameFileSourceDownloader.State), typeof(GoogleDriveDownloader.State), diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 38a1639f..6073d2b9 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -74,8 +74,12 @@ namespace Wabbajack.Lib.Downloaders } catch (Exception) { - Utils.Error($"Error getting mod info for Nexus mod with {general.modID}"); - throw; + return new State + { + Game = GameRegistry.GetByFuzzyName((string)general.gameName).Game, + ModID = long.Parse(general.modID), + FileID = long.Parse(general.fileID), + }; } try @@ -220,15 +224,8 @@ namespace Wabbajack.Lib.Downloaders var nclient = DownloadDispatcher.GetInstance(); await nclient.Prepare(); var client = nclient.Client!; - - var modInfo = await client.GetModInfo(Game, ModID); - if (!modInfo.available) return false; - var modFiles = await client.GetModFiles(Game, ModID); - - var found = modFiles.files - .FirstOrDefault(file => file.file_id == FileID && file.category_name != null); - - return found != null; + var file = await client.GetModFile(Game, ModID, FileID); + return file?.category_name != null; } catch (Exception ex) { @@ -245,7 +242,7 @@ namespace Wabbajack.Lib.Downloaders public override string GetManifestURL(Archive a) { - return $"http://nexusmods.com/{Game.MetaData().NexusName}/mods/{ModID}"; + return $"https://www.nexusmods.com/{Game.MetaData().NexusName}/mods/{ModID}/?tab=files&file_id={FileID}"; } public override string[] GetMetaIni() diff --git a/Wabbajack.Lib/NexusApi/INexusApi.cs b/Wabbajack.Lib/NexusApi/INexusApi.cs index a6d60907..bc4e65ad 100644 --- a/Wabbajack.Lib/NexusApi/INexusApi.cs +++ b/Wabbajack.Lib/NexusApi/INexusApi.cs @@ -8,6 +8,7 @@ namespace Wabbajack.Lib.NexusApi { public Task GetNexusDownloadLink(NexusDownloader.State archive); public Task GetModFiles(Game game, long modid, bool useCache = true); + public Task GetModFile(Game game, long modid, long fileId, bool useCache = true); public Task GetModInfo(Game game, long modId, bool useCache = true); public Task GetUserStatus(); diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index b636a31a..a18a27fd 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -357,6 +357,13 @@ namespace Wabbajack.Lib.NexusApi return result; } + public async Task GetModFile(Game game, long modId, long fileId, bool useCache = true) + { + var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modId}/files/{fileId}.json"; + var result = useCache ? await GetCached(url) : await Get(url); + return result; + } + public async Task> GetModInfoFromMD5(Game game, string md5Hash) { var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/md5_search/{md5Hash}.json"; diff --git a/Wabbajack.Server.Test/NexusCacheTests.cs b/Wabbajack.Server.Test/NexusCacheTests.cs index 0c3ae82c..f4f8d97a 100644 --- a/Wabbajack.Server.Test/NexusCacheTests.cs +++ b/Wabbajack.Server.Test/NexusCacheTests.cs @@ -103,23 +103,13 @@ namespace Wabbajack.BuildServer.Test h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449); Assert.True(found != default); - Assert.True(found.LastChecked > startTime && found.LastChecked < DateTime.UtcNow); - // Delete with exactly the same date, shouldn't clear out the record - await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, found.LastChecked); var hs2 = await sql.AllNexusFiles(); var found2 = hs2.FirstOrDefault(h => h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449); Assert.True(found != default); - Assert.True(found2.LastChecked == found.LastChecked); - - // Delete all the records, it should now be gone - await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow); - var hs3 = await sql.AllNexusFiles(); - Assert.DoesNotContain(hs3, f => f.NexusGameId == gameId && f.ModId == 1137); - } diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index bc2f2e8a..d79ea8e3 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -806,6 +806,29 @@ CREATE TABLE [dbo].[NexusModFiles]( )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] GO + +/****** Object: Table [dbo].[NexusModFile] Script Date: 6/24/2021 2:39:17 PM ******/ +SET ANSI_NULLS ON +GO + +SET QUOTED_IDENTIFIER ON +GO + +CREATE TABLE [dbo].[NexusModFile]( + [Game] [int] NOT NULL, + [ModId] [bigint] NOT NULL, + [FileId] [bigint] NOT NULL, + [Data] [nvarchar](max) NOT NULL, + [LastChecked] [datetime] NOT NULL, + CONSTRAINT [PK_NexusModFile] PRIMARY KEY CLUSTERED +( + [Game] ASC, + [ModId] ASC, + [FileId] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO + /****** Object: Table [dbo].[NexusModFilesSlow] Script Date: 3/9/2021 11:12:53 PM ******/ SET ANSI_NULLS ON GO diff --git a/Wabbajack.Server/Controllers/Heartbeat.cs b/Wabbajack.Server/Controllers/Heartbeat.cs index 2f86afa8..39d1b793 100644 --- a/Wabbajack.Server/Controllers/Heartbeat.cs +++ b/Wabbajack.Server/Controllers/Heartbeat.cs @@ -41,7 +41,7 @@ namespace Wabbajack.BuildServer.Controllers } private const int MAX_LOG_SIZE = 128; - private static List Log = new List(); + private static List Log = new(); private GlobalInformation _globalInformation; private SqlService _sql; private ILogger _logger; diff --git a/Wabbajack.Server/Controllers/NexusCache.cs b/Wabbajack.Server/Controllers/NexusCache.cs index 8829812b..d493fc9b 100644 --- a/Wabbajack.Server/Controllers/NexusCache.cs +++ b/Wabbajack.Server/Controllers/NexusCache.cs @@ -126,6 +126,43 @@ namespace Wabbajack.BuildServer.Controllers Response.Headers.Add("x-cache-result", method); return result; } + + [HttpGet] + [Route("{GameName}/mods/{ModId}/files/{FileId}.json")] + public async Task> GetModFile(string GameName, long ModId, long FileId) + { + try + { + var game = GameRegistry.GetByFuzzyName(GameName).Game; + var result = await _sql.GetModFile(game, ModId, FileId); + + string method = "CACHED"; + if (result == null) + { + var api = await GetClient(); + result = await api.GetModFile(game, ModId, FileId, false); + + var date = result.uploaded_time; + date = date == default ? DateTime.UtcNow : date; + await _sql.AddNexusModFile(game, ModId, FileId, date, result); + + method = "NOT_CACHED"; + Interlocked.Increment(ref ForwardCount); + } + else + { + Interlocked.Increment(ref CachedCount); + } + + Response.Headers.Add("x-cache-result", method); + return result; + } + catch (Exception ex) + { + _logger.LogInformation("Unable to find mod file {GameName} {ModId}, {FileId}", GameName, ModId, FileId); + return NotFound(); + } + } [HttpGet] [Authorize(Roles ="Author")] diff --git a/Wabbajack.Server/DTOs/ValidationData.cs b/Wabbajack.Server/DTOs/ValidationData.cs index 1752f256..1c761058 100644 --- a/Wabbajack.Server/DTOs/ValidationData.cs +++ b/Wabbajack.Server/DTOs/ValidationData.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Threading.Tasks; using Wabbajack.Common; @@ -9,7 +10,7 @@ namespace Wabbajack.Server.DTOs { public class ValidationData { - public ConcurrentHashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; } = new ConcurrentHashSet<(long Game, long ModId, long FileId)>(); + public Dictionary<(long Game, long ModId, long FileId), string> NexusFiles { get; set; } = new (); public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; } public List ModLists { get; set; } diff --git a/Wabbajack.Server/DataLayer/Nexus.cs b/Wabbajack.Server/DataLayer/Nexus.cs index 498f70a3..fbfaae93 100644 --- a/Wabbajack.Server/DataLayer/Nexus.cs +++ b/Wabbajack.Server/DataLayer/Nexus.cs @@ -195,5 +195,32 @@ namespace Wabbajack.Server.DataLayer await tx.CommitAsync(); } + + public async Task GetModFile(Game game, long modId, long fileId) + { + await using var conn = await Open(); + var result = await conn.QueryFirstOrDefaultAsync( + "SELECT Data FROM dbo.NexusModFile WHERE Game = @Game AND @ModId = ModId AND @FileId = FileId", + new {Game = game.MetaData().NexusGameId, ModId = modId, FileId = fileId}); + return result == null ? null : JsonConvert.DeserializeObject(result); + } + + public async Task AddNexusModFile(Game game, long modId, long fileId, DateTime lastCheckedUtc, NexusFileInfo data) + { + await using var conn = await Open(); + + await conn.ExecuteAsync( + @"INSERT INTO dbo.NexusModFile (Game, ModId, FileId, LastChecked, Data) + VALUES (@Game, @ModId, @FileId, @LastChecked, @Data)", + new + { + Game = game.MetaData().NexusGameId, + ModId = modId, + FileId = fileId, + LastChecked = lastCheckedUtc, + Data = JsonConvert.SerializeObject(data) + }); + } + } } diff --git a/Wabbajack.Server/DataLayer/ValidationData.cs b/Wabbajack.Server/DataLayer/ValidationData.cs index a344aab3..3937d089 100644 --- a/Wabbajack.Server/DataLayer/ValidationData.cs +++ b/Wabbajack.Server/DataLayer/ValidationData.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -14,14 +15,15 @@ namespace Wabbajack.Server.DataLayer { public async Task GetValidationData() { - var nexusFiles = AllNexusFiles(); + var archiveStatus = AllModListArchivesStatus(); var modLists = AllModLists(); var mirrors = GetAllMirroredHashes(); var authoredFiles = AllAuthoredFiles(); + var nexusFiles = await AllNexusFiles(); return new ValidationData { - NexusFiles = new ConcurrentHashSet<(long Game, long ModId, long FileId)>((await nexusFiles).Select(f => (f.NexusGameId, f.ModId, f.FileId))), + NexusFiles = nexusFiles.ToDictionary(nf => (nf.NexusGameId, nf.ModId, nf.FileId), nf => nf.category), ArchiveStatus = await archiveStatus, ModLists = await modLists, Mirrors = await mirrors, @@ -39,17 +41,10 @@ namespace Wabbajack.Server.DataLayer return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3); } - public async Task> AllNexusFiles() + public async Task> AllNexusFiles() { await using var conn = await Open(); - var results = await conn.QueryAsync<(long, long, long, DateTime)>(@"SELECT Game, ModId, p.file_id, LastChecked - FROM [NexusModFiles] files - CROSS APPLY - OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p - WHERE p.category is not null - UNION - SELECT GameId, ModId, FileId, LastChecked FROM dbo.NexusModFilesSlow - "); + var results = await conn.QueryAsync<(long, long, long, string)>(@"SELECT Game, ModId, FileId, JSON_VALUE(Data, '$.category') FROM dbo.NexusModFile"); return results.ToHashSet(); } diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 46f9fd80..e0a301be 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -102,6 +102,9 @@ namespace Wabbajack.Server.Services await _sql.StartMirror((archive.Hash, reason)); return (archive, ArchiveStatus.Updating); } + + if (archive.State is NexusDownloader.State) + return (archive, result); return await TryToHeal(data, archive, metadata); } @@ -349,9 +352,9 @@ namespace Wabbajack.Server.Services case GoogleDriveDownloader.State _: // Disabled for now due to GDrive rate-limiting the build server return (archive, ArchiveStatus.Valid); - case NexusDownloader.State nexusState when data.NexusFiles.Contains(( - nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)): - return (archive, ArchiveStatus.Valid); + case NexusDownloader.State nexusState when data.NexusFiles.TryGetValue( + (nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID), out var category): + return (archive, category != null ? ArchiveStatus.Valid : ArchiveStatus.InValid); case NexusDownloader.State ns: return (archive, await FastNexusModStats(ns)); case ManualDownloader.State _: @@ -362,6 +365,10 @@ namespace Wabbajack.Server.Services return (archive, ArchiveStatus.Valid); case MediaFireDownloader.State _: return (archive, ArchiveStatus.Valid); + case DeprecatedLoversLabDownloader.State _: + return (archive, ArchiveStatus.Valid); + case DeprecatedVectorPlexusDownloader.State _: + return (archive, ArchiveStatus.Valid); default: { if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash), @@ -374,94 +381,45 @@ namespace Wabbajack.Server.Services } } } - - private AsyncLock _lock = new(); public async Task FastNexusModStats(NexusDownloader.State ns) { // Check if some other thread has added them - var mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID); - var files = await _sql.GetModFiles(ns.Game, ns.ModID); + var file = await _sql.GetModFile(ns.Game, ns.ModID, ns.FileID); - if (mod == null || files == null) + if (file == null) { - // Acquire the lock - using var lck = await _lock.WaitAsync(); - - // Check again - mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID); - files = await _sql.GetModFiles(ns.Game, ns.ModID); - - if (mod == null || files == null) + try { + NexusApiClient nexusClient = await _nexus.GetClient(); + var queryTime = DateTime.UtcNow; + _logger.Log(LogLevel.Information, "Found missing Nexus file info {Game} {ModID} {FileID}", ns.Game, ns.ModID, ns.FileID); + try + { + file = await nexusClient.GetModFile(ns.Game, ns.ModID, ns.FileID, false); + } + catch + { + file = new NexusFileInfo() {category_name = null}; + } try { - NexusApiClient nexusClient = await _nexus.GetClient(); - var queryTime = DateTime.UtcNow; - - if (mod == null) - { - _logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}"); - try - { - mod = await nexusClient.GetModInfo(ns.Game, ns.ModID, false); - } - catch (Exception ex) - { - Utils.Log("Exception in Nexus Validation " + ex); - mod = new ModInfo - { - mod_id = ns.ModID.ToString(), - game_id = ns.Game.MetaData().NexusGameId, - available = false - }; - } - - try - { - await _sql.AddNexusModInfo(ns.Game, ns.ModID, queryTime, mod); - } - catch (Exception) - { - // Could be a PK constraint failure - } - - } - - if (files == null) - { - _logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}"); - try - { - files = await nexusClient.GetModFiles(ns.Game, ns.ModID, false); - } - catch - { - files = new NexusApiClient.GetModFilesResponse {files = new List()}; - } - - try - { - await _sql.AddNexusModFiles(ns.Game, ns.ModID, queryTime, files); - } - catch (Exception) - { - // Could be a PK constraint failure - } - } + await _sql.AddNexusModFile(ns.Game, ns.ModID, ns.FileID, queryTime, file); } catch (Exception) { - return ArchiveStatus.InValid; + // Could be a PK constraint failure } } + catch (Exception) + { + return ArchiveStatus.InValid; + } } - if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID)) - return ArchiveStatus.Valid; - return ArchiveStatus.InValid; + return file?.category_name != null ? ArchiveStatus.Valid : ArchiveStatus.InValid; } } diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 6675fe40..be921e9e 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -256,6 +256,11 @@ namespace Wabbajack.Test await converted.Download(new Archive(state: null!) { Name = "SkyUI.7z" }, filename.Path); Assert.Equal(Hash.FromBase64("dF2yafV2Oks="), await filename.Path.FileHashAsync()); + + // Verify that we can see a older file + var data = await (await NexusApiClient.Get()).GetModFile(Game.SkyrimSpecialEdition, 45221, 185392, useCache:false); + Assert.Equal("Smooth Combat - Non Combat Animation System 2.3", data.name); + } [Fact]