From 24d10305a4739d3690e5a7fd983cc33a3569a3f5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 4 Jul 2021 15:37:55 -0600 Subject: [PATCH] Re-enable nexus archived file support again --- Wabbajack.Lib/Downloaders/NexusDownloader.cs | 16 +-- Wabbajack.Server.Test/NexusCacheTests.cs | 16 --- Wabbajack.Server/DTOs/ValidationData.cs | 2 +- Wabbajack.Server/DataLayer/Nexus.cs | 1 + Wabbajack.Server/DataLayer/ValidationData.cs | 13 +-- Wabbajack.Server/Services/ListValidator.cs | 104 ++++++------------- Wabbajack.Server/Startup.cs | 2 + 7 files changed, 43 insertions(+), 111 deletions(-) diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index dd097cd7..765189a6 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -221,18 +221,12 @@ namespace Wabbajack.Lib.Downloaders { try { - var nclient = DownloadDispatcher.GetInstance(); - await nclient.Prepare(); - var client = nclient.Client!; + var nexusClient = DownloadDispatcher.GetInstance(); + await nexusClient.Prepare(); + var client = nexusClient.Client!; - var modInfo = await client.GetModInfo(Game, ModID); - if (!modInfo.available) return false; - var modFiles = await client.GetModFiles(Game, ModID); - - var found = modFiles.files - .FirstOrDefault(file => file.file_id == FileID && file.category_name != null); - - return found != null; + var file = await client.GetModFile(Game, ModID, FileID); + return file?.category_name != null; } catch (Exception ex) { diff --git a/Wabbajack.Server.Test/NexusCacheTests.cs b/Wabbajack.Server.Test/NexusCacheTests.cs index 2c3cdfbb..b77078c3 100644 --- a/Wabbajack.Server.Test/NexusCacheTests.cs +++ b/Wabbajack.Server.Test/NexusCacheTests.cs @@ -102,23 +102,7 @@ namespace Wabbajack.BuildServer.Test var found = hs.FirstOrDefault(h => h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449); Assert.True(found != default); - - Assert.True(found.LastChecked > startTime && found.LastChecked < DateTime.UtcNow); - // Delete with exactly the same date, shouldn't clear out the record - await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, found.LastChecked); - var hs2 = await sql.AllNexusFiles(); - - var found2 = hs2.FirstOrDefault(h => - h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449); - Assert.True(found != default); - - Assert.True(found2.LastChecked == found.LastChecked); - - // Delete all the records, it should now be gone - await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow); - var hs3 = await sql.AllNexusFiles(); - Assert.DoesNotContain(hs3, f => f.NexusGameId == gameId && f.ModId == 1137); } diff --git a/Wabbajack.Server/DTOs/ValidationData.cs b/Wabbajack.Server/DTOs/ValidationData.cs index 1752f256..e8bcc11a 100644 --- a/Wabbajack.Server/DTOs/ValidationData.cs +++ b/Wabbajack.Server/DTOs/ValidationData.cs @@ -9,7 +9,7 @@ namespace Wabbajack.Server.DTOs { public class ValidationData { - public ConcurrentHashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; } = new ConcurrentHashSet<(long Game, long ModId, long FileId)>(); + public Dictionary<(long Game, long ModId, long FileId), string> NexusFiles { get; set; } = new (); public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; } public List ModLists { get; set; } diff --git a/Wabbajack.Server/DataLayer/Nexus.cs b/Wabbajack.Server/DataLayer/Nexus.cs index fbfaae93..24042141 100644 --- a/Wabbajack.Server/DataLayer/Nexus.cs +++ b/Wabbajack.Server/DataLayer/Nexus.cs @@ -118,6 +118,7 @@ namespace Wabbajack.Server.DataLayer await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId}); await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId}); await conn.ExecuteAsync("DELETE FROM dbo.NexusModPermissions WHERE ModId = @ModId", new {ModId = modId}); + await conn.ExecuteAsync("DELETE FROM dbo.NexusModFile WHERE ModId = @ModID", new {ModId = modId}); } public async Task> GetNexusPermissions() diff --git a/Wabbajack.Server/DataLayer/ValidationData.cs b/Wabbajack.Server/DataLayer/ValidationData.cs index c647d988..50fff123 100644 --- a/Wabbajack.Server/DataLayer/ValidationData.cs +++ b/Wabbajack.Server/DataLayer/ValidationData.cs @@ -23,7 +23,7 @@ namespace Wabbajack.Server.DataLayer var nexusFiles = await AllNexusFiles(); return new ValidationData { - NexusFiles = new ConcurrentHashSet<(long Game, long ModId, long FileId)>(nexusFiles.Select(f => (f.NexusGameId, f.ModId, f.FileId))), + NexusFiles = nexusFiles.ToDictionary(nf => (nf.NexusGameId, nf.ModId, nf.FileId), nf => nf.category), ArchiveStatus = await archiveStatus, ModLists = await modLists, Mirrors = await mirrors, @@ -41,17 +41,10 @@ namespace Wabbajack.Server.DataLayer return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3); } - public async Task> AllNexusFiles() + public async Task> AllNexusFiles() { await using var conn = await Open(); - var results = await conn.QueryAsync<(long, long, long, DateTime)>(@"SELECT Game, ModId, p.file_id, LastChecked - FROM [NexusModFiles] files - CROSS APPLY - OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p - WHERE p.category is not null - UNION - SELECT GameId, ModId, FileId, LastChecked FROM dbo.NexusModFilesSlow - "); + var results = await conn.QueryAsync<(long, long, long, string)>(@"SELECT Game, ModId, FileId, JSON_VALUE(Data, '$.category_name') FROM dbo.NexusModFile"); return results.ToHashSet(); } diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index aa628035..d44318bb 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -41,6 +41,7 @@ namespace Wabbajack.Server.Services public override async Task Execute() { var data = await _sql.GetValidationData(); + _logger.LogInformation("Found {count} nexus files", data.NexusFiles.Count); using var queue = new WorkQueue(); var oldSummaries = Summaries; @@ -102,6 +103,8 @@ namespace Wabbajack.Server.Services await _sql.StartMirror((archive.Hash, reason)); return (archive, ArchiveStatus.Updating); } + if (archive.State is NexusDownloader.State) + return (archive, result); return await TryToHeal(data, archive, metadata); } @@ -349,9 +352,9 @@ namespace Wabbajack.Server.Services case GoogleDriveDownloader.State _: // Disabled for now due to GDrive rate-limiting the build server return (archive, ArchiveStatus.Valid); - case NexusDownloader.State nexusState when data.NexusFiles.Contains(( - nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)): - return (archive, ArchiveStatus.Valid); + case NexusDownloader.State nexusState when data.NexusFiles.TryGetValue( + (nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID), out var category): + return (archive, category != null ? ArchiveStatus.Valid : ArchiveStatus.InValid); case NexusDownloader.State ns: return (archive, await FastNexusModStats(ns)); case ManualDownloader.State _: @@ -362,6 +365,10 @@ namespace Wabbajack.Server.Services return (archive, ArchiveStatus.Valid); case MediaFireDownloader.State _: return (archive, ArchiveStatus.Valid); + case DeprecatedLoversLabDownloader.State _: + return (archive, ArchiveStatus.InValid); + case DeprecatedVectorPlexusDownloader.State _: + return (archive, ArchiveStatus.InValid); default: { if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash), @@ -375,93 +382,44 @@ namespace Wabbajack.Server.Services } } - private AsyncLock _lock = new(); - public async Task FastNexusModStats(NexusDownloader.State ns) { // Check if some other thread has added them - var mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID); - var files = await _sql.GetModFiles(ns.Game, ns.ModID); + var file = await _sql.GetModFile(ns.Game, ns.ModID, ns.FileID); - if (mod == null || files == null) + if (file == null) { - // Acquire the lock - //using var lck = await _lock.WaitAsync(); - - // Check again - mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID); - files = await _sql.GetModFiles(ns.Game, ns.ModID); - - if (mod == null || files == null) + try { + NexusApiClient nexusClient = await _nexus.GetClient(); + var queryTime = DateTime.UtcNow; + _logger.Log(LogLevel.Information, "Found missing Nexus file info {Game} {ModID} {FileID}", ns.Game, ns.ModID, ns.FileID); + try + { + file = await nexusClient.GetModFile(ns.Game, ns.ModID, ns.FileID, false); + } + catch + { + file = new NexusFileInfo() {category_name = null}; + } try { - NexusApiClient nexusClient = await _nexus.GetClient(); - var queryTime = DateTime.UtcNow; - - if (mod == null) - { - _logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}"); - try - { - mod = await nexusClient.GetModInfo(ns.Game, ns.ModID, false); - } - catch (Exception ex) - { - Utils.Log("Exception in Nexus Validation " + ex); - mod = new ModInfo - { - mod_id = ns.ModID.ToString(), - game_id = ns.Game.MetaData().NexusGameId, - available = false - }; - } - - try - { - await _sql.AddNexusModInfo(ns.Game, ns.ModID, queryTime, mod); - } - catch (Exception) - { - // Could be a PK constraint failure - } - - } - - if (files == null) - { - _logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}"); - try - { - files = await nexusClient.GetModFiles(ns.Game, ns.ModID, false); - } - catch - { - files = new NexusApiClient.GetModFilesResponse {files = new List()}; - } - - try - { - await _sql.AddNexusModFiles(ns.Game, ns.ModID, queryTime, files); - } - catch (Exception) - { - // Could be a PK constraint failure - } - } + await _sql.AddNexusModFile(ns.Game, ns.ModID, ns.FileID, queryTime, file); } catch (Exception) { - return ArchiveStatus.InValid; + // Could be a PK constraint failure } } + catch (Exception) + { + return ArchiveStatus.InValid; + } } - if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID)) - return ArchiveStatus.Valid; - return ArchiveStatus.InValid; + return file?.category_name != null ? ArchiveStatus.Valid : ArchiveStatus.InValid; } } diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 8adc1905..0134e570 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -13,6 +13,7 @@ using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Hosting; using Newtonsoft.Json; using Wabbajack.BuildServer; +using Wabbajack.Common; using Wabbajack.Lib.LibCefHelpers; using Wabbajack.Server.DataLayer; using Wabbajack.Server.Services; @@ -37,6 +38,7 @@ namespace Wabbajack.Server // This method gets called by the runtime. Use this method to add services to the container. public void ConfigureServices(IServiceCollection services) { + Consts.UseNetworkWorkaroundMode = true; Helpers.Init(); /*services.AddSwaggerGen(c => {