diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index 60c74bf1..29183caf 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -165,13 +165,7 @@ namespace Wabbajack.Lib return; var b = await metaState.LoadMetaData(); - Utils.Log(b - ? $"Getting meta data for {a.Name} was successful!" - : $"Getting meta data for {a.Name} failed!"); - } - else - { - Utils.Log($"Archive {a.Name} is not an AbstractMetaState!"); + if (b) Utils.Log($"Getting meta data for {a.Name} was successful!"); } }); diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs index eab8b57a..20a68d19 100644 --- a/Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs @@ -140,7 +140,7 @@ namespace Wabbajack.Lib.Downloaders } var data = await Utils.FromEncryptedJson(EncryptedKeyName); - await data.Refresh(); + await data.Refresh(SiteName); var client = new Http.Client(); client.Headers.Add(("Authorization", $"Bearer {data.AccessToken}")); return client; @@ -240,6 +240,7 @@ namespace Wabbajack.Lib.Downloaders public string? Description { get; set; } public async Task LoadMetaData() { + if (IsAttachment) return false; var data = await TypedDownloader.GetDownloads(IPS4Mod); Name = data.Title; Author = data.Author?.Name; @@ -321,7 +322,7 @@ namespace Wabbajack.Lib.Downloaders } - public async Task Refresh() + public async Task Refresh(string siteName = "") { if (ExpiresAt > DateTime.UtcNow + TimeSpan.FromHours(6)) return true; @@ -333,13 +334,28 @@ namespace Wabbajack.Lib.Downloaders new ("refresh_token", RefreshToken), new ("client_id", ClientID) }; - using var response = await client.PostAsync(TokenEndpoint!.ToString(), new FormUrlEncodedContent(formData.ToList())); - var responseData = (await response.Content.ReadAsStringAsync()).FromJsonString(); + try + { + using var response = await client.PostAsync(TokenEndpoint!.ToString(), + new FormUrlEncodedContent(formData.ToList())); + var responseData = (await response.Content.ReadAsStringAsync()).FromJsonString(); + + AccessToken = responseData.AccessToken; + ExpiresIn = responseData.ExpiresIn; + ExpiresAt = DateTime.UtcNow + TimeSpan.FromSeconds(ExpiresIn); + + return true; + } + catch (HttpException ex) + { + if (ex.Code == 400) + { + throw new CriticalFailureIntervention( + $"You have been logged out of {siteName} for reasons out of our control, please log back in via the settings panel", + $"Logged out of {siteName}"); + } + } - AccessToken = responseData.AccessToken; - ExpiresIn = responseData.ExpiresIn; - ExpiresAt = DateTime.UtcNow + TimeSpan.FromSeconds(ExpiresIn); - return true; } diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 6073d2b9..dd097cd7 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -224,15 +224,21 @@ namespace Wabbajack.Lib.Downloaders var nclient = DownloadDispatcher.GetInstance(); await nclient.Prepare(); var client = nclient.Client!; - var file = await client.GetModFile(Game, ModID, FileID); - return file?.category_name != null; + + var modInfo = await client.GetModInfo(Game, ModID); + if (!modInfo.available) return false; + var modFiles = await client.GetModFiles(Game, ModID); + + var found = modFiles.files + .FirstOrDefault(file => file.file_id == FileID && file.category_name != null); + + return found != null; } catch (Exception ex) { Utils.Log($"{Name} - {Game} - {ModID} - {FileID} - Error getting Nexus download URL - {ex}"); return false; } - } public override IDownloader GetDownloader() diff --git a/Wabbajack.Lib/Http/ClientFactory.cs b/Wabbajack.Lib/Http/ClientFactory.cs index 4f120efd..67aca46b 100644 --- a/Wabbajack.Lib/Http/ClientFactory.cs +++ b/Wabbajack.Lib/Http/ClientFactory.cs @@ -25,7 +25,6 @@ namespace Wabbajack.Lib.Http AutomaticDecompression = DecompressionMethods.All, }; - Utils.Log($"Configuring with SSL {_socketsHandler.SslOptions.EnabledSslProtocols}"); ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, errors) => diff --git a/Wabbajack.Server.Test/NexusCacheTests.cs b/Wabbajack.Server.Test/NexusCacheTests.cs index f4f8d97a..2c3cdfbb 100644 --- a/Wabbajack.Server.Test/NexusCacheTests.cs +++ b/Wabbajack.Server.Test/NexusCacheTests.cs @@ -103,12 +103,22 @@ namespace Wabbajack.BuildServer.Test h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449); Assert.True(found != default); + Assert.True(found.LastChecked > startTime && found.LastChecked < DateTime.UtcNow); + // Delete with exactly the same date, shouldn't clear out the record + await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, found.LastChecked); var hs2 = await sql.AllNexusFiles(); - + var found2 = hs2.FirstOrDefault(h => h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449); Assert.True(found != default); + + Assert.True(found2.LastChecked == found.LastChecked); + + // Delete all the records, it should now be gone + await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow); + var hs3 = await sql.AllNexusFiles(); + Assert.DoesNotContain(hs3, f => f.NexusGameId == gameId && f.ModId == 1137); } diff --git a/Wabbajack.Server/DTOs/ValidationData.cs b/Wabbajack.Server/DTOs/ValidationData.cs index 1c761058..1752f256 100644 --- a/Wabbajack.Server/DTOs/ValidationData.cs +++ b/Wabbajack.Server/DTOs/ValidationData.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Threading.Tasks; using Wabbajack.Common; @@ -10,7 +9,7 @@ namespace Wabbajack.Server.DTOs { public class ValidationData { - public Dictionary<(long Game, long ModId, long FileId), string> NexusFiles { get; set; } = new (); + public ConcurrentHashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; } = new ConcurrentHashSet<(long Game, long ModId, long FileId)>(); public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; } public List ModLists { get; set; } diff --git a/Wabbajack.Server/DataLayer/ValidationData.cs b/Wabbajack.Server/DataLayer/ValidationData.cs index 50fff123..c647d988 100644 --- a/Wabbajack.Server/DataLayer/ValidationData.cs +++ b/Wabbajack.Server/DataLayer/ValidationData.cs @@ -23,7 +23,7 @@ namespace Wabbajack.Server.DataLayer var nexusFiles = await AllNexusFiles(); return new ValidationData { - NexusFiles = nexusFiles.ToDictionary(nf => (nf.NexusGameId, nf.ModId, nf.FileId), nf => nf.category), + NexusFiles = new ConcurrentHashSet<(long Game, long ModId, long FileId)>(nexusFiles.Select(f => (f.NexusGameId, f.ModId, f.FileId))), ArchiveStatus = await archiveStatus, ModLists = await modLists, Mirrors = await mirrors, @@ -41,10 +41,17 @@ namespace Wabbajack.Server.DataLayer return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3); } - public async Task> AllNexusFiles() + public async Task> AllNexusFiles() { await using var conn = await Open(); - var results = await conn.QueryAsync<(long, long, long, string)>(@"SELECT Game, ModId, FileId, JSON_VALUE(Data, '$.category_name') FROM dbo.NexusModFile"); + var results = await conn.QueryAsync<(long, long, long, DateTime)>(@"SELECT Game, ModId, p.file_id, LastChecked + FROM [NexusModFiles] files + CROSS APPLY + OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p + WHERE p.category is not null + UNION + SELECT GameId, ModId, FileId, LastChecked FROM dbo.NexusModFilesSlow + "); return results.ToHashSet(); } diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index e0a301be..aa628035 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -102,9 +102,6 @@ namespace Wabbajack.Server.Services await _sql.StartMirror((archive.Hash, reason)); return (archive, ArchiveStatus.Updating); } - - if (archive.State is NexusDownloader.State) - return (archive, result); return await TryToHeal(data, archive, metadata); } @@ -352,9 +349,9 @@ namespace Wabbajack.Server.Services case GoogleDriveDownloader.State _: // Disabled for now due to GDrive rate-limiting the build server return (archive, ArchiveStatus.Valid); - case NexusDownloader.State nexusState when data.NexusFiles.TryGetValue( - (nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID), out var category): - return (archive, category != null ? ArchiveStatus.Valid : ArchiveStatus.InValid); + case NexusDownloader.State nexusState when data.NexusFiles.Contains(( + nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)): + return (archive, ArchiveStatus.Valid); case NexusDownloader.State ns: return (archive, await FastNexusModStats(ns)); case ManualDownloader.State _: @@ -365,10 +362,6 @@ namespace Wabbajack.Server.Services return (archive, ArchiveStatus.Valid); case MediaFireDownloader.State _: return (archive, ArchiveStatus.Valid); - case DeprecatedLoversLabDownloader.State _: - return (archive, ArchiveStatus.Valid); - case DeprecatedVectorPlexusDownloader.State _: - return (archive, ArchiveStatus.Valid); default: { if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash), @@ -381,45 +374,94 @@ namespace Wabbajack.Server.Services } } } + + private AsyncLock _lock = new(); public async Task FastNexusModStats(NexusDownloader.State ns) { // Check if some other thread has added them - var file = await _sql.GetModFile(ns.Game, ns.ModID, ns.FileID); + var mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID); + var files = await _sql.GetModFiles(ns.Game, ns.ModID); - if (file == null) + if (mod == null || files == null) { - try + // Acquire the lock + //using var lck = await _lock.WaitAsync(); + + // Check again + mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID); + files = await _sql.GetModFiles(ns.Game, ns.ModID); + + if (mod == null || files == null) { - NexusApiClient nexusClient = await _nexus.GetClient(); - var queryTime = DateTime.UtcNow; - _logger.Log(LogLevel.Information, "Found missing Nexus file info {Game} {ModID} {FileID}", ns.Game, ns.ModID, ns.FileID); - try - { - file = await nexusClient.GetModFile(ns.Game, ns.ModID, ns.FileID, false); - } - catch - { - file = new NexusFileInfo() {category_name = null}; - } try { - await _sql.AddNexusModFile(ns.Game, ns.ModID, ns.FileID, queryTime, file); + NexusApiClient nexusClient = await _nexus.GetClient(); + var queryTime = DateTime.UtcNow; + + if (mod == null) + { + _logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}"); + try + { + mod = await nexusClient.GetModInfo(ns.Game, ns.ModID, false); + } + catch (Exception ex) + { + Utils.Log("Exception in Nexus Validation " + ex); + mod = new ModInfo + { + mod_id = ns.ModID.ToString(), + game_id = ns.Game.MetaData().NexusGameId, + available = false + }; + } + + try + { + await _sql.AddNexusModInfo(ns.Game, ns.ModID, queryTime, mod); + } + catch (Exception) + { + // Could be a PK constraint failure + } + + } + + if (files == null) + { + _logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}"); + try + { + files = await nexusClient.GetModFiles(ns.Game, ns.ModID, false); + } + catch + { + files = new NexusApiClient.GetModFilesResponse {files = new List()}; + } + + try + { + await _sql.AddNexusModFiles(ns.Game, ns.ModID, queryTime, files); + } + catch (Exception) + { + // Could be a PK constraint failure + } + } } catch (Exception) { - // Could be a PK constraint failure + return ArchiveStatus.InValid; } } - catch (Exception) - { - return ArchiveStatus.InValid; - } } - return file?.category_name != null ? ArchiveStatus.Valid : ArchiveStatus.InValid; + if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID)) + return ArchiveStatus.Valid; + return ArchiveStatus.InValid; } } diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 1c8654b4..4dbecfe1 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -384,6 +384,10 @@ namespace Wabbajack.Test await converted.Download(new Archive(state: null!) { Name = "LoversLab Test.txt" }, filename.Path); Assert.Equal(Hash.FromBase64("gLJDxGDaeQ0="), await filename.Path.FileHashAsync()); + + Assert.False(await ((LoversLabOAuthDownloader.State) state).LoadMetaData()); + + }