diff --git a/Compression.BSA/BSABuilder.cs b/Compression.BSA/BSABuilder.cs index 04246c72..dcaa2b49 100644 --- a/Compression.BSA/BSABuilder.cs +++ b/Compression.BSA/BSABuilder.cs @@ -280,7 +280,7 @@ namespace Compression.BSA case VersionType.SSE: { var r = new MemoryStream(); - await using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L12_MAX}, true)) + await using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L08_HC}, true)) { await _srcData.CopyToWithStatusAsync(_srcData.Length, w, $"Compressing {_path}"); } diff --git a/Wabbajack.App.Test/Wabbajack.App.Test.csproj b/Wabbajack.App.Test/Wabbajack.App.Test.csproj index fe0caef1..c3720f5f 100644 --- a/Wabbajack.App.Test/Wabbajack.App.Test.csproj +++ b/Wabbajack.App.Test/Wabbajack.App.Test.csproj @@ -7,7 +7,7 @@ - + diff --git a/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj b/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj index fe3d65bd..af882a93 100644 --- a/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj +++ b/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj @@ -7,7 +7,7 @@ - + diff --git a/Wabbajack.BuildServer/BackendServices/ValidateNonNexusArchives.cs b/Wabbajack.BuildServer/BackendServices/ValidateNonNexusArchives.cs index 4d79c299..c144a9e0 100644 --- a/Wabbajack.BuildServer/BackendServices/ValidateNonNexusArchives.cs +++ b/Wabbajack.BuildServer/BackendServices/ValidateNonNexusArchives.cs @@ -1,6 +1,8 @@ using System; using System.Linq; using System.Threading.Tasks; +using Microsoft.AspNetCore.Identity; +using Wabbajack.BuildServer.Controllers; using Wabbajack.BuildServer.Model.Models; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; @@ -24,7 +26,18 @@ namespace Wabbajack.BuildServer.BackendServices { try { - var isValid = await archive.State.Verify(archive); + bool isValid; + switch (archive.State) + { + case GoogleDriveDownloader.State _: + case ManualDownloader.State _: + case HTTPDownloader.State s when new Uri(s.Url).Host.StartsWith("wabbajackpush"): + isValid = true; + break; + default: + isValid = await archive.State.Verify(archive); + break; + } return (Archive: archive, IsValid: isValid); } catch (Exception ex) diff --git a/Wabbajack.BuildServer/Controllers/Heartbeat.cs b/Wabbajack.BuildServer/Controllers/Heartbeat.cs index 6e97518a..b79f2d48 100644 --- a/Wabbajack.BuildServer/Controllers/Heartbeat.cs +++ b/Wabbajack.BuildServer/Controllers/Heartbeat.cs @@ -44,7 +44,8 @@ namespace Wabbajack.BuildServer.Controllers return Ok(new HeartbeatResult { Uptime = DateTime.Now - _startTime, - LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync + LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync, + LastListValidation = DateTime.UtcNow - ListValidation.SummariesLastChecked }); } @@ -53,6 +54,8 @@ namespace Wabbajack.BuildServer.Controllers { public TimeSpan Uptime { get; set; } public TimeSpan LastNexusUpdate { get; set; } + + public TimeSpan LastListValidation { get; set; } } [HttpGet("only-authenticated")] diff --git a/Wabbajack.BuildServer/Controllers/ListValidation.cs b/Wabbajack.BuildServer/Controllers/ListValidation.cs index 48f66f2b..8722d6fa 100644 --- a/Wabbajack.BuildServer/Controllers/ListValidation.cs +++ b/Wabbajack.BuildServer/Controllers/ListValidation.cs @@ -17,6 +17,7 @@ using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.ModListRegistry; +using Wabbajack.Lib.NexusApi; namespace Wabbajack.BuildServer.Controllers { @@ -24,7 +25,7 @@ namespace Wabbajack.BuildServer.Controllers [Route("/lists")] public class ListValidation : AControllerBase { - enum ArchiveStatus + public enum ArchiveStatus { Valid, InValid, @@ -37,6 +38,8 @@ namespace Wabbajack.BuildServer.Controllers _updater = new ModlistUpdater(null, sql, settings); _settings = settings; Cache = cache; + _nexusClient = NexusApiClient.Get(); + } public static IMemoryCache Cache { get; set; } @@ -44,76 +47,97 @@ namespace Wabbajack.BuildServer.Controllers public static void ResetCache() { - Cache?.Remove(ModListSummariesKey); + SummariesLastChecked = DateTime.UnixEpoch; + ModListSummaries = null; } + private static IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> ModListSummaries = null; + public static DateTime SummariesLastChecked = DateTime.UnixEpoch; + private static AsyncLock UpdateLock = new AsyncLock(); public async Task> GetSummaries() { - - if (Cache.TryGetValue(ModListSummariesKey, out object result)) + static bool TimesUp() { - return (IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)>)result; + return DateTime.UtcNow - SummariesLastChecked > TimeSpan.FromMinutes(5); + } + + if (ModListSummaries != null && !TimesUp()) + { + return ModListSummaries; } - - var data = await SQL.GetValidationData(); - - using var queue = new WorkQueue(); - - var results = await data.ModLists.PMap(queue, async list => + var task = Task.Run(async () => { - var (metadata, modList) = list; - var archives = await modList.Archives.PMap(queue, async archive => + using var _ = await UpdateLock.WaitAsync(); + if (ModListSummaries != null && !TimesUp()) { - var (_, result) = ValidateArchive(data, archive); - if (result == ArchiveStatus.InValid) + return ModListSummaries; + } + SummariesLastChecked = DateTime.UtcNow; + + + var data = await SQL.GetValidationData(); + + using var queue = new WorkQueue(); + + var results = await data.ModLists.PMap(queue, async list => + { + var (metadata, modList) = list; + var archives = await modList.Archives.PMap(queue, async archive => { - var fixResult = await TryToFix(data, archive); - - return fixResult; + var (_, result) = await ValidateArchive(data, archive); + if (result != ArchiveStatus.InValid) return (archive, result); - } + return await TryToFix(data, archive); - return (archive, result); + }); + + var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid); + var passCount = archives.Count(f => + f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated); + var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating); + + var summary = new ModListSummary + { + Checked = DateTime.UtcNow, + Failed = failedCount, + Passed = passCount, + Updating = updatingCount, + MachineURL = metadata.Links.MachineURL, + Name = metadata.Title, + }; + + var detailed = new DetailedStatus + { + Name = metadata.Title, + Checked = DateTime.UtcNow, + DownloadMetaData = metadata.DownloadMetadata, + HasFailures = failedCount > 0, + MachineName = metadata.Links.MachineURL, + Archives = archives.Select(a => new DetailedStatusItem + { + Archive = a.Item1, + IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating + }).ToList() + }; + + return (summary, detailed); }); - var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid); - var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated); - var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating); - var summary = new ModListSummary - { - Checked = DateTime.UtcNow, - Failed = failedCount, - Passed = passCount, - Updating = updatingCount, - MachineURL = metadata.Links.MachineURL, - Name = metadata.Title, - }; - - var detailed = new DetailedStatus - { - Name = metadata.Title, - Checked = DateTime.UtcNow, - DownloadMetaData = metadata.DownloadMetadata, - HasFailures = failedCount > 0, - MachineName = metadata.Links.MachineURL, - Archives = archives.Select(a => new DetailedStatusItem - { - Archive = a.Item1, IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating - }).ToList() - }; - - return (summary, detailed); + var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1)); + Cache.Set(ModListSummariesKey, results, cacheOptions); + + ModListSummaries = results; + return results; }); - - - var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1)); - Cache.Set(ModListSummariesKey, results, cacheOptions); - return results; + var data = ModListSummaries; + if (data == null) + return await task; + return data; } - private static (Archive archive, ArchiveStatus) ValidateArchive(SqlService.ValidationData data, Archive archive) + private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(SqlService.ValidationData data, Archive archive) { switch (archive.State) { @@ -123,8 +147,10 @@ namespace Wabbajack.BuildServer.Controllers case NexusDownloader.State nexusState when data.NexusFiles.Contains(( nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)): return (archive, ArchiveStatus.Valid); - case NexusDownloader.State _: - return (archive, ArchiveStatus.InValid); + case NexusDownloader.State ns: + return (archive, await FastNexusModStats(ns)); + case HTTPDownloader.State s when new Uri(s.Url).Host.StartsWith("wabbajackpush"): + return (archive, ArchiveStatus.Valid); case ManualDownloader.State _: return (archive, ArchiveStatus.Valid); default: @@ -140,6 +166,47 @@ namespace Wabbajack.BuildServer.Controllers } } + private async Task FastNexusModStats(NexusDownloader.State ns) + { + + var mod = await SQL.GetNexusModInfoString(ns.Game, ns.ModID); + var files = await SQL.GetModFiles(ns.Game, ns.ModID); + + if (mod == null) + { + Utils.Log($"Found missing Nexus mod info {ns.Game} {ns.ModID}"); + mod = await (await _nexusClient).GetModInfo(ns.Game, ns.ModID, false); + try + { + await SQL.AddNexusModInfo(ns.Game, ns.ModID, mod.updated_time, mod); + } + catch (Exception _) + { + // Could be a PK constraint failure + } + } + + if (files == null) + { + Utils.Log($"Found missing Nexus mod file infos {ns.Game} {ns.ModID}"); + files = await (await _nexusClient).GetModFiles(ns.Game, ns.ModID, false); + + try + { + await SQL.AddNexusModFiles(ns.Game, ns.ModID, mod.updated_time, files); + } + catch (Exception _) + { + // Could be a PK constraint failure + } + } + + if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID)) + return ArchiveStatus.Valid; + return ArchiveStatus.InValid; + + } + private static AsyncLock _findPatchLock = new AsyncLock(); private async Task<(Archive, ArchiveStatus)> TryToFix(SqlService.ValidationData data, Archive archive) { @@ -219,6 +286,7 @@ namespace Wabbajack.BuildServer.Controllers private AppSettings _settings; private ModlistUpdater _updater; + private Task _nexusClient; [HttpGet] [Route("status/{Name}.html")] diff --git a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs index f6216adb..af3dbd01 100644 --- a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs +++ b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs @@ -88,14 +88,12 @@ namespace Wabbajack.BuildServer.Controllers public async Task GetAlternative(string xxHash) { var startingHash = Hash.FromHex(xxHash); - Utils.Log($"Alternative requested for {startingHash}"); await Metric("requested_upgrade", startingHash.ToString()); var archive = await SQL.GetStateByHash(startingHash); if (archive == null) { - Utils.Log($"No original state for {startingHash}"); return NotFound("Original state not found"); } @@ -120,7 +118,6 @@ namespace Wabbajack.BuildServer.Controllers } - Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}"); if (newArchive.Hash == Hash.Empty) { await SQL.EnqueueJob(new Job @@ -138,6 +135,7 @@ namespace Wabbajack.BuildServer.Controllers } } }); + Utils.Log($"Enqueued Index and Upgrade for {startingHash} -> {newArchive.State.PrimaryKeyString}"); return Accepted("Enqueued for Processing"); } @@ -155,6 +153,7 @@ namespace Wabbajack.BuildServer.Controllers DestPK = newArchive.State.PrimaryKeyString } }); + Utils.Log($"Enqueued Upgrade for {startingHash} -> {newArchive.State.PrimaryKeyString}"); } return Ok(newArchive.ToJson()); } diff --git a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj index c0f5b361..bb7a9a6b 100644 --- a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj +++ b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj @@ -18,7 +18,7 @@ - + @@ -28,7 +28,7 @@ - + diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 82b17462..ac2bc5e4 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -32,7 +32,7 @@ - + diff --git a/Wabbajack.Lib/Downloaders/MEGADownloader.cs b/Wabbajack.Lib/Downloaders/MEGADownloader.cs index ecf60c8e..ed0fbd7e 100644 --- a/Wabbajack.Lib/Downloaders/MEGADownloader.cs +++ b/Wabbajack.Lib/Downloaders/MEGADownloader.cs @@ -96,27 +96,30 @@ namespace Wabbajack.Lib.Downloaders private static MegaApiClient MegaApiClient => DownloadDispatcher.GetInstance().MegaApiClient; - private static void MegaLogin() + private static AsyncLock _loginLock = new AsyncLock(); + private static async Task MegaLogin() { + using var _ = await _loginLock.WaitAsync(); + if (MegaApiClient.IsLoggedIn) return; if (!Utils.HaveEncryptedJson(DataName)) { Utils.Status("Logging into MEGA (as anonymous)"); - MegaApiClient.LoginAnonymous(); + await MegaApiClient.LoginAnonymousAsync(); } else { Utils.Status("Logging into MEGA with saved credentials."); var authInfo = Utils.FromEncryptedJson(DataName); - MegaApiClient.Login(authInfo); + await MegaApiClient.LoginAsync(authInfo); } } public override async Task Download(Archive a, AbsolutePath destination) { - MegaLogin(); + await MegaLogin(); var fileLink = new Uri(Url); Utils.Status($"Downloading MEGA file: {a.Name}"); @@ -126,7 +129,7 @@ namespace Wabbajack.Lib.Downloaders public override async Task Verify(Archive a) { - MegaLogin(); + await MegaLogin(); var fileLink = new Uri(Url); try diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 989e4aa2..f13f5686 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -35,10 +35,10 @@ 2.1.0 - 11.3.1 + 11.3.8 - 11.3.1 + 11.3.8 0.25.0 @@ -65,7 +65,7 @@ 1.0.0 - 5.0.1 + 5.0.2 diff --git a/Wabbajack.Test/Wabbajack.Test.csproj b/Wabbajack.Test/Wabbajack.Test.csproj index c5ad8b23..c81e88e4 100644 --- a/Wabbajack.Test/Wabbajack.Test.csproj +++ b/Wabbajack.Test/Wabbajack.Test.csproj @@ -27,7 +27,7 @@ - + diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index c5935f64..323218e7 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -57,7 +57,7 @@ - + all @@ -72,9 +72,9 @@ - - - + + +