diff --git a/Compression.BSA/BSABuilder.cs b/Compression.BSA/BSABuilder.cs index 04246c72..dcaa2b49 100644 --- a/Compression.BSA/BSABuilder.cs +++ b/Compression.BSA/BSABuilder.cs @@ -280,7 +280,7 @@ namespace Compression.BSA case VersionType.SSE: { var r = new MemoryStream(); - await using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L12_MAX}, true)) + await using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L08_HC}, true)) { await _srcData.CopyToWithStatusAsync(_srcData.Length, w, $"Compressing {_path}"); } diff --git a/Wabbajack.App.Test/Wabbajack.App.Test.csproj b/Wabbajack.App.Test/Wabbajack.App.Test.csproj index fe0caef1..c3720f5f 100644 --- a/Wabbajack.App.Test/Wabbajack.App.Test.csproj +++ b/Wabbajack.App.Test/Wabbajack.App.Test.csproj @@ -7,7 +7,7 @@ - + diff --git a/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj b/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj index fe3d65bd..af882a93 100644 --- a/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj +++ b/Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj @@ -7,7 +7,7 @@ - + diff --git a/Wabbajack.BuildServer/BackendServices/ValidateNonNexusArchives.cs b/Wabbajack.BuildServer/BackendServices/ValidateNonNexusArchives.cs index 4d79c299..c144a9e0 100644 --- a/Wabbajack.BuildServer/BackendServices/ValidateNonNexusArchives.cs +++ b/Wabbajack.BuildServer/BackendServices/ValidateNonNexusArchives.cs @@ -1,6 +1,8 @@ using System; using System.Linq; using System.Threading.Tasks; +using Microsoft.AspNetCore.Identity; +using Wabbajack.BuildServer.Controllers; using Wabbajack.BuildServer.Model.Models; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; @@ -24,7 +26,18 @@ namespace Wabbajack.BuildServer.BackendServices { try { - var isValid = await archive.State.Verify(archive); + bool isValid; + switch (archive.State) + { + case GoogleDriveDownloader.State _: + case ManualDownloader.State _: + case HTTPDownloader.State s when new Uri(s.Url).Host.StartsWith("wabbajackpush"): + isValid = true; + break; + default: + isValid = await archive.State.Verify(archive); + break; + } return (Archive: archive, IsValid: isValid); } catch (Exception ex) diff --git a/Wabbajack.BuildServer/Controllers/Heartbeat.cs b/Wabbajack.BuildServer/Controllers/Heartbeat.cs index 6e97518a..b79f2d48 100644 --- a/Wabbajack.BuildServer/Controllers/Heartbeat.cs +++ b/Wabbajack.BuildServer/Controllers/Heartbeat.cs @@ -44,7 +44,8 @@ namespace Wabbajack.BuildServer.Controllers return Ok(new HeartbeatResult { Uptime = DateTime.Now - _startTime, - LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync + LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync, + LastListValidation = DateTime.UtcNow - ListValidation.SummariesLastChecked }); } @@ -53,6 +54,8 @@ namespace Wabbajack.BuildServer.Controllers { public TimeSpan Uptime { get; set; } public TimeSpan LastNexusUpdate { get; set; } + + public TimeSpan LastListValidation { get; set; } } [HttpGet("only-authenticated")] diff --git a/Wabbajack.BuildServer/Controllers/ListValidation.cs b/Wabbajack.BuildServer/Controllers/ListValidation.cs index 48f66f2b..8722d6fa 100644 --- a/Wabbajack.BuildServer/Controllers/ListValidation.cs +++ b/Wabbajack.BuildServer/Controllers/ListValidation.cs @@ -17,6 +17,7 @@ using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.ModListRegistry; +using Wabbajack.Lib.NexusApi; namespace Wabbajack.BuildServer.Controllers { @@ -24,7 +25,7 @@ namespace Wabbajack.BuildServer.Controllers [Route("/lists")] public class ListValidation : AControllerBase { - enum ArchiveStatus + public enum ArchiveStatus { Valid, InValid, @@ -37,6 +38,8 @@ namespace Wabbajack.BuildServer.Controllers _updater = new ModlistUpdater(null, sql, settings); _settings = settings; Cache = cache; + _nexusClient = NexusApiClient.Get(); + } public static IMemoryCache Cache { get; set; } @@ -44,76 +47,97 @@ namespace Wabbajack.BuildServer.Controllers public static void ResetCache() { - Cache?.Remove(ModListSummariesKey); + SummariesLastChecked = DateTime.UnixEpoch; + ModListSummaries = null; } + private static IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> ModListSummaries = null; + public static DateTime SummariesLastChecked = DateTime.UnixEpoch; + private static AsyncLock UpdateLock = new AsyncLock(); public async Task> GetSummaries() { - - if (Cache.TryGetValue(ModListSummariesKey, out object result)) + static bool TimesUp() { - return (IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)>)result; + return DateTime.UtcNow - SummariesLastChecked > TimeSpan.FromMinutes(5); + } + + if (ModListSummaries != null && !TimesUp()) + { + return ModListSummaries; } - - var data = await SQL.GetValidationData(); - - using var queue = new WorkQueue(); - - var results = await data.ModLists.PMap(queue, async list => + var task = Task.Run(async () => { - var (metadata, modList) = list; - var archives = await modList.Archives.PMap(queue, async archive => + using var _ = await UpdateLock.WaitAsync(); + if (ModListSummaries != null && !TimesUp()) { - var (_, result) = ValidateArchive(data, archive); - if (result == ArchiveStatus.InValid) + return ModListSummaries; + } + SummariesLastChecked = DateTime.UtcNow; + + + var data = await SQL.GetValidationData(); + + using var queue = new WorkQueue(); + + var results = await data.ModLists.PMap(queue, async list => + { + var (metadata, modList) = list; + var archives = await modList.Archives.PMap(queue, async archive => { - var fixResult = await TryToFix(data, archive); - - return fixResult; + var (_, result) = await ValidateArchive(data, archive); + if (result != ArchiveStatus.InValid) return (archive, result); - } + return await TryToFix(data, archive); - return (archive, result); + }); + + var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid); + var passCount = archives.Count(f => + f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated); + var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating); + + var summary = new ModListSummary + { + Checked = DateTime.UtcNow, + Failed = failedCount, + Passed = passCount, + Updating = updatingCount, + MachineURL = metadata.Links.MachineURL, + Name = metadata.Title, + }; + + var detailed = new DetailedStatus + { + Name = metadata.Title, + Checked = DateTime.UtcNow, + DownloadMetaData = metadata.DownloadMetadata, + HasFailures = failedCount > 0, + MachineName = metadata.Links.MachineURL, + Archives = archives.Select(a => new DetailedStatusItem + { + Archive = a.Item1, + IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating + }).ToList() + }; + + return (summary, detailed); }); - var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid); - var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated); - var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating); - var summary = new ModListSummary - { - Checked = DateTime.UtcNow, - Failed = failedCount, - Passed = passCount, - Updating = updatingCount, - MachineURL = metadata.Links.MachineURL, - Name = metadata.Title, - }; - - var detailed = new DetailedStatus - { - Name = metadata.Title, - Checked = DateTime.UtcNow, - DownloadMetaData = metadata.DownloadMetadata, - HasFailures = failedCount > 0, - MachineName = metadata.Links.MachineURL, - Archives = archives.Select(a => new DetailedStatusItem - { - Archive = a.Item1, IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating - }).ToList() - }; - - return (summary, detailed); + var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1)); + Cache.Set(ModListSummariesKey, results, cacheOptions); + + ModListSummaries = results; + return results; }); - - - var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1)); - Cache.Set(ModListSummariesKey, results, cacheOptions); - return results; + var data = ModListSummaries; + if (data == null) + return await task; + return data; } - private static (Archive archive, ArchiveStatus) ValidateArchive(SqlService.ValidationData data, Archive archive) + private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(SqlService.ValidationData data, Archive archive) { switch (archive.State) { @@ -123,8 +147,10 @@ namespace Wabbajack.BuildServer.Controllers case NexusDownloader.State nexusState when data.NexusFiles.Contains(( nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)): return (archive, ArchiveStatus.Valid); - case NexusDownloader.State _: - return (archive, ArchiveStatus.InValid); + case NexusDownloader.State ns: + return (archive, await FastNexusModStats(ns)); + case HTTPDownloader.State s when new Uri(s.Url).Host.StartsWith("wabbajackpush"): + return (archive, ArchiveStatus.Valid); case ManualDownloader.State _: return (archive, ArchiveStatus.Valid); default: @@ -140,6 +166,47 @@ namespace Wabbajack.BuildServer.Controllers } } + private async Task FastNexusModStats(NexusDownloader.State ns) + { + + var mod = await SQL.GetNexusModInfoString(ns.Game, ns.ModID); + var files = await SQL.GetModFiles(ns.Game, ns.ModID); + + if (mod == null) + { + Utils.Log($"Found missing Nexus mod info {ns.Game} {ns.ModID}"); + mod = await (await _nexusClient).GetModInfo(ns.Game, ns.ModID, false); + try + { + await SQL.AddNexusModInfo(ns.Game, ns.ModID, mod.updated_time, mod); + } + catch (Exception _) + { + // Could be a PK constraint failure + } + } + + if (files == null) + { + Utils.Log($"Found missing Nexus mod file infos {ns.Game} {ns.ModID}"); + files = await (await _nexusClient).GetModFiles(ns.Game, ns.ModID, false); + + try + { + await SQL.AddNexusModFiles(ns.Game, ns.ModID, mod.updated_time, files); + } + catch (Exception _) + { + // Could be a PK constraint failure + } + } + + if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID)) + return ArchiveStatus.Valid; + return ArchiveStatus.InValid; + + } + private static AsyncLock _findPatchLock = new AsyncLock(); private async Task<(Archive, ArchiveStatus)> TryToFix(SqlService.ValidationData data, Archive archive) { @@ -219,6 +286,7 @@ namespace Wabbajack.BuildServer.Controllers private AppSettings _settings; private ModlistUpdater _updater; + private Task _nexusClient; [HttpGet] [Route("status/{Name}.html")] diff --git a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs index f6216adb..af3dbd01 100644 --- a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs +++ b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs @@ -88,14 +88,12 @@ namespace Wabbajack.BuildServer.Controllers public async Task GetAlternative(string xxHash) { var startingHash = Hash.FromHex(xxHash); - Utils.Log($"Alternative requested for {startingHash}"); await Metric("requested_upgrade", startingHash.ToString()); var archive = await SQL.GetStateByHash(startingHash); if (archive == null) { - Utils.Log($"No original state for {startingHash}"); return NotFound("Original state not found"); } @@ -120,7 +118,6 @@ namespace Wabbajack.BuildServer.Controllers } - Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}"); if (newArchive.Hash == Hash.Empty) { await SQL.EnqueueJob(new Job @@ -138,6 +135,7 @@ namespace Wabbajack.BuildServer.Controllers } } }); + Utils.Log($"Enqueued Index and Upgrade for {startingHash} -> {newArchive.State.PrimaryKeyString}"); return Accepted("Enqueued for Processing"); } @@ -155,6 +153,7 @@ namespace Wabbajack.BuildServer.Controllers DestPK = newArchive.State.PrimaryKeyString } }); + Utils.Log($"Enqueued Upgrade for {startingHash} -> {newArchive.State.PrimaryKeyString}"); } return Ok(newArchive.ToJson()); } diff --git a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj index c0f5b361..bb7a9a6b 100644 --- a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj +++ b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj @@ -18,7 +18,7 @@ - + @@ -28,7 +28,7 @@ - + diff --git a/Wabbajack.Common/StoreHandlers/StoreHandler.cs b/Wabbajack.Common/StoreHandlers/StoreHandler.cs index dd1220c5..295c2225 100644 --- a/Wabbajack.Common/StoreHandlers/StoreHandler.cs +++ b/Wabbajack.Common/StoreHandlers/StoreHandler.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; namespace Wabbajack.Common.StoreHandlers { @@ -13,7 +14,7 @@ namespace Wabbajack.Common.StoreHandlers public class StoreHandler { - private static readonly Lazy _instance = new Lazy(() => new StoreHandler(), true); + private static readonly Lazy _instance = new Lazy(() => new StoreHandler(), isThreadSafe: true); public static StoreHandler Instance => _instance.Value; private static readonly Lazy _steamHandler = new Lazy(() => new SteamHandler()); @@ -72,6 +73,11 @@ namespace Wabbajack.Common.StoreHandlers { return StoreGames.FirstOrDefault(g => g.Game == game)?.Path; } + + public static void Warmup() + { + Task.Run(() => _instance.Value).FireAndForget(); + } } public abstract class AStoreGame diff --git a/Wabbajack.Common/Util/TempFolder.cs b/Wabbajack.Common/Util/TempFolder.cs index 9ce128d3..70862216 100644 --- a/Wabbajack.Common/Util/TempFolder.cs +++ b/Wabbajack.Common/Util/TempFolder.cs @@ -18,7 +18,10 @@ namespace Wabbajack.Common _cleanTask = Task.Run(() => "tmp_files".RelativeTo(AbsolutePath.EntryPoint).DeleteDirectory()); } - public static void Init() + /// + /// Starts the initialization in a background task + /// + public static void Warmup() { // Nothing to do, as work is done in static ctor } diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 82b17462..ac2bc5e4 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -32,7 +32,7 @@ - + diff --git a/Wabbajack.Lib/Downloaders/MEGADownloader.cs b/Wabbajack.Lib/Downloaders/MEGADownloader.cs index 74137a42..cac854fe 100644 --- a/Wabbajack.Lib/Downloaders/MEGADownloader.cs +++ b/Wabbajack.Lib/Downloaders/MEGADownloader.cs @@ -133,28 +133,31 @@ namespace Wabbajack.Lib.Downloaders private static MegaApiClient MegaApiClient => DownloadDispatcher.GetInstance().MegaApiClient; - private static void MegaLogin() + private static AsyncLock _loginLock = new AsyncLock(); + private static async Task MegaLogin() { + using var _ = await _loginLock.WaitAsync(); + if (MegaApiClient.IsLoggedIn) return; if (!Utils.HaveEncryptedJson(DataName)) { Utils.Status("Logging into MEGA (as anonymous)"); - MegaApiClient.LoginAnonymous(); + await MegaApiClient.LoginAnonymousAsync(); } else { Utils.Status("Logging into MEGA with saved credentials."); var infos = Utils.FromEncryptedJson(DataName); var authInfo = infos.ToAuthInfos(); - MegaApiClient.Login(authInfo); + await MegaApiClient.LoginAsync(authInfo); } } public override async Task Download(Archive a, AbsolutePath destination) { - MegaLogin(); + await MegaLogin(); var fileLink = new Uri(Url); Utils.Status($"Downloading MEGA file: {a.Name}"); @@ -164,7 +167,7 @@ namespace Wabbajack.Lib.Downloaders public override async Task Verify(Archive a) { - MegaLogin(); + await MegaLogin(); var fileLink = new Uri(Url); try diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 989e4aa2..f13f5686 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -35,10 +35,10 @@ 2.1.0 - 11.3.1 + 11.3.8 - 11.3.1 + 11.3.8 0.25.0 @@ -65,7 +65,7 @@ 1.0.0 - 5.0.1 + 5.0.2 diff --git a/Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs b/Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs index 48e73d50..d2b39217 100644 --- a/Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs +++ b/Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs @@ -18,6 +18,9 @@ namespace Wabbajack.Lib.WebAutomation public CefSharpWrapper(IWebBrowser browser) { _browser = browser; + + _browser.DownloadHandler = new DownloadHandler(this); + _browser.LifeSpanHandler = new PopupBlocker(this); } public Task NavigateTo(Uri uri) @@ -33,11 +36,9 @@ namespace Wabbajack.Lib.WebAutomation tcs.SetResult(true); } }; - _browser.LoadingStateChanged += handler; _browser.Load(uri.ToString()); - _browser.DownloadHandler = new DownloadHandler(this); - _browser.LifeSpanHandler = new PopupBlocker(this); + return tcs.Task; } @@ -80,7 +81,7 @@ namespace Wabbajack.Lib.WebAutomation IWindowInfo windowInfo, IBrowserSettings browserSettings, ref bool noJavascriptAccess, out IWebBrowser? newBrowser) { // Block popups - newBrowser = null; + newBrowser = chromiumWebBrowser; return true; } @@ -90,7 +91,7 @@ namespace Wabbajack.Lib.WebAutomation public bool DoClose(IWebBrowser chromiumWebBrowser, IBrowser browser) { - return true; + return false; } public void OnBeforeClose(IWebBrowser chromiumWebBrowser, IBrowser browser) diff --git a/Wabbajack.Test/Wabbajack.Test.csproj b/Wabbajack.Test/Wabbajack.Test.csproj index c5ad8b23..c81e88e4 100644 --- a/Wabbajack.Test/Wabbajack.Test.csproj +++ b/Wabbajack.Test/Wabbajack.Test.csproj @@ -27,7 +27,7 @@ - + diff --git a/Wabbajack/App.xaml.cs b/Wabbajack/App.xaml.cs index b2986871..7d8a7492 100644 --- a/Wabbajack/App.xaml.cs +++ b/Wabbajack/App.xaml.cs @@ -1,15 +1,6 @@ using System; -using System.Collections.Generic; -using System.Configuration; -using System.Data; -using System.Linq; -using System.Threading.Tasks; using System.Windows; -using System.Windows.Interop; -using System.Windows.Media; using Wabbajack.Common; -using Wabbajack.Common.StoreHandlers; -using Wabbajack.Util; namespace Wabbajack { @@ -23,7 +14,6 @@ namespace Wabbajack CLIOld.ParseOptions(Environment.GetCommandLineArgs()); if (CLIArguments.Help) CLIOld.DisplayHelpText(); - var storeHandler = new StoreHandler(); } } } diff --git a/Wabbajack/Views/MainWindow.xaml.cs b/Wabbajack/Views/MainWindow.xaml.cs index c6572535..8c48ffec 100644 --- a/Wabbajack/Views/MainWindow.xaml.cs +++ b/Wabbajack/Views/MainWindow.xaml.cs @@ -6,6 +6,7 @@ using System.Windows; using MahApps.Metro.Controls; using Newtonsoft.Json; using Wabbajack.Common; +using Wabbajack.Common.StoreHandlers; using Wabbajack.Lib.LibCefHelpers; using Wabbajack.Util; using Application = System.Windows.Application; @@ -23,8 +24,6 @@ namespace Wabbajack public MainWindow() { - TempFolder.Init(); - Helpers.Init(); // Wire any unhandled crashing exceptions to log before exiting AppDomain.CurrentDomain.UnhandledException += (sender, e) => { @@ -44,22 +43,7 @@ namespace Wabbajack Utils.Log( $"System settings - ({p.SystemMemorySize.ToFileSizeString()} RAM), Display: {p.ScreenWidth} x {p.ScreenHeight} ({p.VideoMemorySize.ToFileSizeString()} VRAM - VideoMemorySizeMb={p.EnbLEVRAMSize})"); - // Run logic to associate wabbajack lists with this app in the background - Task.Run(async () => - { - var appPath = System.Reflection.Assembly.GetExecutingAssembly().Location; - try - { - if (!ModListAssociationManager.IsAssociated() || ModListAssociationManager.NeedsUpdating(appPath)) - { - ModListAssociationManager.Associate(appPath); - } - } - catch (Exception e) - { - Utils.Log($"ExtensionManager had an exception:\n{e}"); - } - }).FireAndForget(); + Warmup(); // Load settings if (CLIArguments.NoSettings || !MainSettings.TryLoadTypicalSettings(out var settings)) @@ -100,6 +84,40 @@ namespace Wabbajack _settings = settings; } + /// + /// Starts some background initialization tasks spinning so they're already prepped when actually needed + /// + private void Warmup() + { + TempFolder.Warmup(); + // ToDo + // Currently this is a blocking call. Perhaps upgrade to be run in a background task. + // Would first need to ensure users of CEF properly await the background initialization before use + Helpers.Init(); + StoreHandler.Warmup(); + + Task.Run(AssociateListsWithWabbajack).FireAndForget(); + } + + /// + /// Run logic to associate wabbajack lists with this app in the background + /// + private void AssociateListsWithWabbajack() + { + var appPath = System.Reflection.Assembly.GetExecutingAssembly().Location; + try + { + if (!ModListAssociationManager.IsAssociated() || ModListAssociationManager.NeedsUpdating(appPath)) + { + ModListAssociationManager.Associate(appPath); + } + } + catch (Exception e) + { + Utils.Log($"ExtensionManager had an exception:\n{e}"); + } + } + private void RunWhenLoaded(Action a) { if (IsLoaded) diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index c5935f64..323218e7 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -57,7 +57,7 @@ - + all @@ -72,9 +72,9 @@ - - - + + +