From 7f603e9c855c2c7b9739ad682328f3ca85f8043b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 2 Nov 2020 18:55:54 -0700 Subject: [PATCH 01/33] Store game file hashes on GitHub --- Wabbajack.CLI/OptionsDefinition.cs | 4 +- Wabbajack.CLI/Verbs/ExportServerGameFiles.cs | 32 +++++++++++ Wabbajack.CLI/Verbs/HashGameFiles.cs | 54 +++++++++++++++++++ Wabbajack.Common/Json.cs | 26 +++++++-- Wabbajack.Lib/ClientAPI.cs | 29 ++++++++-- Wabbajack.Server/Controllers/GameFiles.cs | 8 +++ .../DataLayer/ArchiveDownloads.cs | 11 ++++ 7 files changed, 156 insertions(+), 8 deletions(-) create mode 100644 Wabbajack.CLI/Verbs/ExportServerGameFiles.cs create mode 100644 Wabbajack.CLI/Verbs/HashGameFiles.cs diff --git a/Wabbajack.CLI/OptionsDefinition.cs b/Wabbajack.CLI/OptionsDefinition.cs index 50398100..e0193872 100644 --- a/Wabbajack.CLI/OptionsDefinition.cs +++ b/Wabbajack.CLI/OptionsDefinition.cs @@ -30,7 +30,9 @@ namespace Wabbajack.CLI typeof(HashVariants), typeof(ParseMeta), typeof(NoPatch), - typeof(NexusPermissions) + typeof(NexusPermissions), + typeof(ExportServerGameFiles), + typeof(HashGamefiles) }; } } diff --git a/Wabbajack.CLI/Verbs/ExportServerGameFiles.cs b/Wabbajack.CLI/Verbs/ExportServerGameFiles.cs new file mode 100644 index 00000000..9c1887b1 --- /dev/null +++ b/Wabbajack.CLI/Verbs/ExportServerGameFiles.cs @@ -0,0 +1,32 @@ +using System.Linq; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Lib; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("export-server-game-files", HelpText = "Exports all the game file data from the server to the output folder")] + public class ExportServerGameFiles : AVerb + { + [Option('o', "output", Required = true, HelpText = @"Output folder in which the files will be placed")] + public string OutputFolder { get; set; } = ""; + + private AbsolutePath _outputFolder => (AbsolutePath)OutputFolder; + + protected override async Task Run() + { + var games = await ClientAPI.GetServerGamesAndVersions(); + foreach (var (game, version) in games) + { + Utils.Log($"Exporting {game} {version}"); + var file = _outputFolder.Combine(game.ToString(), version).WithExtension(new Extension(".json")); + file.Parent.CreateDirectory(); + var files = await ClientAPI.GetGameFilesFromServer(game, version); + await files.ToJsonAsync(file, prettyPrint:true); + } + + return ExitCode.Ok; + } + } +} diff --git a/Wabbajack.CLI/Verbs/HashGameFiles.cs b/Wabbajack.CLI/Verbs/HashGameFiles.cs new file mode 100644 index 00000000..e632a488 --- /dev/null +++ b/Wabbajack.CLI/Verbs/HashGameFiles.cs @@ -0,0 +1,54 @@ +using System.Linq; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("hash-game-files", HelpText = "Hashes a game's files for inclusion in the public github repo")] + public class HashGamefiles : AVerb + { + [Option('o', "output", Required = true, HelpText = @"Output folder in which the file will be placed")] + public string OutputFolder { get; set; } = ""; + + private AbsolutePath _outputFolder => (AbsolutePath)OutputFolder; + + [Option('g', "game", Required = true, HelpText = @"WJ Game to index")] + public string Game { get; set; } = ""; + + private Game _game => GameRegistry.GetByFuzzyName(Game).Game; + + protected override async Task Run() + { + var version = _game.MetaData().InstalledVersion; + var file = _outputFolder.Combine(_game.ToString(), version).WithExtension(new Extension(".json")); + file.Parent.CreateDirectory(); + + using var queue = new WorkQueue(); + var gameLocation = _game.MetaData().GameLocation(); + + Utils.Log($"Hashing files for {_game} {version}"); + + var indexed = await gameLocation + .EnumerateFiles() + .PMap(queue, async f => + { + var hash = await f.FileHashCachedAsync(); + return new GameFileSourceDownloader.State + { + Game = _game, + GameFile = f.RelativeTo(gameLocation), + Hash = hash, + GameVersion = version + }; + + }); + + Utils.Log($"Found and hashed {indexed.Length} files"); + await indexed.ToJsonAsync(file, prettyPrint: true); + return ExitCode.Ok; + } + } +} diff --git a/Wabbajack.Common/Json.cs b/Wabbajack.Common/Json.cs index 006421e9..0de935bb 100644 --- a/Wabbajack.Common/Json.cs +++ b/Wabbajack.Common/Json.cs @@ -36,6 +36,15 @@ namespace Wabbajack.Common Converters = Converters, DateTimeZoneHandling = DateTimeZoneHandling.Utc }; + + public static JsonSerializerSettings JsonSettingsPretty => + new JsonSerializerSettings { + TypeNameHandling = TypeNameHandling.Objects, + SerializationBinder = new JsonNameSerializationBinder(), + Converters = Converters, + DateTimeZoneHandling = DateTimeZoneHandling.Utc, + Formatting = Formatting.Indented + }; public static JsonSerializerSettings GenericJsonSettings => new JsonSerializerSettings @@ -51,18 +60,27 @@ namespace Wabbajack.Common File.WriteAllText(filename, JsonConvert.SerializeObject(obj, Formatting.Indented, JsonSettings)); } - public static void ToJson(this T obj, Stream stream, bool useGenericSettings = false) + public static void ToJson(this T obj, Stream stream, bool useGenericSettings = false, bool prettyPrint = false) { using var tw = new StreamWriter(stream, Encoding.UTF8, bufferSize: 1024, leaveOpen: true); using var writer = new JsonTextWriter(tw); - var ser = JsonSerializer.Create(useGenericSettings ? GenericJsonSettings : JsonSettings); + + JsonSerializerSettings settings = (useGenericSettings, prettyPrint) switch + { + (true, true) => GenericJsonSettings, + (false, true) => JsonSettingsPretty, + (false, false) => JsonSettings, + (true, false) => GenericJsonSettings + }; + + var ser = JsonSerializer.Create(settings); ser.Serialize(writer, obj); } - public static async ValueTask ToJsonAsync(this T obj, AbsolutePath path, bool useGenericSettings = false) + public static async ValueTask ToJsonAsync(this T obj, AbsolutePath path, bool useGenericSettings = false, bool prettyPrint = false) { await using var fs = await path.Create(); - obj.ToJson(fs, useGenericSettings); + obj.ToJson(fs, useGenericSettings, prettyPrint: prettyPrint); } public static string ToJson(this T obj, bool useGenericSettings = false) diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index 8ec1aa54..fea3de11 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -157,9 +157,7 @@ using Wabbajack.Lib.Downloaders; return new Archive[0]; var client = await GetClient(); var metaData = game.MetaData(); - var results = - await client.GetJsonAsync( - $"{Consts.WabbajackBuildServerUri}game_files/{game}/{metaData.InstalledVersion}"); + var results = await GetGameFilesFromGithub(game, metaData.InstalledVersion); return (await results.PMap(queue, async file => (await file.State.Verify(file), file))).Where(f => f.Item1) .Select(f => @@ -169,6 +167,22 @@ using Wabbajack.Lib.Downloaders; }) .ToArray(); } + + public static async Task GetGameFilesFromGithub(Game game, string version) + { + var url = + $"https://raw.githubusercontent.com/wabbajack-tools/indexed-game-files/master/{game}/{version}.json"; + Utils.Log($"Loading game file definition from {url}"); + var client = await GetClient(); + return await client.GetJsonAsync(url); + } + + public static async Task GetGameFilesFromServer(Game game, string version) + { + var client = await GetClient(); + return await client.GetJsonAsync( + $"{Consts.WabbajackBuildServerUri}game_files/{game}/{version}"); + } public static async Task InferDownloadState(Hash hash) { @@ -262,5 +276,14 @@ using Wabbajack.Lib.Downloaders; return await client.GetJsonAsync( $"{Consts.WabbajackBuildServerUri}site-integration/auth-info/{key}"); } + + public static async Task> GetServerGamesAndVersions() + { + var client = await GetClient(); + var results = + await client.GetJsonAsync<(Game, string)[]>( + $"{Consts.WabbajackBuildServerUri}game_files"); + return results; + } } } diff --git a/Wabbajack.Server/Controllers/GameFiles.cs b/Wabbajack.Server/Controllers/GameFiles.cs index a4c695be..27bb7174 100644 --- a/Wabbajack.Server/Controllers/GameFiles.cs +++ b/Wabbajack.Server/Controllers/GameFiles.cs @@ -67,6 +67,14 @@ namespace Wabbajack.BuildServer.Controllers return Ok(files.ToJson()); } + [Authorize(Roles = "User")] + [HttpGet] + public async Task GetAllGames() + { + var registeredGames = await _sql.GetAllRegisteredGames(); + return Ok(registeredGames.ToArray().ToJson()); + } + } diff --git a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs index f988d750..ee328419 100644 --- a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs +++ b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs @@ -255,5 +255,16 @@ namespace Wabbajack.Server.DataLayer return files.ToArray(); } + + public async Task> GetAllRegisteredGames() + { + await using var conn = await Open(); + var pks = (await conn.QueryAsync( + @"SELECT PrimaryKeyString FROM dbo.ArchiveDownloads WHERE PrimaryKeyString like 'GameFileSourceDownloader+State|%'") + ); + return pks.Select(p => p.Split("|")) + .Select(t => (GameRegistry.GetByFuzzyName(t[1]).Game, t[2])) + .Distinct(); + } } } From 90e9af05b1ea42eff3ad15d93db2d4242afeb95e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 3 Nov 2020 07:45:08 -0700 Subject: [PATCH 02/33] More fine-grained status updates --- Wabbajack.Common/StatusUpdateTracker.cs | 7 +++++-- Wabbajack.Lib/AInstaller.cs | 9 +++------ Wabbajack.Lib/MO2Compiler.cs | 2 +- Wabbajack.Lib/MO2Installer.cs | 6 +++--- Wabbajack.VirtualFileSystem/Context.cs | 5 +++-- Wabbajack/View Models/Installers/InstallerVM.cs | 3 +++ 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Wabbajack.Common/StatusUpdateTracker.cs b/Wabbajack.Common/StatusUpdateTracker.cs index 8f6d94f0..23d72c20 100644 --- a/Wabbajack.Common/StatusUpdateTracker.cs +++ b/Wabbajack.Common/StatusUpdateTracker.cs @@ -7,7 +7,7 @@ namespace Wabbajack.Common public class StatusUpdateTracker { private Subject _stepName = new Subject(); - public IObservable StepName => _stepName; + public IObservable StepName => _stepName.Debounce(TimeSpan.FromMilliseconds(100)); private Subject _step = new Subject(); public IObservable Step => _step; @@ -20,6 +20,7 @@ namespace Wabbajack.Common private int _internalCurrentStep; private int _internalMaxStep; + private string _currentStepName = ""; public StatusUpdateTracker(int maxStep) { @@ -33,10 +34,11 @@ namespace Wabbajack.Common public void NextStep(string name) { + _currentStepName = name; _internalCurrentStep += 1; Utils.Log(name); _step.OnNext(_internalCurrentStep); - _stepName.OnNext(name); + _stepName.OnNext($"({_internalCurrentStep}/{_internalMaxStep}) {_currentStepName}"); MakeUpdate(Percent.Zero); } @@ -62,6 +64,7 @@ namespace Wabbajack.Common public void MakeUpdate(int max, int curr) { MakeUpdate(Percent.FactoryPutInRange(curr, max == 0 ? 1 : max)); + _stepName.OnNext($"({_internalCurrentStep}/{_internalMaxStep}) {_currentStepName}, {curr} of {max}"); } } diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index c2ec0b4b..e53ec1e4 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -181,7 +181,7 @@ namespace Wabbajack.Lib await file.To.RelativeTo(OutputFolder).Compact(FileCompaction.Algorithm.XPRESS16K); } } - }, tempFolder: OutputFolder); + }, tempFolder: OutputFolder, updateTracker: UpdateTracker); } public async Task DownloadArchives() @@ -211,7 +211,7 @@ namespace Wabbajack.Lib DesiredThreads.OnNext(DownloadThreads); await missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State)) - .PMap(Queue, async archive => + .PMap(Queue, UpdateTracker, async archive => { Info($"Downloading {archive.Name}"); var outputPath = DownloadFolder.Combine(archive.Name); @@ -268,7 +268,7 @@ namespace Wabbajack.Lib var hashResults = await toHash - .PMap(Queue, async e => (await e.FileHashCachedAsync(), e)); + .PMap(Queue, UpdateTracker,async e => (await e.FileHashCachedAsync(), e)); HashedArchives.SetTo(hashResults .OrderByDescending(e => e.Item2.LastModified) @@ -385,9 +385,6 @@ namespace Wabbajack.Lib var path = OutputFolder.Combine(d.To); if (!existingfiles.Contains(path)) return null; - if (path.Size != d.Size) return null; - Status($"Optimizing {d.To}"); - return await path.FileHashCachedAsync() == d.Hash ? d : null; })) .Do(d => diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 165ce073..eccdf22f 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -202,7 +202,7 @@ namespace Wabbajack.Lib // Find all Downloads IndexedArchives = (await DownloadsPath.EnumerateFiles() .Where(f => f.WithExtension(Consts.MetaFileExtension).Exists) - .PMap(Queue, + .PMap(Queue, UpdateTracker, async f => new IndexedArchive(VFS.Index.ByRootPath[f]) { Name = (string)f.FileName, diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index 255dc434..f1bf7a82 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -257,7 +257,7 @@ namespace Wabbajack.Lib private async Task InstallIncludedDownloadMetas() { await ModList.Archives - .PMap(Queue, async archive => + .PMap(Queue, UpdateTracker, async archive => { if (HashedArchives.TryGetValue(archive.Hash, out var paths)) { @@ -313,7 +313,7 @@ namespace Wabbajack.Lib var bsaSize = bsa.FileStates.Select(state => sourceDir.Combine(state.Path).Size).Sum(); await using var a = await bsa.State.MakeBuilder(bsaSize); - var streams = await bsa.FileStates.PMap(Queue, async state => + var streams = await bsa.FileStates.PMap(Queue, UpdateTracker, async state => { Status($"Adding {state.Path} to BSA"); var fs = await sourceDir.Combine(state.Path).OpenRead(); @@ -344,7 +344,7 @@ namespace Wabbajack.Lib Info("Writing inline files"); await ModList.Directives .OfType() - .PMap(Queue, async directive => + .PMap(Queue, UpdateTracker, async directive => { Status($"Writing included file {directive.To}"); var outPath = OutputFolder.Combine(directive.To); diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index 3aea8ffb..33ee0ad4 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -209,7 +209,7 @@ namespace Wabbajack.VirtualFileSystem /// /// /// - public async Task Extract(WorkQueue queue, HashSet files, Func callback, AbsolutePath? tempFolder = null) + public async Task Extract(WorkQueue queue, HashSet files, Func callback, AbsolutePath? tempFolder = null, StatusUpdateTracker updateTracker = null) { var top = new VirtualFile(); var filesByParent = files.SelectMany(f => f.FilesInFullPath) @@ -251,7 +251,8 @@ namespace Wabbajack.VirtualFileSystem } } - await filesByParent[top].PMap(queue, async file => await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName) {CanMove = false})); + updateTracker ??= new StatusUpdateTracker(1); + await filesByParent[top].PMap(queue, updateTracker, async file => await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName) {CanMove = false})); } #region KnownFiles diff --git a/Wabbajack/View Models/Installers/InstallerVM.cs b/Wabbajack/View Models/Installers/InstallerVM.cs index 2b50cd06..50271e1e 100644 --- a/Wabbajack/View Models/Installers/InstallerVM.cs +++ b/Wabbajack/View Models/Installers/InstallerVM.cs @@ -306,6 +306,9 @@ namespace Wabbajack if (err) return "Corrupted Modlist"; return name; }) + .Merge(this.WhenAny(x => x.Installer.ActiveInstallation) + .Where(c => c != null) + .SelectMany(c => c.TextStatus)) .ToGuiProperty(this, nameof(ModListName)); ShowManifestCommand = ReactiveCommand.Create(() => From 98e0f2503cc7a258dd6b720a310aa15ecff14f64 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 5 Nov 2020 14:30:00 -0700 Subject: [PATCH 03/33] Rework how we handle API limits. --- Wabbajack.Lib/AInstaller.cs | 7 ++++ Wabbajack.Lib/Downloaders/NexusDownloader.cs | 14 ++++++-- Wabbajack.Lib/NexusApi/INexusApi.cs | 2 ++ Wabbajack.Lib/NexusApi/NexusApi.cs | 36 ++++++-------------- 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index e53ec1e4..37bb8147 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -195,6 +195,12 @@ namespace Wabbajack.Lib await Task.WhenAll(dispatchers.Select(d => d.Prepare())); + var nexusDownloader = dispatchers.OfType().FirstOrDefault(); + if (nexusDownloader != null && !await nexusDownloader.HaveEnoughAPICalls(missing)) + { + throw new Exception($"Not enough Nexus API calls to download this list, please try again after midnight GMT when your API limits reset"); + } + await DownloadMissingArchives(missing); } @@ -210,6 +216,7 @@ namespace Wabbajack.Lib } DesiredThreads.OnNext(DownloadThreads); + await missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State)) .PMap(Queue, UpdateTracker, async archive => { diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 1967584f..8fcd50b2 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.Linq; using System.Reactive; @@ -142,6 +143,16 @@ namespace Wabbajack.Lib.Downloaders } } + public async Task HaveEnoughAPICalls(IEnumerable archives) + { + if (await Client!.IsPremium()) + return true; + + var count = archives.Select(a => a.State).OfType().Count(); + + return count < Client!.RemainingAPICalls; + } + [JsonName("NexusDownloader")] public class State : AbstractDownloadState, IMetaState, IUpgradingState { @@ -191,12 +202,9 @@ namespace Wabbajack.Lib.Downloaders } catch (Exception ex) { - Utils.Log($"{a.Name} - Error getting Nexus download URL - {ex.Message}"); return false; } - Utils.Log($"Downloading Nexus Archive - {a.Name} - {Game} - {ModID} - {FileID}"); - return await new HTTPDownloader.State(url).Download(a, destination); } diff --git a/Wabbajack.Lib/NexusApi/INexusApi.cs b/Wabbajack.Lib/NexusApi/INexusApi.cs index e6bb9efd..a6d60907 100644 --- a/Wabbajack.Lib/NexusApi/INexusApi.cs +++ b/Wabbajack.Lib/NexusApi/INexusApi.cs @@ -13,5 +13,7 @@ namespace Wabbajack.Lib.NexusApi public Task GetUserStatus(); public Task IsPremium(); public bool IsAuthenticated { get; } + + public int RemainingAPICalls { get; } } } diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index c2be42d9..bf7a0e02 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -27,6 +27,7 @@ namespace Wabbajack.Lib.NexusApi public static string? ApiKey { get; set; } public bool IsAuthenticated => ApiKey != null; + public int RemainingAPICalls => Math.Max(HourlyRemaining, DailyRemaining); private Task? _userStatus; public Task UserStatus @@ -213,24 +214,12 @@ namespace Wabbajack.Lib.NexusApi } - protected virtual async Task UpdateRemaining(HttpResponseMessage response) { try { - var oldDaily = _dailyRemaining; - var oldHourly = _hourlyRemaining; - var dailyRemaining = int.Parse(response.Headers.GetValues("x-rl-daily-remaining").First()); - var hourlyRemaining = int.Parse(response.Headers.GetValues("x-rl-hourly-remaining").First()); - - lock (RemainingLock) - { - _dailyRemaining = Math.Min(_dailyRemaining, dailyRemaining); - _hourlyRemaining = Math.Min(_hourlyRemaining, hourlyRemaining); - } - - if (oldDaily != _dailyRemaining || oldHourly != _hourlyRemaining) - Utils.Log($"Nexus requests remaining: {_dailyRemaining} daily - {_hourlyRemaining} hourly"); + _dailyRemaining = int.Parse(response.Headers.GetValues("x-rl-daily-remaining").First()); + _hourlyRemaining = int.Parse(response.Headers.GetValues("x-rl-hourly-remaining").First()); this.RaisePropertyChanged(nameof(DailyRemaining)); this.RaisePropertyChanged(nameof(HourlyRemaining)); @@ -317,20 +306,17 @@ namespace Wabbajack.Lib.NexusApi var info = await GetModInfo(archive.Game, archive.ModID); if (!info.available) throw new Exception("Mod unavailable"); - - var url = $"https://api.nexusmods.com/v1/games/{archive.Game.MetaData().NexusName}/mods/{archive.ModID}/files/{archive.FileID}/download_link.json"; - try - { - return (await Get>(url)).First().URI; - } - catch (HttpException ex) - { - - if (ex.Code != 403 || await IsPremium()) + if (await IsPremium()) + { + if (HourlyRemaining <= 0 && DailyRemaining <= 0) { - throw; + throw new Exception($"You have run out of Nexus API requests, please try again after midnight GMT when the API limits reset"); } + + var url = + $"https://api.nexusmods.com/v1/games/{archive.Game.MetaData().NexusName}/mods/{archive.ModID}/files/{archive.FileID}/download_link.json"; + return (await Get>(url)).First().URI; } try From 6078926d015c2ba8a02eacec94a4f566265ce1cf Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 5 Nov 2020 15:45:22 -0700 Subject: [PATCH 04/33] Version bump --- CHANGELOG.md | 8 ++++++++ Wabbajack.CLI/Wabbajack.CLI.csproj | 4 ++-- Wabbajack.Launcher/Wabbajack.Launcher.csproj | 4 ++-- Wabbajack.Server/Wabbajack.Server.csproj | 4 ++-- Wabbajack/Wabbajack.csproj | 4 ++-- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 302c97c2..f1354926 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ### Changelog +#### Version - 2.3.3.0 - 11/5/2020 +* Game file hashes are now stored on Github instead of on the build server +* Added CLI Verb to produce these hash files for the Github repo +* When a user runs out of Nexus API calls we no longer bombard the Nexus with download attempts +* Check API limits before attempting a modlist download +* Logger is less chatty about recoverable download errors +* Display integer progress values during install so users know how far along in the process they are #issue-1156 + #### Version - 2.3.2.0 - 11/2/2020 * 7Zip errors now re-hash the extracted file to check for file corruption issues. Should provide better feedback in cases that a file is modified after being downloaded (perhaps by a disk failure) diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index 5e3d9ea2..9de67964 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -6,8 +6,8 @@ wabbajack-cli Wabbajack x64 - 2.3.2.0 - 2.3.2.0 + 2.3.3.0 + 2.3.3.0 Copyright © 2019-2020 An automated ModList installer true diff --git a/Wabbajack.Launcher/Wabbajack.Launcher.csproj b/Wabbajack.Launcher/Wabbajack.Launcher.csproj index a2e4372b..2c013949 100644 --- a/Wabbajack.Launcher/Wabbajack.Launcher.csproj +++ b/Wabbajack.Launcher/Wabbajack.Launcher.csproj @@ -4,8 +4,8 @@ WinExe netcoreapp3.1 true - 2.3.2.0 - 2.3.2.0 + 2.3.3.0 + 2.3.3.0 Copyright © 2019-2020 Wabbajack Application Launcher true diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index 547c4e4a..18f915b0 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -3,8 +3,8 @@ Exe netcoreapp3.1 - 2.3.2.0 - 2.3.2.0 + 2.3.3.0 + 2.3.3.0 Copyright © 2019-2020 Wabbajack Server win-x64 diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 22a48ba1..605c6f0d 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -6,8 +6,8 @@ true x64 win10-x64 - 2.3.2.0 - 2.3.2.0 + 2.3.3.0 + 2.3.3.0 Copyright © 2019-2020 An automated ModList installer true From fd74b1b4dcc67d86897533400f855ff9a8f40bfc Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 8 Nov 2020 17:41:17 -0700 Subject: [PATCH 05/33] Add backup/restore CLI options (not useful for most people) --- Wabbajack.CLI/OptionsDefinition.cs | 4 +- Wabbajack.CLI/Verbs/Backup.cs | 97 ++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 Wabbajack.CLI/Verbs/Backup.cs diff --git a/Wabbajack.CLI/OptionsDefinition.cs b/Wabbajack.CLI/OptionsDefinition.cs index e0193872..3eba22bd 100644 --- a/Wabbajack.CLI/OptionsDefinition.cs +++ b/Wabbajack.CLI/OptionsDefinition.cs @@ -32,7 +32,9 @@ namespace Wabbajack.CLI typeof(NoPatch), typeof(NexusPermissions), typeof(ExportServerGameFiles), - typeof(HashGamefiles) + typeof(HashGamefiles), + typeof(Backup), + typeof(Restore) }; } } diff --git a/Wabbajack.CLI/Verbs/Backup.cs b/Wabbajack.CLI/Verbs/Backup.cs new file mode 100644 index 00000000..75e35acd --- /dev/null +++ b/Wabbajack.CLI/Verbs/Backup.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Common.IO; + +namespace Wabbajack.CLI.Verbs +{ + + [Verb("backup", HelpText = @"Copy all encrypted and personal info into a location for transferring to a new machine", Hidden = true)] + public class Backup : AVerb + { + public static Dictionary Keys = new Dictionary() + { + {"bunnycdn", true}, + {"nexusapikey", true}, + {"nexus-cookies", true}, + {"x-metrics-key", true}, + {"author-api-key.txt", false} + }; + + [Option('o', "output", Required = true, HelpText = @"Output folder for the decrypted data")] + public string Output { get; set; } = ""; + + public AbsolutePath OutputPath => (AbsolutePath)Output; + protected override async Task Run() + { + foreach(var (name, encrypted) in Keys) + { + byte[] data; + var src = name.RelativeTo(Consts.LocalAppDataPath); + if (!src.Exists) + { + Console.WriteLine($"{name} doesn't exist, skipping"); + continue; + } + + if (encrypted) + { + data = await Utils.FromEncryptedData(name); + } + else + { + data = await src.ReadAllBytesAsync(); + } + + await name.RelativeTo(OutputPath).WriteAllBytesAsync(data); + } + + return ExitCode.Ok; + } + } + + [Verb("restore", HelpText = @"Copy all encrypted and personal info into a location for transferring to a new machine", Hidden = true)] + public class Restore : AVerb + { + public static Dictionary Keys = new Dictionary() + { + {"bunnycdn", true}, + {"nexusapikey", true}, + {"nexus-cookies", true}, + {"x-metrics-key", true}, + {"author-api-key.txt", false} + }; + + [Option('i', "input", Required = true, HelpText = @"Input folder for the decrypted data")] + public string Input { get; set; } = ""; + + public AbsolutePath InputPath => (AbsolutePath)Input; + protected override async Task Run() + { + foreach(var (name, encrypted) in Keys) + { + var src = name.RelativeTo(InputPath); + if (!src.Exists) + { + Console.WriteLine($"{name} doesn't exist, skipping"); + continue; + } + + var data = await src.ReadAllBytesAsync(); + + if (encrypted) + { + await data.ToEcryptedData(name); + } + else + { + await name.RelativeTo(Consts.LocalAppDataPath).WriteAllBytesAsync(data); + } + } + + return ExitCode.Ok; + } + } +} From ad1a0843deb777b2e72c70cc356d1126534f1985 Mon Sep 17 00:00:00 2001 From: erri120 Date: Mon, 9 Nov 2020 12:26:07 +0100 Subject: [PATCH 06/33] Removed the internal Manifest Viewer --- .../View Models/Installers/InstallerVM.cs | 2 +- Wabbajack/View Models/ManifestVM.cs | 102 ------------- Wabbajack/Views/ManifestView.xaml | 131 ---------------- Wabbajack/Views/ManifestView.xaml.cs | 143 ------------------ Wabbajack/Views/ManifestWindow.xaml | 22 --- Wabbajack/Views/ManifestWindow.xaml.cs | 22 --- 6 files changed, 1 insertion(+), 421 deletions(-) delete mode 100644 Wabbajack/View Models/ManifestVM.cs delete mode 100644 Wabbajack/Views/ManifestView.xaml delete mode 100644 Wabbajack/Views/ManifestView.xaml.cs delete mode 100644 Wabbajack/Views/ManifestWindow.xaml delete mode 100644 Wabbajack/Views/ManifestWindow.xaml.cs diff --git a/Wabbajack/View Models/Installers/InstallerVM.cs b/Wabbajack/View Models/Installers/InstallerVM.cs index 50271e1e..a56e3a66 100644 --- a/Wabbajack/View Models/Installers/InstallerVM.cs +++ b/Wabbajack/View Models/Installers/InstallerVM.cs @@ -313,7 +313,7 @@ namespace Wabbajack ShowManifestCommand = ReactiveCommand.Create(() => { - new ManifestWindow(ModList.SourceModList).Show(); + Utils.OpenWebsite(new Uri("https://www.wabbajack.org/#/modlists/manifest")); }, this.WhenAny(x => x.ModList) .Select(x => x?.SourceModList != null) .ObserveOnGuiThread()); diff --git a/Wabbajack/View Models/ManifestVM.cs b/Wabbajack/View Models/ManifestVM.cs deleted file mode 100644 index dd52745a..00000000 --- a/Wabbajack/View Models/ManifestVM.cs +++ /dev/null @@ -1,102 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reactive; -using System.Reactive.Linq; -using ReactiveUI; -using ReactiveUI.Fody.Helpers; -using Wabbajack.Common; -using Wabbajack.Lib; - -namespace Wabbajack -{ - public enum SortBy { Name, Size } - - public class ManifestVM : ViewModel - { - public Manifest Manifest { get; set; } - - public string Name => !string.IsNullOrWhiteSpace(Manifest.Name) ? Manifest.Name : "Wabbajack Modlist"; - public string Author => !string.IsNullOrWhiteSpace(Manifest.Author) ? $"Created by {Manifest.Author}" : "Created by Jyggalag"; - public string Description => !string.IsNullOrWhiteSpace(Manifest.Description) ? Manifest.Description : ""; - public string InstallSize => $"Install Size: {Manifest.InstallSize.ToFileSizeString()}"; - public string DownloadSize => $"Download Size: {Manifest.DownloadSize.ToFileSizeString()}"; - - public IEnumerable Archives => Manifest.Archives; - - [Reactive] - public string SearchTerm { get; set; } - - private readonly ObservableAsPropertyHelper> _searchResults; - public IEnumerable SearchResults => _searchResults.Value; - - [Reactive] - public bool SortAscending { get; set; } = true; - - [Reactive] - public SortBy SortEnum { get; set; } = SortBy.Name; - - public ReactiveCommand SortByNameCommand; - public ReactiveCommand SortBySizeCommand; - - private IEnumerable Order(IEnumerable list) - { - if (SortAscending) - { - return SortEnum switch - { - SortBy.Name => list.OrderBy(x => x.Name), - SortBy.Size => list.OrderBy(x => x.Size), - _ => throw new ArgumentOutOfRangeException() - }; - } - - return SortEnum switch - { - SortBy.Name => list.OrderByDescending(x => x.Name), - SortBy.Size => list.OrderByDescending(x => x.Size), - _ => throw new ArgumentOutOfRangeException() - }; - } - - private void Swap(SortBy to) - { - if (SortEnum != to) - SortEnum = to; - else - SortAscending = !SortAscending; - } - - public ManifestVM(Manifest manifest) - { - Manifest = manifest; - - SortByNameCommand = ReactiveCommand.Create(() => Swap(SortBy.Name)); - - SortBySizeCommand = ReactiveCommand.Create(() => Swap(SortBy.Size)); - - _searchResults = - this.WhenAnyValue(x => x.SearchTerm) - .CombineLatest( - this.WhenAnyValue(x => x.SortAscending), - this.WhenAnyValue(x => x.SortEnum), - (term, ascending, sort) => term) - .Throttle(TimeSpan.FromMilliseconds(800)) - .Select(term => term?.Trim()) - //.DistinctUntilChanged() - .Select(term => - { - if (string.IsNullOrWhiteSpace(term)) - return Order(Archives); - - return Order(Archives.Where(x => - { - if (term.StartsWith("hash:")) - return x.Hash.ToString().StartsWith(term.Replace("hash:", "")); - return x.Name.StartsWith(term); - })); - }) - .ToGuiProperty(this, nameof(SearchResults), Order(Archives)); - } - } -} diff --git a/Wabbajack/Views/ManifestView.xaml b/Wabbajack/Views/ManifestView.xaml deleted file mode 100644 index 00aa995f..00000000 --- a/Wabbajack/Views/ManifestView.xaml +++ /dev/null @@ -1,131 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - Order by: - - - - - - - - - - - - - - - - - - - - - - - - - - Link - - - - - - - - - - - - diff --git a/Wabbajack/Views/ManifestView.xaml.cs b/Wabbajack/Views/ManifestView.xaml.cs deleted file mode 100644 index e6162108..00000000 --- a/Wabbajack/Views/ManifestView.xaml.cs +++ /dev/null @@ -1,143 +0,0 @@ -using System; -using System.Diagnostics; -using System.Reactive.Disposables; -using System.Windows; -using System.Windows.Controls; -using System.Windows.Documents; -using System.Windows.Input; -using System.Windows.Navigation; -using ReactiveUI; -using Wabbajack.Lib; - -namespace Wabbajack -{ - public partial class ManifestView - { - public ModList Modlist { get; set; } - - public ManifestView(ModList modlist) - { - Modlist = modlist; - - var manifest = new Manifest(modlist); - ViewModel ??= new ManifestVM(manifest); - - InitializeComponent(); - - this.WhenActivated(disposable => - { - this.OneWayBind(ViewModel, x => x.Name, x => x.Name.Text) - .DisposeWith(disposable); - this.OneWayBind(ViewModel, x => x.Manifest.Version, x => x.Version.Text) - .DisposeWith(disposable); - this.OneWayBind(ViewModel, x => x.Author, x => x.Author.Text) - .DisposeWith(disposable); - this.OneWayBind(ViewModel, x => x.Description, x => x.Description.Text) - .DisposeWith(disposable); - this.OneWayBind(ViewModel, x => x.SearchResults, x => x.ModsList.ItemsSource) - .DisposeWith(disposable); - this.OneWayBind(ViewModel, x => x.InstallSize, x => x.InstallSize.Text) - .DisposeWith(disposable); - this.OneWayBind(ViewModel, x => x.DownloadSize, x => x.DownloadSize.Text) - .DisposeWith(disposable); - this.Bind(ViewModel, x => x.SearchTerm, x => x.SearchBar.Text) - .DisposeWith(disposable); - this.BindCommand(ViewModel, x => x.SortByNameCommand, x => x.OrderByNameButton) - .DisposeWith(disposable); - this.BindCommand(ViewModel, x => x.SortBySizeCommand, x => x.OrderBySizeButton) - .DisposeWith(disposable); - }); - } - - private void Hyperlink_OnRequestNavigate(object sender, RequestNavigateEventArgs e) - { - if (!(sender is Hyperlink hyperlink)) return; - if (!(hyperlink.DataContext is Archive archive)) return; - - var url = archive.State.GetManifestURL(archive); - if (string.IsNullOrWhiteSpace(url)) return; - - if (url.StartsWith("https://github.com/")) - url = url.Substring(0, url.IndexOf("release", StringComparison.Ordinal)); - - //url = url.Replace("&", "^&"); - url = url.Replace(" ", "%20"); - Process.Start(new ProcessStartInfo("cmd", $"/c start {url}") {CreateNoWindow = true}); - - e.Handled = true; - } - - //solution from https://stackoverflow.com/questions/5426232/how-can-i-make-wpf-scrollviewer-middle-click-scroll/5446307#5446307 - - private bool _isMoving; //False - ignore mouse movements and don't scroll - private bool _isDeferredMovingStarted; //True - Mouse down -> Mouse up without moving -> Move; False - Mouse down -> Move - private Point? _startPosition; - private const double Slowdown = 10; //smaller = faster - - private void ScrollViewer_MouseDown(object sender, MouseButtonEventArgs e) - { - if (_isMoving) - CancelScrolling(); - else if (e.ChangedButton == MouseButton.Middle && e.ButtonState == MouseButtonState.Pressed) - { - if (_isMoving) return; - - _isMoving = true; - _startPosition = e.GetPosition(sender as IInputElement); - _isDeferredMovingStarted = true; - - AddScrollSign(e.GetPosition(TopLayer).X, e.GetPosition(TopLayer).Y); - } - } - - private void ScrollViewer_MouseUp(object sender, MouseButtonEventArgs e) - { - if(e.ChangedButton == MouseButton.Middle && e.ButtonState == MouseButtonState.Released && _isDeferredMovingStarted != true) - CancelScrolling(); - } - - private void ScrollViewer_MouseMove(object sender, MouseEventArgs e) - { - if (!_isMoving || !(sender is ScrollViewer sv)) - return; - - _isDeferredMovingStarted = false; - - var currentPosition = e.GetPosition(sv); - if (_startPosition == null) - return; - - var offset = currentPosition - _startPosition.Value; - offset.Y /= Slowdown; - offset.X /= Slowdown; - - sv.ScrollToVerticalOffset(sv.VerticalOffset + offset.Y); - sv.ScrollToHorizontalOffset(sv.HorizontalOffset + offset.X); - } - - private void CancelScrolling() - { - _isMoving = false; - _startPosition = null; - _isDeferredMovingStarted = false; - RemoveScrollSign(); - } - - private void AddScrollSign(double x, double y) - { - const double size = 50.0; - var img = ResourceLinks.MiddleMouseButton.Value; - var icon = new Image {Source = img, Width = size, Height = size}; - //var icon = new Ellipse { Stroke = Brushes.Red, StrokeThickness = 2.0, Width = 20, Height = 20 }; - - TopLayer.Children.Add(icon); - Canvas.SetLeft(icon, x - size / 2); - Canvas.SetTop(icon, y - size / 2); - } - - private void RemoveScrollSign() - { - TopLayer.Children.Clear(); - } - } -} diff --git a/Wabbajack/Views/ManifestWindow.xaml b/Wabbajack/Views/ManifestWindow.xaml deleted file mode 100644 index 2f686a82..00000000 --- a/Wabbajack/Views/ManifestWindow.xaml +++ /dev/null @@ -1,22 +0,0 @@ - - - diff --git a/Wabbajack/Views/ManifestWindow.xaml.cs b/Wabbajack/Views/ManifestWindow.xaml.cs deleted file mode 100644 index 532754c6..00000000 --- a/Wabbajack/Views/ManifestWindow.xaml.cs +++ /dev/null @@ -1,22 +0,0 @@ -using Wabbajack.Lib; - -namespace Wabbajack -{ - public partial class ManifestWindow - { - public ModList Modlist { get; set; } - - public ManifestWindow(ModList modlist) - { - Modlist = modlist; - - InitializeComponent(); - - var manifestView = new ManifestView(Modlist); - - Grid.Children.Add(manifestView); - - Title = $"{Modlist.Name} by {Modlist.Author}"; - } - } -} From 17dbc216cbdada86a5e20753e524cd608475aea2 Mon Sep 17 00:00:00 2001 From: erri120 Date: Mon, 9 Nov 2020 12:26:54 +0100 Subject: [PATCH 07/33] Updated CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1354926..f08c247f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ### Changelog +#### Version - next +* Removed the internal Manifest Viewer, you can still view the Manifest of any Modlist on the website + #### Version - 2.3.3.0 - 11/5/2020 * Game file hashes are now stored on Github instead of on the build server * Added CLI Verb to produce these hash files for the Github repo From 01872fb62b661fb6cda52321051863b5e7bfee7e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 11 Nov 2020 06:25:57 -0700 Subject: [PATCH 08/33] More logging information for server backend --- Wabbajack.Lib/Downloaders/NexusDownloader.cs | 3 +++ Wabbajack.Lib/NexusApi/NexusApi.cs | 2 +- Wabbajack.Server/Services/NexusKeyMaintainance.cs | 10 +++++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 8fcd50b2..9f215713 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -282,6 +282,7 @@ namespace Wabbajack.Lib.Downloaders return (newArchive, new TempFile()); } + Utils.Log($"Downloading possible upgrade {newArchive.State.PrimaryKeyString}"); var tempFile = new TempFile(); await newArchive.State.Download(newArchive, tempFile.Path); @@ -289,6 +290,8 @@ namespace Wabbajack.Lib.Downloaders newArchive.Size = tempFile.Path.Size; newArchive.Hash = await tempFile.Path.FileHashAsync(); + Utils.Log($"Possible upgrade {newArchive.State.PrimaryKeyString} downloaded"); + return (newArchive, tempFile); } diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index bf7a0e02..f4bb739e 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -321,7 +321,7 @@ namespace Wabbajack.Lib.NexusApi try { - Utils.Log($"Requesting manual download for {archive.Name}"); + Utils.Log($"Requesting manual download for {archive.Name} {archive.PrimaryKeyString}"); return (await Utils.Log(await ManuallyDownloadNexusFile.Create(archive)).Task).ToString(); } catch (TaskCanceledException ex) diff --git a/Wabbajack.Server/Services/NexusKeyMaintainance.cs b/Wabbajack.Server/Services/NexusKeyMaintainance.cs index 4f37ef30..bb35fe60 100644 --- a/Wabbajack.Server/Services/NexusKeyMaintainance.cs +++ b/Wabbajack.Server/Services/NexusKeyMaintainance.cs @@ -25,7 +25,15 @@ namespace Wabbajack.Server.Services var keys = await _sql.GetNexusApiKeysWithCounts(1500); foreach (var key in keys.Where(k => k.Key != _selfKey)) { - return new TrackingClient(_sql, key); + var client = new TrackingClient(_sql, key); + if (!await client.IsPremium()) + { + _logger.LogWarning($"Purging non premium key"); + await _sql.DeleteNexusAPIKey(key.Key); + continue; + } + + return client; } return await NexusApiClient.Get(); From e1725bb80881db8b3f684ac1a47a671498027d9a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 14 Nov 2020 07:21:13 -0700 Subject: [PATCH 09/33] Throw critical errors on nexus API quota issues --- Wabbajack.Lib/Downloaders/NexusDownloader.cs | 7 +++++++ Wabbajack.Lib/NexusApi/NexusApi.cs | 2 +- .../StatusMessages/NexusAPIQuotaExceeded.cs | 12 ++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 Wabbajack.Lib/StatusMessages/NexusAPIQuotaExceeded.cs diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 9f215713..7ce2ff33 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -200,6 +200,11 @@ namespace Wabbajack.Lib.Downloaders var client = await NexusApiClient.Get(); url = await client.GetNexusDownloadLink(this); } + catch (NexusAPIQuotaExceeded ex) + { + Utils.Log(ex.ExtendedDescription); + throw; + } catch (Exception ex) { return false; @@ -248,6 +253,8 @@ namespace Wabbajack.Lib.Downloaders public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func> downloadResolver) { var client = await NexusApiClient.Get(); + if (client.RemainingAPICalls <= 0) + throw new NexusAPIQuotaExceeded(); var mod = await client.GetModInfo(Game, ModID); if (!mod.available) diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index f4bb739e..56a078f6 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -311,7 +311,7 @@ namespace Wabbajack.Lib.NexusApi { if (HourlyRemaining <= 0 && DailyRemaining <= 0) { - throw new Exception($"You have run out of Nexus API requests, please try again after midnight GMT when the API limits reset"); + throw new NexusAPIQuotaExceeded(); } var url = diff --git a/Wabbajack.Lib/StatusMessages/NexusAPIQuotaExceeded.cs b/Wabbajack.Lib/StatusMessages/NexusAPIQuotaExceeded.cs new file mode 100644 index 00000000..99273290 --- /dev/null +++ b/Wabbajack.Lib/StatusMessages/NexusAPIQuotaExceeded.cs @@ -0,0 +1,12 @@ +using Wabbajack.Common.StatusFeed; + +namespace Wabbajack.Lib +{ + public class NexusAPIQuotaExceeded : AErrorMessage + { + public override string ShortDescription => $"You have exceeded your Nexus API limit for the day"; + + public override string ExtendedDescription => + "You have exceeded your Nexus API limit for the day, please try again after midnight GMT"; + } +} From 802448c934427b8179737f49a1cb1996e9d61cfc Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 14 Nov 2020 07:26:04 -0700 Subject: [PATCH 10/33] Deps version bump --- Compression.BSA/Compression.BSA.csproj | 2 +- Wabbajack.CLI/Wabbajack.CLI.csproj | 4 ++-- Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj | 2 +- Wabbajack.Common/Extensions/RxExt.cs | 2 +- Wabbajack.Common/Wabbajack.Common.csproj | 10 +++++----- Wabbajack.Lib/ClientAPI.cs | 2 +- Wabbajack.Lib/CompilationSteps/IncludePatches.cs | 1 - Wabbajack.Lib/Downloaders/NexusDownloader.cs | 2 +- Wabbajack.Lib/MO2Compiler.cs | 2 -- Wabbajack.Lib/Wabbajack.Lib.csproj | 8 ++++---- Wabbajack.Server/Wabbajack.Server.csproj | 1 - Wabbajack.Test/Wabbajack.Test.csproj | 2 +- Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs | 2 +- .../Wabbajack.VirtualFileSystem.Test.csproj | 2 +- Wabbajack.VirtualFileSystem/Context.cs | 2 +- Wabbajack.VirtualFileSystem/VirtualFile.cs | 4 ++-- .../Wabbajack.VirtualFileSystem.csproj | 2 +- Wabbajack/Wabbajack.csproj | 2 +- 18 files changed, 24 insertions(+), 28 deletions(-) diff --git a/Compression.BSA/Compression.BSA.csproj b/Compression.BSA/Compression.BSA.csproj index 3b725ae7..7b51c3b9 100644 --- a/Compression.BSA/Compression.BSA.csproj +++ b/Compression.BSA/Compression.BSA.csproj @@ -22,7 +22,7 @@ - + diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index 9de67964..aaa1e540 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -20,8 +20,8 @@ - - + + diff --git a/Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj b/Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj index 622e38a9..6286f90b 100644 --- a/Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj +++ b/Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj @@ -6,7 +6,7 @@ win10-x64 - + diff --git a/Wabbajack.Common/Extensions/RxExt.cs b/Wabbajack.Common/Extensions/RxExt.cs index 1eb06521..788fc0ab 100644 --- a/Wabbajack.Common/Extensions/RxExt.cs +++ b/Wabbajack.Common/Extensions/RxExt.cs @@ -103,7 +103,7 @@ namespace Wabbajack { // We have another value that came in to fire. // Reregister for callback - dueTimeDisposable.Disposable = scheduler.Schedule(interval, internalCallback); + dueTimeDisposable.Disposable = scheduler!.Schedule(interval, internalCallback); o.OnNext(value!); value = default; hasValue = false; diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 9cd2a59d..3bebdf1c 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -49,9 +49,9 @@ - + - + @@ -59,9 +59,9 @@ - - - + + + diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index fea3de11..3984284c 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -264,7 +264,7 @@ using Wabbajack.Lib.Downloaders; await client.GetStringAsync($"{Consts.WabbajackBuildServerUri}mirror/{archiveHash.ToHex()}"); return new Uri(result); } - catch (HttpException ex) + catch (HttpException) { return null; } diff --git a/Wabbajack.Lib/CompilationSteps/IncludePatches.cs b/Wabbajack.Lib/CompilationSteps/IncludePatches.cs index 136884a6..140b1785 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludePatches.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludePatches.cs @@ -16,7 +16,6 @@ namespace Wabbajack.Lib.CompilationSteps private readonly Dictionary> _indexed; private VirtualFile? _bsa; private Dictionary> _indexedByName; - private ACompiler _compiler; private bool _isGenericGame; public IncludePatches(ACompiler compiler, VirtualFile? constructingFromBSA = null) : base(compiler) diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 7ce2ff33..27c223cb 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -205,7 +205,7 @@ namespace Wabbajack.Lib.Downloaders Utils.Log(ex.ExtendedDescription); throw; } - catch (Exception ex) + catch (Exception) { return false; } diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index eccdf22f..f0340d12 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -15,8 +15,6 @@ namespace Wabbajack.Lib { public class MO2Compiler : ACompiler { - private AbsolutePath _mo2DownloadsFolder; - public MO2Compiler(AbsolutePath sourcePath, AbsolutePath downloadsPath, string mo2Profile, AbsolutePath outputFile) : base(21, mo2Profile, sourcePath, downloadsPath, outputFile) { diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index ebec699c..5f42ead1 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -25,7 +25,7 @@ 2.2.2.1 - 1.11.26 + 1.11.28 1.8.2 @@ -46,16 +46,16 @@ 0.26.0 - 5.0.0-preview.8.20407.11 + 5.0.0 - 5.0.0-preview.8.20407.11 + 5.0.0 4.3.4 - 5.0.0-preview.8.20407.11 + 5.0.0 1.0.1 diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index 18f915b0..5e7c0ffe 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -14,7 +14,6 @@ - diff --git a/Wabbajack.Test/Wabbajack.Test.csproj b/Wabbajack.Test/Wabbajack.Test.csproj index e8bd6668..4ef2cd16 100644 --- a/Wabbajack.Test/Wabbajack.Test.csproj +++ b/Wabbajack.Test/Wabbajack.Test.csproj @@ -30,7 +30,7 @@ - + diff --git a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs index 79283ec1..a601d3c6 100644 --- a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs @@ -90,7 +90,7 @@ namespace Wabbajack.VirtualFileSystem.Test return await s.xxHashAsync(); }); - Assert.Equal(1, results.Count); + Assert.Single(results); foreach (var (path, hash) in results) { Assert.Equal(await temp.Dir.Combine(path).FileHashAsync(), hash); diff --git a/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj index 28ac0759..0fa7d79b 100644 --- a/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj +++ b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj @@ -8,7 +8,7 @@ - + diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index 33ee0ad4..a9fb03cc 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -238,7 +238,7 @@ namespace Wabbajack.VirtualFileSystem tempFolder: tempFolder, onlyFiles: fileNames.Keys.ToHashSet()); } - catch (_7zipReturnError ex) + catch (_7zipReturnError) { await using var stream = await sfn.GetStream(); var hash = await stream.xxHashAsync(); diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 224be3af..a038f099 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -224,11 +224,11 @@ namespace Wabbajack.VirtualFileSystem self.Children = list.Values.ToImmutableList(); } - catch (EndOfStreamException ex) + catch (EndOfStreamException) { return self; } - catch (Exception ex) + catch (Exception) { Utils.Log($"Error while examining the contents of {relPath.FileName}"); throw; diff --git a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj index 17d43932..505d0256 100644 --- a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj +++ b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj @@ -18,7 +18,7 @@ - + diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 605c6f0d..b3e619f5 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -67,7 +67,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + From 73d3168d2cff559a19d71f5df2332ecb97213359 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 14 Nov 2020 12:16:11 -0700 Subject: [PATCH 11/33] Implement utility list filtering for the UI --- Wabbajack.Lib/ModListRegistry/ModListMetadata.cs | 3 +++ .../View Models/Gallery/ModListGalleryVM.cs | 16 ++++++++++++++++ Wabbajack/Views/ModListGalleryView.xaml | 8 ++++++++ 3 files changed, 27 insertions(+) diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index 3b54d352..20636c5c 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -34,6 +34,9 @@ namespace Wabbajack.Lib.ModListRegistry [JsonProperty("nsfw")] public bool NSFW { get; set; } + + [JsonProperty("utility_list")] + public bool UtilityList { get; set; } [JsonProperty("links")] public LinksObject Links { get; set; } = new LinksObject(); diff --git a/Wabbajack/View Models/Gallery/ModListGalleryVM.cs b/Wabbajack/View Models/Gallery/ModListGalleryVM.cs index 7d914fa8..81125437 100644 --- a/Wabbajack/View Models/Gallery/ModListGalleryVM.cs +++ b/Wabbajack/View Models/Gallery/ModListGalleryVM.cs @@ -37,6 +37,9 @@ namespace Wabbajack [Reactive] public bool ShowNSFW { get; set; } + + [Reactive] + public bool ShowUtilityLists { get; set; } [Reactive] public string GameType { get; set; } @@ -77,6 +80,7 @@ namespace Wabbajack { OnlyInstalled = false; ShowNSFW = false; + ShowUtilityLists = false; Search = string.Empty; GameType = ALL_GAME_TYPE; }); @@ -150,6 +154,12 @@ namespace Wabbajack if (!vm.Metadata.NSFW) return true; return vm.Metadata.NSFW && showNSFW; })) + .Filter(this.WhenAny(x => x.ShowUtilityLists) + .Select>(showNSFW => vm => + { + if (!vm.Metadata.UtilityList) return true; + return vm.Metadata.UtilityList && showNSFW; + })) // Filter by Game .Filter(this.WhenAny(x => x.GameType) .Debounce(TimeSpan.FromMilliseconds(150), RxApp.MainThreadScheduler) @@ -169,6 +179,12 @@ namespace Wabbajack if (!vm.Metadata.NSFW) return true; return vm.Metadata.NSFW && showNSFW; })) + .Filter(this.WhenAny(x => x.ShowUtilityLists) + .Select>(showUtilityLists => vm => + { + if (!vm.Metadata.UtilityList) return true; + return vm.Metadata.UtilityList && showUtilityLists; + })) // Put broken lists at bottom .Sort(Comparer.Create((a, b) => a.IsBroken.CompareTo(b.IsBroken))) .Bind(ModLists) diff --git a/Wabbajack/Views/ModListGalleryView.xaml b/Wabbajack/Views/ModListGalleryView.xaml index 0d698ec3..152090b9 100644 --- a/Wabbajack/Views/ModListGalleryView.xaml +++ b/Wabbajack/Views/ModListGalleryView.xaml @@ -116,6 +116,14 @@ VerticalAlignment="Center" Content="Show NSFW" Foreground="{StaticResource ForegroundBrush}" /> + + + Date: Sat, 14 Nov 2020 12:30:28 -0700 Subject: [PATCH 12/33] Binding fixes for utility lists --- Wabbajack/Settings.cs | 1 + .../View Models/Gallery/ModListGalleryVM.cs | 20 +++---------------- Wabbajack/Views/ModListGalleryView.xaml.cs | 2 ++ 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/Wabbajack/Settings.cs b/Wabbajack/Settings.cs index 84736319..d4990ac8 100644 --- a/Wabbajack/Settings.cs +++ b/Wabbajack/Settings.cs @@ -107,6 +107,7 @@ namespace Wabbajack private bool _useCompression = false; public bool UseCompression { get => _useCompression; set => RaiseAndSetIfChanged(ref _useCompression, value); } + public bool ShowUtilityLists { get; set; } } [JsonName("PerformanceSettings")] diff --git a/Wabbajack/View Models/Gallery/ModListGalleryVM.cs b/Wabbajack/View Models/Gallery/ModListGalleryVM.cs index 81125437..12d6903d 100644 --- a/Wabbajack/View Models/Gallery/ModListGalleryVM.cs +++ b/Wabbajack/View Models/Gallery/ModListGalleryVM.cs @@ -64,6 +64,7 @@ namespace Wabbajack { GameType = !string.IsNullOrEmpty(settings.Game) ? settings.Game : ALL_GAME_TYPE; ShowNSFW = settings.ShowNSFW; + ShowUtilityLists = settings.ShowUtilityLists; OnlyInstalled = settings.OnlyInstalled; Search = settings.Search; } @@ -155,11 +156,7 @@ namespace Wabbajack return vm.Metadata.NSFW && showNSFW; })) .Filter(this.WhenAny(x => x.ShowUtilityLists) - .Select>(showNSFW => vm => - { - if (!vm.Metadata.UtilityList) return true; - return vm.Metadata.UtilityList && showNSFW; - })) + .Select>(showUtilityLists => vm => showUtilityLists ? vm.Metadata.UtilityList : !vm.Metadata.UtilityList)) // Filter by Game .Filter(this.WhenAny(x => x.GameType) .Debounce(TimeSpan.FromMilliseconds(150), RxApp.MainThreadScheduler) @@ -173,18 +170,6 @@ namespace Wabbajack return GameType == vm.Metadata.Game.GetDescription().ToString(); })) - .Filter(this.WhenAny(x => x.ShowNSFW) - .Select>(showNSFW => vm => - { - if (!vm.Metadata.NSFW) return true; - return vm.Metadata.NSFW && showNSFW; - })) - .Filter(this.WhenAny(x => x.ShowUtilityLists) - .Select>(showUtilityLists => vm => - { - if (!vm.Metadata.UtilityList) return true; - return vm.Metadata.UtilityList && showUtilityLists; - })) // Put broken lists at bottom .Sort(Comparer.Create((a, b) => a.IsBroken.CompareTo(b.IsBroken))) .Bind(ModLists) @@ -220,6 +205,7 @@ namespace Wabbajack settings.Game = GameType; settings.Search = Search; settings.ShowNSFW = ShowNSFW; + settings.ShowUtilityLists = ShowUtilityLists; settings.OnlyInstalled = OnlyInstalled; } } diff --git a/Wabbajack/Views/ModListGalleryView.xaml.cs b/Wabbajack/Views/ModListGalleryView.xaml.cs index 863d7234..d5c5e7f3 100644 --- a/Wabbajack/Views/ModListGalleryView.xaml.cs +++ b/Wabbajack/Views/ModListGalleryView.xaml.cs @@ -63,6 +63,8 @@ namespace Wabbajack .DisposeWith(dispose); this.BindStrict(ViewModel, vm => vm.ShowNSFW, x => x.ShowNSFW.IsChecked) .DisposeWith(dispose); + this.BindStrict(ViewModel, vm => vm.ShowUtilityLists, x => x.ShowUtilityLists.IsChecked) + .DisposeWith(dispose); this.WhenAny(x => x.ViewModel.ClearFiltersCommand) .BindToStrict(this, x => x.ClearFiltersButton.Command) From 6ee701262a60e196f130aff701b7351eebcf3736 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 14 Nov 2020 23:20:05 -0700 Subject: [PATCH 13/33] Version bump to 2.3.4.0 --- CHANGELOG.md | 4 +++- Wabbajack.CLI/Wabbajack.CLI.csproj | 4 ++-- Wabbajack.Launcher/Wabbajack.Launcher.csproj | 4 ++-- Wabbajack.Lib/NexusApi/NexusApi.cs | 9 +++++++-- Wabbajack.Server/Wabbajack.Server.csproj | 4 ++-- Wabbajack/Wabbajack.csproj | 4 ++-- 6 files changed, 18 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f08c247f..62b3eecd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ ### Changelog -#### Version - next +#### Version - 2.3.4.0 - 11/15/2020 * Removed the internal Manifest Viewer, you can still view the Manifest of any Modlist on the website +* Improved Nexus warnings about being low on API calls +* Added marker for "utility modlists" we will expand on this feature further in later releases #### Version - 2.3.3.0 - 11/5/2020 * Game file hashes are now stored on Github instead of on the build server diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index aaa1e540..ac77aa43 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -6,8 +6,8 @@ wabbajack-cli Wabbajack x64 - 2.3.3.0 - 2.3.3.0 + 2.3.4.0 + 2.3.4.0 Copyright © 2019-2020 An automated ModList installer true diff --git a/Wabbajack.Launcher/Wabbajack.Launcher.csproj b/Wabbajack.Launcher/Wabbajack.Launcher.csproj index 2c013949..1688fc1c 100644 --- a/Wabbajack.Launcher/Wabbajack.Launcher.csproj +++ b/Wabbajack.Launcher/Wabbajack.Launcher.csproj @@ -4,8 +4,8 @@ WinExe netcoreapp3.1 true - 2.3.3.0 - 2.3.3.0 + 2.3.4.0 + 2.3.4.0 Copyright © 2019-2020 Wabbajack Application Launcher true diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index 56a078f6..b430a1df 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -71,7 +71,7 @@ namespace Wabbajack.Lib.NexusApi { return env_key; } - + return await RequestAndCacheAPIKey(); } } @@ -155,7 +155,12 @@ namespace Wabbajack.Lib.NexusApi public async Task GetUserStatus() { var url = "https://api.nexusmods.com/v1/users/validate.json"; - return await Get(url); + var result = await Get(url); + + Utils.Log($"Logged into the nexus as {result.name}"); + Utils.Log($"Nexus calls remaining: {DailyRemaining} daily, {HourlyRemaining} hourly"); + + return result; } public async Task<(int, int)> GetRemainingApiCalls() diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index 5e7c0ffe..4a65f933 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -3,8 +3,8 @@ Exe netcoreapp3.1 - 2.3.3.0 - 2.3.3.0 + 2.3.4.0 + 2.3.4.0 Copyright © 2019-2020 Wabbajack Server win-x64 diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index b3e619f5..01f94cce 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -6,8 +6,8 @@ true x64 win10-x64 - 2.3.3.0 - 2.3.3.0 + 2.3.4.0 + 2.3.4.0 Copyright © 2019-2020 An automated ModList installer true From 247d19cba0b26e5729f9acc1f6e76f4c7c253b67 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 15 Nov 2020 17:40:49 -0700 Subject: [PATCH 14/33] Version bump to 2.3.4.1 - fix nexus client creation --- CHANGELOG.md | 3 +++ Wabbajack.CLI/Wabbajack.CLI.csproj | 4 ++-- Wabbajack.Launcher/Wabbajack.Launcher.csproj | 4 ++-- Wabbajack.Lib/Downloaders/NexusDownloader.cs | 4 +++- Wabbajack.Server/Wabbajack.Server.csproj | 4 ++-- Wabbajack/Wabbajack.csproj | 4 ++-- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62b3eecd..9051b3f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ### Changelog +#### Version - 2.3.4.1 - 11/15/2020 +* Tell the mod updater to use the existing Nexus Client instead of creating a new one + #### Version - 2.3.4.0 - 11/15/2020 * Removed the internal Manifest Viewer, you can still view the Manifest of any Modlist on the website * Improved Nexus warnings about being low on API calls diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index ac77aa43..169fe589 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -6,8 +6,8 @@ wabbajack-cli Wabbajack x64 - 2.3.4.0 - 2.3.4.0 + 2.3.4.1 + 2.3.4.1 Copyright © 2019-2020 An automated ModList installer true diff --git a/Wabbajack.Launcher/Wabbajack.Launcher.csproj b/Wabbajack.Launcher/Wabbajack.Launcher.csproj index 1688fc1c..c7402451 100644 --- a/Wabbajack.Launcher/Wabbajack.Launcher.csproj +++ b/Wabbajack.Launcher/Wabbajack.Launcher.csproj @@ -4,8 +4,8 @@ WinExe netcoreapp3.1 true - 2.3.4.0 - 2.3.4.0 + 2.3.4.1 + 2.3.4.1 Copyright © 2019-2020 Wabbajack Application Launcher true diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 27c223cb..c16476a0 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -252,7 +252,9 @@ namespace Wabbajack.Lib.Downloaders public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func> downloadResolver) { - var client = await NexusApiClient.Get(); + var client = DownloadDispatcher.GetInstance().Client ?? await NexusApiClient.Get(); + await client.IsPremium(); + if (client.RemainingAPICalls <= 0) throw new NexusAPIQuotaExceeded(); diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index 4a65f933..cb1a86b3 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -3,8 +3,8 @@ Exe netcoreapp3.1 - 2.3.4.0 - 2.3.4.0 + 2.3.4.1 + 2.3.4.1 Copyright © 2019-2020 Wabbajack Server win-x64 diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 01f94cce..67a566fd 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -6,8 +6,8 @@ true x64 win10-x64 - 2.3.4.0 - 2.3.4.0 + 2.3.4.1 + 2.3.4.1 Copyright © 2019-2020 An automated ModList installer true From c37248c2214857311f0a85f2abbea317b3864ad0 Mon Sep 17 00:00:00 2001 From: erri120 Date: Wed, 18 Nov 2020 12:37:31 +0100 Subject: [PATCH 15/33] Fixed build errors --- Wabbajack.Common/AsyncBlockingCollection.cs | 2 +- Wabbajack.Common/Error States/GetResponse.cs | 4 ++-- Wabbajack.Common/Extensions/RxExt.cs | 2 +- Wabbajack.Lib/Extensions/ReactiveUIExt.cs | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Wabbajack.Common/AsyncBlockingCollection.cs b/Wabbajack.Common/AsyncBlockingCollection.cs index 27225081..953bf1ad 100644 --- a/Wabbajack.Common/AsyncBlockingCollection.cs +++ b/Wabbajack.Common/AsyncBlockingCollection.cs @@ -30,7 +30,7 @@ namespace Wabbajack.Common } if (DateTime.Now - startTime > timeout || token.IsCancellationRequested || isDisposed) - return (false, default); + return (false, default)!; await Task.Delay(100); } } diff --git a/Wabbajack.Common/Error States/GetResponse.cs b/Wabbajack.Common/Error States/GetResponse.cs index 8b80bad4..2c82b2dc 100644 --- a/Wabbajack.Common/Error States/GetResponse.cs +++ b/Wabbajack.Common/Error States/GetResponse.cs @@ -33,7 +33,7 @@ namespace Wabbajack string? reason = null, Exception? ex = null) { - Value = val; + Value = val!; Succeeded = succeeded; _reason = reason ?? string.Empty; Exception = ex; @@ -128,7 +128,7 @@ namespace Wabbajack public static GetResponse Create(bool successful, T val = default(T), string? reason = null) { - return new GetResponse(successful, val, reason); + return new GetResponse(successful, val!, reason); } #endregion } diff --git a/Wabbajack.Common/Extensions/RxExt.cs b/Wabbajack.Common/Extensions/RxExt.cs index 788fc0ab..3b5500e7 100644 --- a/Wabbajack.Common/Extensions/RxExt.cs +++ b/Wabbajack.Common/Extensions/RxExt.cs @@ -204,7 +204,7 @@ namespace Wabbajack var prev = prevStorage; prevStorage = i; return (prev, i); - }); + })!; } public static IObservable DelayInitial(this IObservable source, TimeSpan delay, IScheduler scheduler) diff --git a/Wabbajack.Lib/Extensions/ReactiveUIExt.cs b/Wabbajack.Lib/Extensions/ReactiveUIExt.cs index e608cf3b..f052bda4 100644 --- a/Wabbajack.Lib/Extensions/ReactiveUIExt.cs +++ b/Wabbajack.Lib/Extensions/ReactiveUIExt.cs @@ -105,7 +105,7 @@ namespace Wabbajack { return source .ToProperty(vm, property, initialValue, deferSubscription, RxApp.MainThreadScheduler) - .DisposeWith(vm.CompositeDisposable); + .DisposeWith(vm.CompositeDisposable)!; } public static void ToGuiProperty( @@ -116,7 +116,7 @@ namespace Wabbajack TRet initialValue = default, bool deferSubscription = false) { - source.ToProperty(vm, property, out result, initialValue, deferSubscription, RxApp.MainThreadScheduler) + source.ToProperty(vm, property, out result!, initialValue, deferSubscription, RxApp.MainThreadScheduler) .DisposeWith(vm.CompositeDisposable); } From 35b910be98c9c13b08bc0efe857c3414790084ac Mon Sep 17 00:00:00 2001 From: Chris Bessent Date: Wed, 18 Nov 2020 20:04:43 -0700 Subject: [PATCH 16/33] Make more path comparisons ignore case Paths for Windows should be case-agnostic. The primary impetus for this change is to prevent WJ from deleting downloads if the case of the folder name doesn't match the case of the install setting. Normally this would be caught by the GUI validation but several users have managed to get around that. --- Wabbajack.Common/Paths/AbsolutePath.cs | 4 ++-- Wabbajack.Common/Paths/RelativePath.cs | 6 +++--- Wabbajack.Common/Utils.cs | 2 +- Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Wabbajack.Common/Paths/AbsolutePath.cs b/Wabbajack.Common/Paths/AbsolutePath.cs index 90297883..e7f4a5c6 100644 --- a/Wabbajack.Common/Paths/AbsolutePath.cs +++ b/Wabbajack.Common/Paths/AbsolutePath.cs @@ -275,7 +275,7 @@ namespace Wabbajack.Common public bool InFolder(AbsolutePath folder) { - return _path.StartsWith(folder._path + Path.DirectorySeparator); + return _path.StartsWith(folder._path + Path.DirectorySeparator, StringComparison.OrdinalIgnoreCase); } public async Task ReadAllBytesAsync() @@ -386,7 +386,7 @@ namespace Wabbajack.Common public int CompareTo(AbsolutePath other) { - return string.Compare(_path, other._path, StringComparison.Ordinal); + return string.Compare(_path, other._path, StringComparison.OrdinalIgnoreCase); } public string ReadAllText() diff --git a/Wabbajack.Common/Paths/RelativePath.cs b/Wabbajack.Common/Paths/RelativePath.cs index 21aa1238..01b62614 100644 --- a/Wabbajack.Common/Paths/RelativePath.cs +++ b/Wabbajack.Common/Paths/RelativePath.cs @@ -136,12 +136,12 @@ namespace Wabbajack.Common public bool StartsWith(string s) { - return _path.StartsWith(s); + return _path.StartsWith(s, StringComparison.OrdinalIgnoreCase); } public bool StartsWith(RelativePath s) { - return _path.StartsWith(s._path); + return _path.StartsWith(s._path, StringComparison.OrdinalIgnoreCase); } public RelativePath Combine(params RelativePath[] paths ) @@ -156,7 +156,7 @@ namespace Wabbajack.Common public int CompareTo(RelativePath other) { - return string.Compare(_path, other._path, StringComparison.Ordinal); + return string.Compare(_path, other._path, StringComparison.OrdinalIgnoreCase); } } } diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index e1459e84..c1a9b6d5 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -866,7 +866,7 @@ namespace Wabbajack.Common public static bool IsInPath(this string path, string parent) { - return path.ToLower().TrimEnd('\\').StartsWith(parent.ToLower().TrimEnd('\\') + "\\"); + return path.ToLower().TrimEnd('\\').StartsWith(parent.ToLower().TrimEnd('\\') + "\\", StringComparison.OrdinalIgnoreCase); } public static async Task CopyToLimitAsync(this Stream frm, Stream tw, long limit) diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs b/Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs index 47f423d0..73b34d8c 100644 --- a/Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs +++ b/Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs @@ -15,7 +15,7 @@ namespace Wabbajack.Lib.CompilationSteps public override async ValueTask Run(RawSourceFile source) { - if (!((string)source.Path).StartsWith(_startDir)) return null; + if (!((string)source.Path).StartsWith(_startDir, System.StringComparison.OrdinalIgnoreCase)) return null; var i = source.EvolveTo(); i.Reason = "Default game file"; return i; From 9717d4ac4214cdb1c343685d61d80a4eedf64f8a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 19 Nov 2020 15:56:30 -0700 Subject: [PATCH 17/33] Improve backend services, add watchdog service, and add utility_modlists.json --- Wabbajack.Common/Consts.cs | 1 + .../ModListRegistry/ModListMetadata.cs | 5 ++ .../ModListValidationTests.cs | 3 + Wabbajack.Server/Controllers/Heartbeat.cs | 59 ++++++++++++++++++- Wabbajack.Server/Services/AbstractService.cs | 19 +++++- Wabbajack.Server/Services/ListValidator.cs | 40 +++++++++++-- Wabbajack.Server/Services/QuickSync.cs | 16 +++++ Wabbajack.Server/Services/Watchdog.cs | 33 +++++++++++ Wabbajack.Server/Startup.cs | 2 + .../View Models/Gallery/ModListMetadataVM.cs | 2 +- 10 files changed, 172 insertions(+), 8 deletions(-) create mode 100644 Wabbajack.Server/Services/Watchdog.cs diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index f85d2dfa..ea905001 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -84,6 +84,7 @@ namespace Wabbajack.Common public static string ServerWhitelistURL = "https://raw.githubusercontent.com/wabbajack-tools/opt-out-lists/master/ServerWhitelist.yml"; public static string ModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/modlists.json"; + public static string UtilityModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/utility_modlists.json"; public static string UnlistedModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/unlisted_modlists.json"; public static string ModlistSummaryURL = "http://build.wabbajack.org/lists/status.json"; public static string UserAgent diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index 20636c5c..adc265ea 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -38,6 +38,9 @@ namespace Wabbajack.Lib.ModListRegistry [JsonProperty("utility_list")] public bool UtilityList { get; set; } + [JsonProperty("force_down")] + public bool ForceDown { get; set; } + [JsonProperty("links")] public LinksObject Links { get; set; } = new LinksObject(); @@ -68,9 +71,11 @@ namespace Wabbajack.Lib.ModListRegistry var client = new Http.Client(); Utils.Log("Loading ModLists from GitHub"); var metadataResult = client.GetStringAsync(Consts.ModlistMetadataURL); + var utilityResult = client.GetStringAsync(Consts.UtilityModlistMetadataURL); var summaryResult = client.GetStringAsync(Consts.ModlistSummaryURL); var metadata = (await metadataResult).FromJsonString>(); + metadata = metadata.Concat((await utilityResult).FromJsonString>()).ToList(); try { var summaries = (await summaryResult).FromJsonString>().ToDictionary(d => d.MachineURL); diff --git a/Wabbajack.Server.Test/ModListValidationTests.cs b/Wabbajack.Server.Test/ModListValidationTests.cs index 961e3a01..f2759bcc 100644 --- a/Wabbajack.Server.Test/ModListValidationTests.cs +++ b/Wabbajack.Server.Test/ModListValidationTests.cs @@ -189,6 +189,9 @@ namespace Wabbajack.BuildServer.Test var statusRss = await _client.GetHtmlAsync(MakeURL("lists/status/test_list/broken.rss")); Assert.Equal(failed, statusRss.DocumentNode.SelectNodes("//item")?.Count ?? 0); + + var heartBeat = await _client.GetHtmlAsync(MakeURL("heartbeat/report")); + Assert.Contains(heartBeat.DocumentNode.Descendants(), c => c.InnerText.StartsWith("test_list")); } diff --git a/Wabbajack.Server/Controllers/Heartbeat.cs b/Wabbajack.Server/Controllers/Heartbeat.cs index 7914b388..ab12025a 100644 --- a/Wabbajack.Server/Controllers/Heartbeat.cs +++ b/Wabbajack.Server/Controllers/Heartbeat.cs @@ -1,12 +1,16 @@ using System; using System.Collections.Generic; +using System.Linq; +using System.Net; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; +using Nettle; using Wabbajack.Common.StatusFeed; using Wabbajack.Server; using Wabbajack.Server.DataLayer; using Wabbajack.Server.DTOs; +using Wabbajack.Server.Services; namespace Wabbajack.BuildServer.Controllers { @@ -19,12 +23,18 @@ namespace Wabbajack.BuildServer.Controllers } private static DateTime _startTime; + + private QuickSync _quickSync; + private ListValidator _listValidator; - public Heartbeat(ILogger logger, SqlService sql, GlobalInformation globalInformation) + + public Heartbeat(ILogger logger, SqlService sql, GlobalInformation globalInformation, QuickSync quickSync, ListValidator listValidator) { _globalInformation = globalInformation; _sql = sql; _logger = logger; + _quickSync = quickSync; + _listValidator = listValidator; } private const int MAX_LOG_SIZE = 128; @@ -52,6 +62,53 @@ namespace Wabbajack.BuildServer.Controllers LastNexusUpdate = _globalInformation.TimeSinceLastNexusSync, }); } + + private static readonly Func HandleGetReport = NettleEngine.GetCompiler().Compile(@" + +

Server Status

+ +

Service Overview ({{services.Length}}):

+
    + {{each $.services }} + {{if $.IsLate}} +
  • {{$.Name}} - {{$.Time}} - {{$.MaxTime}}
    • + {{else}} +
  • {{$.Name}} - {{$.Time}} - {{$.MaxTime}}
    • + {{/if}} + {{/each}} +
+ + +

Lists ({{lists.Length}}):

+
    + {{each $.lists }} +
  • {{$.Name}} - {{$.Time}}
    • + {{/each}} +
+ + "); + + [HttpGet("report")] + public async Task Report() + { + var response = HandleGetReport(new + { + services = (await _quickSync.Report()) + .Select(s => new {Name = s.Key, Time = s.Value.LastRunTime, MaxTime = s.Value.Delay, IsLate = s.Value.LastRunTime > s.Value.Delay}) + .OrderBy(s => s.Name) + .ToArray(), + lists = _listValidator.ValidationInfo.Select(s => new {Name = s.Key, Time = s.Value.ValidationTime}) + .OrderBy(l => l.Name) + .ToArray() + }); + return new ContentResult + { + ContentType = "text/html", + StatusCode = (int) HttpStatusCode.OK, + Content = response + }; + + } diff --git a/Wabbajack.Server/Services/AbstractService.cs b/Wabbajack.Server/Services/AbstractService.cs index 297bc3c9..5f854c07 100644 --- a/Wabbajack.Server/Services/AbstractService.cs +++ b/Wabbajack.Server/Services/AbstractService.cs @@ -10,14 +10,27 @@ namespace Wabbajack.Server.Services { public void Start(); } + + public interface IReportingService + { + public TimeSpan Delay { get; } + public DateTime LastStart { get; } + public DateTime LastEnd { get; } + + + } - public abstract class AbstractService : IStartable + public abstract class AbstractService : IStartable, IReportingService { protected AppSettings _settings; private TimeSpan _delay; protected ILogger _logger; protected QuickSync _quickSync; + public TimeSpan Delay => _delay; + public DateTime LastStart { get; private set; } + public DateTime LastEnd { get; private set; } + public AbstractService(ILogger logger, AppSettings settings, QuickSync quickSync, TimeSpan delay) { _settings = settings; @@ -40,7 +53,7 @@ namespace Wabbajack.Server.Services Task.Run(async () => { await Setup(); - + await _quickSync.Register(this); while (true) { @@ -48,7 +61,9 @@ namespace Wabbajack.Server.Services try { _logger.LogInformation($"Running: {GetType().Name}"); + LastStart = DateTime.UtcNow; await Execute(); + LastEnd = DateTime.UtcNow; } catch (Exception ex) { diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 3e98aff6..49acc614 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Linq; @@ -27,8 +28,9 @@ namespace Wabbajack.Server.Services private NexusKeyMaintainance _nexus; private ArchiveMaintainer _archives; - public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries { get; private set; } = - new (ModListSummary Summary, DetailedStatus Detailed)[0]; + public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries => ValidationInfo.Values.Select(e => (e.Summary, e.Detailed)); + + public ConcurrentDictionary ValidationInfo = new ConcurrentDictionary(); public ListValidator(ILogger logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives, QuickSync quickSync) @@ -50,14 +52,21 @@ namespace Wabbajack.Server.Services var stopwatch = new Stopwatch(); stopwatch.Start(); - var results = await data.ModLists.PMap(queue, async metadata => + var results = await data.ModLists.Where(m => !m.ForceDown).PMap(queue, async metadata => { + var timer = new Stopwatch(); + timer.Start(); var oldSummary = oldSummaries.FirstOrDefault(s => s.Summary.MachineURL == metadata.Links.MachineURL); var listArchives = await _sql.ModListArchives(metadata.Links.MachineURL); var archives = await listArchives.PMap(queue, async archive => { + if (timer.Elapsed > Delay) + { + return (archive, ArchiveStatus.InValid); + } + try { var (_, result) = await ValidateArchive(data, archive); @@ -109,6 +118,24 @@ namespace Wabbajack.Server.Services }).ToList() }; + if (timer.Elapsed > Delay) + { + await _discord.Send(Channel.Ham, + new DiscordMessage + { + Embeds = new[] + { + new DiscordEmbed + { + Title = + $"Failing {summary.Name} (`{summary.MachineURL}`) because the max validation time expired", + Url = new Uri( + $"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html") + } + } + }); + } + if (oldSummary != default && oldSummary.Summary.Failed != summary.Failed) { _logger.Log(LogLevel.Information, $"Number of failures {oldSummary.Summary.Failed} -> {summary.Failed}"); @@ -151,9 +178,14 @@ namespace Wabbajack.Server.Services } + timer.Stop(); + + + + ValidationInfo[summary.MachineURL] = (summary, detailed, timer.Elapsed); + return (summary, detailed); }); - Summaries = results; stopwatch.Stop(); _logger.LogInformation($"Finished Validation in {stopwatch.Elapsed}"); diff --git a/Wabbajack.Server/Services/QuickSync.cs b/Wabbajack.Server/Services/QuickSync.cs index 3a772c21..3bdd182f 100644 --- a/Wabbajack.Server/Services/QuickSync.cs +++ b/Wabbajack.Server/Services/QuickSync.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -11,6 +12,7 @@ namespace Wabbajack.Server.Services public class QuickSync { private Dictionary _syncs = new Dictionary(); + private Dictionary _services = new Dictionary(); private AsyncLock _lock = new AsyncLock(); private ILogger _logger; @@ -19,6 +21,20 @@ namespace Wabbajack.Server.Services _logger = logger; } + public async Task> Report() + { + using var _ = await _lock.WaitAsync(); + return _services.ToDictionary(s => s.Key, + s => (s.Value.Delay, DateTime.UtcNow - s.Value.LastEnd)); + } + + public async Task Register(T service) + where T : IReportingService + { + using var _ = await _lock.WaitAsync(); + _services[typeof(T)] = service; + } + public async Task GetToken() { using var _ = await _lock.WaitAsync(); diff --git a/Wabbajack.Server/Services/Watchdog.cs b/Wabbajack.Server/Services/Watchdog.cs new file mode 100644 index 00000000..f1cca33c --- /dev/null +++ b/Wabbajack.Server/Services/Watchdog.cs @@ -0,0 +1,33 @@ +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.BuildServer; +using Wabbajack.Server.DTOs; + +namespace Wabbajack.Server.Services +{ + public class Watchdog : AbstractService + { + private DiscordWebHook _discord; + + public Watchdog(ILogger logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discordWebHook) : base(logger, settings, quickSync, TimeSpan.FromMinutes(5)) + { + _discord = discordWebHook; + } + + public override async Task Execute() + { + var report = await _quickSync.Report(); + foreach (var service in report) + { + if (service.Value.LastRunTime >= service.Value.Delay * 2) + { + await _discord.Send(Channel.Spam, + new DiscordMessage {Content = $"Service {service.Key.Name} has missed it's scheduled execution window"}); + } + } + + return report.Count; + } + } +} diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 381f9eea..a747ffc8 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -71,6 +71,7 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddMvc(); services.AddControllers() @@ -129,6 +130,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); + app.UseService(); app.Use(next => { diff --git a/Wabbajack/View Models/Gallery/ModListMetadataVM.cs b/Wabbajack/View Models/Gallery/ModListMetadataVM.cs index 963fb136..bfa3f715 100644 --- a/Wabbajack/View Models/Gallery/ModListMetadataVM.cs +++ b/Wabbajack/View Models/Gallery/ModListMetadataVM.cs @@ -83,7 +83,7 @@ namespace Wabbajack }); DownloadSizeText = "Download size : " + UIUtils.FormatBytes(Metadata.DownloadMetadata.SizeOfArchives); InstallSizeText = "Installation size : " + UIUtils.FormatBytes(Metadata.DownloadMetadata.SizeOfInstalledFiles); - IsBroken = metadata.ValidationSummary.HasFailures; + IsBroken = metadata.ValidationSummary.HasFailures || metadata.ForceDown; //https://www.wabbajack.org/#/modlists/info?machineURL=eldersouls OpenWebsiteCommand = ReactiveCommand.Create(() => Utils.OpenWebsite(new Uri($"https://www.wabbajack.org/#/modlists/info?machineURL={Metadata.Links.MachineURL}"))); ExecuteCommand = ReactiveCommand.CreateFromObservable( From a494839a09edfa4f103853e25479d7eba978062f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 19 Nov 2020 22:00:07 -0700 Subject: [PATCH 18/33] Updates to metrics and watchdog systems --- Wabbajack.Server/Controllers/Heartbeat.cs | 2 +- Wabbajack.Server/Controllers/Metrics.cs | 20 +++++++++---- Wabbajack.Server/DataLayer/Metrics.cs | 35 ++++++++++++----------- Wabbajack.Server/Services/QuickSync.cs | 2 +- Wabbajack.Server/Services/Watchdog.cs | 2 +- Wabbajack.Server/public/metrics.html | 4 +-- 6 files changed, 38 insertions(+), 27 deletions(-) diff --git a/Wabbajack.Server/Controllers/Heartbeat.cs b/Wabbajack.Server/Controllers/Heartbeat.cs index ab12025a..a46aacbf 100644 --- a/Wabbajack.Server/Controllers/Heartbeat.cs +++ b/Wabbajack.Server/Controllers/Heartbeat.cs @@ -94,7 +94,7 @@ namespace Wabbajack.BuildServer.Controllers var response = HandleGetReport(new { services = (await _quickSync.Report()) - .Select(s => new {Name = s.Key, Time = s.Value.LastRunTime, MaxTime = s.Value.Delay, IsLate = s.Value.LastRunTime > s.Value.Delay}) + .Select(s => new {Name = s.Key.Name, Time = s.Value.LastRunTime, MaxTime = s.Value.Delay, IsLate = s.Value.LastRunTime > s.Value.Delay}) .OrderBy(s => s.Name) .ToArray(), lists = _listValidator.ValidationInfo.Select(s => new {Name = s.Key, Time = s.Value.ValidationTime}) diff --git a/Wabbajack.Server/Controllers/Metrics.cs b/Wabbajack.Server/Controllers/Metrics.cs index f9550df1..62963a97 100644 --- a/Wabbajack.Server/Controllers/Metrics.cs +++ b/Wabbajack.Server/Controllers/Metrics.cs @@ -43,13 +43,23 @@ namespace Wabbajack.BuildServer.Controllers [ResponseCache(Duration = 60 * 60)] public async Task MetricsReport(string subject) { - var results = (await _sql.MetricsReport(subject)) + var metrics = (await _sql.MetricsReport(subject)).ToList(); + var labels = metrics.GroupBy(m => m.Date) + .OrderBy(m => m.Key) + .Select(m => m.Key) + .ToArray(); + var labelStrings = labels.Select(l => l.ToString("MM-dd-yyy")).ToList(); + var results = metrics .GroupBy(m => m.Subject) - .Select(g => new MetricResult + .Select(g => { - SeriesName = g.Key, - Labels = g.Select(m => m.Date.ToString(CultureInfo.InvariantCulture)).ToList(), - Values = g.Select(m => m.Count).ToList() + var indexed = g.ToDictionary(m => m.Date, m => m.Count); + return new MetricResult + { + SeriesName = g.Key, + Labels = labelStrings, + Values = labels.Select(l => indexed.TryGetValue(l, out var found) ? found : 0).ToList() + }; }); return Ok(results.ToList()); } diff --git a/Wabbajack.Server/DataLayer/Metrics.cs b/Wabbajack.Server/DataLayer/Metrics.cs index 1d6d5e7a..968286e3 100644 --- a/Wabbajack.Server/DataLayer/Metrics.cs +++ b/Wabbajack.Server/DataLayer/Metrics.cs @@ -30,23 +30,24 @@ namespace Wabbajack.Server.DataLayer { await using var conn = await Open(); return (await conn.QueryAsync(@" - SELECT d.Date, d.GroupingSubject as Subject, Count(*) as Count FROM - (select DISTINCT CONVERT(date, Timestamp) as Date, GroupingSubject, Action, MetricsKey from dbo.Metrics) m - RIGHT OUTER JOIN - (SELECT CONVERT(date, DATEADD(DAY, number + 1, dbo.MinMetricDate())) as Date, GroupingSubject, Action - FROM master..spt_values - CROSS JOIN ( - SELECT DISTINCT GroupingSubject, Action FROM dbo.Metrics - WHERE MetricsKey is not null - AND Subject != 'Default' - AND TRY_CONVERT(uniqueidentifier, Subject) is null) as keys - WHERE type = 'P' - AND DATEADD(DAY, number+1, dbo.MinMetricDate()) <= dbo.MaxMetricDate()) as d - ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action - WHERE d.Action = @action - AND d.Date >= DATEADD(month, -1, GETUTCDATE()) - group by d.Date, d.GroupingSubject, d.Action - ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action})) + select + datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)) as Date, + GroupingSubject as Subject, + count(*) as Count + from dbo.metrics where + Action = @Action + AND GroupingSubject in (select DISTINCT GroupingSubject from dbo.Metrics + WHERE action = @Action + AND MetricsKey is not null + AND Subject != 'Default' + AND Subject != 'untitled' + AND TRY_CONVERT(uniqueidentifier, Subject) is null + AND Timestamp >= DATEADD(DAY, -1, GETUTCDATE())) + group by + datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)), + GroupingSubject + Order by datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)) asc", + new {Action = action})) .ToList(); } diff --git a/Wabbajack.Server/Services/QuickSync.cs b/Wabbajack.Server/Services/QuickSync.cs index 3bdd182f..fa4308d5 100644 --- a/Wabbajack.Server/Services/QuickSync.cs +++ b/Wabbajack.Server/Services/QuickSync.cs @@ -32,7 +32,7 @@ namespace Wabbajack.Server.Services where T : IReportingService { using var _ = await _lock.WaitAsync(); - _services[typeof(T)] = service; + _services[service.GetType()] = service; } public async Task GetToken() diff --git a/Wabbajack.Server/Services/Watchdog.cs b/Wabbajack.Server/Services/Watchdog.cs index f1cca33c..ef7e873c 100644 --- a/Wabbajack.Server/Services/Watchdog.cs +++ b/Wabbajack.Server/Services/Watchdog.cs @@ -20,7 +20,7 @@ namespace Wabbajack.Server.Services var report = await _quickSync.Report(); foreach (var service in report) { - if (service.Value.LastRunTime >= service.Value.Delay * 2) + if (service.Value.LastRunTime != default && service.Value.LastRunTime >= service.Value.Delay * 2) { await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Service {service.Key.Name} has missed it's scheduled execution window"}); diff --git a/Wabbajack.Server/public/metrics.html b/Wabbajack.Server/public/metrics.html index a20497de..d81e4c0b 100644 --- a/Wabbajack.Server/public/metrics.html +++ b/Wabbajack.Server/public/metrics.html @@ -51,7 +51,7 @@ return { label: series.seriesName, fill: false, - data: _.last(series.values, 30) + data: _.last(series.values, 90) }}); var ctx = document.getElementById(ele).getContext('2d'); var chart = new Chart(ctx, { @@ -60,7 +60,7 @@ // The data for our dataset data: { - labels: _.last(labels, 30), + labels: _.last(labels, 90), datasets: datasets}, // Configuration options go here From d93367d351266ac3e2a1a58eeccbfb5928578356 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 24 Nov 2020 10:19:26 -0700 Subject: [PATCH 19/33] Discord integration --- Wabbajack.Server.Test/DiscordFrontentTests.cs | 25 ++++ Wabbajack.Server/Services/DiscordFrontend.cs | 121 ++++++++++++++++++ Wabbajack.Server/Startup.cs | 2 + Wabbajack.Server/Wabbajack.Server.csproj | 1 + 4 files changed, 149 insertions(+) create mode 100644 Wabbajack.Server.Test/DiscordFrontentTests.cs create mode 100644 Wabbajack.Server/Services/DiscordFrontend.cs diff --git a/Wabbajack.Server.Test/DiscordFrontentTests.cs b/Wabbajack.Server.Test/DiscordFrontentTests.cs new file mode 100644 index 00000000..c476d885 --- /dev/null +++ b/Wabbajack.Server.Test/DiscordFrontentTests.cs @@ -0,0 +1,25 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Wabbajack.BuildServer.Test; +using Wabbajack.Server.Services; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.Server.Test +{ + public class DiscordFrontentTests: ABuildServerSystemTest + { + public DiscordFrontentTests(ITestOutputHelper output, SingletonAdaptor fixture) : base(output, fixture) + { + } + + [Fact] + public async Task CanLogIn() + { + var frontend = Fixture.GetService(); + frontend.Start(); + } + + } +} diff --git a/Wabbajack.Server/Services/DiscordFrontend.cs b/Wabbajack.Server/Services/DiscordFrontend.cs new file mode 100644 index 00000000..9d7061af --- /dev/null +++ b/Wabbajack.Server/Services/DiscordFrontend.cs @@ -0,0 +1,121 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Discord; +using Discord.WebSocket; +using Microsoft.Extensions.Logging; +using OMODFramework; +using Wabbajack.BuildServer; +using Wabbajack.Server.DataLayer; +using Utils = Wabbajack.Common.Utils; + +namespace Wabbajack.Server.Services +{ + public class DiscordFrontend : IStartable + { + private ILogger _logger; + private AppSettings _settings; + private QuickSync _quickSync; + private DiscordSocketClient _client; + private SqlService _sql; + + public DiscordFrontend(ILogger logger, AppSettings settings, QuickSync quickSync, SqlService sql) + { + _logger = logger; + _settings = settings; + _quickSync = quickSync; + + _client = new DiscordSocketClient(); + + _client.Log += LogAsync; + _client.Ready += ReadyAsync; + _client.MessageReceived += MessageReceivedAsync; + + _sql = sql; + } + + private async Task MessageReceivedAsync(SocketMessage arg) + { + _logger.LogInformation(arg.Content); + if (arg.Content.StartsWith("!dervenin")) + { + var parts = arg.Content.Split(" ", StringSplitOptions.RemoveEmptyEntries); + if (parts[0] != "!dervenin") + return; + + if (parts.Length == 1) + { + await ReplyTo(arg, "Wat?"); + } + + if (parts[1] == "purge-nexus-cache") + { + if (parts.Length != 3) + { + await ReplyTo(arg, "Welp you did that wrong, gotta give me a mod-id or url"); + return; + } + await PurgeNexusCache(arg, parts[2]); + } + } + } + + private async Task PurgeNexusCache(SocketMessage arg, string mod) + { + if (Uri.TryCreate(mod, UriKind.Absolute, out var url)) + { + mod = url.AbsolutePath.Split("/", StringSplitOptions.RemoveEmptyEntries).Last(); + } + + if (int.TryParse(mod, out var mod_id)) + { + await _sql.PurgeNexusCache(mod_id); + await _quickSync.Notify(); + await ReplyTo(arg, $"It is done, {mod_id} has been purged, list validation has been triggered"); + } + } + + private async Task ReplyTo(SocketMessage socketMessage, string message) + { + await socketMessage.Channel.SendMessageAsync(message); + } + + private async Task ReadyAsync() + { + } + + private async Task LogAsync(LogMessage arg) + { + switch (arg.Severity) + { + case LogSeverity.Info: + _logger.LogInformation(arg.Message); + break; + case LogSeverity.Warning: + _logger.LogWarning(arg.Message); + break; + case LogSeverity.Critical: + _logger.LogCritical(arg.Message); + break; + case LogSeverity.Error: + _logger.LogError(arg.Exception, arg.Message); + break; + case LogSeverity.Verbose: + _logger.LogTrace(arg.Message); + break; + case LogSeverity.Debug: + _logger.LogDebug(arg.Message); + break; + default: + throw new ArgumentOutOfRangeException(); + } + } + + public void Start() + { + _client.LoginAsync(TokenType.Bot, Utils.FromEncryptedJson("discord-key").Result).Wait(); + _client.StartAsync().Wait(); + } + + } +} diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index a747ffc8..17e940bf 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -72,6 +72,7 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddMvc(); services.AddControllers() @@ -131,6 +132,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); + app.UseService(); app.Use(next => { diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index cb1a86b3..6bc25d73 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -15,6 +15,7 @@ + From f97fcd5aa0b09a5209f1b4da4a58e5261e876d6b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 24 Nov 2020 15:14:09 -0700 Subject: [PATCH 20/33] 2.3.4.2 --- CHANGELOG.md | 4 ++++ Wabbajack.CLI/Wabbajack.CLI.csproj | 4 ++-- Wabbajack.Common/GameMetaData.cs | 19 ++++++++++++++++++- Wabbajack.Launcher/Wabbajack.Launcher.csproj | 4 ++-- Wabbajack.Lib/AInstaller.cs | 10 ++++++++++ Wabbajack.Server/Wabbajack.Server.csproj | 4 ++-- Wabbajack/Wabbajack.csproj | 4 ++-- 7 files changed, 40 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9051b3f1..f36033dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ### Changelog +#### Version - 2.3.4.2 - 11/24/2020 +* Add Support for Kingdom Come : Deliverance (via MO2) +* Several other small bug fixes and deps updates + #### Version - 2.3.4.1 - 11/15/2020 * Tell the mod updater to use the existing Nexus Client instead of creating a new one diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index 169fe589..111105c0 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -6,8 +6,8 @@ wabbajack-cli Wabbajack x64 - 2.3.4.1 - 2.3.4.1 + 2.3.4.2 + 2.3.4.2 Copyright © 2019-2020 An automated ModList installer true diff --git a/Wabbajack.Common/GameMetaData.cs b/Wabbajack.Common/GameMetaData.cs index bef77305..c8bf3203 100644 --- a/Wabbajack.Common/GameMetaData.cs +++ b/Wabbajack.Common/GameMetaData.cs @@ -37,7 +37,8 @@ namespace Wabbajack.Common Dishonored, Witcher3, [Description("Stardew Valley")] - StardewValley + StardewValley, + KingdomComeDeliverance } public static class GameExtensions @@ -490,6 +491,22 @@ namespace Wabbajack.Common }, MainExecutable = "Stardew Valley.exe" } + }, + { + Game.KingdomComeDeliverance, new GameMetaData + { + Game = Game.KingdomComeDeliverance, + NexusName = "kingdomcomedeliverance", + MO2Name = "Kingdom Come: Deliverance", + NexusGameId = 2298, + SteamIDs = new List{379430}, + IsGenericMO2Plugin = true, + RequiredFiles = new List + { + @"bin\Win64\KingdomCome.exe" + }, + MainExecutable = @"bin\Win64\KingdomCome.exe" + } } }; diff --git a/Wabbajack.Launcher/Wabbajack.Launcher.csproj b/Wabbajack.Launcher/Wabbajack.Launcher.csproj index c7402451..acd5d1ac 100644 --- a/Wabbajack.Launcher/Wabbajack.Launcher.csproj +++ b/Wabbajack.Launcher/Wabbajack.Launcher.csproj @@ -4,8 +4,8 @@ WinExe netcoreapp3.1 true - 2.3.4.1 - 2.3.4.1 + 2.3.4.2 + 2.3.4.2 Copyright © 2019-2020 Wabbajack Application Launcher true diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 37bb8147..7685fb18 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -208,6 +208,7 @@ namespace Wabbajack.Lib { if (download) { + var result = SendDownloadMetrics(missing); foreach (var a in missing.Where(a => a.State.GetType() == typeof(ManualDownloader.State))) { var outputPath = DownloadFolder.Combine(a.Name); @@ -242,6 +243,15 @@ namespace Wabbajack.Lib } + private async Task SendDownloadMetrics(List missing) + { + var grouped = missing.GroupBy(m => m.State.GetType()); + foreach (var group in grouped) + { + await Metrics.Send($"downloading_{group.Key.Name}", group.Sum(g => g.Size).ToString()); + } + } + public async Task DownloadArchive(Archive archive, bool download, AbsolutePath? destination = null) { try diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index 6bc25d73..a9854d4e 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -3,8 +3,8 @@ Exe netcoreapp3.1 - 2.3.4.1 - 2.3.4.1 + 2.3.4.2 + 2.3.4.2 Copyright © 2019-2020 Wabbajack Server win-x64 diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 67a566fd..58ff96eb 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -6,8 +6,8 @@ true x64 win10-x64 - 2.3.4.1 - 2.3.4.1 + 2.3.4.2 + 2.3.4.2 Copyright © 2019-2020 An automated ModList installer true From 693497711b4ea128c0e85ca3840dac4773cb38ee Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 24 Nov 2020 17:22:24 -0700 Subject: [PATCH 21/33] Fix a bug in download stats --- Wabbajack.Lib/AInstaller.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 7685fb18..b218647c 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -248,7 +248,7 @@ namespace Wabbajack.Lib var grouped = missing.GroupBy(m => m.State.GetType()); foreach (var group in grouped) { - await Metrics.Send($"downloading_{group.Key.Name}", group.Sum(g => g.Size).ToString()); + await Metrics.Send($"downloading_{group.Key.FullName!.Split(".").Last().Split("+").First()}", group.Sum(g => g.Size).ToString()); } } From 4d03e07b942af82abb35aeeed5084633630127b3 Mon Sep 17 00:00:00 2001 From: Unnoen Date: Sun, 29 Nov 2020 18:42:50 +1100 Subject: [PATCH 22/33] -net to .net Someone made an oopsie. --- Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs index 62842633..fa3430ae 100644 --- a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs +++ b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs @@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders { {"wabbajack.b-cdn.net", "authored-files.wabbajack.org"}, {"wabbajack-mirror.b-cdn.net", "mirror.wabbajack.org"}, - {"wabbajack-patches.b-cdn-net", "patches.wabbajack.org"}, + {"wabbajack-patches.b-cdn.net", "patches.wabbajack.org"}, {"wabbajacktest.b-cdn.net", "test-files.wabbajack.org"} }; From 164a8a4f4fb630a132e706fa1b10b3b212cfb771 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 30 Nov 2020 05:49:54 -0700 Subject: [PATCH 23/33] Fix game file indexing --- Wabbajack.CLI/Verbs/HashGameFiles.cs | 7 ++++++- Wabbajack.Common/GameMetaData.cs | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Wabbajack.CLI/Verbs/HashGameFiles.cs b/Wabbajack.CLI/Verbs/HashGameFiles.cs index e632a488..096bd1be 100644 --- a/Wabbajack.CLI/Verbs/HashGameFiles.cs +++ b/Wabbajack.CLI/Verbs/HashGameFiles.cs @@ -36,12 +36,17 @@ namespace Wabbajack.CLI.Verbs .PMap(queue, async f => { var hash = await f.FileHashCachedAsync(); - return new GameFileSourceDownloader.State + return new Archive(new GameFileSourceDownloader.State { Game = _game, GameFile = f.RelativeTo(gameLocation), Hash = hash, GameVersion = version + }) + { + Name = f.FileName.ToString(), + Hash = hash, + Size = f.Size }; }); diff --git a/Wabbajack.Common/GameMetaData.cs b/Wabbajack.Common/GameMetaData.cs index c8bf3203..fc4bcded 100644 --- a/Wabbajack.Common/GameMetaData.cs +++ b/Wabbajack.Common/GameMetaData.cs @@ -498,6 +498,7 @@ namespace Wabbajack.Common Game = Game.KingdomComeDeliverance, NexusName = "kingdomcomedeliverance", MO2Name = "Kingdom Come: Deliverance", + MO2ArchiveName = "kingdomcomedeliverance", NexusGameId = 2298, SteamIDs = new List{379430}, IsGenericMO2Plugin = true, From 021cb61563e7b3df591aace188a4a9a53b68f21e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 1 Dec 2020 15:58:07 -0700 Subject: [PATCH 24/33] MW5 Support --- Compression.BSA/Compression.BSA.csproj | 2 +- Wabbajack.Common/GameMetaData.cs | 34 +++++++- .../StoreHandlers/EpicGameStoreHandler.cs | 80 +++++++++++++++++++ .../StoreHandlers/StoreHandler.cs | 18 ++++- Wabbajack.Common/Wabbajack.Common.csproj | 4 +- Wabbajack.Lib/Wabbajack.Lib.csproj | 4 +- Wabbajack.Test/Wabbajack.Test.csproj | 4 +- .../Wabbajack.VirtualFileSystem.csproj | 2 +- Wabbajack/Wabbajack.csproj | 6 +- 9 files changed, 140 insertions(+), 14 deletions(-) create mode 100644 Wabbajack.Common/StoreHandlers/EpicGameStoreHandler.cs diff --git a/Compression.BSA/Compression.BSA.csproj b/Compression.BSA/Compression.BSA.csproj index 7b51c3b9..25f52db0 100644 --- a/Compression.BSA/Compression.BSA.csproj +++ b/Compression.BSA/Compression.BSA.csproj @@ -21,7 +21,7 @@ - + diff --git a/Wabbajack.Common/GameMetaData.cs b/Wabbajack.Common/GameMetaData.cs index fc4bcded..d4d0e397 100644 --- a/Wabbajack.Common/GameMetaData.cs +++ b/Wabbajack.Common/GameMetaData.cs @@ -4,6 +4,7 @@ using System.ComponentModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; +using System.Net.Http.Headers; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using Wabbajack.Common.StoreHandlers; @@ -38,7 +39,8 @@ namespace Wabbajack.Common Witcher3, [Description("Stardew Valley")] StardewValley, - KingdomComeDeliverance + KingdomComeDeliverance, + MechWarrior5Mercenaries } public static class GameExtensions @@ -67,6 +69,8 @@ namespace Wabbajack.Common // to get gog ids: https://www.gogdb.org public List? GOGIDs { get; internal set; } + + public List EpicGameStoreIDs { get; internal set; } = new List(); // to get BethNet IDs: check the registry public int BethNetID { get; internal set; } @@ -99,7 +103,16 @@ namespace Wabbajack.Common if (MainExecutable == null) throw new NotImplementedException(); - return FileVersionInfo.GetVersionInfo((string)gameLoc.Combine(MainExecutable)).ProductVersion; + var info = FileVersionInfo.GetVersionInfo((string)gameLoc.Combine(MainExecutable)); + var version = info.ProductVersion; + if (string.IsNullOrWhiteSpace(version)) + { + version = + $"{info.ProductMajorPart}.{info.ProductMinorPart}.{info.ProductBuildPart}.{info.ProductPrivatePart}"; + return version; + } + + return version; } } @@ -508,6 +521,23 @@ namespace Wabbajack.Common }, MainExecutable = @"bin\Win64\KingdomCome.exe" } + }, + { + Game.MechWarrior5Mercenaries, new GameMetaData + { + Game = Game.MechWarrior5Mercenaries, + NexusName = "mechwarrior5mercenaries", + MO2Name = "Mechwarrior 5: Mercenaries", + MO2ArchiveName = "mechwarrior5mercenaries", + NexusGameId = 3099, + EpicGameStoreIDs = new List {"9fd39d8ac72946a2a10a887ce86e6c35"}, + IsGenericMO2Plugin = true, + RequiredFiles = new List + { + @"MW5Mercs\Binaries\Win64\MechWarrior-Win64-Shipping.exe" + }, + MainExecutable = @"MW5Mercs\Binaries\Win64\MechWarrior-Win64-Shipping.exe" + } } }; diff --git a/Wabbajack.Common/StoreHandlers/EpicGameStoreHandler.cs b/Wabbajack.Common/StoreHandlers/EpicGameStoreHandler.cs new file mode 100644 index 00000000..67cf1852 --- /dev/null +++ b/Wabbajack.Common/StoreHandlers/EpicGameStoreHandler.cs @@ -0,0 +1,80 @@ +using System; +using System.Linq; +using Microsoft.Win32; + +namespace Wabbajack.Common.StoreHandlers +{ + public class EpicGameStoreHandler : AStoreHandler + { + public override StoreType Type { get; internal set; } + + public string BaseRegKey = @"SOFTWARE\Epic Games\EOS"; + public override bool Init() + { + return true; + } + + public override bool LoadAllGames() + { + using var eosKey = Registry.CurrentUser.OpenSubKey(BaseRegKey); + if (eosKey == null) + { + Utils.Log("Epic Game Store is not installed"); + return false; + } + + var name = eosKey.GetValue("ModSdkMetadataDir"); + if (name == null) + { + Utils.Log("Registry key entry does not exist for Epic Game store"); + return false; + } + + var byID = GameRegistry.Games.SelectMany(g => g.Value.EpicGameStoreIDs + .Select(id => (id, g.Value.Game))) + .GroupBy(t => t.id) + .ToDictionary(t => t.Key, t => t.First().Game); + + foreach (var itm in ((AbsolutePath)(string)(name!)).EnumerateFiles(false, "*.item")) + { + var item = itm.FromJson(); + Console.WriteLine($"Found Epic Game Store Game: {item.DisplayName} at {item.InstallLocation}"); + + if (byID.TryGetValue(item.CatalogItemId, out var game)) + { + Games.Add(new EpicStoreGame(game, item)); + } + + } + + + + return true; + } + + public class EpicStoreGame : AStoreGame + { + public EpicStoreGame(Game game, EpicGameItem item) + { + Type = StoreType.EpicGameStore; + Game = game; + Path = (AbsolutePath)item.InstallLocation; + Name = game.MetaData().HumanFriendlyGameName; + + } + + public override Game Game { get; internal set; } + public override StoreType Type { get; internal set; } + } + + public class EpicGameItem + { + public string DisplayName { get; set; } = ""; + public string InstallationGuid { get; set; } = ""; + public string CatalogItemId { get; set; } = ""; + public string CatalogNamespace { get; set; } = ""; + public string InstallSessionId { get; set; } = ""; + public string InstallLocation { get; set; } = ""; + } + } +} diff --git a/Wabbajack.Common/StoreHandlers/StoreHandler.cs b/Wabbajack.Common/StoreHandlers/StoreHandler.cs index 295c2225..68d3653c 100644 --- a/Wabbajack.Common/StoreHandlers/StoreHandler.cs +++ b/Wabbajack.Common/StoreHandlers/StoreHandler.cs @@ -9,7 +9,8 @@ namespace Wabbajack.Common.StoreHandlers { STEAM, GOG, - BethNet + BethNet, + EpicGameStore } public class StoreHandler @@ -25,6 +26,9 @@ namespace Wabbajack.Common.StoreHandlers private static readonly Lazy _bethNetHandler = new Lazy(() => new BethNetHandler()); public BethNetHandler BethNetHandler = _bethNetHandler.Value; + + private static readonly Lazy _epicGameStoreHandler = new Lazy(() => new EpicGameStoreHandler()); + public EpicGameStoreHandler EpicGameStoreHandler = _epicGameStoreHandler.Value; public List StoreGames; @@ -67,6 +71,18 @@ namespace Wabbajack.Common.StoreHandlers { Utils.Error(new StoreException("Could not Init the BethNetHandler, check previous error messages!")); } + + if (EpicGameStoreHandler.Init()) + { + if (EpicGameStoreHandler.LoadAllGames()) + StoreGames.AddRange(EpicGameStoreHandler.Games); + else + Utils.Error(new StoreException("Could not load all Games from the EpicGameStoreHandler, check previous error messages!")); + } + else + { + Utils.Error(new StoreException("Could not Init the EpicGameStoreHandler, check previous error messages!")); + } } public AbsolutePath? TryGetGamePath(Game game) diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 3bebdf1c..7a349848 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -56,13 +56,13 @@ - + - + diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 5f42ead1..a00d3242 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -8,10 +8,10 @@ - 85.3.130 + 86.0.241 - 85.3.130 + 86.0.241 3.1.0 diff --git a/Wabbajack.Test/Wabbajack.Test.csproj b/Wabbajack.Test/Wabbajack.Test.csproj index 4ef2cd16..bc2acc5a 100644 --- a/Wabbajack.Test/Wabbajack.Test.csproj +++ b/Wabbajack.Test/Wabbajack.Test.csproj @@ -28,8 +28,8 @@ - - + + diff --git a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj index 505d0256..bc1043c9 100644 --- a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj +++ b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj @@ -16,7 +16,7 @@ - + diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 58ff96eb..c1035ef7 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -55,8 +55,8 @@ - - + + all @@ -67,7 +67,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + From 053a71795a888a7d28c09ab4708c92f0299ca7b2 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 6 Dec 2020 21:26:02 -0700 Subject: [PATCH 25/33] disable back button and 2.3.4.3 --- CHANGELOG.md | 3 +++ Compression.BSA/Compression.BSA.csproj | 2 +- Wabbajack.CLI/Wabbajack.CLI.csproj | 6 ++--- Wabbajack.Common/Wabbajack.Common.csproj | 2 +- Wabbajack.Launcher/Wabbajack.Launcher.csproj | 4 ++-- Wabbajack.Lib/Wabbajack.Lib.csproj | 4 ++-- Wabbajack.Server/Services/DiscordFrontend.cs | 22 ++++++++++++++++++- Wabbajack.Server/Wabbajack.Server.csproj | 4 ++-- Wabbajack.Test/Wabbajack.Test.csproj | 6 ++--- .../Wabbajack.VirtualFileSystem.Test.csproj | 2 +- .../Wabbajack.VirtualFileSystem.csproj | 2 +- Wabbajack/View Models/BackNavigatingVM.cs | 11 +++++++++- Wabbajack/View Models/Compilers/CompilerVM.cs | 13 ++++++----- .../View Models/Installers/InstallerVM.cs | 18 ++++++++++----- Wabbajack/View Models/WebBrowserVM.cs | 6 +++++ Wabbajack/Wabbajack.csproj | 8 +++---- 16 files changed, 80 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f36033dc..97f3b900 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ### Changelog +#### Version - 2.3.4.3 - 12/6/2020 +* Disable the back button during install/compilation + #### Version - 2.3.4.2 - 11/24/2020 * Add Support for Kingdom Come : Deliverance (via MO2) * Several other small bug fixes and deps updates diff --git a/Compression.BSA/Compression.BSA.csproj b/Compression.BSA/Compression.BSA.csproj index 7b51c3b9..25f52db0 100644 --- a/Compression.BSA/Compression.BSA.csproj +++ b/Compression.BSA/Compression.BSA.csproj @@ -21,7 +21,7 @@ - + diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index 111105c0..16237353 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -6,8 +6,8 @@ wabbajack-cli Wabbajack x64 - 2.3.4.2 - 2.3.4.2 + 2.3.4.3 + 2.3.4.3 Copyright © 2019-2020 An automated ModList installer true @@ -19,7 +19,7 @@ - + diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 3bebdf1c..ef4895e7 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -56,7 +56,7 @@ - + diff --git a/Wabbajack.Launcher/Wabbajack.Launcher.csproj b/Wabbajack.Launcher/Wabbajack.Launcher.csproj index acd5d1ac..9bdb83d4 100644 --- a/Wabbajack.Launcher/Wabbajack.Launcher.csproj +++ b/Wabbajack.Launcher/Wabbajack.Launcher.csproj @@ -4,8 +4,8 @@ WinExe netcoreapp3.1 true - 2.3.4.2 - 2.3.4.2 + 2.3.4.3 + 2.3.4.3 Copyright © 2019-2020 Wabbajack Application Launcher true diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 5f42ead1..a00d3242 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -8,10 +8,10 @@ - 85.3.130 + 86.0.241 - 85.3.130 + 86.0.241 3.1.0 diff --git a/Wabbajack.Server/Services/DiscordFrontend.cs b/Wabbajack.Server/Services/DiscordFrontend.cs index 9d7061af..d324debb 100644 --- a/Wabbajack.Server/Services/DiscordFrontend.cs +++ b/Wabbajack.Server/Services/DiscordFrontend.cs @@ -6,6 +6,7 @@ using Discord.WebSocket; using Microsoft.Extensions.Logging; using OMODFramework; using Wabbajack.BuildServer; +using Wabbajack.Common; using Wabbajack.Server.DataLayer; using Utils = Wabbajack.Common.Utils; @@ -57,6 +58,25 @@ namespace Wabbajack.Server.Services } await PurgeNexusCache(arg, parts[2]); } + else if (parts[1] == "cyberpunk") + { + var random = new Random(); + var releaseDate = new DateTime(2020, 12, 10, 0, 0, 0, DateTimeKind.Utc); + var r = releaseDate - DateTime.UtcNow; + if (r < TimeSpan.Zero) + { + await ReplyTo(arg, "It's out, what are you doing here?"); + } + else + { + var msgs = (await "cyberpunk_message.txt".RelativeTo(AbsolutePath.EntryPoint) + .ReadAllLinesAsync()).ToArray(); + var msg = msgs[random.Next(0, msgs.Length)]; + var fullmsg = String.Format(msg, + $"{r.Days} days, {r.Hours} hours, {r.Minutes} minutes, {r.Seconds} seconds"); + await ReplyTo(arg, fullmsg); + } + } } } @@ -64,7 +84,7 @@ namespace Wabbajack.Server.Services { if (Uri.TryCreate(mod, UriKind.Absolute, out var url)) { - mod = url.AbsolutePath.Split("/", StringSplitOptions.RemoveEmptyEntries).Last(); + mod = Enumerable.Last(url.AbsolutePath.Split("/", StringSplitOptions.RemoveEmptyEntries)); } if (int.TryParse(mod, out var mod_id)) diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index a9854d4e..9eb7a5f2 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -3,8 +3,8 @@ Exe netcoreapp3.1 - 2.3.4.2 - 2.3.4.2 + 2.3.4.3 + 2.3.4.3 Copyright © 2019-2020 Wabbajack Server win-x64 diff --git a/Wabbajack.Test/Wabbajack.Test.csproj b/Wabbajack.Test/Wabbajack.Test.csproj index 4ef2cd16..9ebc4f4d 100644 --- a/Wabbajack.Test/Wabbajack.Test.csproj +++ b/Wabbajack.Test/Wabbajack.Test.csproj @@ -28,9 +28,9 @@ - - - + + + diff --git a/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj index 0fa7d79b..c5041fcd 100644 --- a/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj +++ b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj @@ -8,7 +8,7 @@ - + diff --git a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj index 505d0256..bc1043c9 100644 --- a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj +++ b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj @@ -16,7 +16,7 @@ - + diff --git a/Wabbajack/View Models/BackNavigatingVM.cs b/Wabbajack/View Models/BackNavigatingVM.cs index 8dad3ebc..b4cef5e5 100644 --- a/Wabbajack/View Models/BackNavigatingVM.cs +++ b/Wabbajack/View Models/BackNavigatingVM.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Reactive; using System.Reactive.Linq; +using System.Reactive.Subjects; using System.Text; using System.Threading.Tasks; using System.Windows.Input; @@ -17,6 +18,9 @@ namespace Wabbajack { ViewModel NavigateBackTarget { get; set; } ReactiveCommand BackCommand { get; } + + Subject IsBackEnabledSubject { get; } + IObservable IsBackEnabled { get; } } public class BackNavigatingVM : ViewModel, IBackNavigatingVM @@ -27,9 +31,13 @@ namespace Wabbajack private readonly ObservableAsPropertyHelper _IsActive; public bool IsActive => _IsActive.Value; + + public Subject IsBackEnabledSubject { get; } = new Subject(); + public IObservable IsBackEnabled { get; } public BackNavigatingVM(MainWindowVM mainWindowVM) { + IsBackEnabled = IsBackEnabledSubject.StartWith(true); BackCommand = ReactiveCommand.Create( execute: () => Utils.CatchAndLog(() => { @@ -53,7 +61,8 @@ namespace Wabbajack public static IObservable ConstructCanNavigateBack(this IBackNavigatingVM vm) { return vm.WhenAny(x => x.NavigateBackTarget) - .Select(x => x != null); + .CombineLatest(vm.IsBackEnabled) + .Select(x => x.First != null && x.Second); } public static IObservable ConstructIsActive(this IBackNavigatingVM vm, MainWindowVM mwvm) diff --git a/Wabbajack/View Models/Compilers/CompilerVM.cs b/Wabbajack/View Models/Compilers/CompilerVM.cs index 6ac4d3aa..38f78933 100644 --- a/Wabbajack/View Models/Compilers/CompilerVM.cs +++ b/Wabbajack/View Models/Compilers/CompilerVM.cs @@ -10,6 +10,7 @@ using System.Linq; using System.Reactive; using System.Reactive.Disposables; using System.Reactive.Linq; +using System.Reactive.Subjects; using System.Windows.Input; using System.Windows.Media.Imaging; using Wabbajack.Common; @@ -18,16 +19,13 @@ using Wabbajack.Lib; namespace Wabbajack { - public class CompilerVM : ViewModel, IBackNavigatingVM, ICpuStatusVM + public class CompilerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM { public MainWindowVM MWVM { get; } private readonly ObservableAsPropertyHelper _image; public BitmapImage Image => _image.Value; - [Reactive] - public ViewModel NavigateBackTarget { get; set; } - [Reactive] public ModManager SelectedCompilerType { get; set; } @@ -69,7 +67,7 @@ namespace Wabbajack private readonly ObservableAsPropertyHelper<(int CurrentCPUs, int DesiredCPUs)> _CurrentCpuCount; public (int CurrentCPUs, int DesiredCPUs) CurrentCpuCount => _CurrentCpuCount.Value; - public CompilerVM(MainWindowVM mainWindowVM) + public CompilerVM(MainWindowVM mainWindowVM) : base(mainWindowVM) { MWVM = mainWindowVM; @@ -178,6 +176,7 @@ namespace Wabbajack { try { + IsBackEnabledSubject.OnNext(false); var modList = await this.Compiler.Compile(); Completed = ErrorResponse.Create(modList.Succeeded); } @@ -187,6 +186,10 @@ namespace Wabbajack while (ex.InnerException != null) ex = ex.InnerException; Utils.Error(ex, $"Compiler error"); } + finally + { + IsBackEnabledSubject.OnNext(true); + } }); // When sub compiler begins a compile, mark state variable diff --git a/Wabbajack/View Models/Installers/InstallerVM.cs b/Wabbajack/View Models/Installers/InstallerVM.cs index a56e3a66..ab8de50d 100644 --- a/Wabbajack/View Models/Installers/InstallerVM.cs +++ b/Wabbajack/View Models/Installers/InstallerVM.cs @@ -19,13 +19,14 @@ using DynamicData.Binding; using Wabbajack.Common.StatusFeed; using System.Reactive; using System.Collections.Generic; +using System.Reactive.Subjects; using System.Windows.Input; using Microsoft.WindowsAPICodePack.Dialogs; using Wabbajack.Common.IO; namespace Wabbajack { - public class InstallerVM : ViewModel, IBackNavigatingVM, ICpuStatusVM + public class InstallerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM { public SlideShow Slideshow { get; } @@ -36,9 +37,6 @@ namespace Wabbajack public FilePickerVM ModListLocation { get; } - [Reactive] - public ViewModel NavigateBackTarget { get; set; } - private readonly ObservableAsPropertyHelper _installer; public ISubInstallerVM Installer => _installer.Value; @@ -95,12 +93,15 @@ namespace Wabbajack public ReactiveCommand OpenReadmeCommand { get; } public ReactiveCommand VisitModListWebsiteCommand { get; } public ReactiveCommand BackCommand { get; } + + public ReactiveCommand CloseWhenCompleteCommand { get; } public ReactiveCommand GoToInstallCommand { get; } public ReactiveCommand BeginCommand { get; } - public InstallerVM(MainWindowVM mainWindowVM) + public InstallerVM(MainWindowVM mainWindowVM) : base(mainWindowVM) { + if (Path.GetDirectoryName(Assembly.GetEntryAssembly().Location.ToLower()) == KnownFolders.Downloads.Path.ToLower()) { Utils.Error(new CriticalFailureIntervention( @@ -369,6 +370,7 @@ namespace Wabbajack try { Utils.Log($"Starting to install {ModList.Name}"); + IsBackEnabledSubject.OnNext(false); var success = await this.Installer.Install(); Completed = ErrorResponse.Create(success); try @@ -381,11 +383,15 @@ namespace Wabbajack } } catch (Exception ex) - { + { Utils.Error(ex, $"Encountered error, can't continue"); while (ex.InnerException != null) ex = ex.InnerException; Completed = ErrorResponse.Fail(ex); } + finally + { + IsBackEnabledSubject.OnNext(true); + } }); // When sub installer begins an install, mark state variable diff --git a/Wabbajack/View Models/WebBrowserVM.cs b/Wabbajack/View Models/WebBrowserVM.cs index aac18c58..2885c8df 100644 --- a/Wabbajack/View Models/WebBrowserVM.cs +++ b/Wabbajack/View Models/WebBrowserVM.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Linq; using System.Reactive; +using System.Reactive.Linq; +using System.Reactive.Subjects; using System.Text; using System.Threading.Tasks; using CefSharp; @@ -28,8 +30,12 @@ namespace Wabbajack [Reactive] public ReactiveCommand BackCommand { get; set; } + public Subject IsBackEnabledSubject { get; } = new Subject(); + public IObservable IsBackEnabled { get; } + private WebBrowserVM(string url = "http://www.wabbajack.org") { + IsBackEnabled = IsBackEnabledSubject.StartWith(true); Instructions = "Wabbajack Web Browser"; } diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 58ff96eb..0580401c 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -6,8 +6,8 @@ true x64 win10-x64 - 2.3.4.2 - 2.3.4.2 + 2.3.4.3 + 2.3.4.3 Copyright © 2019-2020 An automated ModList installer true @@ -55,7 +55,7 @@ - + @@ -67,7 +67,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + From d302e2f5f3e8d97266ad490e7f539d1751709487 Mon Sep 17 00:00:00 2001 From: Pierre Despereaux <60482452+PierreDespereaux@users.noreply.github.com> Date: Tue, 8 Dec 2020 15:55:21 -0500 Subject: [PATCH 26/33] Update README.md Updated Modlists that host all their stuff on GitHub --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3ae79b2d..7bcbc57e 100644 --- a/README.md +++ b/README.md @@ -192,12 +192,13 @@ Example structure for a good README: Some Modlists that host all their stuff on GitHub: - the OG GitHub Modlist: [Lotus](https://github.com/erri120/lotus) +- [Keizaal](https://github.com/PierreDespereaux/Keizaal) - [Living Skyrim](https://github.com/ForgottenGlory/Living-Skyrim-2) - [Eldersouls](https://github.com/jdsmith2816/eldersouls) - [Total Visual Overhaul](https://github.com/NotTotal/Total-Visual-Overhaul) - [Serenity](https://github.com/ixanza/serenity) - [MOISE](https://github.com/ForgottenGlory/MOISE) -- [Cupid](https://github.com/ForgottenGlory/MOISE) +- [Dungeons & Deviousness](https://github.com/ForgottenGlory/MOISE) ### Meta Files From 51bbc01661658f760af017511160e58d22787814 Mon Sep 17 00:00:00 2001 From: Pierre Despereaux <60482452+PierreDespereaux@users.noreply.github.com> Date: Tue, 8 Dec 2020 15:59:02 -0500 Subject: [PATCH 27/33] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7bcbc57e..1b1bfcc4 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ Some Modlists that host all their stuff on GitHub: - [Total Visual Overhaul](https://github.com/NotTotal/Total-Visual-Overhaul) - [Serenity](https://github.com/ixanza/serenity) - [MOISE](https://github.com/ForgottenGlory/MOISE) -- [Dungeons & Deviousness](https://github.com/ForgottenGlory/MOISE) +- [Dungeons & Deviousness](https://github.com/ForgottenGlory/Dungeons-Deviousness) ### Meta Files From b79830ad54a78628127465d69af78b2e7a869460 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 9 Dec 2020 14:46:08 -0700 Subject: [PATCH 28/33] Added purge-archive verb --- Wabbajack.CLI/OptionsDefinition.cs | 3 +- Wabbajack.CLI/Verbs/PurgeArchive.cs | 65 ++++++++++++++++++++++++++ Wabbajack.Common/Paths/AbsolutePath.cs | 2 +- 3 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 Wabbajack.CLI/Verbs/PurgeArchive.cs diff --git a/Wabbajack.CLI/OptionsDefinition.cs b/Wabbajack.CLI/OptionsDefinition.cs index 3eba22bd..cb61728d 100644 --- a/Wabbajack.CLI/OptionsDefinition.cs +++ b/Wabbajack.CLI/OptionsDefinition.cs @@ -34,7 +34,8 @@ namespace Wabbajack.CLI typeof(ExportServerGameFiles), typeof(HashGamefiles), typeof(Backup), - typeof(Restore) + typeof(Restore), + typeof(PurgeArchive) }; } } diff --git a/Wabbajack.CLI/Verbs/PurgeArchive.cs b/Wabbajack.CLI/Verbs/PurgeArchive.cs new file mode 100644 index 00000000..e005dbaf --- /dev/null +++ b/Wabbajack.CLI/Verbs/PurgeArchive.cs @@ -0,0 +1,65 @@ +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Lib; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("purge-archive", HelpText = "Purges an archive and all directives from a .wabbajack file")] + public class PurgeArchive : AVerb + { + [Option('i', "input", Required = true, HelpText = "Input .wabbajack file")] + public string Input { get; set; } = ""; + private AbsolutePath _Input => (AbsolutePath)Input; + + [Option('o', "output", Required = true, HelpText = "Output .wabbajack file")] + public string Output { get; set; } = ""; + private AbsolutePath _Output => (AbsolutePath)Output; + + [Option('h', "hash", Required = true, HelpText = "Hash to purge")] + public string ArchiveHash { get; set; } = ""; + private Hash _Hash => Hash.Interpret(ArchiveHash); + + protected override async Task Run() + { + Utils.Log("Copying .wabbajack file"); + await _Input.CopyToAsync(_Output); + + Utils.Log("Loading modlist"); + + await using var fs = await _Output.OpenWrite(); + using var ar = new ZipArchive(fs, ZipArchiveMode.Update); + ModList modlist; + await using (var entry = ar.Entries.First(e => e.Name == "modlist").Open()) + { + modlist = entry.FromJson(); + } + + Utils.Log("Purging archives"); + modlist.Archives = modlist.Archives.Where(a => a.Hash != _Hash).ToList(); + modlist.Directives = modlist.Directives.Select(d => + { + if (d is FromArchive a) + { + if (a.ArchiveHashPath.BaseHash == _Hash) return (false, d); + } + return (true, d); + }).Where(d => d.Item1) + .Select(d => d.d) + .ToList(); + + Utils.Log("Writing modlist"); + + await using (var entry = ar.Entries.First(e => e.Name == "modlist").Open()) + { + entry.SetLength(0); + entry.Position = 0; + modlist.ToJson(entry); + } + + return ExitCode.Ok; + } + } +} diff --git a/Wabbajack.Common/Paths/AbsolutePath.cs b/Wabbajack.Common/Paths/AbsolutePath.cs index e7f4a5c6..26d8a27e 100644 --- a/Wabbajack.Common/Paths/AbsolutePath.cs +++ b/Wabbajack.Common/Paths/AbsolutePath.cs @@ -95,7 +95,7 @@ namespace Wabbajack.Common public ValueTask OpenWrite() { var path = _path; - return CircuitBreaker.WithAutoRetryAsync(async () => File.OpenWrite(path)); + return CircuitBreaker.WithAutoRetryAsync(async () => File.Open(path, FileMode.OpenOrCreate, FileAccess.ReadWrite)); } public async Task WriteAllTextAsync(string text) From aa591bafa7c4ec7600152b0d4b6e173f921c527d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 15 Dec 2020 17:09:59 -0700 Subject: [PATCH 29/33] Add TES4RU support back in --- Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs | 11 +++++++++++ Wabbajack.Test/DownloaderTests.cs | 4 +--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs index 4ad33f64..8c87bfe4 100644 --- a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs @@ -68,6 +68,16 @@ namespace Wabbajack.Lib.Downloaders IsAttachment = true }; } + + + if (url.PathAndQuery.StartsWith("/files/download/") && long.TryParse(url.PathAndQuery.Split("/").Last(), out var fileId)) + { + return new TState + { + FullURL = url.ToString(), + IsAttachment = true + }; + } if (!url.PathAndQuery.StartsWith("/files/file/")) { @@ -76,6 +86,7 @@ namespace Wabbajack.Lib.Downloaders absolute = false; } + var id = HttpUtility.ParseQueryString(url.Query)["r"]; var file = absolute ? url.AbsolutePath.Split('/').Last(s => s != "") diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 0bd46741..a418dcce 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -407,13 +407,12 @@ namespace Wabbajack.Test Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); } - /* Site is down [Fact] public async Task TESAllDownloader() { await DownloadDispatcher.GetInstance().Prepare(); const string ini = "[General]\n" + - "directURL=https://tesall.ru/files/getdownload/594545-wabbajack-test-file/"; + "directURL=https://tesall.ru/files/download/594545"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); @@ -431,7 +430,6 @@ namespace Wabbajack.Test Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); } - */ /* WAITING FOR APPROVAL BY MODERATOR [Fact] From e441d78720fe773329908f522c11c7b47bb7775c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 15 Dec 2020 18:26:41 -0700 Subject: [PATCH 30/33] Fixes issue#2112 --- Wabbajack.Lib/ModListRegistry/ModListMetadata.cs | 3 +++ Wabbajack/Views/ModListTileView.xaml.cs | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index adc265ea..06f78199 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -37,6 +37,9 @@ namespace Wabbajack.Lib.ModListRegistry [JsonProperty("utility_list")] public bool UtilityList { get; set; } + + [JsonProperty("image_contains_title")] + public bool ImageContainsTitle { get; set; } [JsonProperty("force_down")] public bool ForceDown { get; set; } diff --git a/Wabbajack/Views/ModListTileView.xaml.cs b/Wabbajack/Views/ModListTileView.xaml.cs index 014485b8..787e6a35 100644 --- a/Wabbajack/Views/ModListTileView.xaml.cs +++ b/Wabbajack/Views/ModListTileView.xaml.cs @@ -36,12 +36,17 @@ namespace Wabbajack .Select(p => p.Value) .BindToStrict(this, x => x.DownloadProgressBar.Value) .DisposeWith(dispose); - this.WhenAny(x => x.ViewModel.Metadata.Title) + this.WhenAny(x => x.ViewModel.Metadata) + .Where(x => !x.ImageContainsTitle) + .Select(x => x.Title) .BindToStrict(this, x => x.DescriptionTextShadow.Text) .DisposeWith(dispose); - this.WhenAny(x => x.ViewModel.Metadata.Title) + this.WhenAny(x => x.ViewModel.Metadata) + .Where(x => !x.ImageContainsTitle) + .Select(x => x.Title) .BindToStrict(this, x => x.ModListTitleShadow.Text) .DisposeWith(dispose); + this.WhenAny(x => x.ViewModel.IsBroken) .Select(x => x ? Visibility.Visible : Visibility.Collapsed) .BindToStrict(this, x => x.Overlay.Visibility) From 364bfe6cea56a7f7a292fac68b67fcfffda10cda Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 16 Dec 2020 14:41:28 -0700 Subject: [PATCH 31/33] 2.3.5.0 Bump and MW5 support --- CHANGELOG.md | 6 ++ Wabbajack.CLI/Wabbajack.CLI.csproj | 4 +- Wabbajack.Common/GameMetaData.cs | 34 +++++++- .../StoreHandlers/EpicGameStoreHandler.cs | 80 +++++++++++++++++++ .../StoreHandlers/StoreHandler.cs | 18 ++++- Wabbajack.Launcher/Wabbajack.Launcher.csproj | 4 +- Wabbajack.Server/Wabbajack.Server.csproj | 4 +- Wabbajack/Wabbajack.csproj | 4 +- 8 files changed, 143 insertions(+), 11 deletions(-) create mode 100644 Wabbajack.Common/StoreHandlers/EpicGameStoreHandler.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 97f3b900..8387f3b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ### Changelog +#### Version - 2.3.5.0 - 12/16/2020 +* Fix tesall.ru download support +* Implement MechWarrior 5 support as a native compiler game +* Make the title in the WJ gallery (in app) optional for games that want the title to be in the splash screen +* Worked a few kinks out of the native game compiler + #### Version - 2.3.4.3 - 12/6/2020 * Disable the back button during install/compilation diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index 16237353..2842fa10 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -6,8 +6,8 @@ wabbajack-cli Wabbajack x64 - 2.3.4.3 - 2.3.4.3 + 2.3.5.0 + 2.3.5.0 Copyright © 2019-2020 An automated ModList installer true diff --git a/Wabbajack.Common/GameMetaData.cs b/Wabbajack.Common/GameMetaData.cs index fc4bcded..d4d0e397 100644 --- a/Wabbajack.Common/GameMetaData.cs +++ b/Wabbajack.Common/GameMetaData.cs @@ -4,6 +4,7 @@ using System.ComponentModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; +using System.Net.Http.Headers; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using Wabbajack.Common.StoreHandlers; @@ -38,7 +39,8 @@ namespace Wabbajack.Common Witcher3, [Description("Stardew Valley")] StardewValley, - KingdomComeDeliverance + KingdomComeDeliverance, + MechWarrior5Mercenaries } public static class GameExtensions @@ -67,6 +69,8 @@ namespace Wabbajack.Common // to get gog ids: https://www.gogdb.org public List? GOGIDs { get; internal set; } + + public List EpicGameStoreIDs { get; internal set; } = new List(); // to get BethNet IDs: check the registry public int BethNetID { get; internal set; } @@ -99,7 +103,16 @@ namespace Wabbajack.Common if (MainExecutable == null) throw new NotImplementedException(); - return FileVersionInfo.GetVersionInfo((string)gameLoc.Combine(MainExecutable)).ProductVersion; + var info = FileVersionInfo.GetVersionInfo((string)gameLoc.Combine(MainExecutable)); + var version = info.ProductVersion; + if (string.IsNullOrWhiteSpace(version)) + { + version = + $"{info.ProductMajorPart}.{info.ProductMinorPart}.{info.ProductBuildPart}.{info.ProductPrivatePart}"; + return version; + } + + return version; } } @@ -508,6 +521,23 @@ namespace Wabbajack.Common }, MainExecutable = @"bin\Win64\KingdomCome.exe" } + }, + { + Game.MechWarrior5Mercenaries, new GameMetaData + { + Game = Game.MechWarrior5Mercenaries, + NexusName = "mechwarrior5mercenaries", + MO2Name = "Mechwarrior 5: Mercenaries", + MO2ArchiveName = "mechwarrior5mercenaries", + NexusGameId = 3099, + EpicGameStoreIDs = new List {"9fd39d8ac72946a2a10a887ce86e6c35"}, + IsGenericMO2Plugin = true, + RequiredFiles = new List + { + @"MW5Mercs\Binaries\Win64\MechWarrior-Win64-Shipping.exe" + }, + MainExecutable = @"MW5Mercs\Binaries\Win64\MechWarrior-Win64-Shipping.exe" + } } }; diff --git a/Wabbajack.Common/StoreHandlers/EpicGameStoreHandler.cs b/Wabbajack.Common/StoreHandlers/EpicGameStoreHandler.cs new file mode 100644 index 00000000..67cf1852 --- /dev/null +++ b/Wabbajack.Common/StoreHandlers/EpicGameStoreHandler.cs @@ -0,0 +1,80 @@ +using System; +using System.Linq; +using Microsoft.Win32; + +namespace Wabbajack.Common.StoreHandlers +{ + public class EpicGameStoreHandler : AStoreHandler + { + public override StoreType Type { get; internal set; } + + public string BaseRegKey = @"SOFTWARE\Epic Games\EOS"; + public override bool Init() + { + return true; + } + + public override bool LoadAllGames() + { + using var eosKey = Registry.CurrentUser.OpenSubKey(BaseRegKey); + if (eosKey == null) + { + Utils.Log("Epic Game Store is not installed"); + return false; + } + + var name = eosKey.GetValue("ModSdkMetadataDir"); + if (name == null) + { + Utils.Log("Registry key entry does not exist for Epic Game store"); + return false; + } + + var byID = GameRegistry.Games.SelectMany(g => g.Value.EpicGameStoreIDs + .Select(id => (id, g.Value.Game))) + .GroupBy(t => t.id) + .ToDictionary(t => t.Key, t => t.First().Game); + + foreach (var itm in ((AbsolutePath)(string)(name!)).EnumerateFiles(false, "*.item")) + { + var item = itm.FromJson(); + Console.WriteLine($"Found Epic Game Store Game: {item.DisplayName} at {item.InstallLocation}"); + + if (byID.TryGetValue(item.CatalogItemId, out var game)) + { + Games.Add(new EpicStoreGame(game, item)); + } + + } + + + + return true; + } + + public class EpicStoreGame : AStoreGame + { + public EpicStoreGame(Game game, EpicGameItem item) + { + Type = StoreType.EpicGameStore; + Game = game; + Path = (AbsolutePath)item.InstallLocation; + Name = game.MetaData().HumanFriendlyGameName; + + } + + public override Game Game { get; internal set; } + public override StoreType Type { get; internal set; } + } + + public class EpicGameItem + { + public string DisplayName { get; set; } = ""; + public string InstallationGuid { get; set; } = ""; + public string CatalogItemId { get; set; } = ""; + public string CatalogNamespace { get; set; } = ""; + public string InstallSessionId { get; set; } = ""; + public string InstallLocation { get; set; } = ""; + } + } +} diff --git a/Wabbajack.Common/StoreHandlers/StoreHandler.cs b/Wabbajack.Common/StoreHandlers/StoreHandler.cs index 295c2225..68d3653c 100644 --- a/Wabbajack.Common/StoreHandlers/StoreHandler.cs +++ b/Wabbajack.Common/StoreHandlers/StoreHandler.cs @@ -9,7 +9,8 @@ namespace Wabbajack.Common.StoreHandlers { STEAM, GOG, - BethNet + BethNet, + EpicGameStore } public class StoreHandler @@ -25,6 +26,9 @@ namespace Wabbajack.Common.StoreHandlers private static readonly Lazy _bethNetHandler = new Lazy(() => new BethNetHandler()); public BethNetHandler BethNetHandler = _bethNetHandler.Value; + + private static readonly Lazy _epicGameStoreHandler = new Lazy(() => new EpicGameStoreHandler()); + public EpicGameStoreHandler EpicGameStoreHandler = _epicGameStoreHandler.Value; public List StoreGames; @@ -67,6 +71,18 @@ namespace Wabbajack.Common.StoreHandlers { Utils.Error(new StoreException("Could not Init the BethNetHandler, check previous error messages!")); } + + if (EpicGameStoreHandler.Init()) + { + if (EpicGameStoreHandler.LoadAllGames()) + StoreGames.AddRange(EpicGameStoreHandler.Games); + else + Utils.Error(new StoreException("Could not load all Games from the EpicGameStoreHandler, check previous error messages!")); + } + else + { + Utils.Error(new StoreException("Could not Init the EpicGameStoreHandler, check previous error messages!")); + } } public AbsolutePath? TryGetGamePath(Game game) diff --git a/Wabbajack.Launcher/Wabbajack.Launcher.csproj b/Wabbajack.Launcher/Wabbajack.Launcher.csproj index 9bdb83d4..67ced83f 100644 --- a/Wabbajack.Launcher/Wabbajack.Launcher.csproj +++ b/Wabbajack.Launcher/Wabbajack.Launcher.csproj @@ -4,8 +4,8 @@ WinExe netcoreapp3.1 true - 2.3.4.3 - 2.3.4.3 + 2.3.5.0 + 2.3.5.0 Copyright © 2019-2020 Wabbajack Application Launcher true diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index 9eb7a5f2..540bddb7 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -3,8 +3,8 @@ Exe netcoreapp3.1 - 2.3.4.3 - 2.3.4.3 + 2.3.5.0 + 2.3.5.0 Copyright © 2019-2020 Wabbajack Server win-x64 diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 0580401c..32ede8d1 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -6,8 +6,8 @@ true x64 win10-x64 - 2.3.4.3 - 2.3.4.3 + 2.3.5.0 + 2.3.5.0 Copyright © 2019-2020 An automated ModList installer true From 5d8a76f265b604cd1aa580ea0c56f4ac7cd07d1f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 16 Dec 2020 16:28:35 -0700 Subject: [PATCH 32/33] Downgrade YAML --- Wabbajack.Common/Wabbajack.Common.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 7a349848..ef4895e7 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -62,7 +62,7 @@ - + From b22943c2b54f4c047eb0fa65219a8e9443d3c940 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 23 Dec 2020 06:55:13 -0700 Subject: [PATCH 33/33] 2.3.5.1 HOTFIX --- CHANGELOG.md | 3 + Wabbajack.CLI/Wabbajack.CLI.csproj | 4 +- .../StoreHandlers/EpicGameStoreHandler.cs | 60 ++++++++++--------- Wabbajack.Launcher/Wabbajack.Launcher.csproj | 4 +- Wabbajack.Server/Wabbajack.Server.csproj | 4 +- Wabbajack/Wabbajack.csproj | 4 +- 6 files changed, 44 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8387f3b6..7e29d960 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ### Changelog +#### Version - 2.3.5.1 - 12/23/2020 +* HOTFIX : Recover from errors in the EGS location detector + #### Version - 2.3.5.0 - 12/16/2020 * Fix tesall.ru download support * Implement MechWarrior 5 support as a native compiler game diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index 2842fa10..f3fc98c3 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -6,8 +6,8 @@ wabbajack-cli Wabbajack x64 - 2.3.5.0 - 2.3.5.0 + 2.3.5.1 + 2.3.5.1 Copyright © 2019-2020 An automated ModList installer true diff --git a/Wabbajack.Common/StoreHandlers/EpicGameStoreHandler.cs b/Wabbajack.Common/StoreHandlers/EpicGameStoreHandler.cs index 67cf1852..1e8b83cf 100644 --- a/Wabbajack.Common/StoreHandlers/EpicGameStoreHandler.cs +++ b/Wabbajack.Common/StoreHandlers/EpicGameStoreHandler.cs @@ -16,38 +16,44 @@ namespace Wabbajack.Common.StoreHandlers public override bool LoadAllGames() { - using var eosKey = Registry.CurrentUser.OpenSubKey(BaseRegKey); - if (eosKey == null) + try { - Utils.Log("Epic Game Store is not installed"); - return false; - } - - var name = eosKey.GetValue("ModSdkMetadataDir"); - if (name == null) - { - Utils.Log("Registry key entry does not exist for Epic Game store"); - return false; - } - - var byID = GameRegistry.Games.SelectMany(g => g.Value.EpicGameStoreIDs - .Select(id => (id, g.Value.Game))) - .GroupBy(t => t.id) - .ToDictionary(t => t.Key, t => t.First().Game); - - foreach (var itm in ((AbsolutePath)(string)(name!)).EnumerateFiles(false, "*.item")) - { - var item = itm.FromJson(); - Console.WriteLine($"Found Epic Game Store Game: {item.DisplayName} at {item.InstallLocation}"); - - if (byID.TryGetValue(item.CatalogItemId, out var game)) + using var eosKey = Registry.CurrentUser.OpenSubKey(BaseRegKey); + if (eosKey == null) { - Games.Add(new EpicStoreGame(game, item)); + Utils.Log("Epic Game Store is not installed"); + return false; } - } - + var name = eosKey.GetValue("ModSdkMetadataDir"); + if (name == null) + { + Utils.Log("Registry key entry does not exist for Epic Game store"); + return false; + } + var byID = GameRegistry.Games.SelectMany(g => g.Value.EpicGameStoreIDs + .Select(id => (id, g.Value.Game))) + .GroupBy(t => t.id) + .ToDictionary(t => t.Key, t => t.First().Game); + + foreach (var itm in ((AbsolutePath)(string)(name!)).EnumerateFiles(false, "*.item")) + { + var item = itm.FromJson(); + Utils.Log($"Found Epic Game Store Game: {item.DisplayName} at {item.InstallLocation}"); + + if (byID.TryGetValue(item.CatalogItemId, out var game)) + { + Games.Add(new EpicStoreGame(game, item)); + } + + } + } + catch (NullReferenceException ex) + { + Utils.Log("Epic Game Store is does not appear to be installed"); + return false; + } return true; } diff --git a/Wabbajack.Launcher/Wabbajack.Launcher.csproj b/Wabbajack.Launcher/Wabbajack.Launcher.csproj index 67ced83f..97bad912 100644 --- a/Wabbajack.Launcher/Wabbajack.Launcher.csproj +++ b/Wabbajack.Launcher/Wabbajack.Launcher.csproj @@ -4,8 +4,8 @@ WinExe netcoreapp3.1 true - 2.3.5.0 - 2.3.5.0 + 2.3.5.1 + 2.3.5.1 Copyright © 2019-2020 Wabbajack Application Launcher true diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index 540bddb7..fccdfd09 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -3,8 +3,8 @@ Exe netcoreapp3.1 - 2.3.5.0 - 2.3.5.0 + 2.3.5.1 + 2.3.5.1 Copyright © 2019-2020 Wabbajack Server win-x64 diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 32ede8d1..f72be72d 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -6,8 +6,8 @@ true x64 win10-x64 - 2.3.5.0 - 2.3.5.0 + 2.3.5.1 + 2.3.5.1 Copyright © 2019-2020 An automated ModList installer true