From 09cf00263718641a032918efe7653ecef3eec0c8 Mon Sep 17 00:00:00 2001 From: erri120 Date: Tue, 17 Dec 2019 17:25:15 +0100 Subject: [PATCH] Update Vortex Installer&Compiler with recent changes --- Wabbajack.Lib/VortexCompiler.cs | 261 +++++++++++++++++-------------- Wabbajack.Lib/VortexInstaller.cs | 12 +- 2 files changed, 152 insertions(+), 121 deletions(-) diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs index 68626892..fdc5c16e 100644 --- a/Wabbajack.Lib/VortexCompiler.cs +++ b/Wabbajack.Lib/VortexCompiler.cs @@ -6,13 +6,14 @@ using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; using System.Threading; -using DynamicData; using Microsoft.WindowsAPICodePack.Shell; using Newtonsoft.Json; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.NexusApi; +using Wabbajack.Lib.Validation; using File = Alphaleonis.Win32.Filesystem.File; +using Game = Wabbajack.Common.Game; namespace Wabbajack.Lib { @@ -37,7 +38,7 @@ namespace Wabbajack.Lib public override ModManager ModManager => ModManager.Vortex; public override string GamePath { get; } - public override string ModListOutputFolder { get; } + public override string ModListOutputFolder => "output_folder"; public override string ModListOutputFile { get; } public const string StagingMarkerName = "__vortex_staging_folder"; @@ -56,9 +57,18 @@ namespace Wabbajack.Lib VortexFolder = vortexFolder; DownloadsFolder = downloadsFolder; StagingFolder = stagingFolder; - ModListOutputFolder = "output_folder"; ModListOutputFile = outputFile; + if (string.IsNullOrEmpty(ModListName)) + { + ModListName = $"Vortex ModList for {Game.ToString()}"; + ModListOutputFile = $"{ModListName}{ExtensionManager.Extension}"; + } + + GameName = Game.MetaData().NexusName; + + ActiveArchives = new List(); + // there can be max one game after filtering SteamHandler.Instance.Games.Where(g => g.Game != null && g.Game == game).Do(g => { @@ -67,52 +77,51 @@ namespace Wabbajack.Lib SteamHandler.Instance.LoadWorkshopItems(_steamGame); _hasSteamWorkshopItems = _steamGame.WorkshopItems.Count > 0; }); - - ActiveArchives = new List(); } protected override async Task _Begin(CancellationToken cancel) { if (cancel.IsCancellationRequested) return false; - ConfigureProcessor(10); - if (string.IsNullOrEmpty(ModListName)) - ModListName = $"Vortex ModList for {Game.ToString()}"; + Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}."); - Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}."); + ConfigureProcessor(12); + UpdateTracker.Reset(); if (cancel.IsCancellationRequested) return false; + UpdateTracker.NextStep("Parsing deployment file"); ParseDeploymentFile(); if (cancel.IsCancellationRequested) return false; - Info("Starting pre-compilation steps"); + UpdateTracker.NextStep("Creating metas for archives"); await CreateMetaFiles(); if (cancel.IsCancellationRequested) return false; - Info($"Indexing {StagingFolder}"); - await VFS.AddRoot(StagingFolder); + await VFS.IntegrateFromFile(_vfsCacheName); - Info($"Indexing {GamePath}"); - await VFS.AddRoot(GamePath); - - Info($"Indexing {DownloadsFolder}"); - await VFS.AddRoot(DownloadsFolder); + var roots = new List {StagingFolder, GamePath, DownloadsFolder}; + AddExternalFolder(ref roots); if (cancel.IsCancellationRequested) return false; - await AddExternalFolder(); + UpdateTracker.NextStep("Indexing folders"); + await VFS.AddRoots(roots); + await VFS.WriteToFile(_vfsCacheName); if (cancel.IsCancellationRequested) return false; - Info("Cleaning output folder"); - if (Directory.Exists(ModListOutputFolder)) Utils.DeleteDirectory(ModListOutputFolder); + UpdateTracker.NextStep("Cleaning output folder"); + if (Directory.Exists(ModListOutputFolder)) + Utils.DeleteDirectory(ModListOutputFolder); + Directory.CreateDirectory(ModListOutputFolder); - + + UpdateTracker.NextStep("Finding Install Files"); var vortexStagingFiles = Directory.EnumerateFiles(StagingFolder, "*", SearchOption.AllDirectories) .Where(p => p.FileExists() && p != StagingMarkerName) .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p]) {Path = p.RelativeTo(StagingFolder)}); var vortexDownloads = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.AllDirectories) - .Where(p => p.FileExists()) + .Where(p => p.FileExists() && p != DownloadMarkerName) .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p]) {Path = p.RelativeTo(DownloadsFolder)}); @@ -139,7 +148,6 @@ namespace Wabbajack.Lib .GroupBy(f => f.Hash) .ToDictionary(f => f.Key, f => f.AsEnumerable()); - Info("Searching for mod files"); AllFiles = vortexStagingFiles.Concat(vortexDownloads) .Concat(gameFiles) .DistinctBy(f => f.Path) @@ -148,7 +156,7 @@ namespace Wabbajack.Lib Info($"Found {AllFiles.Count} files to build into mod list"); if (cancel.IsCancellationRequested) return false; - Info("Verifying destinations"); + UpdateTracker.NextStep("Verifying destinations"); var duplicates = AllFiles.GroupBy(f => f.Path) .Where(fs => fs.Count() > 1) .Select(fs => @@ -228,17 +236,16 @@ namespace Wabbajack.Lib InstallDirectives = results.Where(i => !(i is IgnoredDirectly)).ToList(); - // TODO: nexus stuff - /*Info("Getting Nexus api_key, please click authorize if a browser window appears"); + Info("Getting Nexus api_key, please click authorize if a browser window appears"); if (IndexedArchives.Any(a => a.IniData?.General?.gameName != null)) { - var nexusClient = new NexusApiClient(); - if (!nexusClient.IsPremium) Error($"User {nexusClient.Username} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue"); - + var nexusClient = await NexusApiClient.Get(); + if (!await nexusClient.IsPremium()) Error($"User {await nexusClient.Username()} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue"); } - */ + if (cancel.IsCancellationRequested) return false; + UpdateTracker.NextStep("Gathering Archives"); await GatherArchives(); ModList = new ModList @@ -255,15 +262,45 @@ namespace Wabbajack.Lib GameType = Game }; + UpdateTracker.NextStep("Running Validation"); + await ValidateModlist.RunValidation(Queue, ModList); + + UpdateTracker.NextStep("Generating Report"); GenerateReport(); + + UpdateTracker.NextStep("Exporting ModList"); ExportModList(); - Info("Done Building ModList"); + ResetMembers(); ShowReport(); + + UpdateTracker.NextStep("Done Building ModList"); + return true; } + /// + /// Clear references to lists that hold a lot of data. + /// + private void ResetMembers() + { + AllFiles = null; + InstallDirectives = null; + SelectedArchives = null; + } + + private void AddExternalFolder(ref List roots) + { + var currentGame = Game.MetaData(); + if (currentGame.AdditionalFolders == null || currentGame.AdditionalFolders.Count == 0) return; + foreach (var path in currentGame.AdditionalFolders.Select(f => f.Replace("%documents%", KnownFolders.Documents.Path))) + { + if (!Directory.Exists(path)) return; + roots.Add(path); + } + } + private void ParseDeploymentFile() { Info("Searching for vortex.deployment.json..."); @@ -280,10 +317,10 @@ namespace Wabbajack.Lib if (string.IsNullOrEmpty(deploymentFile)) { - Info("vortex.deployment.json not found!"); + Error("vortex.deployment.json not found!"); return; } - Info("vortex.deployment.json found at "+deploymentFile); + Info($"vortex.deployment.json found at {deploymentFile}"); Info("Parsing vortex.deployment.json..."); try @@ -306,100 +343,82 @@ namespace Wabbajack.Lib }); } - /// - /// Some have mods outside their game folder located - /// - private async Task AddExternalFolder() - { - var currentGame = Game.MetaData(); - if (currentGame.AdditionalFolders == null || currentGame.AdditionalFolders.Count == 0) return; - foreach (var f in currentGame.AdditionalFolders) - { - var path = f.Replace("%documents%", KnownFolders.Documents.Path); - if (!Directory.Exists(path)) return; - Info($"Indexing {path}"); - await VFS.AddRoot(path); - } - } - private async Task CreateMetaFiles() { Utils.Log("Getting Nexus api_key, please click authorize if a browser window appears"); var nexusClient = await NexusApiClient.Get(); - await Task.WhenAll( - Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly) - .Where(File.Exists) - .Select(async f => + var archives = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly).Where(f => + File.Exists(f) && Path.GetExtension(f) != ".meta" && Path.GetExtension(f) != ".xxHash" && + !File.Exists($"{f}.meta") && ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f))); + + await archives.PMap(Queue, async f => + { + Info($"Creating meta file for {Path.GetFileName(f)}"); + var metaString = "[General]\n" + + "repository=Nexus\n" + + $"gameName={GameName}\n"; + string hash; + using (var md5 = MD5.Create()) + using (var stream = File.OpenRead(f)) { - if (Path.GetExtension(f) != ".meta" && Path.GetExtension(f) != ".xxHash" && !File.Exists($"{f}.meta") && ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f))) - { - Utils.Log($"Trying to create meta file for {Path.GetFileName(f)}"); - var metaString = "[General]\n" + - "repository=Nexus\n" + - "installed=true\n" + - "uninstalled=false\n" + - "paused=false\n" + - "removed=false\n" + - $"gameName={GameName}\n"; - string hash; - using(var md5 = MD5.Create()) - using (var stream = File.OpenRead(f)) - { - Utils.Log($"Calculating hash for {Path.GetFileName(f)}"); - var cH = md5.ComputeHash(stream); - hash = BitConverter.ToString(cH).Replace("-", "").ToLowerInvariant(); - Utils.Log($"Hash is {hash}"); - } + Utils.Log($"Calculating hash for {Path.GetFileName(f)}"); + var cH = md5.ComputeHash(stream); + hash = BitConverter.ToString(cH).Replace("-", "").ToLowerInvariant(); + Utils.Log($"Hash is {hash}"); + } - var md5Response = await nexusClient.GetModInfoFromMD5(Game, hash); - if (md5Response.Count >= 1) - { - var modInfo = md5Response[0].mod; - metaString += $"modID={modInfo.mod_id}\n" + - $"modName={modInfo.name}\nfileID={md5Response[0].file_details.file_id}"; - File.WriteAllText(f+".meta",metaString, Encoding.UTF8); - } - else - { - Error("Error while getting information from NexusMods via MD5 hash!"); - } - } - else + var md5Response = await nexusClient.GetModInfoFromMD5(Game, hash); + if (md5Response.Count >= 1) + { + var modInfo = md5Response[0].mod; + metaString += $"modID={modInfo.mod_id}\n" + + $"modName={modInfo.name}\n" + + $"fileID={md5Response[0].file_details.file_id}\n" + + $"version={md5Response[0].file_details.version}\n" + + $"hash={hash}\n"; + File.WriteAllText(f + ".meta", metaString, Encoding.UTF8); + } + else + { + Error("Error while getting information from NexusMods via MD5 hash!"); + } + }); + + var otherFiles = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly).Where(f => + Path.GetExtension(f) == ".meta" && !ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f))); + + await otherFiles.PMap(Queue, async f => + { + Info($"File {f} is not in ActiveArchives"); + var lines = File.ReadAllLines(f); + if (lines.Length == 0 || !lines.Any(line => lines.Contains("directURL="))) + { + if (lines.Length == 0) + return; + + lines.Do(line => { - if (Path.GetExtension(f) != ".meta" || - ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f))) + var tag = ""; + if (line.Contains("tag=")) + tag = line.Substring("tag=".Length); + + if (tag != Consts.WABBAJACK_VORTEX_MANUAL) return; - Utils.Log($"File {f} is not in ActiveArchives"); - var lines = File.ReadAllLines(f); + Info($"File {f} contains the {Consts.WABBAJACK_VORTEX_MANUAL} tag, adding to ActiveArchives"); + ActiveArchives.Add(Path.GetFileNameWithoutExtension(f)); + }); + } + else + { + Info($"File {f} appears to not be from the Nexus, adding to ActiveArchives"); + ActiveArchives.Add(Path.GetFileNameWithoutExtension(f)); + } + }); - if (lines.Length == 0) - return; - - lines.Do(line => - { - var tag = ""; - if (line.Contains("tag=")) - tag = line.Substring("tag=".Length); - - if (tag != Consts.WABBAJACK_VORTEX_MANUAL) - return; - - Utils.Log($"File {f} contains the {Consts.WABBAJACK_VORTEX_MANUAL} tag, adding to ActiveArchives"); - ActiveArchives.Add(Path.GetFileNameWithoutExtension(f)); - }); - - if (lines.Any(line => line.Contains("directURL=")) && !ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f))) - { - Utils.Log($"File {f} appears to not come from the Nexus, adding to ActiveArchives"); - ActiveArchives.Add(Path.GetFileNameWithoutExtension(f)); - } - } - })); - - Utils.Log($"Checking for Steam Workshop Items..."); - if (!_isSteamGame || _steamGame == null || _steamGame.WorkshopItems.Count <= 0) + Info("Checking for Steam Workshop Items..."); + if (!_isSteamGame || _steamGame == null || !_hasSteamWorkshopItems) return; _steamGame.WorkshopItems.Do(item => @@ -414,7 +433,6 @@ namespace Wabbajack.Lib Utils.Log($"Creating meta file for {item.ItemID}"); var metaString = "[General]\n" + "repository=Steam\n" + - "installed=true\n" + $"gameName={GameName}\n" + $"steamID={_steamGame.AppId}\n" + $"itemID={item.ItemID}\n" + @@ -439,20 +457,23 @@ namespace Wabbajack.Lib var stack = MakeStack(); + var compilationSteps = stack.ToList(); File.WriteAllText(Path.Combine(s, "_current_compilation_stack.yml"), - Serialization.Serialize(stack)); + Serialization.Serialize(compilationSteps)); - return stack; + return compilationSteps; } public override IEnumerable MakeStack() { - Utils.Log("Generating compilation stack"); + Info("Generating compilation stack"); return new List { new IncludePropertyFiles(this), + new IncludeSteamWorkshopItems(this, _steamGame), _hasSteamWorkshopItems ? new IncludeRegex(this, "^steamWorkshopItem_\\d*\\.meta$") : null, + new IgnoreDisabledVortexMods(this), new IncludeVortexDeployment(this), new IgnoreVortex(this), diff --git a/Wabbajack.Lib/VortexInstaller.cs b/Wabbajack.Lib/VortexInstaller.cs index f0255dff..013ddef0 100644 --- a/Wabbajack.Lib/VortexInstaller.cs +++ b/Wabbajack.Lib/VortexInstaller.cs @@ -49,12 +49,15 @@ namespace Wabbajack.Lib Directory.CreateDirectory(DownloadFolder); if (cancel.IsCancellationRequested) return false; + UpdateTracker.NextStep("Hashing Archives"); await HashArchives(); if (cancel.IsCancellationRequested) return false; + UpdateTracker.NextStep("Downloading Missing Archives"); await DownloadArchives(); if (cancel.IsCancellationRequested) return false; + UpdateTracker.NextStep("Hashing Remaining Archives"); await HashArchives(); if (cancel.IsCancellationRequested) return false; @@ -69,26 +72,33 @@ namespace Wabbajack.Lib Error("Cannot continue, was unable to download one or more archives"); } + if (cancel.IsCancellationRequested) return false; + UpdateTracker.NextStep("Priming VFS"); await PrimeVFS(); if (cancel.IsCancellationRequested) return false; + UpdateTracker.NextStep("Building Folder Structure"); BuildFolderStructure(); if (cancel.IsCancellationRequested) return false; + UpdateTracker.NextStep("Installing Archives"); await InstallArchives(); if (cancel.IsCancellationRequested) return false; + UpdateTracker.NextStep("Installing Included files"); await InstallIncludedFiles(); if (cancel.IsCancellationRequested) return false; + UpdateTracker.NextStep("Installing Manual files"); await InstallManualGameFiles(); if (cancel.IsCancellationRequested) return false; + UpdateTracker.NextStep("Installing SteamWorkshopItems"); await InstallSteamWorkshopItems(); //InstallIncludedDownloadMetas(); var metric2 = Metrics.Send("finish_install", ModList.Name); - Info("Installation complete! You may exit the program."); + UpdateTracker.NextStep("Installation complete! You may exit the program."); return true; }