From 059f2ec96fc81dcd72779a1fa40f4bed00e0a141 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 2 May 2020 15:09:29 -0600 Subject: [PATCH 01/10] Can source game downloads from the game folder --- Wabbajack.Common/GameMetaData.cs | 2 +- .../StoreHandlers/SteamHandler.cs | 2 +- Wabbajack.Lib/ACompiler.cs | 10 ++++ .../Downloaders/AbstractDownloadState.cs | 5 ++ .../Downloaders/GameFileSourceDownloader.cs | 8 +++- Wabbajack.Lib/MO2Compiler.cs | 46 ++++++++++++++----- Wabbajack.Test/SanityTests.cs | 21 +++++++++ 7 files changed, 80 insertions(+), 14 deletions(-) diff --git a/Wabbajack.Common/GameMetaData.cs b/Wabbajack.Common/GameMetaData.cs index 3744f62c..e5e2e4f1 100644 --- a/Wabbajack.Common/GameMetaData.cs +++ b/Wabbajack.Common/GameMetaData.cs @@ -85,7 +85,7 @@ namespace Wabbajack.Common public AbsolutePath? TryGetGameLocation() { - return Consts.TestMode ? AbsolutePath.GetCurrentDirectory() : StoreHandler.Instance.TryGetGamePath(Game); + return StoreHandler.Instance.TryGetGamePath(Game); } public bool TryGetGameLocation(out AbsolutePath path) diff --git a/Wabbajack.Common/StoreHandlers/SteamHandler.cs b/Wabbajack.Common/StoreHandlers/SteamHandler.cs index 3fec8415..fda12882 100644 --- a/Wabbajack.Common/StoreHandlers/SteamHandler.cs +++ b/Wabbajack.Common/StoreHandlers/SteamHandler.cs @@ -155,7 +155,7 @@ namespace Wabbajack.Common.StoreHandlers if (!l.ContainsCaseInsensitive("\"installdir\"")) return; - var path = new RelativePath($"common//{GetVdfValue(l)}").RelativeTo(u); + var path = new RelativePath("common").Combine(GetVdfValue(l)).RelativeTo(u); if (path.Exists) game.Path = path; }); diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index 9d0c694b..e11b6c4a 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -4,6 +4,7 @@ using System.IO; using System.IO.Compression; using System.Linq; using System.Reactive.Subjects; +using System.Text; using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; @@ -100,6 +101,15 @@ namespace Wabbajack.Lib return id; } + + internal async Task<(RelativePath, AbsolutePath)> IncludeString(string str) + { + var id = IncludeId(); + var fullPath = ModListOutputFolder.Combine(id); + await fullPath.WriteAllTextAsync(str); + return (id, fullPath); + } + public async Task GatherMetaData() { Utils.Log($"Getting meta data for {SelectedArchives.Count} archives"); diff --git a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs index ebfc5b32..381e9aa3 100644 --- a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs +++ b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs @@ -94,5 +94,10 @@ namespace Wabbajack.Lib.Downloaders public abstract string? GetManifestURL(Archive a); public abstract string[] GetMetaIni(); + + public string GetMetaIniString() + { + return string.Join("\n", GetMetaIni()); + } } } diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs index 4268033e..25f5b5fa 100644 --- a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs @@ -46,13 +46,18 @@ namespace Wabbajack.Lib.Downloaders public Game Game { get; set; } public RelativePath GameFile { get; set; } public Hash Hash { get; set; } - public string GameVersion { get; } + public string GameVersion { get; set; } = ""; public State(string gameVersion) { GameVersion = gameVersion; } + public State() + { + + } + [JsonIgnore] internal AbsolutePath SourcePath => Game.MetaData().GameLocation().Combine(GameFile); @@ -93,6 +98,7 @@ namespace Wabbajack.Lib.Downloaders { return new[] {"[General]", $"gameName={Game.MetaData().MO2ArchiveName}", $"gameFile={GameFile}"}; } + } } } diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 5a7d313a..9cef5c1f 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -1,24 +1,17 @@ using Compression.BSA; using System; -using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Net.Http; using System.Threading; using System.Threading.Tasks; -using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.FileUploader; -using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.Validation; -using Directory = Alphaleonis.Win32.Filesystem.Directory; -using File = Alphaleonis.Win32.Filesystem.File; -using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; -using Game = Wabbajack.Common.Game; +using Wabbajack.VirtualFileSystem; using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib @@ -105,7 +98,7 @@ namespace Wabbajack.Lib var roots = new List { - MO2Folder, GamePath, MO2DownloadsFolder + MO2Folder, GamePath, MO2DownloadsFolder, CompilingGame.GameLocation() }; // TODO: make this generic so we can add more paths @@ -172,6 +165,7 @@ namespace Wabbajack.Lib UpdateTracker.NextStep("Pre-validating Archives"); + // Find all Downloads IndexedArchives = (await MO2DownloadsFolder.EnumerateFiles() .Where(f => f.WithExtension(Consts.MetaFileExtension).Exists) .PMap(Queue, async f => new IndexedArchive(VFS.Index.ByRootPath[f]) @@ -180,9 +174,37 @@ namespace Wabbajack.Lib IniData = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(), Meta = await f.WithExtension(Consts.MetaFileExtension).ReadAllTextAsync() })).ToList(); + + + var stockGameFolder = CompilingGame.GameLocation(); + foreach (var (relativePath, hash) in await ClientAPI.GetGameFiles(CompilingGame.Game, Version.Parse(CompilingGame.InstalledVersion))) + { + if (!VFS.Index.ByRootPath.TryGetValue(relativePath.RelativeTo(stockGameFolder), out var virtualFile)) + continue; + if (virtualFile.Hash != hash) + { + Utils.Log( + $"File {relativePath} int the game folder appears to be modified, it will not be used during compilation"); + continue; + } + var state = new GameFileSourceDownloader.State + { + Game = CompilingGame.Game, GameVersion = CompilingGame.InstalledVersion, GameFile = relativePath + }; + Utils.Log($"Adding Game file: {relativePath}"); + IndexedArchives.Add(new IndexedArchive(virtualFile) + { + Name = (string)relativePath.FileName, + IniData = state.GetMetaIniString().LoadIniString(), + Meta = state.GetMetaIniString() + }); + } + + + await CleanInvalidArchives(); UpdateTracker.NextStep("Finding Install Files"); @@ -395,11 +417,13 @@ namespace Wabbajack.Lib await SelectedArchives.PMap(Queue, async a => { var source = MO2DownloadsFolder.Combine(a.Name + Consts.MetaFileExtension); + var ini = a.State.GetMetaIniString(); + var (id, fullPath) = await IncludeString(ini); InstallDirectives.Add(new ArchiveMeta { - SourceDataID = await IncludeFile(source), + SourceDataID = id, Size = source.Size, - Hash = await source.FileHashAsync(), + Hash = await fullPath.FileHashAsync(), To = source.FileName }); }); diff --git a/Wabbajack.Test/SanityTests.cs b/Wabbajack.Test/SanityTests.cs index f1b58015..f6279b49 100644 --- a/Wabbajack.Test/SanityTests.cs +++ b/Wabbajack.Test/SanityTests.cs @@ -421,6 +421,27 @@ namespace Wabbajack.Test utils.VerifyInstalledFile(mod, @"baz.bsa"); } + + [Fact] + public async Task CanSourceFilesFromStockGameFiles() + { + Consts.TestMode = false; + + var profile = utils.AddProfile(); + var mod = utils.AddMod(); + var skyrimExe = utils.AddModFile(mod, @"Data\test.exe", 10); + + await Game.SkyrimSpecialEdition.MetaData().GameLocation().Combine("SkyrimSE.exe").CopyToAsync(skyrimExe); + + await utils.Configure(); + + await CompileAndInstall(profile); + + utils.VerifyInstalledFile(mod, @"Data\test.exe"); + + Consts.TestMode = true; + + } [Fact] public async Task NoMatchIncludeIncludesNonMatchingFiles() From 3c671272b66d18d5ff106af7c4468ad9ed5d94b9 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 2 May 2020 17:05:05 -0600 Subject: [PATCH 02/10] MO2 remaps game folder to Game Folder Files if the game executable exists in that folder --- Wabbajack.Lib/AInstaller.cs | 9 +++++++-- .../IgnoreGameFilesIfGameFolderFilesExist.cs | 17 +++++++--------- Wabbajack.Lib/MO2Compiler.cs | 3 --- Wabbajack.Lib/MO2Installer.cs | 19 +++++++++++------- Wabbajack.Test/DownloaderTests.cs | 2 +- Wabbajack.Test/EndToEndTests.cs | 1 - Wabbajack.Test/SanityTests.cs | 20 ++++++++++++++++--- 7 files changed, 44 insertions(+), 27 deletions(-) diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 89eb8d5d..72dfc8cc 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -29,9 +29,11 @@ namespace Wabbajack.Lib public ModList ModList { get; private set; } public Dictionary HashedArchives { get; } = new Dictionary(); + public GameMetaData Game { get; } + public SystemParameters? SystemParameters { get; set; } - public AInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters? parameters, int steps) + public AInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters? parameters, int steps, Game game) : base(steps) { ModList = modList; @@ -39,6 +41,7 @@ namespace Wabbajack.Lib OutputFolder = outputFolder; DownloadFolder = downloadFolder; SystemParameters = parameters; + Game = game.MetaData(); } public void Info(string msg) @@ -280,7 +283,9 @@ namespace Wabbajack.Lib public async Task HashArchives() { - var hashResults = await DownloadFolder.EnumerateFiles() + var hashResults = await + DownloadFolder.EnumerateFiles() + .Concat(Game.GameLocation().EnumerateFiles()) .Where(e => e.Extension != Consts.HashFileExtension) .PMap(Queue, async e => (await e.FileHashCachedAsync(), e)); HashedArchives.SetTo(hashResults diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreGameFilesIfGameFolderFilesExist.cs b/Wabbajack.Lib/CompilationSteps/IgnoreGameFilesIfGameFolderFilesExist.cs index 6f86f783..4191184e 100644 --- a/Wabbajack.Lib/CompilationSteps/IgnoreGameFilesIfGameFolderFilesExist.cs +++ b/Wabbajack.Lib/CompilationSteps/IgnoreGameFilesIfGameFolderFilesExist.cs @@ -17,17 +17,14 @@ namespace Wabbajack.Lib.CompilationSteps public override async ValueTask Run(RawSourceFile source) { - if (_gameFolderFilesExists) - { - if (source.AbsolutePath.InFolder(_gameFolder)) - { - var result = source.EvolveTo(); - result.Reason = $"Ignoring game files because {Consts.GameFolderFilesDir} exists"; - return result; - } - } + if (!_gameFolderFilesExists) return null; + + if (!source.AbsolutePath.InFolder(_gameFolder)) return null; + + var result = source.EvolveTo(); + result.Reason = $"Ignoring game files because {Consts.GameFolderFilesDir} exists"; + return result; - return null; } public override IState GetState() diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 9cef5c1f..9e0829a4 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -547,9 +547,6 @@ namespace Wabbajack.Lib // Ignore the ModOrganizer.ini file it contains info created by MO2 on startup new IncludeStubbedConfigFiles(this), new IncludeLootFiles(this), - new IgnoreStartsWith(this, Path.Combine((string)Consts.GameFolderFilesDir, "Data")), - new IgnoreStartsWith(this, Path.Combine((string)Consts.GameFolderFilesDir, "Papyrus Compiler")), - new IgnoreStartsWith(this, Path.Combine((string)Consts.GameFolderFilesDir, "Skyrim")), new IgnoreRegex(this, Consts.GameFolderFilesDir + "\\\\.*\\.bsa"), new IncludeRegex(this, "^[^\\\\]*\\.bat$"), new IncludeModIniData(this), diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index 996cecee..205130d4 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -32,8 +32,6 @@ namespace Wabbajack.Lib public AbsolutePath? GameFolder { get; set; } - public GameMetaData Game { get; } - public MO2Installer(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters parameters) : base( archive: archive, @@ -41,11 +39,15 @@ namespace Wabbajack.Lib outputFolder: outputFolder, downloadFolder: downloadFolder, parameters: parameters, - steps: 20) + steps: 20, + game: modList.GameType) { - Game = ModList.GameType.MetaData(); + var gameExe = Consts.GameFolderFilesDir.Combine(modList.GameType.MetaData().MainExecutable!); + RedirectGamePath = modList.Directives.Any(d => d.To == gameExe); } + public bool RedirectGamePath { get; } + protected override async Task _Begin(CancellationToken cancel) { if (cancel.IsCancellationRequested) return false; @@ -379,9 +381,12 @@ namespace Wabbajack.Lib { var data = Encoding.UTF8.GetString(await LoadBytesFromPath(directive.SourceDataID)); - data = data.Replace(Consts.GAME_PATH_MAGIC_BACK, (string)GameFolder!); - data = data.Replace(Consts.GAME_PATH_MAGIC_DOUBLE_BACK, ((string)GameFolder!).Replace("\\", "\\\\")); - data = data.Replace(Consts.GAME_PATH_MAGIC_FORWARD, ((string)GameFolder).Replace("\\", "/")); + var gameFolder = (string)(RedirectGamePath ? Consts.GameFolderFilesDir.RelativeTo(OutputFolder) : GameFolder!); + + + data = data.Replace(Consts.GAME_PATH_MAGIC_BACK, gameFolder); + data = data.Replace(Consts.GAME_PATH_MAGIC_DOUBLE_BACK, gameFolder.Replace("\\", "\\\\")); + data = data.Replace(Consts.GAME_PATH_MAGIC_FORWARD, gameFolder.Replace("\\", "/")); data = data.Replace(Consts.MO2_PATH_MAGIC_BACK, (string)OutputFolder); data = data.Replace(Consts.MO2_PATH_MAGIC_DOUBLE_BACK, ((string)OutputFolder).Replace("\\", "\\\\")); diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 3f086371..a4006df1 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -551,7 +551,7 @@ namespace Wabbajack.Test class TestInstaller : AInstaller { public TestInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters parameters) - : base(archive, modList, outputFolder, downloadFolder, parameters, steps: 1) + : base(archive, modList, outputFolder, downloadFolder, parameters, steps: 1, modList.GameType) { Queue.SetActiveThreadsObservable(Observable.Return(1)); } diff --git a/Wabbajack.Test/EndToEndTests.cs b/Wabbajack.Test/EndToEndTests.cs index 86b0e12a..d2f74495 100644 --- a/Wabbajack.Test/EndToEndTests.cs +++ b/Wabbajack.Test/EndToEndTests.cs @@ -61,7 +61,6 @@ namespace Wabbajack.Test DownloadAndInstall(Game.SkyrimSpecialEdition, 12604, "SkyUI"), DownloadAndInstall(Game.Fallout4, 11925, "Anti-Tank Rifle"), DownloadAndInstall(Game.SkyrimSpecialEdition, 4783, "Frost Armor UNP"), - DownloadAndInstall(Game.Fallout4, 43474, "EM 2 Rifle No.9 Mk1"), DownloadAndInstall(Game.SkyrimSpecialEdition, 32359, "Frost Armor HDT")); // We're going to fully patch this mod from another source. diff --git a/Wabbajack.Test/SanityTests.cs b/Wabbajack.Test/SanityTests.cs index f6279b49..b712c8aa 100644 --- a/Wabbajack.Test/SanityTests.cs +++ b/Wabbajack.Test/SanityTests.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -431,7 +432,12 @@ namespace Wabbajack.Test var mod = utils.AddMod(); var skyrimExe = utils.AddModFile(mod, @"Data\test.exe", 10); - await Game.SkyrimSpecialEdition.MetaData().GameLocation().Combine("SkyrimSE.exe").CopyToAsync(skyrimExe); + var gameFolder = Consts.GameFolderFilesDir.RelativeTo(utils.MO2Folder); + gameFolder.CreateDirectory(); + + var gameMeta = Game.SkyrimSpecialEdition.MetaData(); + await gameMeta.GameLocation().Combine(gameMeta.MainExecutable!).CopyToAsync(skyrimExe); + await gameMeta.GameLocation().Combine(gameMeta.MainExecutable!).CopyToAsync(gameFolder.Combine(gameMeta.MainExecutable!)); await utils.Configure(); @@ -439,8 +445,16 @@ namespace Wabbajack.Test utils.VerifyInstalledFile(mod, @"Data\test.exe"); - Consts.TestMode = true; + Assert.False("SkyrimSE.exe".RelativeTo(utils.DownloadsFolder).Exists, "File should not appear in the download folder because it should be copied from the game folder"); + var file = "ModOrganizer.ini".RelativeTo(utils.InstallFolder); + Assert.True(file.Exists); + + var ini = file.LoadIniFile(); + Assert.Equal(((AbsolutePath)(string)ini?.General?.gamePath).Combine(gameMeta.MainExecutable), + Consts.GameFolderFilesDir.Combine(gameMeta.MainExecutable).RelativeTo(utils.InstallFolder)); + + Consts.TestMode = true; } [Fact] From 0c0cbbab4f4bf9e56d3d03f98c6b2027610ab248 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 2 May 2020 21:41:33 -0600 Subject: [PATCH 03/10] Implement basics of game folder remapping --- Wabbajack.Common/Paths.cs | 5 +++ Wabbajack.Lib/Tasks/MigrateGameFolder.cs | 53 ++++++++++++++++++++++++ Wabbajack.Test/TasksTests.cs | 35 ++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 Wabbajack.Lib/Tasks/MigrateGameFolder.cs create mode 100644 Wabbajack.Test/TasksTests.cs diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index 2d786dbf..1a6ec6b2 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -410,6 +410,11 @@ namespace Wabbajack.Common await file.CopyToAsync(dest); } } + + public async Task CopyOrLinkIfOverSizeAsync(AbsolutePath newFile) + { + await CopyToAsync(newFile); + } } [JsonConverter(typeof(Utils.RelativePathConverter))] diff --git a/Wabbajack.Lib/Tasks/MigrateGameFolder.cs b/Wabbajack.Lib/Tasks/MigrateGameFolder.cs new file mode 100644 index 00000000..ec44215a --- /dev/null +++ b/Wabbajack.Lib/Tasks/MigrateGameFolder.cs @@ -0,0 +1,53 @@ +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.Lib.Tasks +{ + public class MigrateGameFolder + { + public static async Task Execute(AbsolutePath mo2Folder) + { + var iniPath = mo2Folder.Combine(Consts.ModOrganizer2Ini); + if (iniPath.Exists) + { + Utils.Log($"Game folder conversion failed, {Consts.ModOrganizer2Ini} does not exist in {mo2Folder}"); + return false; + } + + var newGamePath = mo2Folder.Combine(Consts.GameFolderFilesDir); + newGamePath.CreateDirectory(); + var gameIni = iniPath.LoadIniFile(); + + if (!GameRegistry.TryGetByFuzzyName((string)gameIni.General.gameName, out var gameMeta)) + { + Utils.Log($"Could not locate game for {gameIni.General.gameName}"); + return false; + } + + + var orginGamePath = gameMeta.GameLocation(); + foreach (var file in gameMeta.GameLocation().EnumerateFiles()) + { + var relPath = file.RelativeTo(orginGamePath); + var newFile = relPath.RelativeTo(newGamePath); + if (newFile.Exists) + { + Utils.Log($"Skipping {relPath} it already exists in the target path"); + continue; + } + + Utils.Log($"Copying/Linking {relPath}"); + await file.CopyOrLinkIfOverSizeAsync(newFile); + } + + Utils.Log("Remapping INI"); + var iniString = await iniPath.ReadAllTextAsync(); + iniString = iniString.Replace((string)orginGamePath, (string)newGamePath); + iniString = iniString.Replace(((string)orginGamePath).Replace(@"\", @"\\"), ((string)newGamePath).Replace(@"\", @"\\")); + iniString = iniString.Replace(((string)orginGamePath).Replace(@"\", @"/"), ((string)newGamePath).Replace(@"\", @"/")); + await iniPath.WriteAllTextAsync(iniString); + + return true; + } + } +} diff --git a/Wabbajack.Test/TasksTests.cs b/Wabbajack.Test/TasksTests.cs new file mode 100644 index 00000000..6a90e36e --- /dev/null +++ b/Wabbajack.Test/TasksTests.cs @@ -0,0 +1,35 @@ +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Lib.Tasks; +using Xunit; + +namespace Wabbajack.Test +{ + public class TasksTests + { + [Fact] + public async Task CanRemapGameFolder() + { + await using var tempFolder = await TempFolder.Create(); + + await tempFolder.Dir.Combine("some_file.txt").WriteAllTextAsync("some_file"); + await tempFolder.Dir.Combine("steam_api64.dll").WriteAllTextAsync("steam_api"); + + + var meta = Game.SkyrimSpecialEdition.MetaData(); + await tempFolder.Dir.Combine(Consts.ModOrganizer2Ini) + .WriteAllLinesAsync( + "[General]", + $"gameName={meta.MO2Name}", + $"gamePath={meta.GameLocation()}", + $"pathDouble={meta.GameLocation().ToString().Replace(@"\", @"\\")}", + $"pathForward={meta.GameLocation().ToString().Replace(@"\", @"/")}"); + + await MigrateGameFolder.Execute(tempFolder.Dir); + + + + + } + } +} From 2be1771d897ef0318bc759caa8fb5d3c82c6a15f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 4 May 2020 06:11:53 -0600 Subject: [PATCH 04/10] Can remap game folder files --- Wabbajack.Common.Test/PathTests.cs | 15 +++++++++++++++ Wabbajack.Common/GameMetaData.cs | 10 ++++++++++ Wabbajack.Common/Paths.cs | 12 ++++++------ Wabbajack.Lib/Tasks/MigrateGameFolder.cs | 2 +- Wabbajack.Test/TasksTests.cs | 24 +++++++++++++++++++----- 5 files changed, 51 insertions(+), 12 deletions(-) diff --git a/Wabbajack.Common.Test/PathTests.cs b/Wabbajack.Common.Test/PathTests.cs index 89bac5e1..1408d3dd 100644 --- a/Wabbajack.Common.Test/PathTests.cs +++ b/Wabbajack.Common.Test/PathTests.cs @@ -24,5 +24,20 @@ namespace Wabbajack.Common.Test tempFile.Path.MoveTo(tempFile2.Path); } + + [Fact] + public void CanGetTopParentOfPath() + { + var path = (RelativePath)"foo/bar"; + Assert.Equal((RelativePath)"foo", path.TopParent); + + } + + [Fact] + public void CanGetTopParentOfSinglePath() + { + var path = (RelativePath)"foo"; + Assert.Equal((RelativePath)"foo", path.TopParent); + } } } diff --git a/Wabbajack.Common/GameMetaData.cs b/Wabbajack.Common/GameMetaData.cs index e5e2e4f1..02715f42 100644 --- a/Wabbajack.Common/GameMetaData.cs +++ b/Wabbajack.Common/GameMetaData.cs @@ -180,10 +180,20 @@ namespace Wabbajack.Common result = GetByMO2ArchiveName(someName); if (result != null) return result; + + result = GetByMO2Name(someName); + if (result != null) return result; + return int.TryParse(someName, out int id) ? GetBySteamID(id) : null; } + private static GameMetaData? GetByMO2Name(string gameName) + { + gameName = gameName.ToLower(); + return Games.Values.FirstOrDefault(g => g.MO2Name?.ToLower() == gameName); + } + public static bool TryGetByFuzzyName(string someName, [MaybeNullWhen(false)] out GameMetaData gameMetaData) { var result = TryGetByFuzzyName(someName); diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index 1a6ec6b2..8a4da9d6 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -208,12 +208,10 @@ namespace Wabbajack.Common public RelativePath RelativeTo(AbsolutePath p) { - if (_path.Substring(0, p._path.Length + 1) != p._path + "\\") - { - throw new InvalidDataException("Not a parent path"); - } - - return new RelativePath(_path.Substring(p._path.Length + 1)); + var relPath = Path.GetRelativePath(p._path, _path); + if (relPath == _path) + throw new ArgumentException($"{_path} is not a subpath of {p._path}"); + return new RelativePath(relPath); } public async Task ReadAllTextAsync() @@ -413,6 +411,8 @@ namespace Wabbajack.Common public async Task CopyOrLinkIfOverSizeAsync(AbsolutePath newFile) { + if (newFile.Parent != default) + newFile.Parent.CreateDirectory(); await CopyToAsync(newFile); } } diff --git a/Wabbajack.Lib/Tasks/MigrateGameFolder.cs b/Wabbajack.Lib/Tasks/MigrateGameFolder.cs index ec44215a..10905607 100644 --- a/Wabbajack.Lib/Tasks/MigrateGameFolder.cs +++ b/Wabbajack.Lib/Tasks/MigrateGameFolder.cs @@ -8,7 +8,7 @@ namespace Wabbajack.Lib.Tasks public static async Task Execute(AbsolutePath mo2Folder) { var iniPath = mo2Folder.Combine(Consts.ModOrganizer2Ini); - if (iniPath.Exists) + if (!iniPath.Exists) { Utils.Log($"Game folder conversion failed, {Consts.ModOrganizer2Ini} does not exist in {mo2Folder}"); return false; diff --git a/Wabbajack.Test/TasksTests.cs b/Wabbajack.Test/TasksTests.cs index 6a90e36e..be0d0525 100644 --- a/Wabbajack.Test/TasksTests.cs +++ b/Wabbajack.Test/TasksTests.cs @@ -2,18 +2,21 @@ using Wabbajack.Common; using Wabbajack.Lib.Tasks; using Xunit; +using Xunit.Abstractions; namespace Wabbajack.Test { - public class TasksTests + public class TasksTests : ACompilerTest { [Fact] public async Task CanRemapGameFolder() { await using var tempFolder = await TempFolder.Create(); + var gameff = tempFolder.Dir.Combine(Consts.GameFolderFilesDir); + gameff.CreateDirectory(); - await tempFolder.Dir.Combine("some_file.txt").WriteAllTextAsync("some_file"); - await tempFolder.Dir.Combine("steam_api64.dll").WriteAllTextAsync("steam_api"); + await gameff.Combine("some_file.txt").WriteAllTextAsync("some_file"); + await gameff.Combine("steam_api64.dll").WriteAllTextAsync("steam_api"); var meta = Game.SkyrimSpecialEdition.MetaData(); @@ -25,11 +28,22 @@ namespace Wabbajack.Test $"pathDouble={meta.GameLocation().ToString().Replace(@"\", @"\\")}", $"pathForward={meta.GameLocation().ToString().Replace(@"\", @"/")}"); - await MigrateGameFolder.Execute(tempFolder.Dir); - + Assert.True(await MigrateGameFolder.Execute(tempFolder.Dir)); + Assert.Equal("some_file", await gameff.Combine("some_file.txt").ReadAllTextAsync()); + Assert.Equal("steam_api", await gameff.Combine("steam_api64.dll").ReadAllTextAsync()); + Assert.Equal(Hash.FromBase64("k5EWx/9Woqg="), await gameff.Combine(@"Data\Skyrim - Interface.bsa").FileHashAsync()); + + var ini = tempFolder.Dir.Combine(Consts.ModOrganizer2Ini).LoadIniFile(); + Assert.Equal(gameff, (AbsolutePath)(string)ini.General.gamePath); + Assert.Equal(gameff, (AbsolutePath)(string)ini.General.pathDouble); + Assert.Equal(gameff, (AbsolutePath)(string)ini.General.pathForward); } + + public TasksTests(ITestOutputHelper helper) : base(helper) + { + } } } From 314c77a8301af5ff75d86b3f6c95bad8a05d8f28 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 4 May 2020 14:33:45 -0600 Subject: [PATCH 05/10] Fix tests and implement basic hard link support --- Wabbajack.Common/Paths.cs | 24 ++++++++++++++++++++ Wabbajack.Lib/AInstaller.cs | 9 ++++++-- Wabbajack.Lib/WebAutomation/WebAutomation.cs | 1 + Wabbajack.Test/DownloaderTests.cs | 2 +- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index 8a4da9d6..e892b47c 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Net; using System.Reflection; +using System.Runtime.InteropServices; using System.Text; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; @@ -340,6 +341,29 @@ namespace Wabbajack.Common { File.Copy(_path, dest._path); } + + + + [DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Auto)] + private static extern bool CreateHardLink(string lpFileName, string lpExistingFileName, IntPtr lpSecurityAttributes); + + public bool HardLinkTo(AbsolutePath destination) + { + return CreateHardLink((string)destination, (string)this, IntPtr.Zero); + } + + public static long HARDLINK_THRESHOLD = 2 ^ 29; // 512 MB + + public async ValueTask HardLinkIfOversize(AbsolutePath destination) + { + if (Root == destination.Root || Size >= HARDLINK_THRESHOLD) + { + if (HardLinkTo(destination)) + return; + } + + await CopyToAsync(destination); + } public async Task> ReadAllLinesAsync() { diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 72dfc8cc..7b11f55e 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -141,7 +141,7 @@ namespace Wabbajack.Lib Status($"Copying files for {archive.Name}"); - async ValueTask CopyFile(AbsolutePath from, AbsolutePath to, bool useMove) + async ValueTask CopyFile(VirtualFile? vf, AbsolutePath from, AbsolutePath to, bool useMove) { if (to.Exists) { @@ -156,6 +156,11 @@ namespace Wabbajack.Lib from.IsReadOnly = false; } + if (vf!.IsNative && vf.AbsoluteName == from) + { + await from.HardLinkIfOversize(to); + return; + } if (useMove) from.MoveTo(to); @@ -180,7 +185,7 @@ namespace Wabbajack.Lib foreach (var copy in group.Skip(1)) { - await CopyFile(firstDest, OutputFolder.Combine(copy.To), false); + await CopyFile(group.Key, firstDest, OutputFolder.Combine(copy.To), false); } }); diff --git a/Wabbajack.Lib/WebAutomation/WebAutomation.cs b/Wabbajack.Lib/WebAutomation/WebAutomation.cs index d906c968..60d2f4f4 100644 --- a/Wabbajack.Lib/WebAutomation/WebAutomation.cs +++ b/Wabbajack.Lib/WebAutomation/WebAutomation.cs @@ -15,6 +15,7 @@ namespace Wabbajack.Lib.WebAutomation public Driver() { _browser = new ChromiumWebBrowser(); + _driver = new CefSharpWrapper(_browser); } public static async Task Create() diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index a4006df1..3b752249 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -505,7 +505,7 @@ namespace Wabbajack.Test await folder.DeleteDirectory(); folder.CreateDirectory(); - var inst = new TestInstaller(default, null, default, folder, null); + var inst = new TestInstaller(default, new ModList {GameType = Game.SkyrimSpecialEdition}, default, folder, null); await inst.DownloadMissingArchives(archivesa, true); await inst.DownloadMissingArchives(archivesb, true); From 287eebdd3d56bfee4761b755f2316cd6d49ae0c5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 4 May 2020 14:39:43 -0600 Subject: [PATCH 06/10] Fix tests and implement hard link support --- Wabbajack.Common/Paths.cs | 13 +++++-------- Wabbajack.Lib/Tasks/MigrateGameFolder.cs | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index e892b47c..e006d552 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -349,6 +349,7 @@ namespace Wabbajack.Common public bool HardLinkTo(AbsolutePath destination) { + Utils.Log($"Hard Linking {_path} to {destination}"); return CreateHardLink((string)destination, (string)this, IntPtr.Zero); } @@ -356,7 +357,10 @@ namespace Wabbajack.Common public async ValueTask HardLinkIfOversize(AbsolutePath destination) { - if (Root == destination.Root || Size >= HARDLINK_THRESHOLD) + if (!destination.Parent.Exists) + destination.Parent.CreateDirectory(); + + if (Root == destination.Root && Size >= HARDLINK_THRESHOLD) { if (HardLinkTo(destination)) return; @@ -432,13 +436,6 @@ namespace Wabbajack.Common await file.CopyToAsync(dest); } } - - public async Task CopyOrLinkIfOverSizeAsync(AbsolutePath newFile) - { - if (newFile.Parent != default) - newFile.Parent.CreateDirectory(); - await CopyToAsync(newFile); - } } [JsonConverter(typeof(Utils.RelativePathConverter))] diff --git a/Wabbajack.Lib/Tasks/MigrateGameFolder.cs b/Wabbajack.Lib/Tasks/MigrateGameFolder.cs index 10905607..06feb725 100644 --- a/Wabbajack.Lib/Tasks/MigrateGameFolder.cs +++ b/Wabbajack.Lib/Tasks/MigrateGameFolder.cs @@ -37,7 +37,7 @@ namespace Wabbajack.Lib.Tasks } Utils.Log($"Copying/Linking {relPath}"); - await file.CopyOrLinkIfOverSizeAsync(newFile); + await file.HardLinkIfOversize(newFile); } Utils.Log("Remapping INI"); From 223b6d4e92b23704785e6eef7ecd3c17c6783b07 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 4 May 2020 16:49:24 -0600 Subject: [PATCH 07/10] Few more fixes --- Wabbajack.Lib/AInstaller.cs | 6 +++--- Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 7b11f55e..9a152081 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -156,16 +156,16 @@ namespace Wabbajack.Lib from.IsReadOnly = false; } - if (vf!.IsNative && vf.AbsoluteName == from) + if (vf!.IsNative) { await from.HardLinkIfOversize(to); return; } if (useMove) - from.MoveTo(to); + await @from.MoveToAsync(to); else - from.CopyTo(to); + await @from.CopyToAsync(to); // If we don't do this, the file will use the last-modified date of the file when it was compressed // into an archive, which isn't really what we want in the case of files installed archives to.LastModified = DateTime.Now; diff --git a/Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs b/Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs index d2b39217..f4b538d7 100644 --- a/Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs +++ b/Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs @@ -81,7 +81,7 @@ namespace Wabbajack.Lib.WebAutomation IWindowInfo windowInfo, IBrowserSettings browserSettings, ref bool noJavascriptAccess, out IWebBrowser? newBrowser) { // Block popups - newBrowser = chromiumWebBrowser; + newBrowser = null; return true; } From aa2c81eb4885f6f6343be1d519f719c86a47a232 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 4 May 2020 17:03:50 -0600 Subject: [PATCH 08/10] Add migration task as cli verb --- Wabbajack.CLI/OptionsDefinition.cs | 3 ++- Wabbajack.CLI/Program.cs | 1 + Wabbajack.CLI/Verbs/MigrateGameFolderFiles.cs | 25 +++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 Wabbajack.CLI/Verbs/MigrateGameFolderFiles.cs diff --git a/Wabbajack.CLI/OptionsDefinition.cs b/Wabbajack.CLI/OptionsDefinition.cs index b14a52dd..65af0719 100644 --- a/Wabbajack.CLI/OptionsDefinition.cs +++ b/Wabbajack.CLI/OptionsDefinition.cs @@ -19,7 +19,8 @@ namespace Wabbajack.CLI typeof(DeleteFile), typeof(Changelog), typeof(FindSimilar), - typeof(BSADump) + typeof(BSADump), + typeof(MigrateGameFolderFiles) }; } } diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index c8f4412f..eea8793d 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -22,6 +22,7 @@ namespace Wabbajack.CLI (Changelog opts) => opts.Execute(), (FindSimilar opts) => opts.Execute(), (BSADump opts) => opts.Execute(), + (MigrateGameFolderFiles opts) => opts.Execute(), errs => 1); } } diff --git a/Wabbajack.CLI/Verbs/MigrateGameFolderFiles.cs b/Wabbajack.CLI/Verbs/MigrateGameFolderFiles.cs new file mode 100644 index 00000000..6da6dc68 --- /dev/null +++ b/Wabbajack.CLI/Verbs/MigrateGameFolderFiles.cs @@ -0,0 +1,25 @@ +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Lib.Tasks; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("migrate-game-folder", HelpText = "Migrates game files into the 'Game Folder Files' in a MO2 directory")] + public class MigrateGameFolderFiles : AVerb + { + [IsDirectory(CustomMessage = "Downloads folder at %1 does not exist!")] + [Option('i', "input", HelpText = "Input Mod Organizer 2 Folder", Required = true)] + public string MO2Folder { get; set; } = ""; + + + protected override async Task Run() + { + if (await MigrateGameFolder.Execute((AbsolutePath)MO2Folder)) + { + return ExitCode.Ok; + }; + return ExitCode.Error; + } + } +} From bac8900df762917bb793ea599255313ba7d708a5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 4 May 2020 17:55:51 -0600 Subject: [PATCH 09/10] Fix for Update.esm getting deleted during installation --- Wabbajack.Common/Paths.cs | 4 +--- Wabbajack.Lib/AInstaller.cs | 26 ++++++++++++-------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index e006d552..cdd6a2b8 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -353,14 +353,12 @@ namespace Wabbajack.Common return CreateHardLink((string)destination, (string)this, IntPtr.Zero); } - public static long HARDLINK_THRESHOLD = 2 ^ 29; // 512 MB - public async ValueTask HardLinkIfOversize(AbsolutePath destination) { if (!destination.Parent.Exists) destination.Parent.CreateDirectory(); - if (Root == destination.Root && Size >= HARDLINK_THRESHOLD) + if (Root == destination.Root && Consts.SupportedBSAs.Contains(Extension)) { if (HardLinkTo(destination)) return; diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 9a152081..436a30b2 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -141,7 +141,7 @@ namespace Wabbajack.Lib Status($"Copying files for {archive.Name}"); - async ValueTask CopyFile(VirtualFile? vf, AbsolutePath from, AbsolutePath to, bool useMove) + async ValueTask CopyFile(AbsolutePath from, AbsolutePath to) { if (to.Exists) { @@ -156,16 +156,7 @@ namespace Wabbajack.Lib from.IsReadOnly = false; } - if (vf!.IsNative) - { - await from.HardLinkIfOversize(to); - return; - } - - if (useMove) - await @from.MoveToAsync(to); - else - await @from.CopyToAsync(to); + await @from.CopyToAsync(to); // If we don't do this, the file will use the last-modified date of the file when it was compressed // into an archive, which isn't really what we want in the case of files installed archives to.LastModified = DateTime.Now; @@ -181,11 +172,18 @@ namespace Wabbajack.Lib } var firstDest = OutputFolder.Combine(group.First().To); - await group.Key.StagedFile.MoveTo(firstDest); - + if (group.Key.IsNative) + { + await group.Key.AbsoluteName.HardLinkIfOversize(firstDest); + } + else + { + await group.Key.StagedFile.MoveTo(firstDest); + } + foreach (var copy in group.Skip(1)) { - await CopyFile(group.Key, firstDest, OutputFolder.Combine(copy.To), false); + await CopyFile(firstDest, OutputFolder.Combine(copy.To)); } }); From 9531d6c81311f01eb6e1ee6d728164cc53019ddb Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 4 May 2020 22:08:00 -0600 Subject: [PATCH 10/10] Remove unused test --- Wabbajack.Common/Paths.cs | 5 +++-- Wabbajack.Test/SanityTests.cs | 25 ------------------------- 2 files changed, 3 insertions(+), 27 deletions(-) diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index cdd6a2b8..e2dd9f3b 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -374,7 +374,8 @@ namespace Wabbajack.Common public byte[] ReadAllBytes() { - return File.ReadAllBytes(_path); + using var file = OpenShared(); + return file.ReadAll(); } public static AbsolutePath GetCurrentDirectory() @@ -417,7 +418,7 @@ namespace Wabbajack.Common public FileStream OpenShared() { - return File.Open(_path, FileMode.Open, FileAccess.Read); + return File.Open(_path, FileMode.Open, FileAccess.Read, FileShare.Read); } public FileStream WriteShared() diff --git a/Wabbajack.Test/SanityTests.cs b/Wabbajack.Test/SanityTests.cs index b712c8aa..bdbff997 100644 --- a/Wabbajack.Test/SanityTests.cs +++ b/Wabbajack.Test/SanityTests.cs @@ -160,31 +160,6 @@ namespace Wabbajack.Test Assert.False(extraFolder.Exists); } - - [Fact] - public async Task CleanedESMTest() - { - var profile = utils.AddProfile(); - var mod = utils.AddMod("Cleaned ESMs"); - var updateEsm = utils.AddModFile(mod, @"Update.esm", 10); - - await utils.Configure(); - - var gameFile = utils.GameFolder.Combine("Data", "Update.esm"); - utils.GenerateRandomFileData(gameFile, 20); - - var modlist = await CompileAndInstall(profile); - - utils.VerifyInstalledFile(mod, @"Update.esm"); - - var compiler = await ConfigureAndRunCompiler(profile); - - // Update the file and verify that it throws an error. - utils.GenerateRandomFileData(gameFile, 20); - var exception = await Assert.ThrowsAsync(async () => await Install(compiler)); - Assert.IsAssignableFrom(exception); - } - [Fact] public async Task SetScreenSizeTest() {