diff --git a/.gitignore b/.gitignore index 02139e36..79d51ade 100644 --- a/.gitignore +++ b/.gitignore @@ -365,3 +365,4 @@ MigrationBackup/ # Ionide (cross platform F# VS Code tools) working folder .ionide/ +/.idea diff --git a/Compression.BSA.Test/Compression.BSA.Test.csproj b/Compression.BSA.Test/Compression.BSA.Test.csproj index 191e765e..288dc349 100644 --- a/Compression.BSA.Test/Compression.BSA.Test.csproj +++ b/Compression.BSA.Test/Compression.BSA.Test.csproj @@ -89,10 +89,10 @@ 2.2.6 - 1.3.2 + 2.0.0 - 1.3.2 + 2.0.0 diff --git a/VirtualFileSystem/VirtualFileSystem.cs b/VirtualFileSystem/VirtualFileSystem.cs index 6c441a64..acbcfdcf 100644 --- a/VirtualFileSystem/VirtualFileSystem.cs +++ b/VirtualFileSystem/VirtualFileSystem.cs @@ -44,7 +44,10 @@ namespace VFS public static void Reconfigure(string root) { RootFolder = root; - _stagedRoot = Path.Combine(RootFolder, "vfs_staged_files"); + if (RootFolder != null) + _stagedRoot = Path.Combine(RootFolder, "vfs_staged_files"); + else + _stagedRoot = "vfs_staged_files"; } public static void Clean() diff --git a/VirtualFileSystem/VirtualFileSystem.csproj b/VirtualFileSystem/VirtualFileSystem.csproj index b365484b..facaa800 100644 --- a/VirtualFileSystem/VirtualFileSystem.csproj +++ b/VirtualFileSystem/VirtualFileSystem.csproj @@ -112,7 +112,7 @@ 1.2.0 - 1.5.0 + 1.6.0 diff --git a/Wabbajack.Common.CSP/ManyToManyChannel.cs b/Wabbajack.Common.CSP/ManyToManyChannel.cs index 464d2840..25239c94 100644 --- a/Wabbajack.Common.CSP/ManyToManyChannel.cs +++ b/Wabbajack.Common.CSP/ManyToManyChannel.cs @@ -122,7 +122,7 @@ namespace Wabbajack.Common.CSP Monitor.Exit(this); throw new TooManyHanldersException(); } - _puts.Unshift((handler, val)); + _puts.UnboundedUnshift((handler, val)); } Monitor.Exit(this); return (AsyncResult.Enqueued, true); @@ -191,7 +191,7 @@ namespace Wabbajack.Common.CSP throw new TooManyHanldersException(); } - _takes.Unshift(handler); + _takes.UnboundedUnshift(handler); } Monitor.Exit(this); return (AsyncResult.Enqueued, default); diff --git a/Wabbajack.Common.CSP/PIpelines.cs b/Wabbajack.Common.CSP/PIpelines.cs index 86fd826c..01c3b795 100644 --- a/Wabbajack.Common.CSP/PIpelines.cs +++ b/Wabbajack.Common.CSP/PIpelines.cs @@ -121,6 +121,59 @@ namespace Wabbajack.Common.CSP } + public static IReadPort UnorderedPipelineRx( + this IReadPort from, + Func, IObservable> f, + bool propagateClose = true) + { + var parallelism = Environment.ProcessorCount; + var to = Channel.Create(parallelism * 2); + var pipeline = from.UnorderedPipeline(parallelism, to, f); + return to; + + } + + public static IReadPort UnorderedPipelineSync( + this IReadPort from, + Func f, + bool propagateClose = true) + { + var parallelism = Environment.ProcessorCount; + var to = Channel.Create(parallelism * 2); + + async Task Pump() + { + while (true) + { + var (is_open, job) = await from.Take(); + if (!is_open) break; + try + { + var putIsOpen = await to.Put(f(job)); + if (!putIsOpen) return; + } + catch (Exception ex) + { + + } + } + } + + Task.Run(async () => + { + await Task.WhenAll(Enumerable.Range(0, parallelism) + .Select(idx => Task.Run(Pump))); + + if (propagateClose) + { + from.Close(); + to.Close(); + } + }); + + return to; + } + public static async Task UnorderedThreadedPipeline( this IReadPort from, int parallelism, diff --git a/Wabbajack.Common.CSP/RingBuffer.cs b/Wabbajack.Common.CSP/RingBuffer.cs index 24344d2c..5b59b522 100644 --- a/Wabbajack.Common.CSP/RingBuffer.cs +++ b/Wabbajack.Common.CSP/RingBuffer.cs @@ -1,6 +1,8 @@ using System; using System.Collections; using System.Collections.Generic; +using System.ComponentModel; +using System.IO; namespace Wabbajack.Common.CSP { @@ -23,7 +25,8 @@ namespace Wabbajack.Common.CSP public T Pop() { - if (_length == 0) return default; + if (_length == 0) + throw new InvalidDataException("Pop on empty buffer"); var val = _arr[_tail]; _arr[_tail] = default; _tail = (_tail + 1) % _size; @@ -45,7 +48,7 @@ namespace Wabbajack.Common.CSP public void UnboundedUnshift(T x) { - if (_length == _size) + if (_length + 1 == _size) Resize(); Unshift(x); } @@ -67,8 +70,8 @@ namespace Wabbajack.Common.CSP } else if (_tail > _head) { - Array.Copy(_arr, _tail, new_arr, 0, _length - _tail); - Array.Copy(_arr, 0, new_arr, (_length - _tail), _head); + Array.Copy(_arr, _tail, new_arr, 0, _size - _tail); + Array.Copy(_arr, 0, new_arr, (_size - _tail), _head); _tail = 0; _head = _length; _arr = new_arr; diff --git a/Wabbajack.Common/Enums/ModManager.cs b/Wabbajack.Common/Enums/ModManager.cs new file mode 100644 index 00000000..cd25ee9e --- /dev/null +++ b/Wabbajack.Common/Enums/ModManager.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Wabbajack.Common +{ + public enum ModManager + { + MO2, + Vortex + } +} diff --git a/Wabbajack/Enums/RunMode.cs b/Wabbajack.Common/Enums/RunMode.cs similarity index 100% rename from Wabbajack/Enums/RunMode.cs rename to Wabbajack.Common/Enums/RunMode.cs diff --git a/Wabbajack.Common/GOGHandler.cs b/Wabbajack.Common/GOGHandler.cs index 3012e625..72ae196b 100644 --- a/Wabbajack.Common/GOGHandler.cs +++ b/Wabbajack.Common/GOGHandler.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using Microsoft.Win32; diff --git a/Wabbajack.Common/GameMetaData.cs b/Wabbajack.Common/GameMetaData.cs index 34b3cf13..203d24a4 100644 --- a/Wabbajack.Common/GameMetaData.cs +++ b/Wabbajack.Common/GameMetaData.cs @@ -1,16 +1,12 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; -using System.Reflection; -using System.Security.Cryptography.X509Certificates; -using System.Text; -using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; using Microsoft.Win32; namespace Wabbajack.Common { public enum Game { + //MO2 GAMES Morrowind, Oblivion, Fallout3, @@ -18,16 +14,33 @@ namespace Wabbajack.Common Skyrim, SkyrimSpecialEdition, Fallout4, - SkyrimVR + SkyrimVR, + //VORTEX GAMES + DarkestDungeon, + DivinityOriginalSin2, + DivinityOriginalSin2DE, //definitive edition has its own nexus page but same Steam/GOG ids + Starbound, + SWKOTOR, + SWKOTOR2, + WITCHER, + WITCHER2, + WITCHER3 } public class GameMetaData { + public ModManager SupportedModManager { get; internal set; } public string MO2ArchiveName { get; internal set; } public Game Game { get; internal set; } public string NexusName { get; internal set; } public string MO2Name { get; internal set; } public string GameLocationRegistryKey { get; internal set; } + // to get steam ids: https://steamdb.info + public List SteamIDs { get; internal set; } + // to get gog ids: https://www.gogdb.org + public List GOGIDs { get; internal set; } + // these are additional folders when a game installs mods outside the game folder + public List AdditionalFolders { get; internal set; } public string GameLocation { @@ -53,6 +66,11 @@ namespace Wabbajack.Common return Games.Values.FirstOrDefault(g => g.MO2ArchiveName?.ToLower() == gamename); } + public static GameMetaData GetByNexusName(string gameName) + { + return Games.Values.FirstOrDefault(g => g.NexusName == gameName.ToLower()); + } + public static Dictionary Games = new Dictionary { @@ -62,72 +80,195 @@ namespace Wabbajack.Common { Game.Oblivion, new GameMetaData { + SupportedModManager = ModManager.MO2, Game = Game.Oblivion, NexusName = "oblivion", MO2Name = "Oblivion", MO2ArchiveName = "oblivion", - GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Oblivion" + GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Oblivion", + SteamIDs = new List {22330} } }, { Game.Fallout3, new GameMetaData { + SupportedModManager = ModManager.MO2, Game = Game.Fallout3, NexusName = "fallout3", MO2Name = "fallout3", MO2ArchiveName = "fallout3", - GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Fallout3" + GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Fallout3", + SteamIDs = new List {22300, 22370} // base game and GotY } }, { Game.FalloutNewVegas, new GameMetaData { + SupportedModManager = ModManager.MO2, Game = Game.FalloutNewVegas, NexusName = "newvegas", MO2Name = "New Vegas", MO2ArchiveName = "falloutnv", - GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\falloutnv" + GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\falloutnv", + SteamIDs = new List {22380} } }, { Game.Skyrim, new GameMetaData { + SupportedModManager = ModManager.MO2, Game = Game.Skyrim, NexusName = "skyrim", MO2Name = "Skyrim", MO2ArchiveName = "skyrim", - GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\skyrim" + GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\skyrim", + SteamIDs = new List {72850} } }, { Game.SkyrimSpecialEdition, new GameMetaData { + SupportedModManager = ModManager.MO2, Game = Game.SkyrimSpecialEdition, NexusName = "skyrimspecialedition", MO2Name = "Skyrim Special Edition", MO2ArchiveName = "skyrimse", - GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Skyrim Special Edition" + GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Skyrim Special Edition", + SteamIDs = new List {489830} } }, { Game.Fallout4, new GameMetaData { + SupportedModManager = ModManager.MO2, Game = Game.Fallout4, NexusName = "fallout4", MO2Name = "Fallout 4", MO2ArchiveName = "fallout4", - GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Fallout4" + GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Fallout4", + SteamIDs = new List {377160} } }, + /*{ + Game.Fallout4VR, new GameMetaData + { + SupportedModManager = ModManager.MO2, + Game = Game.Fallout4VR, + NexusName = "fallout4", + MO2Name = "Fallout 4", + MO2ArchiveName = "fallout4", + SteamIDs = new List{611660} + } + },*/ { Game.SkyrimVR, new GameMetaData { + SupportedModManager = ModManager.MO2, Game = Game.SkyrimVR, NexusName = "skyrimspecialedition", MO2Name = "Skyrim VR", MO2ArchiveName = "skyrimse", - GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Skyrim VR" + GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Skyrim VR", + SteamIDs = new List {611670} + } + }, + { + Game.DarkestDungeon, new GameMetaData + { + SupportedModManager = ModManager.Vortex, + Game = Game.DarkestDungeon, + NexusName = "darkestdungeon", + SteamIDs = new List {262060}, + GOGIDs = new List{1450711444} + } + }, + { + Game.DivinityOriginalSin2, new GameMetaData + { + SupportedModManager = ModManager.Vortex, + Game = Game.DivinityOriginalSin2, + NexusName = "divinityoriginalsin2", + SteamIDs = new List {435150}, + GOGIDs = new List{1584823040}, + AdditionalFolders = new List + { + "%documents%\\Larian Studios\\Divinity Original Sin 2\\Mods\\", + } + } + }, + { + Game.DivinityOriginalSin2DE, new GameMetaData + { + SupportedModManager = ModManager.Vortex, + Game = Game.DivinityOriginalSin2DE, + NexusName = "divinityoriginalsin2definitiveedition", + SteamIDs = new List {435150}, + GOGIDs = new List{1584823040}, + AdditionalFolders = new List + { + "%documents%\\Larian Studios\\Divinity Original Sin 2 Definitive Edition\\Mods\\" + } + } + }, + { + Game.Starbound, new GameMetaData + { + SupportedModManager = ModManager.Vortex, + Game = Game.Starbound, + NexusName = "starbound", + SteamIDs = new List{211820}, + GOGIDs = new List{1452598881} + } + }, + { + Game.SWKOTOR, new GameMetaData + { + SupportedModManager = ModManager.Vortex, + Game = Game.SWKOTOR, + NexusName = "kotor", + SteamIDs = new List{32370}, + GOGIDs = new List{1207666283} + } + }, + { + Game.SWKOTOR2, new GameMetaData + { + SupportedModManager = ModManager.Vortex, + Game = Game.SWKOTOR2, + NexusName = "kotor2", + SteamIDs = new List{208580}, + GOGIDs = new List{1421404581} + } + }, + { + Game.WITCHER, new GameMetaData + { + SupportedModManager = ModManager.Vortex, + Game = Game.WITCHER, + NexusName = "witcher", + SteamIDs = new List{20900}, + GOGIDs = new List{1207658924} + } + }, + { + Game.WITCHER2, new GameMetaData + { + SupportedModManager = ModManager.Vortex, + Game = Game.WITCHER2, + NexusName = "witcher2", + SteamIDs = new List{20920}, + GOGIDs = new List{1207658930} + } + }, + { + Game.WITCHER3, new GameMetaData + { + SupportedModManager = ModManager.Vortex, + Game = Game.WITCHER3, + NexusName = "witcher3", + SteamIDs = new List{292030, 499450}, // normal and GotY + GOGIDs = new List{1207664643, 1495134320, 1207664663, 1640424747} // normal, GotY and both in packages } } }; diff --git a/Wabbajack.Common/SteamHandler.cs b/Wabbajack.Common/SteamHandler.cs index cf542ca9..541ba286 100644 --- a/Wabbajack.Common/SteamHandler.cs +++ b/Wabbajack.Common/SteamHandler.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index ed87903d..183230fd 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -125,6 +125,28 @@ namespace Wabbajack.Common } } + public static async Task FileHashAsync(this string file, bool nullOnIOError = false) + { + try + { + var hash = new xxHashConfig(); + hash.HashSizeInBits = 64; + hash.Seed = 0x42; + using (var fs = File.OpenRead(file)) + { + var config = new xxHashConfig(); + config.HashSizeInBits = 64; + var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(fs); + return value.AsBase64String(); + } + } + catch (IOException ex) + { + if (nullOnIOError) return null; + throw ex; + } + } + public static void CopyToWithStatus(this Stream istream, long maxSize, Stream ostream, string status) { var buffer = new byte[1024 * 64]; diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 213dc82b..4469b164 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -91,6 +91,8 @@ + + diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs new file mode 100644 index 00000000..9c9c8561 --- /dev/null +++ b/Wabbajack.Lib/ACompiler.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using VFS; +using Wabbajack.Common; +using Wabbajack.Lib.CompilationSteps; + +namespace Wabbajack.Lib +{ + public abstract class ACompiler + { + public ModManager ModManager; + + public string GamePath; + + public string ModListOutputFolder; + public string ModListOutputFile; + + public List SelectedArchives; + public List InstallDirectives; + public List AllFiles; + public ModList ModList; + public VirtualFileSystem VFS; + public List IndexedArchives; + public Dictionary> IndexedFiles; + + public abstract void Info(string msg); + public abstract void Status(string msg); + public abstract void Error(string msg); + + internal abstract string IncludeFile(byte[] data); + internal abstract string IncludeFile(string data); + + public abstract bool Compile(); + + public abstract Directive RunStack(IEnumerable stack, RawSourceFile source); + public abstract IEnumerable GetStack(); + public abstract IEnumerable MakeStack(); + } +} diff --git a/Wabbajack.Lib/CompilationSteps/ACompilationStep.cs b/Wabbajack.Lib/CompilationSteps/ACompilationStep.cs index a27e41ed..257e41fb 100644 --- a/Wabbajack.Lib/CompilationSteps/ACompilationStep.cs +++ b/Wabbajack.Lib/CompilationSteps/ACompilationStep.cs @@ -2,9 +2,9 @@ { public abstract class ACompilationStep : ICompilationStep { - protected Compiler _compiler; + protected ACompiler _compiler; - public ACompilationStep(Compiler compiler) + public ACompilationStep(ACompiler compiler) { _compiler = compiler; } diff --git a/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs b/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs index bc8a8e03..8ff21b10 100644 --- a/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs +++ b/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs @@ -13,10 +13,12 @@ namespace Wabbajack.Lib.CompilationSteps private readonly IEnumerable _include_directly; private readonly List _microstack; private readonly List _microstackWithInclude; + private readonly Compiler _mo2Compiler; - public DeconstructBSAs(Compiler compiler) : base(compiler) + public DeconstructBSAs(ACompiler compiler) : base(compiler) { - _include_directly = _compiler.ModInis.Where(kv => + _mo2Compiler = (Compiler) compiler; + _include_directly = _mo2Compiler.ModInis.Where(kv => { var general = kv.Value.General; if (general.notes != null && general.notes.Contains(Consts.WABBAJACK_INCLUDE)) return true; @@ -28,16 +30,16 @@ namespace Wabbajack.Lib.CompilationSteps _microstack = new List { - new DirectMatch(_compiler), - new IncludePatches(_compiler), - new DropAll(_compiler) + new DirectMatch(_mo2Compiler), + new IncludePatches(_mo2Compiler), + new DropAll(_mo2Compiler) }; _microstackWithInclude = new List { - new DirectMatch(_compiler), - new IncludePatches(_compiler), - new IncludeAll(_compiler) + new DirectMatch(_mo2Compiler), + new IncludePatches(_mo2Compiler), + new IncludeAll(_mo2Compiler) }; } @@ -61,7 +63,7 @@ namespace Wabbajack.Lib.CompilationSteps var id = Guid.NewGuid().ToString(); - var matches = source_files.PMap(e => Compiler.RunStack(stack, new RawSourceFile(e) + var matches = source_files.PMap(e => _mo2Compiler.RunStack(stack, new RawSourceFile(e) { Path = Path.Combine(Consts.BSACreationDir, id, e.Paths.Last()) })); @@ -71,7 +73,7 @@ namespace Wabbajack.Lib.CompilationSteps { if (match is IgnoredDirectly) Utils.Error($"File required for BSA {source.Path} creation doesn't exist: {match.To}"); - _compiler.ExtraFiles.Add(match); + _mo2Compiler.ExtraFiles.Add(match); } CreateBSA directive; @@ -92,7 +94,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("DeconstructBSAs")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new DeconstructBSAs(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/DirectMatch.cs b/Wabbajack.Lib/CompilationSteps/DirectMatch.cs index 0b840082..289625f6 100644 --- a/Wabbajack.Lib/CompilationSteps/DirectMatch.cs +++ b/Wabbajack.Lib/CompilationSteps/DirectMatch.cs @@ -6,7 +6,7 @@ namespace Wabbajack.Lib.CompilationSteps { public class DirectMatch : ACompilationStep { - public DirectMatch(Compiler compiler) : base(compiler) + public DirectMatch(ACompiler compiler) : base(compiler) { } @@ -34,7 +34,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("DirectMatch")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new DirectMatch(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/DropAll.cs b/Wabbajack.Lib/CompilationSteps/DropAll.cs index d818be90..33ed357f 100644 --- a/Wabbajack.Lib/CompilationSteps/DropAll.cs +++ b/Wabbajack.Lib/CompilationSteps/DropAll.cs @@ -5,7 +5,7 @@ namespace Wabbajack.Lib.CompilationSteps { public class DropAll : ACompilationStep { - public DropAll(Compiler compiler) : base(compiler) + public DropAll(ACompiler compiler) : base(compiler) { } @@ -25,7 +25,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("DropAll")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new DropAll(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/IStackStep.cs b/Wabbajack.Lib/CompilationSteps/IStackStep.cs index 9ea53d4c..107bab97 100644 --- a/Wabbajack.Lib/CompilationSteps/IStackStep.cs +++ b/Wabbajack.Lib/CompilationSteps/IStackStep.cs @@ -8,6 +8,6 @@ public interface IState { - ICompilationStep CreateStep(Compiler compiler); + ICompilationStep CreateStep(ACompiler compiler); } } \ No newline at end of file diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreDisabledMods.cs b/Wabbajack.Lib/CompilationSteps/IgnoreDisabledMods.cs index bea34d6d..4c01a8ed 100644 --- a/Wabbajack.Lib/CompilationSteps/IgnoreDisabledMods.cs +++ b/Wabbajack.Lib/CompilationSteps/IgnoreDisabledMods.cs @@ -9,13 +9,15 @@ namespace Wabbajack.Lib.CompilationSteps public class IgnoreDisabledMods : ACompilationStep { private readonly IEnumerable _allEnabledMods; + private readonly Compiler _mo2Compiler; - public IgnoreDisabledMods(Compiler compiler) : base(compiler) + public IgnoreDisabledMods(ACompiler compiler) : base(compiler) { - var alwaysEnabled = _compiler.ModInis.Where(f => IsAlwaysEnabled(f.Value)).Select(f => f.Key).ToHashSet(); + _mo2Compiler = (Compiler) compiler; + var alwaysEnabled = _mo2Compiler.ModInis.Where(f => IsAlwaysEnabled(f.Value)).Select(f => f.Key).ToHashSet(); - _allEnabledMods = _compiler.SelectedProfiles - .SelectMany(p => File.ReadAllLines(Path.Combine(_compiler.MO2Folder, "profiles", p, "modlist.txt"))) + _allEnabledMods = _mo2Compiler.SelectedProfiles + .SelectMany(p => File.ReadAllLines(Path.Combine(_mo2Compiler.MO2Folder, "profiles", p, "modlist.txt"))) .Where(line => line.StartsWith("+") || line.EndsWith("_separator")) .Select(line => line.Substring(1)) .Concat(alwaysEnabled) @@ -55,7 +57,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("IgnoreDisabledMods")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IgnoreDisabledMods(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreEndsWith.cs b/Wabbajack.Lib/CompilationSteps/IgnoreEndsWith.cs index 501c4b8c..966d019d 100644 --- a/Wabbajack.Lib/CompilationSteps/IgnoreEndsWith.cs +++ b/Wabbajack.Lib/CompilationSteps/IgnoreEndsWith.cs @@ -7,7 +7,7 @@ namespace Wabbajack.Lib.CompilationSteps private readonly string _postfix; private readonly string _reason; - public IgnoreEndsWith(Compiler compiler, string postfix) : base(compiler) + public IgnoreEndsWith(ACompiler compiler, string postfix) : base(compiler) { _postfix = postfix; _reason = $"Ignored because path ends with {postfix}"; @@ -40,7 +40,7 @@ namespace Wabbajack.Lib.CompilationSteps public string Postfix { get; set; } - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IgnoreEndsWith(compiler, Postfix); } diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs b/Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs index 829403fe..26e25db6 100644 --- a/Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs +++ b/Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs @@ -7,7 +7,7 @@ namespace Wabbajack.Lib.CompilationSteps { private readonly string _startDir; - public IgnoreGameFiles(Compiler compiler) : base(compiler) + public IgnoreGameFiles(ACompiler compiler) : base(compiler) { _startDir = Consts.GameFolderFilesDir + "\\"; } @@ -28,7 +28,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("IgnoreGameFiles")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IgnoreGameFiles(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/IgnorePathContains.cs b/Wabbajack.Lib/CompilationSteps/IgnorePathContains.cs index ac34717a..9b37faca 100644 --- a/Wabbajack.Lib/CompilationSteps/IgnorePathContains.cs +++ b/Wabbajack.Lib/CompilationSteps/IgnorePathContains.cs @@ -7,7 +7,7 @@ namespace Wabbajack.Lib.CompilationSteps private readonly string _pattern; private readonly string _reason; - public IgnorePathContains(Compiler compiler, string pattern) : base(compiler) + public IgnorePathContains(ACompiler compiler, string pattern) : base(compiler) { _pattern = $"\\{pattern.Trim('\\')}\\"; _reason = $"Ignored because path contains {_pattern}"; @@ -40,7 +40,7 @@ namespace Wabbajack.Lib.CompilationSteps public string Pattern { get; set; } - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IgnorePathContains(compiler, Pattern); } diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreRegex.cs b/Wabbajack.Lib/CompilationSteps/IgnoreRegex.cs index 04d76386..8c4bf0f0 100644 --- a/Wabbajack.Lib/CompilationSteps/IgnoreRegex.cs +++ b/Wabbajack.Lib/CompilationSteps/IgnoreRegex.cs @@ -9,7 +9,7 @@ namespace Wabbajack.Lib.CompilationSteps private readonly Regex _regex; private readonly string _pattern; - public IgnoreRegex(Compiler compiler, string pattern) : base(compiler) + public IgnoreRegex(ACompiler compiler, string pattern) : base(compiler) { _pattern = pattern; _reason = $"Ignored because path matches regex {pattern}"; @@ -43,7 +43,7 @@ namespace Wabbajack.Lib.CompilationSteps public string Pattern { get; set; } - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IgnoreRegex(compiler, Pattern); } diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreStartsWith.cs b/Wabbajack.Lib/CompilationSteps/IgnoreStartsWith.cs index a551627f..61070bc1 100644 --- a/Wabbajack.Lib/CompilationSteps/IgnoreStartsWith.cs +++ b/Wabbajack.Lib/CompilationSteps/IgnoreStartsWith.cs @@ -7,7 +7,7 @@ namespace Wabbajack.Lib.CompilationSteps private readonly string _prefix; private readonly string _reason; - public IgnoreStartsWith(Compiler compiler, string prefix) : base(compiler) + public IgnoreStartsWith(ACompiler compiler, string prefix) : base(compiler) { _prefix = prefix; _reason = string.Format("Ignored because path starts with {0}", _prefix); @@ -44,7 +44,7 @@ namespace Wabbajack.Lib.CompilationSteps public string Prefix { get; set; } - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IgnoreStartsWith(compiler, Prefix); } diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreVortex.cs b/Wabbajack.Lib/CompilationSteps/IgnoreVortex.cs new file mode 100644 index 00000000..b7e01122 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IgnoreVortex.cs @@ -0,0 +1,40 @@ +using System; +using System.IO; +using Newtonsoft.Json; +using Wabbajack.Common; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IgnoreVortex : ACompilationStep + { + private readonly VortexCompiler _vortex; + + public IgnoreVortex(ACompiler compiler) : base(compiler) + { + _vortex = (VortexCompiler) compiler; + } + + public override Directive Run(RawSourceFile source) + { + if (Path.GetDirectoryName(source.AbsolutePath) != _vortex.DownloadsFolder) return null; + var result = source.EvolveTo(); + result.Reason = "Ignored because it is a Vortex file"; + return result; + + } + + public override IState GetState() + { + return new State(); + } + + [JsonObject("IgnoreVortex")] + public class State : IState + { + public ICompilationStep CreateStep(ACompiler compiler) + { + return new IgnoreVortex(compiler); + } + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreWabbajackInstallCruft.cs b/Wabbajack.Lib/CompilationSteps/IgnoreWabbajackInstallCruft.cs index 73cb13dc..d772f179 100644 --- a/Wabbajack.Lib/CompilationSteps/IgnoreWabbajackInstallCruft.cs +++ b/Wabbajack.Lib/CompilationSteps/IgnoreWabbajackInstallCruft.cs @@ -9,7 +9,7 @@ namespace Wabbajack.Lib.CompilationSteps { private readonly HashSet _cruftFiles; - public IgnoreWabbajackInstallCruft(Compiler compiler) : base(compiler) + public IgnoreWabbajackInstallCruft(ACompiler compiler) : base(compiler) { _cruftFiles = new HashSet { @@ -34,7 +34,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("IgnoreWabbajackInstallCruft")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IgnoreWabbajackInstallCruft(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/IncludeAll.cs b/Wabbajack.Lib/CompilationSteps/IncludeAll.cs index 8903d448..980ad796 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludeAll.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludeAll.cs @@ -5,7 +5,7 @@ namespace Wabbajack.Lib.CompilationSteps { public class IncludeAll : ACompilationStep { - public IncludeAll(Compiler compiler) : base(compiler) + public IncludeAll(ACompiler compiler) : base(compiler) { } @@ -24,7 +24,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("IncludeAll")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IncludeAll(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/IncludeAllConfigs.cs b/Wabbajack.Lib/CompilationSteps/IncludeAllConfigs.cs index 1f669666..41fe7adb 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludeAllConfigs.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludeAllConfigs.cs @@ -6,7 +6,7 @@ namespace Wabbajack.Lib.CompilationSteps { public class IncludeAllConfigs : ACompilationStep { - public IncludeAllConfigs(Compiler compiler) : base(compiler) + public IncludeAllConfigs(ACompiler compiler) : base(compiler) { } @@ -26,7 +26,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("IncludeAllConfigs")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IncludeAllConfigs(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/IncludeDummyESPs.cs b/Wabbajack.Lib/CompilationSteps/IncludeDummyESPs.cs index 63b0c1e8..d8fa4d0c 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludeDummyESPs.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludeDummyESPs.cs @@ -5,7 +5,7 @@ namespace Wabbajack.Lib.CompilationSteps { public class IncludeDummyESPs : ACompilationStep { - public IncludeDummyESPs(Compiler compiler) : base(compiler) + public IncludeDummyESPs(ACompiler compiler) : base(compiler) { } @@ -36,7 +36,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("IncludeDummyESPs")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IncludeDummyESPs(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/IncludeLOOTFiles.cs b/Wabbajack.Lib/CompilationSteps/IncludeLOOTFiles.cs index 567af7f1..342ed413 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludeLOOTFiles.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludeLOOTFiles.cs @@ -8,7 +8,7 @@ namespace Wabbajack.Lib.CompilationSteps { private readonly string _prefix; - public IncludeLootFiles(Compiler compiler) : base(compiler) + public IncludeLootFiles(ACompiler compiler) : base(compiler) { _prefix = Consts.LOOTFolderFilesDir + "\\"; } @@ -29,7 +29,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("IncludeLootFiles")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IncludeLootFiles(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/IncludeModIniData.cs b/Wabbajack.Lib/CompilationSteps/IncludeModIniData.cs index 101205ed..c9dbc4b3 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludeModIniData.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludeModIniData.cs @@ -5,7 +5,7 @@ namespace Wabbajack.Lib.CompilationSteps { public class IncludeModIniData : ACompilationStep { - public IncludeModIniData(Compiler compiler) : base(compiler) + public IncludeModIniData(ACompiler compiler) : base(compiler) { } @@ -25,7 +25,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("IncludeModIniData")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IncludeModIniData(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/IncludeOtherProfiles.cs b/Wabbajack.Lib/CompilationSteps/IncludeOtherProfiles.cs index 82442db7..096f87b9 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludeOtherProfiles.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludeOtherProfiles.cs @@ -8,10 +8,13 @@ namespace Wabbajack.Lib.CompilationSteps public class IgnoreOtherProfiles : ACompilationStep { private readonly IEnumerable _profiles; + private readonly Compiler _mo2Compiler; - public IgnoreOtherProfiles(Compiler compiler) : base(compiler) + public IgnoreOtherProfiles(ACompiler compiler) : base(compiler) { - _profiles = _compiler.SelectedProfiles + _mo2Compiler = (Compiler) compiler; + + _profiles = _mo2Compiler.SelectedProfiles .Select(p => Path.Combine("profiles", p) + "\\") .ToList(); } @@ -33,7 +36,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("IgnoreOtherProfiles")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IgnoreOtherProfiles(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/IncludePatches.cs b/Wabbajack.Lib/CompilationSteps/IncludePatches.cs index 71c4fd44..a4a6e9c4 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludePatches.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludePatches.cs @@ -11,7 +11,7 @@ namespace Wabbajack.Lib.CompilationSteps { private readonly Dictionary> _indexed; - public IncludePatches(Compiler compiler) : base(compiler) + public IncludePatches(ACompiler compiler) : base(compiler) { _indexed = _compiler.IndexedFiles.Values .SelectMany(f => f) @@ -47,7 +47,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("IncludePatches")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IncludePatches(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs b/Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs index dfa7f607..fa6f3d78 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs @@ -7,31 +7,34 @@ namespace Wabbajack.Lib.CompilationSteps { public class IncludePropertyFiles : ACompilationStep { - public IncludePropertyFiles(Compiler compiler) : base(compiler) + private readonly Compiler _mo2Compiler; + + public IncludePropertyFiles(ACompiler compiler) : base(compiler) { + _mo2Compiler = (Compiler) compiler; } public override Directive Run(RawSourceFile source) { var files = new HashSet { - _compiler.ModListImage, _compiler.ModListReadme + _mo2Compiler.ModListImage, _mo2Compiler.ModListReadme }; if (!files.Any(f => source.AbsolutePath.Equals(f))) return null; if (!File.Exists(source.AbsolutePath)) return null; - var isBanner = source.AbsolutePath == _compiler.ModListImage; + var isBanner = source.AbsolutePath == _mo2Compiler.ModListImage; //var isReadme = source.AbsolutePath == ModListReadme; var result = source.EvolveTo(); result.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath)); if (isBanner) { result.Type = PropertyType.Banner; - _compiler.ModListImage = result.SourceDataID; + _mo2Compiler.ModListImage = result.SourceDataID; } else { result.Type = PropertyType.Readme; - _compiler.ModListReadme = result.SourceDataID; + _mo2Compiler.ModListReadme = result.SourceDataID; } return result; @@ -45,7 +48,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("IncludePropertyFiles")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IncludePropertyFiles(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/IncludeRegex.cs b/Wabbajack.Lib/CompilationSteps/IncludeRegex.cs index e53c4875..7f806981 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludeRegex.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludeRegex.cs @@ -9,7 +9,7 @@ namespace Wabbajack.Lib.CompilationSteps private readonly string _pattern; private readonly Regex _regex; - public IncludeRegex(Compiler compiler, string pattern) : base(compiler) + public IncludeRegex(ACompiler compiler, string pattern) : base(compiler) { _pattern = pattern; _regex = new Regex(pattern); @@ -43,7 +43,7 @@ namespace Wabbajack.Lib.CompilationSteps public string Pattern { get; set; } - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IncludeRegex(compiler, Pattern); } diff --git a/Wabbajack.Lib/CompilationSteps/IncludeStubbedConfigfiles.cs b/Wabbajack.Lib/CompilationSteps/IncludeStubbedConfigfiles.cs index a5cede88..75ccee46 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludeStubbedConfigfiles.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludeStubbedConfigfiles.cs @@ -7,8 +7,11 @@ namespace Wabbajack.Lib.CompilationSteps { public class IncludeStubbedConfigFiles : ACompilationStep { - public IncludeStubbedConfigFiles(Compiler compiler) : base(compiler) + private readonly Compiler _mo2Compiler; + + public IncludeStubbedConfigFiles(ACompiler compiler) : base(compiler) { + _mo2Compiler = (Compiler) compiler; } public override Directive Run(RawSourceFile source) @@ -26,18 +29,18 @@ namespace Wabbajack.Lib.CompilationSteps var data = File.ReadAllText(source.AbsolutePath); var originalData = data; - data = data.Replace(_compiler.GamePath, Consts.GAME_PATH_MAGIC_BACK); - data = data.Replace(_compiler.GamePath.Replace("\\", "\\\\"), Consts.GAME_PATH_MAGIC_DOUBLE_BACK); - data = data.Replace(_compiler.GamePath.Replace("\\", "/"), Consts.GAME_PATH_MAGIC_FORWARD); + data = data.Replace(_mo2Compiler.GamePath, Consts.GAME_PATH_MAGIC_BACK); + data = data.Replace(_mo2Compiler.GamePath.Replace("\\", "\\\\"), Consts.GAME_PATH_MAGIC_DOUBLE_BACK); + data = data.Replace(_mo2Compiler.GamePath.Replace("\\", "/"), Consts.GAME_PATH_MAGIC_FORWARD); - data = data.Replace(_compiler.MO2Folder, Consts.MO2_PATH_MAGIC_BACK); - data = data.Replace(_compiler.MO2Folder.Replace("\\", "\\\\"), Consts.MO2_PATH_MAGIC_DOUBLE_BACK); - data = data.Replace(_compiler.MO2Folder.Replace("\\", "/"), Consts.MO2_PATH_MAGIC_FORWARD); + data = data.Replace(_mo2Compiler.MO2Folder, Consts.MO2_PATH_MAGIC_BACK); + data = data.Replace(_mo2Compiler.MO2Folder.Replace("\\", "\\\\"), Consts.MO2_PATH_MAGIC_DOUBLE_BACK); + data = data.Replace(_mo2Compiler.MO2Folder.Replace("\\", "/"), Consts.MO2_PATH_MAGIC_FORWARD); - data = data.Replace(_compiler.MO2DownloadsFolder, Consts.DOWNLOAD_PATH_MAGIC_BACK); - data = data.Replace(_compiler.MO2DownloadsFolder.Replace("\\", "\\\\"), + data = data.Replace(_mo2Compiler.MO2DownloadsFolder, Consts.DOWNLOAD_PATH_MAGIC_BACK); + data = data.Replace(_mo2Compiler.MO2DownloadsFolder.Replace("\\", "\\\\"), Consts.DOWNLOAD_PATH_MAGIC_DOUBLE_BACK); - data = data.Replace(_compiler.MO2DownloadsFolder.Replace("\\", "/"), Consts.DOWNLOAD_PATH_MAGIC_FORWARD); + data = data.Replace(_mo2Compiler.MO2DownloadsFolder.Replace("\\", "/"), Consts.DOWNLOAD_PATH_MAGIC_FORWARD); if (data == originalData) return null; @@ -49,7 +52,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("IncludeStubbedConfigFiles")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IncludeStubbedConfigFiles(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs b/Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs index ea1a8cd6..76611545 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs @@ -9,12 +9,13 @@ namespace Wabbajack.Lib.CompilationSteps { private readonly IEnumerable _includeDirectly; private readonly string _tag; + private readonly Compiler _mo2Compiler; - - public IncludeTaggedMods(Compiler compiler, string tag) : base(compiler) + public IncludeTaggedMods(ACompiler compiler, string tag) : base(compiler) { + _mo2Compiler = (Compiler) compiler; _tag = tag; - _includeDirectly = _compiler.ModInis.Where(kv => + _includeDirectly = _mo2Compiler.ModInis.Where(kv => { var general = kv.Value.General; if (general.notes != null && general.notes.Contains(_tag)) @@ -56,7 +57,7 @@ namespace Wabbajack.Lib.CompilationSteps public string Tag { get; set; } - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IncludeTaggedMods(compiler, Tag); } diff --git a/Wabbajack.Lib/CompilationSteps/IncludeThisProfile.cs b/Wabbajack.Lib/CompilationSteps/IncludeThisProfile.cs index d1c97e39..858ca04b 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludeThisProfile.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludeThisProfile.cs @@ -9,10 +9,12 @@ namespace Wabbajack.Lib.CompilationSteps public class IncludeThisProfile : ACompilationStep { private readonly IEnumerable _correctProfiles; + private readonly Compiler _mo2Compiler; - public IncludeThisProfile(Compiler compiler) : base(compiler) + public IncludeThisProfile(ACompiler compiler) : base(compiler) { - _correctProfiles = _compiler.SelectedProfiles.Select(p => Path.Combine("profiles", p) + "\\").ToList(); + _mo2Compiler = (Compiler) compiler; + _correctProfiles = _mo2Compiler.SelectedProfiles.Select(p => Path.Combine("profiles", p) + "\\").ToList(); } public override Directive Run(RawSourceFile source) @@ -48,7 +50,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("IncludeThisProfile")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IncludeThisProfile(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/IncludeVortexDeployment.cs b/Wabbajack.Lib/CompilationSteps/IncludeVortexDeployment.cs new file mode 100644 index 00000000..774acc50 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IncludeVortexDeployment.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IncludeVortexDeployment : ACompilationStep + { + public IncludeVortexDeployment(ACompiler compiler) : base(compiler) + { + } + + public override Directive Run(RawSourceFile source) + { + if (!source.Path.EndsWith("vortex.deployment.msgpack") && + !source.Path.EndsWith("\\vortex.deployment.json")) return null; + var inline = source.EvolveTo(); + inline.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath)); + return inline; + } + + public override IState GetState() + { + return new State(); + } + + public class State : IState + { + public ICompilationStep CreateStep(ACompiler compiler) + { + return new IncludeVortexDeployment(compiler); + } + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/PatchStockESMs.cs b/Wabbajack.Lib/CompilationSteps/PatchStockESMs.cs index 521ff6d4..804bc56e 100644 --- a/Wabbajack.Lib/CompilationSteps/PatchStockESMs.cs +++ b/Wabbajack.Lib/CompilationSteps/PatchStockESMs.cs @@ -8,14 +8,17 @@ namespace Wabbajack.Lib.CompilationSteps { public class PatchStockESMs : ACompilationStep { - public PatchStockESMs(Compiler compiler) : base(compiler) + private readonly Compiler _mo2Compiler; + + public PatchStockESMs(ACompiler compiler) : base(compiler) { + _mo2Compiler = (Compiler) compiler; } public override Directive Run(RawSourceFile source) { var filename = Path.GetFileName(source.Path); - var gameFile = Path.Combine(_compiler.GamePath, "Data", filename); + var gameFile = Path.Combine(_mo2Compiler.GamePath, "Data", filename); if (!Consts.GameESMs.Contains(filename) || !source.Path.StartsWith("mods\\") || !File.Exists(gameFile)) return null; @@ -44,7 +47,7 @@ namespace Wabbajack.Lib.CompilationSteps [JsonObject("PatchStockESMs")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new PatchStockESMs(compiler); } diff --git a/Wabbajack.Lib/CompilationSteps/Serialization.cs b/Wabbajack.Lib/CompilationSteps/Serialization.cs index e601cf5d..2aff7668 100644 --- a/Wabbajack.Lib/CompilationSteps/Serialization.cs +++ b/Wabbajack.Lib/CompilationSteps/Serialization.cs @@ -13,7 +13,7 @@ namespace Wabbajack.Lib.CompilationSteps .ToJSON(TypeNameHandling.Auto, TypeNameAssemblyFormatHandling.Simple); } - public static List Deserialize(string stack, Compiler compiler) + public static List Deserialize(string stack, ACompiler compiler) { return stack.FromJSONString>(TypeNameHandling.Auto, TypeNameAssemblyFormatHandling.Simple) .Select(s => s.CreateStep(compiler)).ToList(); diff --git a/Wabbajack.Lib/Compiler.cs b/Wabbajack.Lib/Compiler.cs index bf343416..3f395c63 100644 --- a/Wabbajack.Lib/Compiler.cs +++ b/Wabbajack.Lib/Compiler.cs @@ -26,7 +26,7 @@ using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib { - public class Compiler + public class Compiler : ACompiler { private string _mo2DownloadsFolder; @@ -42,9 +42,23 @@ namespace Wabbajack.Lib public Compiler(string mo2_folder) { + ModManager = ModManager.MO2; + MO2Folder = mo2_folder; MO2Ini = Path.Combine(MO2Folder, "ModOrganizer.ini").LoadIniFile(); GamePath = ((string)MO2Ini.General.gamePath).Replace("\\\\", "\\"); + + ModListOutputFolder = "output_folder"; + ModListOutputFile = MO2Profile + ExtensionManager.Extension; + + SelectedArchives = new List(); + InstallDirectives = new List(); + AllFiles = new List(); + ModList = new ModList(); + + VFS = VirtualFileSystem.VFS; + IndexedArchives = new List(); + IndexedFiles = new Dictionary>(); } public dynamic MO2Ini { get; } @@ -70,55 +84,43 @@ namespace Wabbajack.Lib public string MO2ProfileDir => Path.Combine(MO2Folder, "profiles", MO2Profile); - public string ModListOutputFolder => "output_folder"; - public string ModListOutputFile => MO2Profile + ExtensionManager.Extension; - - public List InstallDirectives { get; private set; } internal UserStatus User { get; private set; } - public List SelectedArchives { get; private set; } - public List AllFiles { get; private set; } - public ModList ModList { get; private set; } public ConcurrentBag ExtraFiles { get; private set; } public Dictionary ModInis { get; private set; } - public VirtualFileSystem VFS => VirtualFileSystem.VFS; - - public List IndexedArchives { get; private set; } - public Dictionary> IndexedFiles { get; private set; } - public HashSet SelectedProfiles { get; set; } = new HashSet(); - public void Info(string msg) + public override void Info(string msg) { Utils.Log(msg); } - public void Status(string msg) + public override void Status(string msg) { WorkQueue.Report(msg, 0); } - private void Error(string msg) + public override void Error(string msg) { Utils.Log(msg); throw new Exception(msg); } - internal string IncludeFile(byte[] data) + internal override string IncludeFile(byte[] data) { var id = Guid.NewGuid().ToString(); File.WriteAllBytes(Path.Combine(ModListOutputFolder, id), data); return id; } - internal string IncludeFile(string data) + internal override string IncludeFile(string data) { var id = Guid.NewGuid().ToString(); File.WriteAllText(Path.Combine(ModListOutputFolder, id), data); return id; } - public bool Compile() + public override bool Compile() { VirtualFileSystem.Clean(); Info("Looking for other profiles"); @@ -283,6 +285,7 @@ namespace Wabbajack.Lib GameType = GameRegistry.Games.Values.First(f => f.MO2Name == MO2Ini.General.gameName).Game, WabbajackVersion = WabbajackVersion, Archives = SelectedArchives, + ModManager = ModManager.MO2, Directives = InstallDirectives, Name = ModListName ?? MO2Profile, Author = ModListAuthor ?? "", @@ -533,7 +536,7 @@ namespace Wabbajack.Lib } - public static Directive RunStack(IEnumerable stack, RawSourceFile source) + public override Directive RunStack(IEnumerable stack, RawSourceFile source) { Utils.Status($"Compiling {source.Path}"); foreach (var step in stack) @@ -545,7 +548,7 @@ namespace Wabbajack.Lib throw new InvalidDataException("Data fell out of the compilation stack"); } - public IEnumerable GetStack() + public override IEnumerable GetStack() { var user_config = Path.Combine(MO2ProfileDir, "compilation_stack.yml"); if (File.Exists(user_config)) @@ -566,7 +569,7 @@ namespace Wabbajack.Lib /// result included into the pack /// /// - public IEnumerable MakeStack() + public override IEnumerable MakeStack() { Utils.Log("Generating compilation stack"); return new List diff --git a/Wabbajack.Lib/Data.cs b/Wabbajack.Lib/Data.cs index 7208c4e1..7366ba19 100644 --- a/Wabbajack.Lib/Data.cs +++ b/Wabbajack.Lib/Data.cs @@ -40,6 +40,11 @@ namespace Wabbajack.Lib /// public List Archives; + /// + /// The Mod Manager used to create the modlist + /// + public ModManager ModManager; + /// /// The game variant to which this game applies /// @@ -204,6 +209,7 @@ namespace Wabbajack.Lib /// public string Hash; + /// /// Meta INI for the downloaded archive /// public string Meta; diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 8c8d8d9b..4f66d984 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -19,7 +19,8 @@ namespace Wabbajack.Lib.Downloaders if (general.modID != null && general.fileID != null && general.gameName != null) { var name = (string)general.gameName; - var game = GameRegistry.GetByMO2ArchiveName(name).Game; + var gameMeta = GameRegistry.GetByMO2ArchiveName(name); + var game = gameMeta != null ? GameRegistry.GetByMO2ArchiveName(name).Game : GameRegistry.GetByNexusName(name).Game; var info = new NexusApiClient().GetModInfo(game, general.modID); return new State { @@ -52,14 +53,8 @@ namespace Wabbajack.Lib.Downloaders return; } - if (!status.is_premium) - { - Utils.Error($"Automated installs with Wabbajack requires a premium nexus account. {client.Username} is not a premium account."); - return; - } - - client.ClearUpdatedModsInCache(); - //var updated = client.GetModsUpdatedSince(Game.Skyrim,DateTime.Now - TimeSpan.FromDays(30)); + if (status.is_premium) return; + Utils.Error($"Automated installs with Wabbajack requires a premium nexus account. {client.Username} is not a premium account."); } public class State : AbstractDownloadState diff --git a/Wabbajack.Lib/NexusApi/Dtos.cs b/Wabbajack.Lib/NexusApi/Dtos.cs index d1f66a82..bd948390 100644 --- a/Wabbajack.Lib/NexusApi/Dtos.cs +++ b/Wabbajack.Lib/NexusApi/Dtos.cs @@ -46,6 +46,12 @@ namespace Wabbajack.Lib.NexusApi public bool contains_adult_content; } + public class MD5Response + { + public ModInfo mod; + public NexusFileInfo file_details; + } + public class EndorsementResponse { public string message; diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index b5e7967e..258e785e 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -222,7 +222,6 @@ namespace Wabbajack.Lib.NexusApi } - public string GetNexusDownloadLink(NexusDownloader.State archive, bool cache = false) { if (cache && TryGetCachedLink(archive, out var result)) @@ -269,6 +268,12 @@ namespace Wabbajack.Lib.NexusApi return GetCached(url).files; } + public List GetModInfoFromMD5(Game game, string md5Hash) + { + var url = $"https://api.nexusmods.com/v1/games/{GameRegistry.Games[game].NexusName}/mods/md5_search/{md5Hash}.json"; + return Get>(url); + } + public ModInfo GetModInfo(Game game, string modId) { var url = $"https://api.nexusmods.com/v1/games/{GameRegistry.Games[game].NexusName}/mods/{modId}.json"; @@ -365,4 +370,4 @@ namespace Wabbajack.Lib.NexusApi } } -} \ No newline at end of file +} diff --git a/Wabbajack.Lib/NexusApi/NexusApiUtils.cs b/Wabbajack.Lib/NexusApi/NexusApiUtils.cs index 3446f39c..29915564 100644 --- a/Wabbajack.Lib/NexusApi/NexusApiUtils.cs +++ b/Wabbajack.Lib/NexusApi/NexusApiUtils.cs @@ -1,4 +1,5 @@ -using Wabbajack.Common; +using System.Text.RegularExpressions; +using Wabbajack.Common; namespace Wabbajack.Lib.NexusApi { @@ -6,6 +7,8 @@ namespace Wabbajack.Lib.NexusApi { public static string ConvertGameName(string gameName) { + if (Regex.IsMatch(gameName, @"^[^a-z\s]+\.[^a-z\s]+$")) + return gameName; return GameRegistry.GetByMO2ArchiveName(gameName)?.NexusName ?? gameName.ToLower(); } diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs new file mode 100644 index 00000000..d650c3c2 --- /dev/null +++ b/Wabbajack.Lib/VortexCompiler.cs @@ -0,0 +1,452 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using Microsoft.WindowsAPICodePack.Shell; +using VFS; +using Wabbajack.Common; +using Wabbajack.Lib.CompilationSteps; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.ModListRegistry; +using Wabbajack.Lib.NexusApi; +using File = Alphaleonis.Win32.Filesystem.File; + +namespace Wabbajack.Lib +{ + public class VortexCompiler : ACompiler + { + public Game Game { get; } + public string GameName { get; } + + public string VortexFolder { get; set; } + public string StagingFolder { get; set; } + public string DownloadsFolder { get; set; } + + public bool IgnoreMissingFiles { get; set; } + + public VortexCompiler(string gameName, string gamePath) + { + ModManager = ModManager.Vortex; + + // TODO: only for testing + IgnoreMissingFiles = true; + string[] args = Environment.GetCommandLineArgs(); + + GamePath = gamePath; + GameName = gameName; + Game = GameRegistry.GetByNexusName(GameName).Game; + + //args: wabbajacke.exe gameName gamePath vortexfolder stagingfolder downloadsfolder + VortexFolder = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "Vortex"); + if (File.Exists(Path.Combine(VortexFolder, gameName, "mods", "__vortex_staging_folder"))) + StagingFolder = Path.Combine(VortexFolder, gameName, "mods"); + if (File.Exists(Path.Combine(VortexFolder, "downloads", "__vortex_downloads_folder"))) + DownloadsFolder = Path.Combine(VortexFolder, "downloads", gameName); + + if (args.Length >= 4) + StagingFolder = args[3]; + if (args.Length == 5) + DownloadsFolder = args[4]; + + ModListOutputFolder = "output_folder"; + + // TODO: add custom modlist name + ModListOutputFile = $"VORTEX_TEST_MODLIST{ExtensionManager.Extension}"; + + VFS = VirtualFileSystem.VFS; + + SelectedArchives = new List(); + AllFiles = new List(); + IndexedArchives = new List(); + IndexedFiles = new Dictionary>(); + } + + public override void Info(string msg) + { + Utils.Log(msg); + } + + public override void Status(string msg) + { + WorkQueue.Report(msg, 0); + } + + public override void Error(string msg) + { + Utils.Log(msg); + throw new Exception(msg); + } + + internal override string IncludeFile(byte[] data) + { + var id = Guid.NewGuid().ToString(); + File.WriteAllBytes(Path.Combine(ModListOutputFolder, id), data); + return id; + } + + internal override string IncludeFile(string data) + { + var id = Guid.NewGuid().ToString(); + File.WriteAllText(Path.Combine(ModListOutputFolder, id), data); + return id; + } + + public override bool Compile() + { + VirtualFileSystem.Clean(); + Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}."); + + Info("Starting pre-compilation steps"); + CreateMetaFiles(); + + Info($"Indexing {StagingFolder}"); + VFS.AddRoot(StagingFolder); + + Info($"Indexing {GamePath}"); + VFS.AddRoot(GamePath); + + Info($"Indexing {DownloadsFolder}"); + VFS.AddRoot(DownloadsFolder); + + AddExternalFolder(); + + Info("Cleaning output folder"); + if (Directory.Exists(ModListOutputFolder)) Directory.Delete(ModListOutputFolder, true); + Directory.CreateDirectory(ModListOutputFolder); + + IEnumerable vortexStagingFiles = Directory.EnumerateFiles(StagingFolder, "*", SearchOption.AllDirectories) + .Where(p => p.FileExists() && p != "__vortex_staging_folder") + .Select(p => new RawSourceFile(VFS.Lookup(p)) + {Path = p.RelativeTo(StagingFolder)}); + + IEnumerable vortexDownloads = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.AllDirectories) + .Where(p => p.FileExists()) + .Select(p => new RawSourceFile(VFS.Lookup(p)) + {Path = p.RelativeTo(DownloadsFolder)}); + + IEnumerable gameFiles = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories) + .Where(p => p.FileExists()) + .Select(p => new RawSourceFile(VFS.Lookup(p)) + { Path = Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) }); + + Info("Indexing Archives"); + IndexedArchives = Directory.EnumerateFiles(DownloadsFolder) + .Where(f => File.Exists(f+".meta")) + .Select(f => new IndexedArchive + { + File = VFS.Lookup(f), + Name = Path.GetFileName(f), + IniData = (f+".meta").LoadIniFile(), + Meta = File.ReadAllText(f+".meta") + }) + .ToList(); + + Info("Indexing Files"); + IDictionary> grouped = VFS.GroupedByArchive(); + IndexedFiles = IndexedArchives.Select(f => grouped.TryGetValue(f.File, out var result) ? result : new List()) + .SelectMany(fs => fs) + .Concat(IndexedArchives.Select(f => f.File)) + .OrderByDescending(f => f.TopLevelArchive.LastModified) + .GroupBy(f => f.Hash) + .ToDictionary(f => f.Key, f => f.AsEnumerable()); + + Info("Searching for mod files"); + AllFiles = vortexStagingFiles.Concat(vortexDownloads) + .Concat(gameFiles) + .DistinctBy(f => f.Path) + .ToList(); + + Info($"Found {AllFiles.Count} files to build into mod list"); + + Info("Verifying destinations"); + List> dups = AllFiles.GroupBy(f => f.Path) + .Where(fs => fs.Count() > 1) + .Select(fs => + { + Utils.Log($"Duplicate files installed to {fs.Key} from : {string.Join(", ", fs.Select(f => f.AbsolutePath))}"); + return fs; + }).ToList(); + + if (dups.Count > 0) + { + Error($"Found {dups.Count} duplicates, exiting"); + } + + IEnumerable stack = MakeStack(); + + Info("Running Compilation Stack"); + List results = AllFiles.PMap(f => RunStack(stack.Where(s => s != null), f)).ToList(); + + IEnumerable noMatch = results.OfType().ToList(); + Info($"No match for {noMatch.Count()} files"); + foreach (var file in noMatch) + Info($" {file.To}"); + if (noMatch.Any()) + { + if (IgnoreMissingFiles) + { + Info("Continuing even though files were missing at the request of the user."); + } + else + { + Info("Exiting due to no way to compile these files"); + return false; + } + } + + InstallDirectives = results.Where(i => !(i is IgnoredDirectly)).ToList(); + + // TODO: nexus stuff + /*Info("Getting Nexus api_key, please click authorize if a browser window appears"); + if (IndexedArchives.Any(a => a.IniData?.General?.gameName != null)) + { + var nexusClient = new NexusApiClient(); + if (!nexusClient.IsPremium) Error($"User {nexusClient.Username} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue"); + + } + */ + + GatherArchives(); + + ModList = new ModList + { + Archives = SelectedArchives, + ModManager = ModManager.Vortex, + Directives = InstallDirectives, + GameType = Game + }; + + ExportModList(); + + Info("Done Building ModList"); + return true; + } + + /// + /// Some have mods outside their game folder located + /// + private void AddExternalFolder() + { + var currentGame = GameRegistry.Games[Game]; + if (currentGame.AdditionalFolders == null || currentGame.AdditionalFolders.Count == 0) return; + currentGame.AdditionalFolders.Do(f => + { + var path = f.Replace("%documents%", KnownFolders.Documents.Path); + if (!Directory.Exists(path)) return; + Info($"Indexing {path}"); + VFS.AddRoot(path); + }); + } + + private void ExportModList() + { + Utils.Log($"Exporting ModList to: {ModListOutputFolder}"); + + // using JSON for better debugging + ModList.ToJSON(Path.Combine(ModListOutputFolder, "modlist.json")); + //ModList.ToCERAS(Path.Combine(ModListOutputFolder, "modlist"), ref CerasConfig.Config); + + if(File.Exists(ModListOutputFile)) + File.Delete(ModListOutputFile); + + using (var fs = new FileStream(ModListOutputFile, FileMode.Create)) + { + using (var za = new ZipArchive(fs, ZipArchiveMode.Create)) + { + Directory.EnumerateFiles(ModListOutputFolder, "*.*") + .DoProgress("Compressing ModList", + f => + { + var ze = za.CreateEntry(Path.GetFileName(f)); + using (var os = ze.Open()) + using (var ins = File.OpenRead(f)) + { + ins.CopyTo(os); + } + }); + } + } + + Utils.Log("Exporting ModList metadata"); + var metadata = new ModlistMetadata.DownloadMetadata + { + Size = File.GetSize(ModListOutputFile), + Hash = ModListOutputFile.FileHash(), + NumberOfArchives = ModList.Archives.Count, + SizeOfArchives = ModList.Archives.Sum(a => a.Size), + NumberOfInstalledFiles = ModList.Directives.Count, + SizeOfInstalledFiles = ModList.Directives.Sum(a => a.Size) + }; + metadata.ToJSON(ModListOutputFile + ".meta.json"); + + Utils.Log("Removing ModList staging folder"); + //Directory.Delete(ModListOutputFolder, true); + } + + /*private void GenerateReport() + { + string css; + using (var cssStream = Utils.GetResourceStream("Wabbajack.Lib.css-min.css")) + using (var reader = new StreamReader(cssStream)) + { + css = reader.ReadToEnd(); + } + + using (var fs = File.OpenWrite($"{ModList.Name}.md")) + { + fs.SetLength(0); + using (var reporter = new ReportBuilder(fs, ModListOutputFolder)) + { + reporter.Build(this, ModList); + } + } + }*/ + + private void CreateMetaFiles() + { + Utils.Log("Getting Nexus api_key, please click authorize if a browser window appears"); + var nexusClient = new NexusApiClient(); + + Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly) + .Where(f => File.Exists(f) && Path.GetExtension(f) != ".meta" && !File.Exists(f+".meta")) + .Do(f => + { + Utils.Log($"Trying to create meta file for {Path.GetFileName(f)}"); + var metaString = $"[General]\n" + + $"repository=Nexus\n" + + $"installed=true\n" + + $"uninstalled=false\n" + + $"paused=false\n" + + $"removed=false\n" + + $"gameName={GameName}\n"; + string hash; + using(var md5 = MD5.Create()) + using (var stream = File.OpenRead(f)) + { + Utils.Log($"Calculating hash for {Path.GetFileName(f)}"); + byte[] cH = md5.ComputeHash(stream); + hash = BitConverter.ToString(cH).Replace("-", "").ToLowerInvariant(); + Utils.Log($"Hash is {hash}"); + } + + List md5Response = nexusClient.GetModInfoFromMD5(Game, hash); + if (md5Response.Count >= 1) + { + var modInfo = md5Response[0].mod; + metaString += $"modID={modInfo.mod_id}\ndescription={NexusApiUtils.FixupSummary(modInfo.summary)}\n" + + $"modName={modInfo.name}\nfileID={md5Response[0].file_details.file_id}"; + File.WriteAllText(f+".meta",metaString, Encoding.UTF8); + } + else + { + Error("Error while getting information from nexusmods via MD5 hash!"); + } + + }); + } + + private void GatherArchives() + { + Info("Building a list of archives based on the files required"); + + var shas = InstallDirectives.OfType() + .Select(a => a.ArchiveHashPath[0]) + .Distinct(); + + var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified) + .GroupBy(f => f.File.Hash) + .ToDictionary(f => f.Key, f => f.First()); + + SelectedArchives = shas.PMap(sha => ResolveArchive(sha, archives)); + } + + private Archive ResolveArchive(string sha, IDictionary archives) + { + if (archives.TryGetValue(sha, out var found)) + { + if(found.IniData == null) + Error($"No download metadata found for {found.Name}, please use MO2 to query info or add a .meta file and try again."); + + var result = new Archive(); + result.State = (AbstractDownloadState) DownloadDispatcher.ResolveArchive(found.IniData); + + if (result.State == null) + Error($"{found.Name} could not be handled by any of the downloaders"); + + result.Name = found.Name; + result.Hash = found.File.Hash; + result.Meta = found.Meta; + result.Size = found.File.Size; + + Info($"Checking link for {found.Name}"); + + if (!result.State.Verify()) + Error( + $"Unable to resolve link for {found.Name}. If this is hosted on the Nexus the file may have been removed."); + + return result; + } + + Error($"No match found for Archive sha: {sha} this shouldn't happen"); + return null; + } + + public override Directive RunStack(IEnumerable stack, RawSourceFile source) + { + Utils.Status($"Compiling {source.Path}"); + foreach (var step in stack) + { + var result = step.Run(source); + if (result != null) return result; + } + + throw new InvalidDataException("Data fell out of the compilation stack"); + + } + + public override IEnumerable GetStack() + { + var s = Consts.TestMode ? DownloadsFolder : VortexFolder; + var userConfig = Path.Combine(s, "compilation_stack.yml"); + if (File.Exists(userConfig)) + return Serialization.Deserialize(File.ReadAllText(userConfig), this); + + IEnumerable stack = MakeStack(); + + File.WriteAllText(Path.Combine(s, "_current_compilation_stack.yml"), + Serialization.Serialize(stack)); + + return stack; + } + + public override IEnumerable MakeStack() + { + Utils.Log("Generating compilation stack"); + return new List + { + //new IncludePropertyFiles(this), + new IncludeVortexDeployment(this), + new IncludeRegex(this, "^*\\.meta"), + new IgnoreVortex(this), + + Game == Game.DarkestDungeon ? new IncludeRegex(this, "project\\.xml$") : null, + + new IgnoreStartsWith(this, " __vortex_staging_folder"), + new IgnoreEndsWith(this, "__vortex_staging_folder"), + + new IgnoreGameFiles(this), + + new DirectMatch(this), + + new IgnoreGameFiles(this), + + new IgnoreWabbajackInstallCruft(this), + + new DropAll(this) + }; + } + } +} diff --git a/Wabbajack.Lib/VortexInstaller.cs b/Wabbajack.Lib/VortexInstaller.cs new file mode 100644 index 00000000..f69d4b92 --- /dev/null +++ b/Wabbajack.Lib/VortexInstaller.cs @@ -0,0 +1,307 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using VFS; +using Wabbajack.Common; +using Wabbajack.Lib.Downloaders; +using Directory = Alphaleonis.Win32.Filesystem.Directory; +using File = Alphaleonis.Win32.Filesystem.File; +using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.Lib +{ + public class VortexInstaller + { + public string ModListArchive { get; } + public ModList ModList { get; } + public Dictionary HashedArchives { get; private set; } + + public GameMetaData GameInfo { get; internal set; } + + public string StagingFolder { get; set; } + public string DownloadFolder { get; set; } + + public VirtualFileSystem VFS => VirtualFileSystem.VFS; + + public bool IgnoreMissingFiles { get; internal set; } + + public VortexInstaller(string archive, ModList modList) + { + ModListArchive = archive; + ModList = modList; + + // TODO: only for testing + IgnoreMissingFiles = true; + + GameInfo = GameRegistry.Games[ModList.GameType]; + } + + public void Info(string msg) + { + Utils.Log(msg); + } + + public void Status(string msg) + { + WorkQueue.Report(msg, 0); + } + + private void Error(string msg) + { + Utils.Log(msg); + throw new Exception(msg); + } + + public byte[] LoadBytesFromPath(string path) + { + using (var fs = new FileStream(ModListArchive, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (var ar = new ZipArchive(fs, ZipArchiveMode.Read)) + using (var ms = new MemoryStream()) + { + var entry = ar.GetEntry(path); + using (var e = entry.Open()) + e.CopyTo(ms); + return ms.ToArray(); + } + } + + public static ModList LoadFromFile(string path) + { + using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (var ar = new ZipArchive(fs, ZipArchiveMode.Read)) + { + var entry = ar.GetEntry("modlist"); + if (entry == null) + { + entry = ar.GetEntry("modlist.json"); + using (var e = entry.Open()) + return e.FromJSON(); + } + using (var e = entry.Open()) + return e.FromCERAS(ref CerasConfig.Config); + } + } + + public void Install() + { + Directory.CreateDirectory(DownloadFolder); + + VirtualFileSystem.Clean(); + + HashArchives(); + DownloadArchives(); + HashArchives(); + + var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList(); + if (missing.Count > 0) + { + foreach (var a in missing) + Info($"Unable to download {a.Name}"); + if (IgnoreMissingFiles) + Info("Missing some archives, but continuing anyways at the request of the user"); + else + Error("Cannot continue, was unable to download one or more archives"); + } + + PrimeVFS(); + + BuildFolderStructure(); + InstallArchives(); + InstallIncludedFiles(); + //InctallIncludedDownloadMetas(); + + Info("Installation complete! You may exit the program."); + } + + private void BuildFolderStructure() + { + Info("Building Folder Structure"); + ModList.Directives + .OfType() + .Select(d => Path.Combine(StagingFolder, Path.GetDirectoryName(d.To))) + .ToHashSet() + .Do(f => + { + if (Directory.Exists(f)) return; + Directory.CreateDirectory(f); + }); + } + + private void InstallArchives() + { + Info("Installing Archives"); + Info("Grouping Install Files"); + var grouped = ModList.Directives + .OfType() + .GroupBy(e => e.ArchiveHashPath[0]) + .ToDictionary(k => k.Key); + var archives = ModList.Archives + .Select(a => new { Archive = a, AbsolutePath = HashedArchives.GetOrDefault(a.Hash) }) + .Where(a => a.AbsolutePath != null) + .ToList(); + + Info("Installing Archives"); + archives.PMap(a => InstallArchive(a.Archive, a.AbsolutePath, grouped[a.Archive.Hash])); + } + + private void InstallArchive(Archive archive, string absolutePath, IGrouping grouping) + { + Status($"Extracting {archive.Name}"); + + var vFiles = grouping.Select(g => + { + var file = VFS.FileForArchiveHashPath(g.ArchiveHashPath); + g.FromFile = file; + return g; + }).ToList(); + + var onFinish = VFS.Stage(vFiles.Select(f => f.FromFile).Distinct()); + + Status($"Copying files for {archive.Name}"); + + void CopyFile(string from, string to, bool useMove) + { + if(File.Exists(to)) + File.Delete(to); + if (useMove) + File.Move(from, to); + else + File.Copy(from, to); + } + + vFiles.GroupBy(f => f.FromFile) + .DoIndexed((idx, group) => + { + Utils.Status("Installing files", idx * 100 / vFiles.Count); + var firstDest = Path.Combine(StagingFolder, group.First().To); + CopyFile(group.Key.StagedPath, firstDest, true); + + foreach (var copy in group.Skip(1)) + { + var nextDest = Path.Combine(StagingFolder, copy.To); + CopyFile(firstDest, nextDest, false); + } + }); + + Status("Unstaging files"); + onFinish(); + } + + private void InstallIncludedFiles() + { + Info("Writing inline files"); + ModList.Directives.OfType() + .PMap(directive => + { + Status($"Writing included file {directive.To}"); + var outPath = Path.Combine(StagingFolder, directive.To); + if(File.Exists(outPath)) File.Delete(outPath); + File.WriteAllBytes(outPath, LoadBytesFromPath(directive.SourceDataID)); + }); + } + + private void PrimeVFS() + { + HashedArchives.Do(a => VFS.AddKnown(new VirtualFile + { + Paths = new[] { a.Value }, + Hash = a.Key + })); + VFS.RefreshIndexes(); + + + ModList.Directives + .OfType() + .Do(f => + { + var updated_path = new string[f.ArchiveHashPath.Length]; + f.ArchiveHashPath.CopyTo(updated_path, 0); + updated_path[0] = VFS.HashIndex[updated_path[0]].Where(e => e.IsConcrete).First().FullPath; + VFS.AddKnown(new VirtualFile { Paths = updated_path }); + }); + + VFS.BackfillMissing(); + } + + private void DownloadArchives() + { + var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList(); + Info($"Missing {missing.Count} archives"); + + Info("Getting Nexus API Key, if a browser appears, please accept"); + + var dispatchers = missing.Select(m => m.State.GetDownloader()).Distinct(); + + foreach (var dispatcher in dispatchers) + dispatcher.Prepare(); + + DownloadMissingArchives(missing); + } + + private void DownloadMissingArchives(List missing, bool download = true) + { + if (download) + { + foreach (var a in missing.Where(a => a.State.GetType() == typeof(ManualDownloader.State))) + { + var output_path = Path.Combine(DownloadFolder, a.Name); + a.State.Download(a, output_path); + } + } + + missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State)) + .PMap(archive => + { + Info($"Downloading {archive.Name}"); + var output_path = Path.Combine(DownloadFolder, archive.Name); + + if (!download) return DownloadArchive(archive, download); + if (output_path.FileExists()) + File.Delete(output_path); + + return DownloadArchive(archive, download); + }); + } + + public bool DownloadArchive(Archive archive, bool download) + { + try + { + archive.State.Download(archive, Path.Combine(DownloadFolder, archive.Name)); + } + catch (Exception ex) + { + Utils.Log($"Download error for file {archive.Name}"); + Utils.Log(ex.ToString()); + return false; + } + + return false; + } + + private void HashArchives() + { + HashedArchives = Directory.EnumerateFiles(DownloadFolder) + .Where(e => !e.EndsWith(".sha")) + .PMap(e => (HashArchive(e), e)) + .OrderByDescending(e => File.GetLastWriteTime(e.Item2)) + .GroupBy(e => e.Item1) + .Select(e => e.First()) + .ToDictionary(e => e.Item1, e => e.Item2); + } + + private string HashArchive(string e) + { + var cache = e + ".sha"; + if (cache.FileExists() && new FileInfo(cache).LastWriteTime >= new FileInfo(e).LastWriteTime) + return File.ReadAllText(cache); + + Status($"Hashing {Path.GetFileName(e)}"); + File.WriteAllText(cache, e.FileHash()); + return HashArchive(e); + } + } +} diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 36c710cc..ce26b2cb 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -77,6 +77,7 @@ + @@ -88,6 +89,7 @@ + @@ -101,6 +103,7 @@ + @@ -131,6 +134,8 @@ + + WebAutomationWindow.xaml diff --git a/Wabbajack.Lib/zEditIntegration.cs b/Wabbajack.Lib/zEditIntegration.cs index fa1216ee..fe667e59 100644 --- a/Wabbajack.Lib/zEditIntegration.cs +++ b/Wabbajack.Lib/zEditIntegration.cs @@ -14,9 +14,12 @@ namespace Wabbajack.Lib { public class zEditIntegration { - public static string FindzEditPath(Compiler compiler) + private static Compiler _mo2Compiler; + + public static string FindzEditPath(ACompiler compiler) { - var executables = compiler.MO2Ini.customExecutables; + _mo2Compiler = (Compiler) compiler; + var executables = _mo2Compiler.MO2Ini.customExecutables; if (executables.size == null) return null; foreach (var idx in Enumerable.Range(1, int.Parse(executables.size))) @@ -35,7 +38,7 @@ namespace Wabbajack.Lib { private Dictionary _mergesIndexed; - public IncludeZEditPatches(Compiler compiler) : base(compiler) + public IncludeZEditPatches(ACompiler compiler) : base(compiler) { var zEditPath = FindzEditPath(compiler); var havezEdit = zEditPath != null; @@ -63,7 +66,7 @@ namespace Wabbajack.Lib _mergesIndexed = merges.ToDictionary( - m => Path.Combine(compiler.MO2Folder, "mods", m.Key.name, m.Key.filename), + m => Path.Combine(_mo2Compiler.MO2Folder, "mods", m.Key.name, m.Key.filename), m => m.First()); } @@ -89,12 +92,12 @@ namespace Wabbajack.Lib return new SourcePatch { - RelativePath = abs_path.RelativeTo(_compiler.MO2Folder), + RelativePath = abs_path.RelativeTo(_mo2Compiler.MO2Folder), Hash = _compiler.VFS[abs_path].Hash }; }).ToList(); - var src_data = result.Sources.Select(f => File.ReadAllBytes(Path.Combine(_compiler.MO2Folder, f.RelativePath))) + var src_data = result.Sources.Select(f => File.ReadAllBytes(Path.Combine(_mo2Compiler.MO2Folder, f.RelativePath))) .ConcatArrays(); var dst_data = File.ReadAllBytes(source.AbsolutePath); @@ -117,7 +120,7 @@ namespace Wabbajack.Lib [JsonObject("IncludeZEditPatches")] public class State : IState { - public ICompilationStep CreateStep(Compiler compiler) + public ICompilationStep CreateStep(ACompiler compiler) { return new IncludeZEditPatches(compiler); } diff --git a/Wabbajack.Test.ListValidation/ListValidation.cs b/Wabbajack.Test.ListValidation/ListValidation.cs index 47315975..5ed3a756 100644 --- a/Wabbajack.Test.ListValidation/ListValidation.cs +++ b/Wabbajack.Test.ListValidation/ListValidation.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Windows.Documents; using Microsoft.VisualStudio.TestTools.UnitTesting; using Wabbajack.Common; using Wabbajack.Lib; @@ -14,22 +15,23 @@ namespace Wabbajack.Test.ListValidation [TestClass] public class ListValidation { - [TestInitialize] - public void Setup() + [ClassInitialize] + public static void SetupNexus(TestContext context) { - Directory.CreateDirectory(Consts.ModListDownloadFolder); - Utils.LogMessages.Subscribe(s => TestContext.WriteLine(s)); + Utils.LogMessages.Subscribe(context.WriteLine); var api = new NexusApiClient(); api.ClearUpdatedModsInCache(); } - private TestContext testContextInstance; - public TestContext TestContext + [TestInitialize] + public void SetupTest() { - get { return testContextInstance; } - set { testContextInstance = value; } + Directory.CreateDirectory(Consts.ModListDownloadFolder); + Utils.LogMessages.Subscribe(s => TestContext.WriteLine(s)); } + public TestContext TestContext { get; set; } + [TestCategory("ListValidation")] [DataTestMethod] [DynamicData(nameof(GetModLists), DynamicDataSourceType.Method)] diff --git a/Wabbajack.Test.ListValidation/Wabbajack.Test.ListValidation.csproj b/Wabbajack.Test.ListValidation/Wabbajack.Test.ListValidation.csproj index 55a57841..bb5da801 100644 --- a/Wabbajack.Test.ListValidation/Wabbajack.Test.ListValidation.csproj +++ b/Wabbajack.Test.ListValidation/Wabbajack.Test.ListValidation.csproj @@ -76,10 +76,10 @@ - 1.3.2 + 2.0.0 - 1.3.2 + 2.0.0 12.0.2 diff --git a/Wabbajack.Test/AVortexCompilerTest.cs b/Wabbajack.Test/AVortexCompilerTest.cs new file mode 100644 index 00000000..b170948d --- /dev/null +++ b/Wabbajack.Test/AVortexCompilerTest.cs @@ -0,0 +1,71 @@ +using System; +using System.IO; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using VFS; +using Wabbajack.Common; +using Wabbajack.Lib; + +namespace Wabbajack.Test +{ + public abstract class AVortexCompilerTest + { + public TestContext TestContext { get; set; } + protected TestUtils utils { get; set; } + + + [TestInitialize] + public void TestInitialize() + { + Consts.TestMode = true; + + utils = new TestUtils + { + GameName = "darkestdungeon" + }; + + Utils.LogMessages.Subscribe(f => TestContext.WriteLine(f)); + } + + [TestCleanup] + public void TestCleanup() + { + utils.Dispose(); + } + + protected VortexCompiler ConfigureAndRunCompiler() + { + var vortexCompiler = MakeCompiler(); + vortexCompiler.VFS.Reset(); + vortexCompiler.DownloadsFolder = utils.DownloadsFolder; + vortexCompiler.StagingFolder = utils.InstallFolder; + Directory.CreateDirectory(utils.InstallFolder); + Assert.IsTrue(vortexCompiler.Compile()); + return vortexCompiler; + } + + protected VortexCompiler MakeCompiler() + { + VirtualFileSystem.Reconfigure(utils.TestFolder); + var vortexCompiler = new VortexCompiler(utils.GameName, utils.GameFolder); + return vortexCompiler; + } + + protected ModList CompileAndInstall() + { + var vortexCompiler = ConfigureAndRunCompiler(); + Install(vortexCompiler); + return vortexCompiler.ModList; + } + + protected void Install(VortexCompiler vortexCompiler) + { + var modList = Installer.LoadFromFile(vortexCompiler.ModListOutputFile); + var installer = new Installer(vortexCompiler.ModListOutputFile, modList, utils.InstallFolder) + { + DownloadFolder = utils.DownloadsFolder, + GameFolder = utils.GameFolder, + }; + installer.Install(); + } + } +} diff --git a/Wabbajack.Test/VortexTests.cs b/Wabbajack.Test/VortexTests.cs new file mode 100644 index 00000000..6f2961e9 --- /dev/null +++ b/Wabbajack.Test/VortexTests.cs @@ -0,0 +1,25 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Wabbajack.Lib.CompilationSteps; + +namespace Wabbajack.Test +{ + [TestClass] + public class VortexTests : AVortexCompilerTest + { + [TestMethod] + public void TestVortexStackSerialization() + { + utils.AddMod("test"); + utils.Configure(); + + var vortexCompiler = ConfigureAndRunCompiler(); + var stack = vortexCompiler.MakeStack(); + + var serialized = Serialization.Serialize(stack); + var rounded = Serialization.Serialize(Serialization.Deserialize(serialized, vortexCompiler)); + + Assert.AreEqual(serialized, rounded); + Assert.IsNotNull(vortexCompiler.GetStack()); + } + } +} diff --git a/Wabbajack.Test/Wabbajack.Test.csproj b/Wabbajack.Test/Wabbajack.Test.csproj index b934ad1e..13534d80 100644 --- a/Wabbajack.Test/Wabbajack.Test.csproj +++ b/Wabbajack.Test/Wabbajack.Test.csproj @@ -84,6 +84,7 @@ + @@ -93,6 +94,7 @@ + @@ -104,6 +106,7 @@ + @@ -137,10 +140,10 @@ 2.2.6 - 1.3.2 + 2.0.0 - 1.3.2 + 2.0.0 12.0.2 diff --git a/Wabbajack.VirtualFileSystem.Test/Properties/AssemblyInfo.cs b/Wabbajack.VirtualFileSystem.Test/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..6e7a9eaf --- /dev/null +++ b/Wabbajack.VirtualFileSystem.Test/Properties/AssemblyInfo.cs @@ -0,0 +1,19 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("Wabbajack.VirtualFileSystem.Test")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Wabbajack.VirtualFileSystem.Test")] +[assembly: AssemblyCopyright("Copyright © 2019")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +[assembly: ComVisible(false)] + +[assembly: Guid("51ceb604-985a-45b9-af0d-c5ba8cfa1bf0")] + +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs new file mode 100644 index 00000000..940cd21e --- /dev/null +++ b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs @@ -0,0 +1,209 @@ +using System; +using System.Collections.Generic; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem.Test +{ + [TestClass] + public class VFSTests + { + private const string VFS_TEST_DIR = "vfs_test_dir"; + private static readonly string VFS_TEST_DIR_FULL = Path.Combine(Directory.GetCurrentDirectory(), VFS_TEST_DIR); + private Context context; + + public TestContext TestContext { get; set; } + + [TestInitialize] + public void Setup() + { + Utils.LogMessages.Subscribe(f => TestContext.WriteLine(f)); + if (Directory.Exists(VFS_TEST_DIR)) + Directory.Delete(VFS_TEST_DIR, true); + Directory.CreateDirectory(VFS_TEST_DIR); + context = new Context(); + } + + [TestMethod] + public async Task FilesAreIndexed() + { + AddFile("test.txt", "This is a test"); + await AddTestRoot(); + + var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")]; + Assert.IsNotNull(file); + + Assert.AreEqual(file.Size, 14); + Assert.AreEqual(file.Hash, "qX0GZvIaTKM="); + } + + private async Task AddTestRoot() + { + await context.AddRoot(VFS_TEST_DIR_FULL); + await context.WriteToFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin")); + await context.IntegrateFromFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin")); + } + + + [TestMethod] + public async Task ArchiveContentsAreIndexed() + { + AddFile("archive/test.txt", "This is a test"); + ZipUpFolder("archive", "test.zip"); + await AddTestRoot(); + + var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip"); + var file = context.Index.ByFullPath[abs_path]; + Assert.IsNotNull(file); + + Assert.AreEqual(128, file.Size); + Assert.AreEqual(abs_path.FileHash(), file.Hash); + + Assert.IsTrue(file.IsArchive); + var inner_file = file.Children.First(); + Assert.AreEqual(14, inner_file.Size); + Assert.AreEqual("qX0GZvIaTKM=", inner_file.Hash); + Assert.AreSame(file, file.Children.First().Parent); + } + + [TestMethod] + public async Task DuplicateFileHashes() + { + AddFile("archive/test.txt", "This is a test"); + ZipUpFolder("archive", "test.zip"); + + AddFile("test.txt", "This is a test"); + await AddTestRoot(); + + + var files = context.Index.ByHash["qX0GZvIaTKM="]; + Assert.AreEqual(files.Count(), 2); + } + + [TestMethod] + public async Task DeletedFilesAreRemoved() + { + AddFile("test.txt", "This is a test"); + await AddTestRoot(); + + var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")]; + Assert.IsNotNull(file); + + Assert.AreEqual(file.Size, 14); + Assert.AreEqual(file.Hash, "qX0GZvIaTKM="); + + File.Delete(Path.Combine(VFS_TEST_DIR_FULL, "test.txt")); + + await AddTestRoot(); + + CollectionAssert.DoesNotContain(context.Index.ByFullPath, Path.Combine(VFS_TEST_DIR_FULL, "test.txt")); + } + + [TestMethod] + public async Task UnmodifiedFilesAreNotReIndexed() + { + AddFile("test.txt", "This is a test"); + await AddTestRoot(); + + var old_file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")]; + var old_time = old_file.LastAnalyzed; + + await AddTestRoot(); + + var new_file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")]; + + Assert.AreEqual(old_time, new_file.LastAnalyzed); + } + + [TestMethod] + public async Task CanStageSimpleArchives() + { + AddFile("archive/test.txt", "This is a test"); + ZipUpFolder("archive", "test.zip"); + await AddTestRoot(); + + var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip"); + var file = context.Index.ByFullPath[abs_path + "|test.txt"]; + + var cleanup = context.Stage(new List {file}); + Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath)); + + cleanup(); + } + + [TestMethod] + public async Task CanStageNestedArchives() + { + AddFile("archive/test.txt", "This is a test"); + ZipUpFolder("archive", "test.zip"); + + Directory.CreateDirectory(Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir")); + File.Move(Path.Combine(VFS_TEST_DIR_FULL, "test.zip"), + Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir\nested.zip")); + ZipUpFolder("archive", "test.zip"); + + await AddTestRoot(); + + var files = context.Index.ByHash["qX0GZvIaTKM="]; + + var cleanup = context.Stage(files); + + foreach (var file in files) + Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath)); + + cleanup(); + } + + [TestMethod] + public async Task CanRequestPortableFileTrees() + { + AddFile("archive/test.txt", "This is a test"); + ZipUpFolder("archive", "test.zip"); + + Directory.CreateDirectory(Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir")); + File.Move(Path.Combine(VFS_TEST_DIR_FULL, "test.zip"), + Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir\nested.zip")); + ZipUpFolder("archive", "test.zip"); + + await AddTestRoot(); + + var files = context.Index.ByHash["qX0GZvIaTKM="]; + var archive = context.Index.ByRootPath[Path.Combine(VFS_TEST_DIR_FULL, "test.zip")]; + + var state = context.GetPortableState(files); + + var new_context = new Context(); + + await new_context.IntegrateFromPortable(state, + new Dictionary {{archive.Hash, archive.FullPath}}); + + var new_files = new_context.Index.ByHash["qX0GZvIaTKM="]; + + var close = new_context.Stage(new_files); + + foreach (var file in new_files) + Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath)); + + close(); + } + + private static void AddFile(string filename, string thisIsATest) + { + var fullpath = Path.Combine(VFS_TEST_DIR, filename); + if (!Directory.Exists(Path.GetDirectoryName(fullpath))) + Directory.CreateDirectory(Path.GetDirectoryName(fullpath)); + File.WriteAllText(fullpath, thisIsATest); + } + + private static void ZipUpFolder(string folder, string output) + { + var path = Path.Combine(VFS_TEST_DIR, folder); + ZipFile.CreateFromDirectory(path, Path.Combine(VFS_TEST_DIR, output)); + Directory.Delete(path, true); + } + } +} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj new file mode 100644 index 00000000..b014ded3 --- /dev/null +++ b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj @@ -0,0 +1,99 @@ + + + + + Debug + AnyCPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0} + Library + Properties + Wabbajack.VirtualFileSystem.Test + Wabbajack.VirtualFileSystem.Test + v4.7.2 + 512 + {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + 15.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages + False + UnitTest + + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + true + bin\x64\Debug\ + DEBUG;TRACE + full + x64 + 7.3 + prompt + MinimumRecommendedRules.ruleset + + + bin\x64\Release\ + TRACE + true + pdbonly + x64 + 7.3 + prompt + MinimumRecommendedRules.ruleset + + + + + + + + + + + + + + + + + {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5} + Wabbajack.Common + + + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E} + Wabbajack.VirtualFileSystem + + + + + 2.2.6 + + + 2.0.0 + + + 2.0.0 + + + 4.2.0 + + + + + \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs new file mode 100644 index 00000000..9935ace5 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -0,0 +1,267 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Reactive.Linq; +using System.Text; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using Wabbajack.Common; +using Wabbajack.Common.CSP; +using Directory = Alphaleonis.Win32.Filesystem.Directory; +using File = System.IO.File; +using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.VirtualFileSystem +{ + public class Context + { + public const ulong FileVersion = 0x02; + public const string Magic = "WABBAJACK VFS FILE"; + + private readonly string _stagingFolder = "vfs_staging"; + public IndexRoot Index { get; private set; } = IndexRoot.Empty; + + public TemporaryDirectory GetTemporaryFolder() + { + return new TemporaryDirectory(Path.Combine(_stagingFolder, Guid.NewGuid().ToString())); + } + + public async Task AddRoot(string root) + { + if (!Path.IsPathRooted(root)) + throw new InvalidDataException($"Path is not absolute: {root}"); + + var filtered = await Index.AllFiles + .ToChannel() + .UnorderedPipelineRx(o => o.Where(file => File.Exists(file.Name))) + .TakeAll(); + + var byPath = filtered.ToImmutableDictionary(f => f.Name); + + var results = Channel.Create(1024); + var pipeline = Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive) + .ToChannel() + .UnorderedPipeline(results, async f => + { + if (byPath.TryGetValue(f, out var found)) + { + var fi = new FileInfo(f); + if (found.LastModified == fi.LastWriteTimeUtc.Ticks && found.Size == fi.Length) + return found; + } + + return await VirtualFile.Analyze(this, null, f, f); + }); + + var allFiles = await results.TakeAll(); + + // Should already be done but let's make the async tracker happy + await pipeline; + + var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList()); + + lock (this) + { + Index = newIndex; + } + + return newIndex; + } + + public async Task WriteToFile(string filename) + { + using (var fs = File.OpenWrite(filename)) + using (var bw = new BinaryWriter(fs, Encoding.UTF8, true)) + { + fs.SetLength(0); + + bw.Write(Encoding.ASCII.GetBytes(Magic)); + bw.Write(FileVersion); + bw.Write((ulong) Index.AllFiles.Count); + + var sizes = await Index.AllFiles + .ToChannel() + .UnorderedPipelineSync(f => + { + var ms = new MemoryStream(); + f.Write(ms); + return ms; + }) + .Select(async ms => + { + var size = ms.Position; + ms.Position = 0; + bw.Write((ulong) size); + await ms.CopyToAsync(fs); + return ms.Position; + }) + .TakeAll(); + Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}"); + } + } + + public async Task IntegrateFromFile(string filename) + { + using (var fs = File.OpenRead(filename)) + using (var br = new BinaryReader(fs, Encoding.UTF8, true)) + { + var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length)); + var fileVersion = br.ReadUInt64(); + if (fileVersion != FileVersion || magic != magic) + throw new InvalidDataException("Bad Data Format"); + + var numFiles = br.ReadUInt64(); + + var input = Channel.Create(1024); + var pipeline = input.UnorderedPipelineSync( + data => VirtualFile.Read(this, data)) + .TakeAll(); + + Utils.Log($"Loading {numFiles} files from {filename}"); + + for (ulong idx = 0; idx < numFiles; idx++) + { + var size = br.ReadUInt64(); + var bytes = new byte[size]; + await br.BaseStream.ReadAsync(bytes, 0, (int) size); + await input.Put(bytes); + } + + input.Close(); + + var files = await pipeline; + var newIndex = await Index.Integrate(files); + lock (this) + { + Index = newIndex; + } + } + } + + public Action Stage(IEnumerable files) + { + var grouped = files.SelectMany(f => f.FilesInFullPath) + .Distinct() + .Where(f => f.Parent != null) + .GroupBy(f => f.Parent) + .OrderBy(f => f.Key?.NestingFactor ?? 0) + .ToList(); + + var paths = new List(); + + foreach (var group in grouped) + { + var tmpPath = Path.Combine(_stagingFolder, Guid.NewGuid().ToString()); + FileExtractor.ExtractAll(group.Key.StagedPath, tmpPath).Wait(); + paths.Add(tmpPath); + foreach (var file in group) + file.StagedPath = Path.Combine(tmpPath, file.Name); + } + + return () => + { + paths.Do(p => + { + if (Directory.Exists(p)) + Directory.Delete(p, true, true); + }); + }; + } + + public List GetPortableState(IEnumerable files) + { + return files.SelectMany(f => f.FilesInFullPath) + .Distinct() + .Select(f => new PortableFile + { + Name = f.Parent != null ? f.Name : null, + Hash = f.Hash, + ParentHash = f.Parent?.Hash, + Size = f.Size + }).ToList(); + } + + public async Task IntegrateFromPortable(List state, Dictionary links) + { + var indexedState = state.GroupBy(f => f.ParentHash) + .ToDictionary(f => f.Key ?? "", f => (IEnumerable) f); + var parents = await indexedState[""] + .ToChannel() + .UnorderedPipelineSync(f => VirtualFile.CreateFromPortable(this, indexedState, links, f)) + .TakeAll(); + + var newIndex = await Index.Integrate(parents); + lock (this) + { + Index = newIndex; + } + } + } + + public class IndexRoot + { + public static IndexRoot Empty = new IndexRoot(); + + public IndexRoot(ImmutableList aFiles, + ImmutableDictionary byFullPath, + ImmutableDictionary> byHash, + ImmutableDictionary byRoot) + { + AllFiles = aFiles; + ByFullPath = byFullPath; + ByHash = byHash; + ByRootPath = byRoot; + } + + public IndexRoot() + { + AllFiles = ImmutableList.Empty; + ByFullPath = ImmutableDictionary.Empty; + ByHash = ImmutableDictionary>.Empty; + ByRootPath = ImmutableDictionary.Empty; + } + + public ImmutableList AllFiles { get; } + public ImmutableDictionary ByFullPath { get; } + public ImmutableDictionary> ByHash { get; } + public ImmutableDictionary ByRootPath { get; } + + public async Task Integrate(List files) + { + var allFiles = AllFiles.Concat(files).GroupBy(f => f.Name).Select(g => g.Last()).ToImmutableList(); + + var byFullPath = Task.Run(() => + allFiles.SelectMany(f => f.ThisAndAllChildren) + .ToImmutableDictionary(f => f.FullPath)); + + var byHash = Task.Run(() => + allFiles.SelectMany(f => f.ThisAndAllChildren) + .ToGroupedImmutableDictionary(f => f.Hash)); + + var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.Name)); + + return new IndexRoot(allFiles, + await byFullPath, + await byHash, + await byRootPath); + } + } + + public class TemporaryDirectory : IDisposable + { + public TemporaryDirectory(string name) + { + FullName = name; + } + + public string FullName { get; } + + public void Dispose() + { + Directory.Delete(FullName, true, true); + } + } +} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/Extensions.cs b/Wabbajack.VirtualFileSystem/Extensions.cs new file mode 100644 index 00000000..92021027 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/Extensions.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; + +namespace Wabbajack.VirtualFileSystem +{ + public static class Extensions + { + public static ImmutableDictionary ToImmutableDictionary(this IEnumerable coll, + Func keyFunc) + { + var builder = ImmutableDictionary.Empty.ToBuilder(); + foreach (var itm in coll) + builder.Add(keyFunc(itm), itm); + return builder.ToImmutable(); + } + + public static ImmutableDictionary> ToGroupedImmutableDictionary( + this IEnumerable coll, Func keyFunc) + { + var builder = ImmutableDictionary>.Empty.ToBuilder(); + foreach (var itm in coll) + { + var key = keyFunc(itm); + if (builder.TryGetValue(key, out var prev)) + builder[key] = prev.Push(itm); + else + builder[key] = ImmutableStack.Empty.Push(itm); + } + + return builder.ToImmutable(); + } + } +} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/PortableFile.cs b/Wabbajack.VirtualFileSystem/PortableFile.cs new file mode 100644 index 00000000..8b3a7e78 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/PortableFile.cs @@ -0,0 +1,10 @@ +namespace Wabbajack.VirtualFileSystem +{ + public class PortableFile + { + public string Name { get; set; } + public string Hash { get; set; } + public string ParentHash { get; set; } + public long Size { get; set; } + } +} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/Properties/AssemblyInfo.cs b/Wabbajack.VirtualFileSystem/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..27b88b86 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Wabbajack.VirtualFileSystem")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Wabbajack.VirtualFileSystem")] +[assembly: AssemblyCopyright("Copyright © 2019")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("5d6a2eaf-6604-4c51-8ae2-a746b4bc5e3e")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs new file mode 100644 index 00000000..02729671 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -0,0 +1,251 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Common.CSP; +using Directory = Alphaleonis.Win32.Filesystem.Directory; +using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.VirtualFileSystem +{ + public class VirtualFile + { + private string _fullPath; + + private string _stagedPath; + public string Name { get; internal set; } + + public string FullPath + { + get + { + if (_fullPath != null) return _fullPath; + var cur = this; + var acc = new LinkedList(); + while (cur != null) + { + acc.AddFirst(cur.Name); + cur = cur.Parent; + } + + _fullPath = string.Join("|", acc); + + return _fullPath; + } + } + + public string Hash { get; internal set; } + public long Size { get; internal set; } + + public long LastModified { get; internal set; } + + public long LastAnalyzed { get; internal set; } + + public VirtualFile Parent { get; internal set; } + + public Context Context { get; set; } + + public string StagedPath + { + get + { + if (IsNative) + return Name; + if (_stagedPath == null) + throw new UnstagedFileException(FullPath); + return _stagedPath; + } + internal set + { + if (IsNative) + throw new CannotStageNativeFile("Cannot stage a native file"); + _stagedPath = value; + } + } + + /// + /// Returns the nesting factor for this file. Native files will have a nesting of 1, the factor + /// goes up for each nesting of a file in an archive. + /// + public int NestingFactor + { + get + { + var cnt = 0; + var cur = this; + while (cur != null) + { + cnt += 1; + cur = cur.Parent; + } + + return cnt; + } + } + + public ImmutableList Children { get; internal set; } = ImmutableList.Empty; + + public bool IsArchive => Children != null && Children.Count > 0; + + public bool IsNative => Parent == null; + + public IEnumerable ThisAndAllChildren => + Children.SelectMany(child => child.ThisAndAllChildren).Append(this); + + + /// + /// Returns all the virtual files in the path to this file, starting from the root file. + /// + public IEnumerable FilesInFullPath + { + get + { + var stack = ImmutableStack.Empty; + var cur = this; + while (cur != null) + { + stack = stack.Push(cur); + cur = cur.Parent; + } + + return stack; + } + } + + public static async Task Analyze(Context context, VirtualFile parent, string abs_path, + string rel_path) + { + var hasher = abs_path.FileHashAsync(); + var fi = new FileInfo(abs_path); + var self = new VirtualFile + { + Context = context, + Name = rel_path, + Parent = parent, + Size = fi.Length, + LastModified = fi.LastWriteTimeUtc.Ticks, + LastAnalyzed = DateTime.Now.Ticks + }; + + if (FileExtractor.CanExtract(Path.GetExtension(abs_path))) + using (var tempFolder = context.GetTemporaryFolder()) + { + await FileExtractor.ExtractAll(abs_path, tempFolder.FullName); + + var results = Channel.Create(1024); + var files = Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories) + .ToChannel() + .UnorderedPipeline(results, + async abs_src => await Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName))); + self.Children = (await results.TakeAll()).ToImmutableList(); + } + + self.Hash = await hasher; + return self; + } + + public void Write(MemoryStream ms) + { + using (var bw = new BinaryWriter(ms, Encoding.UTF8, true)) + { + Write(bw); + } + } + + private void Write(BinaryWriter bw) + { + bw.Write(Name); + bw.Write(Hash); + bw.Write(Size); + bw.Write(LastModified); + bw.Write(LastAnalyzed); + bw.Write(Children.Count); + foreach (var child in Children) + child.Write(bw); + } + + public static VirtualFile Read(Context context, byte[] data) + { + using (var ms = new MemoryStream(data)) + using (var br = new BinaryReader(ms)) + { + return Read(context, null, br); + } + } + + private static VirtualFile Read(Context context, VirtualFile parent, BinaryReader br) + { + var vf = new VirtualFile + { + Context = context, + Parent = parent, + Name = br.ReadString(), + Hash = br.ReadString(), + Size = br.ReadInt64(), + LastModified = br.ReadInt64(), + LastAnalyzed = br.ReadInt64(), + Children = ImmutableList.Empty + }; + + var childrenCount = br.ReadInt32(); + for (var idx = 0; idx < childrenCount; idx += 1) vf.Children = vf.Children.Add(Read(context, vf, br)); + + return vf; + } + + public static VirtualFile CreateFromPortable(Context context, + Dictionary> state, Dictionary links, + PortableFile portableFile) + { + var vf = new VirtualFile + { + Parent = null, + Context = context, + Name = links[portableFile.Hash], + Hash = portableFile.Hash, + Size = portableFile.Size + }; + if (state.TryGetValue(portableFile.Hash, out var children)) + vf.Children = children.Select(child => CreateFromPortable(context, vf, state, child)).ToImmutableList(); + return vf; + } + + public static VirtualFile CreateFromPortable(Context context, VirtualFile parent, + Dictionary> state, PortableFile portableFile) + { + var vf = new VirtualFile + { + Parent = parent, + Context = context, + Name = portableFile.Name, + Hash = portableFile.Hash, + Size = portableFile.Size + }; + if (state.TryGetValue(portableFile.Hash, out var children)) + vf.Children = children.Select(child => CreateFromPortable(context, vf, state, child)).ToImmutableList(); + return vf; + } + } + + public class CannotStageNativeFile : Exception + { + public CannotStageNativeFile(string cannotStageANativeFile) : base(cannotStageANativeFile) + { + } + } + + public class UnstagedFileException : Exception + { + private readonly string _fullPath; + + public UnstagedFileException(string fullPath) : base($"File {fullPath} is unstaged, cannot get staged name") + { + _fullPath = fullPath; + } + } +} \ No newline at end of file diff --git a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj new file mode 100644 index 00000000..34328510 --- /dev/null +++ b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj @@ -0,0 +1,91 @@ + + + + + Debug + AnyCPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E} + Library + Properties + Wabbajack.VirtualFileSystem + Wabbajack.VirtualFileSystem + v4.7.2 + 512 + true + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + true + bin\x64\Debug\ + DEBUG;TRACE + full + x64 + 7.3 + prompt + MinimumRecommendedRules.ruleset + + + bin\x64\Release\ + TRACE + true + pdbonly + x64 + 7.3 + prompt + MinimumRecommendedRules.ruleset + + + + + + + + + + + + + + + + + + + + + + + {9e69bc98-1512-4977-b683-6e7e5292c0b8} + Wabbajack.Common.CSP + + + {b3f3fb6e-b9eb-4f49-9875-d78578bc7ae5} + Wabbajack.Common + + + + + 2.2.6 + + + 1.6.0 + + + + \ No newline at end of file diff --git a/Wabbajack.sln b/Wabbajack.sln index c368a714..745a2991 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -32,6 +32,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Compression.BSA.Test", "Com EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Common.CSP", "Wabbajack.Common.CSP\Wabbajack.Common.CSP.csproj", "{9E69BC98-1512-4977-B683-6E7E5292C0B8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VirtualFileSystem", "Wabbajack.VirtualFileSystem\Wabbajack.VirtualFileSystem.csproj", "{5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VirtualFileSystem.Test", "Wabbajack.VirtualFileSystem.Test\Wabbajack.VirtualFileSystem.Test.csproj", "{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug (no commandargs)|Any CPU = Debug (no commandargs)|Any CPU @@ -225,6 +229,42 @@ Global {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x64.Build.0 = Release|Any CPU {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x86.ActiveCfg = Release|Any CPU {9E69BC98-1512-4977-B683-6E7E5292C0B8}.Release|x86.Build.0 = Release|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|x86.ActiveCfg = Debug|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug (no commandargs)|x86.Build.0 = Debug|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|x64.ActiveCfg = Debug|x64 + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|x64.Build.0 = Debug|x64 + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|x86.ActiveCfg = Debug|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Debug|x86.Build.0 = Debug|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|Any CPU.Build.0 = Release|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x64.ActiveCfg = Release|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x64.Build.0 = Release|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x86.ActiveCfg = Release|Any CPU + {5D6A2EAF-6604-4C51-8AE2-A746B4BC5E3E}.Release|x86.Build.0 = Release|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|x86.ActiveCfg = Debug|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug (no commandargs)|x86.Build.0 = Debug|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|x64.ActiveCfg = Debug|x64 + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|x64.Build.0 = Debug|x64 + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|x86.ActiveCfg = Debug|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Debug|x86.Build.0 = Debug|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|Any CPU.Build.0 = Release|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x64.ActiveCfg = Release|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x64.Build.0 = Release|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x86.ActiveCfg = Release|Any CPU + {51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Wabbajack/Settings.cs b/Wabbajack/Settings.cs index a7b51d27..c2b2a28d 100644 --- a/Wabbajack/Settings.cs +++ b/Wabbajack/Settings.cs @@ -53,6 +53,7 @@ namespace Wabbajack public class InstallationSettings { public string InstallationLocation { get; set; } + public string StagingLocation { get; set; } public string DownloadLocation { get; set; } } diff --git a/Wabbajack/View Models/CompilerVM.cs b/Wabbajack/View Models/CompilerVM.cs index 7ac9fd1a..8693e717 100644 --- a/Wabbajack/View Models/CompilerVM.cs +++ b/Wabbajack/View Models/CompilerVM.cs @@ -205,9 +205,30 @@ namespace Wabbajack private async Task ExecuteBegin() { - Compiler compiler; - try + if (false) { + string[] args = Environment.GetCommandLineArgs(); + var compiler = new VortexCompiler(args[1], args[2]); + await Task.Run(() => + { + Compiling = true; + try + { + compiler.Compile(); + } + catch (Exception ex) + { + while (ex.InnerException != null) ex = ex.InnerException; + Utils.Log($"Can't continue: {ex.ExceptionToString()}"); + } + finally + { + Compiling = false; + } + }); + }else{ + Compiler compiler; + try { compiler = new Compiler(this.Mo2Folder) { MO2Profile = this.MOProfile, @@ -218,34 +239,35 @@ namespace Wabbajack ModListWebsite = this.Website, ModListReadme = this.ReadMeText.TargetPath, }; - } - catch (Exception ex) - { - while (ex.InnerException != null) ex = ex.InnerException; - Utils.Log($"Compiler error: {ex.ExceptionToString()}"); - return; - } - await Task.Run(() => - { - Compiling = true; - try - { - compiler.Compile(); - if (compiler.ModList?.ReportHTML != null) - { - this.HTMLReport = compiler.ModList.ReportHTML; - } } catch (Exception ex) { while (ex.InnerException != null) ex = ex.InnerException; Utils.Log($"Compiler error: {ex.ExceptionToString()}"); + return; } - finally + await Task.Run(() => { - Compiling = false; - } - }); + Compiling = true; + try + { + compiler.Compile(); + if (compiler.ModList?.ReportHTML != null) + { + this.HTMLReport = compiler.ModList.ReportHTML; + } + } + catch (Exception ex) + { + while (ex.InnerException != null) ex = ex.InnerException; + Utils.Log($"Compiler error: {ex.ExceptionToString()}"); + } + finally + { + Compiling = false; + } + }); + } } } } diff --git a/Wabbajack/View Models/InstallerVM.cs b/Wabbajack/View Models/InstallerVM.cs index 0506b307..1a0728b1 100644 --- a/Wabbajack/View Models/InstallerVM.cs +++ b/Wabbajack/View Models/InstallerVM.cs @@ -1,4 +1,4 @@ -using Syroot.Windows.IO; + using Syroot.Windows.IO; using System; using ReactiveUI; using System.Diagnostics; @@ -50,10 +50,15 @@ namespace Wabbajack [Reactive] public bool InstallingMode { get; set; } + [Reactive] + public bool IsMO2ModList { get; set; } + public FilePickerVM Location { get; } public FilePickerVM DownloadLocation { get; } + public FilePickerVM StagingLocation { get; } + private readonly ObservableAsPropertyHelper _ProgressPercent; public float ProgressPercent => _ProgressPercent.Value; @@ -114,15 +119,25 @@ namespace Wabbajack this.DownloadLocation.AdditionalError = this.WhenAny(x => x.DownloadLocation.TargetPath) .Select(x => Utils.IsDirectoryPathValid(x)); + StagingLocation = new FilePickerVM + { + DoExistsCheck = true, + PathType = FilePickerVM.PathTypeOptions.Folder, + PromptTitle = "Select your Vortex Staging Folder", + AdditionalError = this.WhenAny(x => x.StagingLocation.TargetPath) + .Select(Utils.IsDirectoryPathValid) + }; + // Load settings - InstallationSettings settings = this.MWVM.Settings.InstallationSettings.TryCreate(source); - this.Location.TargetPath = settings.InstallationLocation; - this.DownloadLocation.TargetPath = settings.DownloadLocation; + var settings = MWVM.Settings.InstallationSettings.TryCreate(source); this.MWVM.Settings.SaveSignal .Subscribe(_ => { - settings.InstallationLocation = this.Location.TargetPath; - settings.DownloadLocation = this.DownloadLocation.TargetPath; + settings.DownloadLocation = DownloadLocation.TargetPath; + if (IsMO2ModList) + settings.InstallationLocation = Location.TargetPath; + else + settings.StagingLocation = StagingLocation.TargetPath; }) .DisposeWith(this.CompositeDisposable); @@ -148,6 +163,38 @@ namespace Wabbajack }); return default(ModListVM); } + if (modList.ModManager == ModManager.Vortex) + { + IsMO2ModList = false; + StagingLocation.TargetPath = settings.StagingLocation; + + var vortexFolder = + Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), + "Vortex"); + var stagingFolder = Path.Combine(vortexFolder, GameRegistry.Games[modList.GameType].NexusName, + "mods"); + var downloadFolder = Path.Combine(vortexFolder, "downloads", + GameRegistry.Games[modList.GameType].NexusName); + MessageBox.Show( + "The ModList you are about to install was compiled from a Vortex installation. " + + "Vortex support is still very bleeding edge and installing this ModList WILL OVERRIDE your existing mods. " + + "If you encounter any errors during installation go to our discord and ping erri120#2285 with your error and a log file.", + "Important information regarding Vortex support", MessageBoxButton.OK, MessageBoxImage.Stop); + + if (!Directory.Exists(vortexFolder)) return new ModListVM(modList, modListPath); + if (Directory.Exists(stagingFolder) && + File.Exists(Path.Combine(stagingFolder, "__vortex_staging_folder"))) + StagingLocation.TargetPath = stagingFolder; + if (Directory.Exists(Path.Combine(vortexFolder, "downloads")) && + File.Exists(Path.Combine(vortexFolder, "downloads", "__vortex_downloads_folder"))) + DownloadLocation.TargetPath = downloadFolder; + } + else + { + Location.TargetPath = settings.InstallationLocation; + DownloadLocation.TargetPath = settings.DownloadLocation; + IsMO2ModList = true; + } return new ModListVM(modList, modListPath); }) .ObserveOnGuiThread() @@ -219,10 +266,13 @@ namespace Wabbajack this.WhenAny(x => x.Installing), this.WhenAny(x => x.Location.InError), this.WhenAny(x => x.DownloadLocation.InError), - resultSelector: (installing, loc, download) => + this.WhenAny(x => x.StagingLocation.InError), + resultSelector: (installing, loc, download, staging) => { if (installing) return false; - return !loc && !download; + if (IsMO2ModList) + return !loc && !download; + return !staging && !download; }) .ObserveOnGuiThread()); this.VisitWebsiteCommand = ReactiveCommand.Create( @@ -289,35 +339,68 @@ namespace Wabbajack private void ExecuteBegin() { - this.Installing = true; - this.InstallingMode = true; - var installer = new Installer(this.ModListPath, this.ModList.SourceModList, Location.TargetPath) + Installing = true; + InstallingMode = true; + if (ModList.ModManager == ModManager.Vortex) { - DownloadFolder = DownloadLocation.TargetPath - }; - var th = new Thread(() => + var installer = new VortexInstaller(ModListPath, ModList.SourceModList) + { + StagingFolder = StagingLocation.TargetPath, + DownloadFolder = DownloadLocation.TargetPath + }; + var th = new Thread(() => + { + try + { + installer.Install(); + } + catch (Exception ex) + { + while (ex.InnerException != null) ex = ex.InnerException; + Utils.Log(ex.StackTrace); + Utils.Log(ex.ToString()); + Utils.Log($"{ex.Message} - Can't continue"); + } + finally + { + Installing = false; + } + }) + { + Priority = ThreadPriority.BelowNormal + }; + th.Start(); + } + else { - try + var installer = new Installer(this.ModListPath, this.ModList.SourceModList, Location.TargetPath) { - installer.Install(); - } - catch (Exception ex) - { - while (ex.InnerException != null) ex = ex.InnerException; - Utils.Log(ex.StackTrace); - Utils.Log(ex.ToString()); - Utils.Log($"{ex.Message} - Can't continue"); - } - finally + DownloadFolder = DownloadLocation.TargetPath + }; + var th = new Thread(() => { + try + { + installer.Install(); + } + catch (Exception ex) + { + while (ex.InnerException != null) ex = ex.InnerException; + Utils.Log(ex.StackTrace); + Utils.Log(ex.ToString()); + Utils.Log($"{ex.Message} - Can't continue"); + } + finally + { - this.Installing = false; - } - }) - { - Priority = ThreadPriority.BelowNormal - }; - th.Start(); + this.Installing = false; + } + }) + { + Priority = ThreadPriority.BelowNormal + }; + th.Start(); + } } } } \ No newline at end of file diff --git a/Wabbajack/View Models/ModListVM.cs b/Wabbajack/View Models/ModListVM.cs index 95768222..c2bd6056 100644 --- a/Wabbajack/View Models/ModListVM.cs +++ b/Wabbajack/View Models/ModListVM.cs @@ -1,13 +1,8 @@ using ReactiveUI; -using ReactiveUI.Fody.Helpers; using System; -using System.Collections.Generic; using System.IO; using System.IO.Compression; -using System.Linq; using System.Reactive.Linq; -using System.Text; -using System.Threading.Tasks; using System.Windows.Media.Imaging; using Wabbajack.Common; using Wabbajack.Lib; @@ -18,13 +13,14 @@ namespace Wabbajack { public ModList SourceModList { get; } public string ModListPath { get; } - public string Name => this.SourceModList.Name; - public string ReportHTML => this.SourceModList.ReportHTML; - public string Readme => this.SourceModList.Readme; - public string ImageURL => this.SourceModList.Image; - public string Author => this.SourceModList.Author; - public string Description => this.SourceModList.Description; - public string Website => this.SourceModList.Website; + public string Name => SourceModList.Name; + public string ReportHTML => SourceModList.ReportHTML; + public string Readme => SourceModList.Readme; + public string ImageURL => SourceModList.Image; + public string Author => SourceModList.Author; + public string Description => SourceModList.Description; + public string Website => SourceModList.Website; + public ModManager ModManager => SourceModList.ModManager; // Image isn't exposed as a direct property, but as an observable. // This acts as a caching mechanism, as interested parties will trigger it to be created, @@ -33,10 +29,10 @@ namespace Wabbajack public ModListVM(ModList sourceModList, string modListPath) { - this.ModListPath = modListPath; - this.SourceModList = sourceModList; + ModListPath = modListPath; + SourceModList = sourceModList; - this.ImageObservable = Observable.Return(this.ImageURL) + ImageObservable = Observable.Return(this.ImageURL) .ObserveOn(RxApp.TaskpoolScheduler) .Select(url => { diff --git a/Wabbajack/Views/InstallationView.xaml b/Wabbajack/Views/InstallationView.xaml index b537fe0c..f5ba65a4 100644 --- a/Wabbajack/Views/InstallationView.xaml +++ b/Wabbajack/Views/InstallationView.xaml @@ -277,43 +277,93 @@ - - + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CompilerView.xaml - DownloadWindow.xaml