diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index ce5c1e3e..1d9deda2 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -4,23 +4,29 @@ using System.Linq; using System.Text; using System.Threading.Tasks; using VFS; +using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; namespace Wabbajack.Lib { public abstract class ACompiler { - protected string GamePath; + public ModManager ModManager; + public Compiler _mo2Compiler; + public VortexCompiler _vortexCompiler; - protected string ModListOutputFolder; - protected string ModListOutputFile; + public string GamePath; - protected List InstallDirectives; - protected List AllFiles; - protected ModList ModList; - protected VirtualFileSystem VFS; - protected List IndexedArchives; - protected Dictionary> IndexedFiles; + public string ModListOutputFolder; + public string ModListOutputFile; + + public List SelectedArchives; + public List InstallDirectives; + public List AllFiles; + public ModList ModList; + public VirtualFileSystem VFS; + public List IndexedArchives; + public Dictionary> IndexedFiles; public abstract void Info(string msg); public abstract void Status(string msg); diff --git a/Wabbajack.Lib/Compiler.cs b/Wabbajack.Lib/Compiler.cs index bf343416..3cf047e0 100644 --- a/Wabbajack.Lib/Compiler.cs +++ b/Wabbajack.Lib/Compiler.cs @@ -26,7 +26,7 @@ using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib { - public class Compiler + public class Compiler : ACompiler { private string _mo2DownloadsFolder; @@ -42,9 +42,25 @@ namespace Wabbajack.Lib public Compiler(string mo2_folder) { + _vortexCompiler = null; + _mo2Compiler = this; + ModManager = ModManager.MO2; + MO2Folder = mo2_folder; MO2Ini = Path.Combine(MO2Folder, "ModOrganizer.ini").LoadIniFile(); GamePath = ((string)MO2Ini.General.gamePath).Replace("\\\\", "\\"); + + ModListOutputFolder = "output_folder"; + ModListOutputFile = MO2Profile + ExtensionManager.Extension; + + SelectedArchives = new List(); + InstallDirectives = new List(); + AllFiles = new List(); + ModList = new ModList(); + + VFS = VirtualFileSystem.VFS; + IndexedArchives = new List(); + IndexedFiles = new Dictionary>(); } public dynamic MO2Ini { get; } @@ -70,55 +86,43 @@ namespace Wabbajack.Lib public string MO2ProfileDir => Path.Combine(MO2Folder, "profiles", MO2Profile); - public string ModListOutputFolder => "output_folder"; - public string ModListOutputFile => MO2Profile + ExtensionManager.Extension; - - public List InstallDirectives { get; private set; } internal UserStatus User { get; private set; } - public List SelectedArchives { get; private set; } - public List AllFiles { get; private set; } - public ModList ModList { get; private set; } public ConcurrentBag ExtraFiles { get; private set; } public Dictionary ModInis { get; private set; } - public VirtualFileSystem VFS => VirtualFileSystem.VFS; - - public List IndexedArchives { get; private set; } - public Dictionary> IndexedFiles { get; private set; } - public HashSet SelectedProfiles { get; set; } = new HashSet(); - public void Info(string msg) + public override void Info(string msg) { Utils.Log(msg); } - public void Status(string msg) + public override void Status(string msg) { WorkQueue.Report(msg, 0); } - private void Error(string msg) + public override void Error(string msg) { Utils.Log(msg); throw new Exception(msg); } - internal string IncludeFile(byte[] data) + internal override string IncludeFile(byte[] data) { var id = Guid.NewGuid().ToString(); File.WriteAllBytes(Path.Combine(ModListOutputFolder, id), data); return id; } - internal string IncludeFile(string data) + internal override string IncludeFile(string data) { var id = Guid.NewGuid().ToString(); File.WriteAllText(Path.Combine(ModListOutputFolder, id), data); return id; } - public bool Compile() + public override bool Compile() { VirtualFileSystem.Clean(); Info("Looking for other profiles"); @@ -283,6 +287,7 @@ namespace Wabbajack.Lib GameType = GameRegistry.Games.Values.First(f => f.MO2Name == MO2Ini.General.gameName).Game, WabbajackVersion = WabbajackVersion, Archives = SelectedArchives, + ModManager = ModManager.MO2, Directives = InstallDirectives, Name = ModListName ?? MO2Profile, Author = ModListAuthor ?? "", @@ -533,7 +538,7 @@ namespace Wabbajack.Lib } - public static Directive RunStack(IEnumerable stack, RawSourceFile source) + public override Directive RunStack(IEnumerable stack, RawSourceFile source) { Utils.Status($"Compiling {source.Path}"); foreach (var step in stack) @@ -545,7 +550,7 @@ namespace Wabbajack.Lib throw new InvalidDataException("Data fell out of the compilation stack"); } - public IEnumerable GetStack() + public override IEnumerable GetStack() { var user_config = Path.Combine(MO2ProfileDir, "compilation_stack.yml"); if (File.Exists(user_config)) @@ -566,7 +571,7 @@ namespace Wabbajack.Lib /// result included into the pack /// /// - public IEnumerable MakeStack() + public override IEnumerable MakeStack() { Utils.Log("Generating compilation stack"); return new List diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs new file mode 100644 index 00000000..c1293974 --- /dev/null +++ b/Wabbajack.Lib/VortexCompiler.cs @@ -0,0 +1,308 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using VFS; +using Wabbajack.Common; +using Wabbajack.Lib.CompilationSteps; + +namespace Wabbajack.Lib +{ + public class VortexCompiler : ACompiler + { + public string GameName { get; } + + public string VortexFolder { get; } + public string StagingFolder { get; } + public string DownloadsFolder { get; } + + public bool IgnoreMissingFiles { get; set; } + + public VortexCompiler(string gameName, string gamePath) + { + _vortexCompiler = this; + _mo2Compiler = null; + ModManager = ModManager.Vortex; + + // TODO: only for testing + IgnoreMissingFiles = true; + + GamePath = gamePath; + GameName = gameName; + + // currently only works if staging and downloads folder is in the standard directory + // aka %APPDATADA%\Vortex\ + VortexFolder = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "Vortex"); + StagingFolder = Path.Combine(VortexFolder, gameName, "mods"); + DownloadsFolder = Path.Combine(VortexFolder, "downloads", gameName); + + ModListOutputFolder = "output_folder"; + + // TODO: add custom modlist name + ModListOutputFile = $"VORTEX_TEST_MODLIST{ExtensionManager.Extension}"; + + VFS = VirtualFileSystem.VFS; + + SelectedArchives = new List(); + AllFiles = new List(); + IndexedArchives = new List(); + IndexedFiles = new Dictionary>(); + } + + public override void Info(string msg) + { + Utils.Log(msg); + } + + public override void Status(string msg) + { + WorkQueue.Report(msg, 0); + } + + public override void Error(string msg) + { + Utils.Log(msg); + throw new Exception(msg); + } + + internal override string IncludeFile(byte[] data) + { + var id = Guid.NewGuid().ToString(); + File.WriteAllBytes(Path.Combine(ModListOutputFolder, id), data); + return id; + } + + internal override string IncludeFile(string data) + { + var id = Guid.NewGuid().ToString(); + File.WriteAllText(Path.Combine(ModListOutputFolder, id), data); + return id; + } + + public override bool Compile() + { + VirtualFileSystem.Clean(); + Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}."); + + Info($"Indexing {GamePath}"); + VFS.AddRoot(GamePath); + + Info($"Indexing {DownloadsFolder}"); + VFS.AddRoot(DownloadsFolder); + + Info("Cleaning output folder"); + if (Directory.Exists(ModListOutputFolder)) Directory.Delete(ModListOutputFolder, true); + Directory.CreateDirectory(ModListOutputFolder); + + IEnumerable game_files = Directory.EnumerateFiles(GamePath, "*", SearchOption.AllDirectories) + .Where(p => p.FileExists()) + .Select(p => new RawSourceFile(VFS.Lookup(p)) + { Path = Alphaleonis.Win32.Filesystem.Path.Combine(Consts.GameFolderFilesDir, p.RelativeTo(GamePath)) }); + + Info("Indexing Archives"); + IndexedArchives = Directory.EnumerateFiles(DownloadsFolder) + .Where(File.Exists) + .Select(f => new IndexedArchive + { + File = VFS.Lookup(f), + Name = Path.GetFileName(f) + }) + .ToList(); + + Info("Indexing Files"); + IDictionary> grouped = VFS.GroupedByArchive(); + IndexedFiles = IndexedArchives.Select(f => grouped.TryGetValue(f.File, out var result) ? result : new List()) + .SelectMany(fs => fs) + .Concat(IndexedArchives.Select(f => f.File)) + .OrderByDescending(f => f.TopLevelArchive.LastModified) + .GroupBy(f => f.Hash) + .ToDictionary(f => f.Key, f => f.AsEnumerable()); + + Info("Searching for mod files"); + AllFiles = game_files.DistinctBy(f => f.Path).ToList(); + + Info($"Found {AllFiles.Count} files to build into mod list"); + + Info("Verifying destinations"); + List> dups = AllFiles.GroupBy(f => f.Path) + .Where(fs => fs.Count() > 1) + .Select(fs => + { + Utils.Log($"Duplicate files installed to {fs.Key} from : {String.Join(", ", fs.Select(f => f.AbsolutePath))}"); + return fs; + }).ToList(); + + if (dups.Count > 0) + { + Error($"Found {dups.Count} duplicates, exiting"); + } + + IEnumerable stack = MakeStack(); + + Info("Running Compilation Stack"); + List results = AllFiles.PMap(f => RunStack(stack, f)).ToList(); + + IEnumerable noMatch = results.OfType().ToList(); + Info($"No match for {noMatch.Count()} files"); + foreach (var file in noMatch) + Info($" {file.To}"); + if (noMatch.Any()) + { + if (IgnoreMissingFiles) + { + Info("Continuing even though files were missing at the request of the user."); + } + else + { + Info("Exiting due to no way to compile these files"); + return false; + } + } + + InstallDirectives = results.Where(i => !(i is IgnoredDirectly)).ToList(); + + // TODO: nexus stuff + /*Info("Getting Nexus api_key, please click authorize if a browser window appears"); + if (IndexedArchives.Any(a => a.IniData?.General?.gameName != null)) + { + var nexusClient = new NexusApiClient(); + if (!nexusClient.IsPremium) Error($"User {nexusClient.Username} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue"); + + } + */ + + GatherArchives(); + + ModList = new ModList + { + Archives = SelectedArchives, + ModManager = ModManager.Vortex, + Directives = InstallDirectives + }; + + ExportModList(); + + Info("Done Building ModList"); + return true; + } + + private void ExportModList() + { + Utils.Log($"Exporting ModList to: {ModListOutputFolder}"); + + // using JSON for better debugging + ModList.ToJSON(Path.Combine(ModListOutputFolder, "modlist.json")); + //ModList.ToCERAS(Path.Combine(ModListOutputFolder, "modlist"), ref CerasConfig.Config); + + if(File.Exists(ModListOutputFile)) + File.Delete(ModListOutputFile); + + using (var fs = new FileStream(ModListOutputFile, FileMode.Create)) + { + using (var za = new ZipArchive(fs, ZipArchiveMode.Create)) + { + Directory.EnumerateFiles(ModListOutputFolder, "*.*") + .DoProgress("Compressing Modlist", + f => + { + var ze = za.CreateEntry(Path.GetFileName(f)); + using (var os = ze.Open()) + using (var ins = File.OpenRead(f)) + { + ins.CopyTo(os); + } + }); + } + } + Utils.Log("Removing ModList staging folder"); + Directory.Delete(ModListOutputFolder, true); + } + + private void GatherArchives() + { + Info("Building a list of archives based on the files required"); + + var shas = InstallDirectives.OfType() + .Select(a => a.ArchiveHashPath[0]) + .Distinct(); + + var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified) + .GroupBy(f => f.File.Hash) + .ToDictionary(f => f.Key, f => f.First()); + + SelectedArchives = shas.PMap(sha => ResolveArchive(sha, archives)); + } + + // TODO: this whole thing + private Archive ResolveArchive(string sha, IDictionary archives) + { + if (archives.TryGetValue(sha, out var found)) + { + var result = new Archive(); + + result.Name = found.Name; + result.Hash = found.File.Hash; + result.Size = found.File.Size; + + return result; + } + + Error($"No match found for Archive sha: {sha} this shouldn't happen"); + return null; + } + + public override Directive RunStack(IEnumerable stack, RawSourceFile source) + { + Utils.Status($"Compiling {source.Path}"); + foreach (var step in stack) + { + var result = step.Run(source); + if (result != null) return result; + } + + throw new InvalidDataException("Data fell out of the compilation stack"); + + } + + public override IEnumerable GetStack() + { + var userConfig = Path.Combine(VortexFolder, "compilation_stack.yml"); + if (File.Exists(userConfig)) + return Serialization.Deserialize(File.ReadAllText(userConfig), this); + + IEnumerable stack = MakeStack(); + + File.WriteAllText(Path.Combine(VortexFolder, "_current_compilation_stack.yml"), + Serialization.Serialize(stack)); + + return stack; + } + + public override IEnumerable MakeStack() + { + Utils.Log("Generating compilation stack"); + return new List + { + //new IncludePropertyFiles(this), + + new IgnoreGameFiles(this), + + new IgnoreStartsWith(this, Path.Combine(Consts.GameFolderFilesDir, "Data")), + new IgnoreStartsWith(this, Path.Combine(Consts.GameFolderFilesDir, "Papyrus Compiler")), + new IgnoreStartsWith(this, Path.Combine(Consts.GameFolderFilesDir, "Skyrim")), + new IgnoreRegex(this, Consts.GameFolderFilesDir + "\\\\.*\\.bsa"), + + new DirectMatch(this), + + new IgnoreGameFiles(this), + + new IgnoreWabbajackInstallCruft(this), + + new DropAll(this) + }; + } + } +} diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 36c710cc..4620a8ff 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -77,6 +77,7 @@ + @@ -131,6 +132,7 @@ + WebAutomationWindow.xaml