From 61b91ad460aaf5092abf86653c30b406e6331a71 Mon Sep 17 00:00:00 2001 From: erri120 Date: Sat, 16 Nov 2019 14:22:40 +0100 Subject: [PATCH 01/18] Created an abstract Installer class and moved core functions over --- Wabbajack.Lib/AInstaller.cs | 293 ++++++++++++++++++++++++++ Wabbajack.Lib/Installer.cs | 321 ++--------------------------- Wabbajack.Lib/VortexInstaller.cs | 243 +--------------------- Wabbajack.Lib/Wabbajack.Lib.csproj | 1 + 4 files changed, 317 insertions(+), 541 deletions(-) create mode 100644 Wabbajack.Lib/AInstaller.cs diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs new file mode 100644 index 00000000..8c566eab --- /dev/null +++ b/Wabbajack.Lib/AInstaller.cs @@ -0,0 +1,293 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using Wabbajack.Common; +using Wabbajack.Lib.Downloaders; +using Wabbajack.VirtualFileSystem; +using Context = Wabbajack.VirtualFileSystem.Context; +using Directory = Alphaleonis.Win32.Filesystem.Directory; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.Lib +{ + public abstract class AInstaller + { + public bool IgnoreMissingFiles { get; internal set; } = false; + + public Context VFS { get; internal set; } = new Context(); + + public string OutputFolder { get; set; } + public string DownloadFolder { get; set; } + + public ModManager ModManager; + + public string ModListArchive { get; internal set; } + public ModList ModList { get; internal set; } + public Dictionary HashedArchives { get; set; } + + public abstract void Install(); + + public void Info(string msg) + { + Utils.Log(msg); + } + + public void Status(string msg) + { + WorkQueue.Report(msg, 0); + } + + public void Error(string msg) + { + Utils.Log(msg); + throw new Exception(msg); + } + + public byte[] LoadBytesFromPath(string path) + { + using (var fs = new FileStream(ModListArchive, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (var ar = new ZipArchive(fs, ZipArchiveMode.Read)) + using (var ms = new MemoryStream()) + { + var entry = ar.GetEntry(path); + using (var e = entry.Open()) + e.CopyTo(ms); + return ms.ToArray(); + } + } + + public static ModList LoadFromFile(string path) + { + using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (var ar = new ZipArchive(fs, ZipArchiveMode.Read)) + { + var entry = ar.GetEntry("modlist"); + if (entry == null) + { + entry = ar.GetEntry("modlist.json"); + using (var e = entry.Open()) + return e.FromJSON(); + } + using (var e = entry.Open()) + return e.FromCERAS(ref CerasConfig.Config); + } + } + + /// + /// We don't want to make the installer index all the archives, that's just a waste of time, so instead + /// we'll pass just enough information to VFS to let it know about the files we have. + /// + public void PrimeVFS() + { + VFS.AddKnown(HashedArchives.Select(a => new KnownFile + { + Paths = new[] { a.Value }, + Hash = a.Key + })); + + + VFS.AddKnown( + ModList.Directives + .OfType() + .Select(f => new KnownFile { Paths = f.ArchiveHashPath})); + + VFS.BackfillMissing(); + } + + public void BuildFolderStructure() + { + Info("Building Folder Structure"); + ModList.Directives + .Select(d => Path.Combine(OutputFolder, Path.GetDirectoryName(d.To))) + .ToHashSet() + .Do(f => + { + if (Directory.Exists(f)) return; + Directory.CreateDirectory(f); + }); + } + + public void InstallArchives() + { + Info("Installing Archives"); + Info("Grouping Install Files"); + var grouped = ModList.Directives + .OfType() + .GroupBy(e => e.ArchiveHashPath[0]) + .ToDictionary(k => k.Key); + var archives = ModList.Archives + .Select(a => new { Archive = a, AbsolutePath = HashedArchives.GetOrDefault(a.Hash) }) + .Where(a => a.AbsolutePath != null) + .ToList(); + + Info("Installing Archives"); + archives.PMap(a => InstallArchive(a.Archive, a.AbsolutePath, grouped[a.Archive.Hash])); + } + + private void InstallArchive(Archive archive, string absolutePath, IGrouping grouping) + { + Status($"Extracting {archive.Name}"); + + List vFiles = grouping.Select(g => + { + var file = VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath); + g.FromFile = file; + return g; + }).ToList(); + + var onFinish = VFS.Stage(vFiles.Select(f => f.FromFile).Distinct()); + + + Status($"Copying files for {archive.Name}"); + + void CopyFile(string from, string to, bool useMove) + { + if (File.Exists(to)) + { + var fi = new FileInfo(to); + if (fi.IsReadOnly) + fi.IsReadOnly = false; + File.Delete(to); + } + + if (File.Exists(from)) + { + var fi = new FileInfo(from); + if (fi.IsReadOnly) + fi.IsReadOnly = false; + } + + + if (useMove) + File.Move(from, to); + else + File.Copy(from, to); + } + + vFiles.GroupBy(f => f.FromFile) + .DoIndexed((idx, group) => + { + Utils.Status("Installing files", idx * 100 / vFiles.Count); + var firstDest = Path.Combine(OutputFolder, group.First().To); + CopyFile(group.Key.StagedPath, firstDest, true); + + foreach (var copy in group.Skip(1)) + { + var nextDest = Path.Combine(OutputFolder, copy.To); + CopyFile(firstDest, nextDest, false); + } + + }); + + Status("Unstaging files"); + onFinish(); + + // Now patch all the files from this archive + foreach (var toPatch in grouping.OfType()) + using (var patchStream = new MemoryStream()) + { + Status($"Patching {Path.GetFileName(toPatch.To)}"); + // Read in the patch data + + byte[] patchData = LoadBytesFromPath(toPatch.PatchID); + + var toFile = Path.Combine(OutputFolder, toPatch.To); + var oldData = new MemoryStream(File.ReadAllBytes(toFile)); + + // Remove the file we're about to patch + File.Delete(toFile); + + // Patch it + using (var outStream = File.OpenWrite(toFile)) + { + BSDiff.Apply(oldData, () => new MemoryStream(patchData), outStream); + } + + Status($"Verifying Patch {Path.GetFileName(toPatch.To)}"); + var resultSha = toFile.FileHash(); + if (resultSha != toPatch.Hash) + throw new InvalidDataException($"Invalid Hash for {toPatch.To} after patching"); + } + } + + public void DownloadArchives() + { + var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList(); + Info($"Missing {missing.Count} archives"); + + Info("Getting Nexus API Key, if a browser appears, please accept"); + + var dispatchers = missing.Select(m => m.State.GetDownloader()).Distinct(); + + foreach (var dispatcher in dispatchers) + dispatcher.Prepare(); + + DownloadMissingArchives(missing); + } + + private void DownloadMissingArchives(List missing, bool download = true) + { + if (download) + { + foreach (var a in missing.Where(a => a.State.GetType() == typeof(ManualDownloader.State))) + { + var outputPath = Path.Combine(DownloadFolder, a.Name); + a.State.Download(a, outputPath); + } + } + + missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State)) + .PMap(archive => + { + Info($"Downloading {archive.Name}"); + var outputPath = Path.Combine(DownloadFolder, archive.Name); + + if (download) + if (outputPath.FileExists()) + File.Delete(outputPath); + + return DownloadArchive(archive, download); + }); + } + + public bool DownloadArchive(Archive archive, bool download) + { + try + { + archive.State.Download(archive, Path.Combine(DownloadFolder, archive.Name)); + } + catch (Exception ex) + { + Utils.Log($"Download error for file {archive.Name}"); + Utils.Log(ex.ToString()); + return false; + } + + return false; + } + + public void HashArchives() + { + HashedArchives = Directory.EnumerateFiles(DownloadFolder) + .Where(e => !e.EndsWith(".sha")) + .PMap(e => (HashArchive(e), e)) + .OrderByDescending(e => File.GetLastWriteTime(e.Item2)) + .GroupBy(e => e.Item1) + .Select(e => e.First()) + .ToDictionary(e => e.Item1, e => e.Item2); + } + + public string HashArchive(string e) + { + var cache = e + ".sha"; + if (cache.FileExists() && new FileInfo(cache).LastWriteTime >= new FileInfo(e).LastWriteTime) + return File.ReadAllText(cache); + + Status($"Hashing {Path.GetFileName(e)}"); + File.WriteAllText(cache, e.FileHash()); + return HashArchive(e); + } + } +} diff --git a/Wabbajack.Lib/Installer.cs b/Wabbajack.Lib/Installer.cs index 4b38d7d5..f3d86826 100644 --- a/Wabbajack.Lib/Installer.cs +++ b/Wabbajack.Lib/Installer.cs @@ -1,16 +1,12 @@ using System; -using System.Collections.Generic; using System.IO; -using System.IO.Compression; using System.Linq; using System.Text; -using System.Threading.Tasks; using System.Windows; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.Validation; -using Wabbajack.VirtualFileSystem; using Directory = Alphaleonis.Win32.Filesystem.Directory; using File = Alphaleonis.Win32.Filesystem.File; using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; @@ -18,90 +14,24 @@ using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib { - public class Installer + public class Installer : AInstaller { - private string _downloadsFolder; - private WorkQueue Queue { get; set; } public Installer(string archive, ModList mod_list, string output_folder) { Queue = new WorkQueue(); VFS = new Context(Queue); + ModManager = ModManager.MO2; ModListArchive = archive; - Outputfolder = output_folder; + OutputFolder = output_folder; + DownloadFolder = Path.Combine(OutputFolder, "downloads"); ModList = mod_list; } - public Context VFS { get; } - - public string Outputfolder { get; } - - public string DownloadFolder - { - get => _downloadsFolder ?? Path.Combine(Outputfolder, "downloads"); - set => _downloadsFolder = value; - } - - public string ModListArchive { get; } - public ModList ModList { get; } - public Dictionary HashedArchives { get; private set; } - - public bool IgnoreMissingFiles { get; internal set; } public string GameFolder { get; set; } - public void Info(string msg) - { - Utils.Log(msg); - } - - public void Status(string msg) - { - Queue.Report(msg, 0); - } - - public void Status(string msg, int progress) - { - Queue.Report(msg, progress); - } - - private void Error(string msg) - { - Utils.Log(msg); - throw new Exception(msg); - } - - public byte[] LoadBytesFromPath(string path) - { - using (var fs = new FileStream(ModListArchive, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (var ar = new ZipArchive(fs, ZipArchiveMode.Read)) - using (var ms = new MemoryStream()) - { - var entry = ar.GetEntry(path); - using (var e = entry.Open()) - e.CopyTo(ms); - return ms.ToArray(); - } - } - - public static ModList LoadFromFile(string path) - { - using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (var ar = new ZipArchive(fs, ZipArchiveMode.Read)) - { - var entry = ar.GetEntry("modlist"); - if (entry == null) - { - entry = ar.GetEntry("modlist.json"); - using (var e = entry.Open()) - return e.FromJSON(); - } - using (var e = entry.Open()) - return e.FromCERAS(ref CerasConfig.Config); - } - } - - public void Install() + public override void Install() { var game = GameRegistry.Games[ModList.GameType]; @@ -121,10 +51,10 @@ namespace Wabbajack.Lib ValidateGameESMs(); ValidateModlist.RunValidation(ModList); - Directory.CreateDirectory(Outputfolder); + Directory.CreateDirectory(OutputFolder); Directory.CreateDirectory(DownloadFolder); - if (Directory.Exists(Path.Combine(Outputfolder, "mods"))) + if (Directory.Exists(Path.Combine(OutputFolder, "mods"))) { if (MessageBox.Show( "There already appears to be a Mod Organizer 2 install in this folder, are you sure you wish to continue" + @@ -159,7 +89,7 @@ namespace Wabbajack.Lib BuildFolderStructure(); InstallArchives(); InstallIncludedFiles(); - InctallIncludedDownloadMetas(); + InstallIncludedDownloadMetas(); BuildBSAs(); zEditIntegration.GenerateMerges(this); @@ -170,7 +100,7 @@ namespace Wabbajack.Lib //AskToEndorse(); } - private void InctallIncludedDownloadMetas() + private void InstallIncludedDownloadMetas() { ModList.Directives .OfType() @@ -233,27 +163,6 @@ namespace Wabbajack.Lib Info("Done! You may now exit the application!"); } - /// - /// We don't want to make the installer index all the archives, that's just a waste of time, so instead - /// we'll pass just enough information to VFS to let it know about the files we have. - /// - private void PrimeVFS() - { - VFS.AddKnown(HashedArchives.Select(a => new KnownFile - { - Paths = new[] { a.Value }, - Hash = a.Key - })); - - - VFS.AddKnown( - ModList.Directives - .OfType() - .Select(f => new KnownFile { Paths = f.ArchiveHashPath})); - - VFS.BackfillMissing(); - } - private void BuildBSAs() { var bsas = ModList.Directives.OfType().ToList(); @@ -262,7 +171,7 @@ namespace Wabbajack.Lib bsas.Do(bsa => { Status($"Building {bsa.To}"); - var source_dir = Path.Combine(Outputfolder, Consts.BSACreationDir, bsa.TempID); + var source_dir = Path.Combine(OutputFolder, Consts.BSACreationDir, bsa.TempID); using (var a = bsa.State.MakeBuilder()) { @@ -276,12 +185,12 @@ namespace Wabbajack.Lib }); Info($"Writing {bsa.To}"); - a.Build(Path.Combine(Outputfolder, bsa.To)); + a.Build(Path.Combine(OutputFolder, bsa.To)); } }); - var bsa_dir = Path.Combine(Outputfolder, Consts.BSACreationDir); + var bsa_dir = Path.Combine(OutputFolder, Consts.BSACreationDir); if (Directory.Exists(bsa_dir)) { Info($"Removing temp folder {Consts.BSACreationDir}"); @@ -297,7 +206,7 @@ namespace Wabbajack.Lib .PMap(Queue, directive => { Status($"Writing included file {directive.To}"); - var out_path = Path.Combine(Outputfolder, directive.To); + var out_path = Path.Combine(OutputFolder, directive.To); if (File.Exists(out_path)) File.Delete(out_path); if (directive is RemappedInlineFile) WriteRemappedFile((RemappedInlineFile)directive); @@ -321,7 +230,7 @@ namespace Wabbajack.Lib $"Cannot patch {filename} from the game folder hashes don't match have you already cleaned the file?"); var patch_data = LoadBytesFromPath(directive.SourceDataID); - var to_file = Path.Combine(Outputfolder, directive.To); + var to_file = Path.Combine(OutputFolder, directive.To); Status($"Patching {filename}"); using (var output = File.OpenWrite(to_file)) using (var input = File.OpenRead(game_file)) @@ -338,209 +247,15 @@ namespace Wabbajack.Lib data = data.Replace(Consts.GAME_PATH_MAGIC_DOUBLE_BACK, GameFolder.Replace("\\", "\\\\")); data = data.Replace(Consts.GAME_PATH_MAGIC_FORWARD, GameFolder.Replace("\\", "/")); - data = data.Replace(Consts.MO2_PATH_MAGIC_BACK, Outputfolder); - data = data.Replace(Consts.MO2_PATH_MAGIC_DOUBLE_BACK, Outputfolder.Replace("\\", "\\\\")); - data = data.Replace(Consts.MO2_PATH_MAGIC_FORWARD, Outputfolder.Replace("\\", "/")); + data = data.Replace(Consts.MO2_PATH_MAGIC_BACK, OutputFolder); + data = data.Replace(Consts.MO2_PATH_MAGIC_DOUBLE_BACK, OutputFolder.Replace("\\", "\\\\")); + data = data.Replace(Consts.MO2_PATH_MAGIC_FORWARD, OutputFolder.Replace("\\", "/")); data = data.Replace(Consts.DOWNLOAD_PATH_MAGIC_BACK, DownloadFolder); data = data.Replace(Consts.DOWNLOAD_PATH_MAGIC_DOUBLE_BACK, DownloadFolder.Replace("\\", "\\\\")); data = data.Replace(Consts.DOWNLOAD_PATH_MAGIC_FORWARD, DownloadFolder.Replace("\\", "/")); - File.WriteAllText(Path.Combine(Outputfolder, directive.To), data); - } - - private void BuildFolderStructure() - { - Info("Building Folder Structure"); - ModList.Directives - .Select(d => Path.Combine(Outputfolder, Path.GetDirectoryName(d.To))) - .ToHashSet() - .Do(f => - { - if (Directory.Exists(f)) return; - Directory.CreateDirectory(f); - }); - } - - private void InstallArchives() - { - Info("Installing Archives"); - Info("Grouping Install Files"); - var grouped = ModList.Directives - .OfType() - .GroupBy(e => e.ArchiveHashPath[0]) - .ToDictionary(k => k.Key); - var archives = ModList.Archives - .Select(a => new { Archive = a, AbsolutePath = HashedArchives.GetOrDefault(a.Hash) }) - .Where(a => a.AbsolutePath != null) - .ToList(); - - Info("Installing Archives"); - archives.PMap(Queue, a => InstallArchive(a.Archive, a.AbsolutePath, grouped[a.Archive.Hash])); - } - - private void InstallArchive(Archive archive, string absolutePath, IGrouping grouping) - { - Status($"Extracting {archive.Name}"); - - var vfiles = grouping.Select(g => - { - var file = VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath); - g.FromFile = file; - return g; - }).ToList(); - - var on_finish = VFS.Stage(vfiles.Select(f => f.FromFile).Distinct()); - - - Status($"Copying files for {archive.Name}"); - - void CopyFile(string from, string to, bool use_move) - { - if (File.Exists(to)) - { - var fi = new FileInfo(to); - if (fi.IsReadOnly) - fi.IsReadOnly = false; - File.Delete(to); - } - - if (File.Exists(from)) - { - var fi = new FileInfo(from); - if (fi.IsReadOnly) - fi.IsReadOnly = false; - } - - - if (use_move) - File.Move(from, to); - else - File.Copy(from, to); - } - - vfiles.GroupBy(f => f.FromFile) - .DoIndexed((idx, group) => - { - Utils.Status("Installing files", idx * 100 / vfiles.Count); - var first_dest = Path.Combine(Outputfolder, group.First().To); - CopyFile(group.Key.StagedPath, first_dest, true); - - foreach (var copy in group.Skip(1)) - { - var next_dest = Path.Combine(Outputfolder, copy.To); - CopyFile(first_dest, next_dest, false); - } - - }); - - Status("Unstaging files"); - on_finish(); - - // Now patch all the files from this archive - foreach (var to_patch in grouping.OfType()) - using (var patch_stream = new MemoryStream()) - { - Status($"Patching {Path.GetFileName(to_patch.To)}"); - // Read in the patch data - - var patch_data = LoadBytesFromPath(to_patch.PatchID); - - var to_file = Path.Combine(Outputfolder, to_patch.To); - var old_data = new MemoryStream(File.ReadAllBytes(to_file)); - - // Remove the file we're about to patch - File.Delete(to_file); - - // Patch it - using (var out_stream = File.OpenWrite(to_file)) - { - BSDiff.Apply(old_data, () => new MemoryStream(patch_data), out_stream); - } - - Status($"Verifying Patch {Path.GetFileName(to_patch.To)}"); - var result_sha = to_file.FileHash(); - if (result_sha != to_patch.Hash) - throw new InvalidDataException($"Invalid Hash for {to_patch.To} after patching"); - } - } - - private void DownloadArchives() - { - var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList(); - Info($"Missing {missing.Count} archives"); - - Info("Getting Nexus API Key, if a browser appears, please accept"); - - var dispatchers = missing.Select(m => m.State.GetDownloader()).Distinct(); - - foreach (var dispatcher in dispatchers) - dispatcher.Prepare(); - - DownloadMissingArchives(missing); - } - - private void DownloadMissingArchives(List missing, bool download = true) - { - if (download) - { - foreach (var a in missing.Where(a => a.State.GetType() == typeof(ManualDownloader.State))) - { - var output_path = Path.Combine(DownloadFolder, a.Name); - a.State.Download(a, output_path); - } - } - - missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State)) - .PMap(Queue, archive => - { - Info($"Downloading {archive.Name}"); - var output_path = Path.Combine(DownloadFolder, archive.Name); - - if (download) - if (output_path.FileExists()) - File.Delete(output_path); - - return DownloadArchive(archive, download); - }); - } - - public bool DownloadArchive(Archive archive, bool download) - { - try - { - archive.State.Download(archive, Path.Combine(DownloadFolder, archive.Name)); - } - catch (Exception ex) - { - Utils.Log($"Download error for file {archive.Name}"); - Utils.Log(ex.ToString()); - return false; - } - - return false; - } - - private void HashArchives() - { - HashedArchives = Directory.EnumerateFiles(DownloadFolder) - .Where(e => !e.EndsWith(".sha")) - .PMap(Queue, e => (HashArchive(e), e)) - .OrderByDescending(e => File.GetLastWriteTime(e.Item2)) - .GroupBy(e => e.Item1) - .Select(e => e.First()) - .ToDictionary(e => e.Item1, e => e.Item2); - } - - private string HashArchive(string e) - { - var cache = e + ".sha"; - if (cache.FileExists() && new FileInfo(cache).LastWriteTime >= new FileInfo(e).LastWriteTime) - return File.ReadAllText(cache); - - Status($"Hashing {Path.GetFileName(e)}"); - File.WriteAllText(cache, e.FileHash()); - return HashArchive(e); + File.WriteAllText(Path.Combine(OutputFolder, directive.To), data); } } } diff --git a/Wabbajack.Lib/VortexInstaller.cs b/Wabbajack.Lib/VortexInstaller.cs index 6fc8cf69..8f1b5f78 100644 --- a/Wabbajack.Lib/VortexInstaller.cs +++ b/Wabbajack.Lib/VortexInstaller.cs @@ -13,27 +13,18 @@ using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib { - public class VortexInstaller + public class VortexInstaller : AInstaller { - public string ModListArchive { get; } - public ModList ModList { get; } - public Dictionary HashedArchives { get; private set; } - public GameMetaData GameInfo { get; internal set; } - public string StagingFolder { get; set; } - public string DownloadFolder { get; set; } - public WorkQueue Queue { get; } - public Context VFS { get; } public bool IgnoreMissingFiles { get; internal set; } public VortexInstaller(string archive, ModList modList) { Queue = new WorkQueue(); - VFS = new Context(Queue); - + ModManager = ModManager.Vortex; ModListArchive = archive; ModList = modList; @@ -43,53 +34,7 @@ namespace Wabbajack.Lib GameInfo = GameRegistry.Games[ModList.GameType]; } - public void Info(string msg) - { - Utils.Log(msg); - } - - public void Status(string msg) - { - Queue.Report(msg, 0); - } - - private void Error(string msg) - { - Utils.Log(msg); - throw new Exception(msg); - } - - public byte[] LoadBytesFromPath(string path) - { - using (var fs = new FileStream(ModListArchive, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (var ar = new ZipArchive(fs, ZipArchiveMode.Read)) - using (var ms = new MemoryStream()) - { - var entry = ar.GetEntry(path); - using (var e = entry.Open()) - e.CopyTo(ms); - return ms.ToArray(); - } - } - - public static ModList LoadFromFile(string path) - { - using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (var ar = new ZipArchive(fs, ZipArchiveMode.Read)) - { - var entry = ar.GetEntry("modlist"); - if (entry == null) - { - entry = ar.GetEntry("modlist.json"); - using (var e = entry.Open()) - return e.FromJSON(); - } - using (var e = entry.Open()) - return e.FromCERAS(ref CerasConfig.Config); - } - } - - public void Install() + public override void Install() { Directory.CreateDirectory(DownloadFolder); @@ -113,85 +58,11 @@ namespace Wabbajack.Lib BuildFolderStructure(); InstallArchives(); InstallIncludedFiles(); - //InctallIncludedDownloadMetas(); + //InstallIncludedDownloadMetas(); Info("Installation complete! You may exit the program."); } - private void BuildFolderStructure() - { - Info("Building Folder Structure"); - ModList.Directives - .OfType() - .Select(d => Path.Combine(StagingFolder, Path.GetDirectoryName(d.To))) - .ToHashSet() - .Do(f => - { - if (Directory.Exists(f)) return; - Directory.CreateDirectory(f); - }); - } - - private void InstallArchives() - { - Info("Installing Archives"); - Info("Grouping Install Files"); - var grouped = ModList.Directives - .OfType() - .GroupBy(e => e.ArchiveHashPath[0]) - .ToDictionary(k => k.Key); - var archives = ModList.Archives - .Select(a => new { Archive = a, AbsolutePath = HashedArchives.GetOrDefault(a.Hash) }) - .Where(a => a.AbsolutePath != null) - .ToList(); - - Info("Installing Archives"); - archives.PMap(Queue,a => InstallArchive(a.Archive, a.AbsolutePath, grouped[a.Archive.Hash])); - } - - private void InstallArchive(Archive archive, string absolutePath, IGrouping grouping) - { - Status($"Extracting {archive.Name}"); - - var vFiles = grouping.Select(g => - { - var file = VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath); - g.FromFile = file; - return g; - }).ToList(); - - var onFinish = VFS.Stage(vFiles.Select(f => f.FromFile).Distinct()); - - Status($"Copying files for {archive.Name}"); - - void CopyFile(string from, string to, bool useMove) - { - if(File.Exists(to)) - File.Delete(to); - if (useMove) - File.Move(from, to); - else - File.Copy(from, to); - } - - vFiles.GroupBy(f => f.FromFile) - .DoIndexed((idx, group) => - { - Utils.Status("Installing files", idx * 100 / vFiles.Count); - var firstDest = Path.Combine(StagingFolder, group.First().To); - CopyFile(group.Key.StagedPath, firstDest, true); - - foreach (var copy in group.Skip(1)) - { - var nextDest = Path.Combine(StagingFolder, copy.To); - CopyFile(firstDest, nextDest, false); - } - }); - - Status("Unstaging files"); - onFinish(); - } - private void InstallIncludedFiles() { Info("Writing inline files"); @@ -199,114 +70,10 @@ namespace Wabbajack.Lib .PMap(Queue,directive => { Status($"Writing included file {directive.To}"); - var outPath = Path.Combine(StagingFolder, directive.To); + var outPath = Path.Combine(OutputFolder, directive.To); if(File.Exists(outPath)) File.Delete(outPath); File.WriteAllBytes(outPath, LoadBytesFromPath(directive.SourceDataID)); }); } - - /// - /// We don't want to make the installer index all the archives, that's just a waste of time, so instead - /// we'll pass just enough information to VFS to let it know about the files we have. - /// - private void PrimeVFS() - { - VFS.AddKnown(HashedArchives.Select(a => new KnownFile - { - Paths = new[] { a.Value }, - Hash = a.Key - })); - - - ModList.Directives - .OfType() - .Select(f => - { - var updated_path = new string[f.ArchiveHashPath.Length]; - f.ArchiveHashPath.CopyTo(updated_path, 0); - updated_path[0] = VFS.Index.ByHash[updated_path[0]].First(e => e.IsNative).FullPath; - return new KnownFile { Paths = updated_path }; - }); - - VFS.BackfillMissing(); - } - - private void DownloadArchives() - { - var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList(); - Info($"Missing {missing.Count} archives"); - - Info("Getting Nexus API Key, if a browser appears, please accept"); - - var dispatchers = missing.Select(m => m.State.GetDownloader()).Distinct(); - - foreach (var dispatcher in dispatchers) - dispatcher.Prepare(); - - DownloadMissingArchives(missing); - } - - private void DownloadMissingArchives(List missing, bool download = true) - { - if (download) - { - foreach (var a in missing.Where(a => a.State.GetType() == typeof(ManualDownloader.State))) - { - var output_path = Path.Combine(DownloadFolder, a.Name); - a.State.Download(a, output_path); - } - } - - missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State)) - .PMap(Queue, archive => - { - Info($"Downloading {archive.Name}"); - var output_path = Path.Combine(DownloadFolder, archive.Name); - - if (!download) return DownloadArchive(archive, download); - if (output_path.FileExists()) - File.Delete(output_path); - - return DownloadArchive(archive, download); - }); - } - - public bool DownloadArchive(Archive archive, bool download) - { - try - { - archive.State.Download(archive, Path.Combine(DownloadFolder, archive.Name)); - } - catch (Exception ex) - { - Utils.Log($"Download error for file {archive.Name}"); - Utils.Log(ex.ToString()); - return false; - } - - return false; - } - - private void HashArchives() - { - HashedArchives = Directory.EnumerateFiles(DownloadFolder) - .Where(e => !e.EndsWith(".sha")) - .PMap(Queue,e => (HashArchive(e), e)) - .OrderByDescending(e => File.GetLastWriteTime(e.Item2)) - .GroupBy(e => e.Item1) - .Select(e => e.First()) - .ToDictionary(e => e.Item1, e => e.Item2); - } - - private string HashArchive(string e) - { - var cache = e + ".sha"; - if (cache.FileExists() && new FileInfo(cache).LastWriteTime >= new FileInfo(e).LastWriteTime) - return File.ReadAllText(cache); - - Status($"Hashing {Path.GetFileName(e)}"); - File.WriteAllText(cache, e.FileHash()); - return HashArchive(e); - } } } diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 6c10d581..b17545b0 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -79,6 +79,7 @@ + From 03251804c1061b0662d6aed10f47a96e2d304c57 Mon Sep 17 00:00:00 2001 From: erri120 Date: Sat, 16 Nov 2019 14:23:41 +0100 Subject: [PATCH 02/18] Fixed typo --- Wabbajack.Lib/zEditIntegration.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Wabbajack.Lib/zEditIntegration.cs b/Wabbajack.Lib/zEditIntegration.cs index 73b7ec10..7991a6f9 100644 --- a/Wabbajack.Lib/zEditIntegration.cs +++ b/Wabbajack.Lib/zEditIntegration.cs @@ -169,12 +169,12 @@ namespace Wabbajack.Lib { Utils.LogStatus($"Generating zEdit merge: {m.To}"); - var src_data = m.Sources.Select(s => File.ReadAllBytes(Path.Combine(installer.Outputfolder, s.RelativePath))) + var src_data = m.Sources.Select(s => File.ReadAllBytes(Path.Combine(installer.OutputFolder, s.RelativePath))) .ConcatArrays(); var patch_data = installer.LoadBytesFromPath(m.PatchID); - using (var fs = File.OpenWrite(Path.Combine(installer.Outputfolder, m.To))) + using (var fs = File.OpenWrite(Path.Combine(installer.OutputFolder, m.To))) BSDiff.Apply(new MemoryStream(src_data), () => new MemoryStream(patch_data), fs); }); } From e845f767e6e267f9a47152f427971ce3a4c359fb Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 13:34:58 +0100 Subject: [PATCH 03/18] Fixed LoadAllGames failing when no GOGIDs List exist --- Wabbajack.Common/GOGHandler.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Wabbajack.Common/GOGHandler.cs b/Wabbajack.Common/GOGHandler.cs index 520eea7f..c85ebcbf 100644 --- a/Wabbajack.Common/GOGHandler.cs +++ b/Wabbajack.Common/GOGHandler.cs @@ -54,7 +54,7 @@ namespace Wabbajack.Common public void LoadAllGames() { Games = new HashSet(); - if (this.GOGKey == null) return; + if (GOGKey == null) return; string[] keys = GOGKey.GetSubKeyNames(); foreach (var key in keys) { @@ -66,7 +66,7 @@ namespace Wabbajack.Common }; game.Game = GameRegistry.Games.Values - .FirstOrDefault(g => g.GOGIDs.Contains(game.GameID))?.Game; + .FirstOrDefault(g => g.GOGIDs != null && g.GOGIDs.Contains(game.GameID))?.Game; Games.Add(game); } From e9a3030c91ae94220d08fdb929845745712131b3 Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 13:35:18 +0100 Subject: [PATCH 04/18] Removed Morrowind from Game enum --- Wabbajack.Common/GameMetaData.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Wabbajack.Common/GameMetaData.cs b/Wabbajack.Common/GameMetaData.cs index c1b6aee1..b0924515 100644 --- a/Wabbajack.Common/GameMetaData.cs +++ b/Wabbajack.Common/GameMetaData.cs @@ -9,7 +9,7 @@ namespace Wabbajack.Common public enum Game { //MO2 GAMES - Morrowind, + //Morrowind, Oblivion, [Description("Fallout 3")] Fallout3, From 801fa746255cc8a66cc5f4867905323b2656f314 Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 13:36:07 +0100 Subject: [PATCH 05/18] Only Vortex supported games will be shown when selecting Vortex compiler --- .../View Models/Compilers/VortexCompilerVM.cs | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Wabbajack/View Models/Compilers/VortexCompilerVM.cs b/Wabbajack/View Models/Compilers/VortexCompilerVM.cs index 94dc23e7..b45923ab 100644 --- a/Wabbajack/View Models/Compilers/VortexCompilerVM.cs +++ b/Wabbajack/View Models/Compilers/VortexCompilerVM.cs @@ -28,13 +28,14 @@ namespace Wabbajack private static ObservableCollectionExtended gameOptions = new ObservableCollectionExtended( EnumExt.GetValues() - .Select(g => new GameVM(g)) - .OrderBy(g => g.DisplayName)); + .Where(g => GameRegistry.Games[g].SupportedModManager == ModManager.Vortex) + .Select(g => new GameVM(g)) + .OrderBy(g => g.DisplayName)); public ObservableCollectionExtended GameOptions => gameOptions; [Reactive] - public GameVM SelectedGame { get; set; } = gameOptions.First(x => x.Game == Game.SkyrimSpecialEdition); + public GameVM SelectedGame { get; set; } [Reactive] public FilePickerVM GameLocation { get; set; } @@ -54,19 +55,19 @@ namespace Wabbajack public VortexCompilerVM(CompilerVM parent) { - this.GameLocation = new FilePickerVM() + GameLocation = new FilePickerVM() { ExistCheckOption = FilePickerVM.ExistCheckOptions.On, PathType = FilePickerVM.PathTypeOptions.Folder, PromptTitle = "Select Game Folder Location" }; - this.DownloadsLocation = new FilePickerVM() + DownloadsLocation = new FilePickerVM() { ExistCheckOption = FilePickerVM.ExistCheckOptions.On, PathType = FilePickerVM.PathTypeOptions.Folder, PromptTitle = "Select Downloads Folder" }; - this.StagingLocation = new FilePickerVM() + StagingLocation = new FilePickerVM() { ExistCheckOption = FilePickerVM.ExistCheckOptions.On, PathType = FilePickerVM.PathTypeOptions.Folder, @@ -74,7 +75,7 @@ namespace Wabbajack }; // Wire start command - this.BeginCommand = ReactiveCommand.CreateFromTask( + BeginCommand = ReactiveCommand.CreateFromTask( canExecute: Observable.CombineLatest( this.WhenAny(x => x.GameLocation.InError), this.WhenAny(x => x.DownloadsLocation.InError), @@ -117,12 +118,12 @@ namespace Wabbajack } }); }); - this._Compiling = this.BeginCommand.IsExecuting + _Compiling = this.BeginCommand.IsExecuting .ToProperty(this, nameof(this.Compiling)); // Load settings - this.settings = parent.MWVM.Settings.Compiler.VortexCompilation; - this.SelectedGame = gameOptions.First(x => x.Game == settings.LastCompiledGame); + settings = parent.MWVM.Settings.Compiler.VortexCompilation; + SelectedGame = gameOptions.FirstOrDefault(x => x.Game == settings.LastCompiledGame) ?? gameOptions[0]; parent.MWVM.Settings.SaveSignal .Subscribe(_ => Unload()) .DisposeWith(this.CompositeDisposable); From 8e920296b693e2861225131f007d1c13947b5c52 Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 13:42:31 +0100 Subject: [PATCH 06/18] VortexCompilerVM cleanup --- .../View Models/Compilers/VortexCompilerVM.cs | 114 ++++++++---------- 1 file changed, 53 insertions(+), 61 deletions(-) diff --git a/Wabbajack/View Models/Compilers/VortexCompilerVM.cs b/Wabbajack/View Models/Compilers/VortexCompilerVM.cs index b45923ab..57e036a9 100644 --- a/Wabbajack/View Models/Compilers/VortexCompilerVM.cs +++ b/Wabbajack/View Models/Compilers/VortexCompilerVM.cs @@ -1,9 +1,7 @@ using System; -using System.Collections.Generic; using System.Linq; using System.Reactive.Disposables; using System.Reactive.Linq; -using System.Text; using System.Threading.Tasks; using System.Windows.Input; using DynamicData.Binding; @@ -16,23 +14,23 @@ namespace Wabbajack { public class VortexCompilerVM : ViewModel, ISubCompilerVM { - private readonly VortexCompilationSettings settings; + private readonly VortexCompilationSettings _settings; public IReactiveCommand BeginCommand { get; } - private readonly ObservableAsPropertyHelper _Compiling; - public bool Compiling => _Compiling.Value; + private readonly ObservableAsPropertyHelper _compiling; + public bool Compiling => _compiling.Value; - private readonly ObservableAsPropertyHelper _ModlistSettings; - public ModlistSettingsEditorVM ModlistSettings => _ModlistSettings.Value; + private readonly ObservableAsPropertyHelper _modListSettings; + public ModlistSettingsEditorVM ModlistSettings => _modListSettings.Value; - private static ObservableCollectionExtended gameOptions = new ObservableCollectionExtended( + private static readonly ObservableCollectionExtended _gameOptions = new ObservableCollectionExtended( EnumExt.GetValues() .Where(g => GameRegistry.Games[g].SupportedModManager == ModManager.Vortex) .Select(g => new GameVM(g)) .OrderBy(g => g.DisplayName)); - public ObservableCollectionExtended GameOptions => gameOptions; + public ObservableCollectionExtended GameOptions => _gameOptions; [Reactive] public GameVM SelectedGame { get; set; } @@ -80,7 +78,7 @@ namespace Wabbajack this.WhenAny(x => x.GameLocation.InError), this.WhenAny(x => x.DownloadsLocation.InError), this.WhenAny(x => x.StagingLocation.InError), - resultSelector: (g, d, s) => !g && !d && !s) + (g, d, s) => !g && !d && !s) .ObserveOnGuiThread(), execute: async () => { @@ -88,11 +86,11 @@ namespace Wabbajack try { compiler = new VortexCompiler( - game: this.SelectedGame.Game, - gamePath: this.GameLocation.TargetPath, - vortexFolder: VortexCompiler.TypicalVortexFolder(), - downloadsFolder: this.DownloadsLocation.TargetPath, - stagingFolder: this.StagingLocation.TargetPath); + SelectedGame.Game, + GameLocation.TargetPath, + VortexCompiler.TypicalVortexFolder(), + DownloadsLocation.TargetPath, + StagingLocation.TargetPath); } catch (Exception ex) { @@ -118,105 +116,99 @@ namespace Wabbajack } }); }); - _Compiling = this.BeginCommand.IsExecuting - .ToProperty(this, nameof(this.Compiling)); + _compiling = BeginCommand.IsExecuting + .ToProperty(this, nameof(Compiling)); // Load settings - settings = parent.MWVM.Settings.Compiler.VortexCompilation; - SelectedGame = gameOptions.FirstOrDefault(x => x.Game == settings.LastCompiledGame) ?? gameOptions[0]; + _settings = parent.MWVM.Settings.Compiler.VortexCompilation; + SelectedGame = _gameOptions.FirstOrDefault(x => x.Game == _settings.LastCompiledGame) ?? _gameOptions[0]; parent.MWVM.Settings.SaveSignal .Subscribe(_ => Unload()) - .DisposeWith(this.CompositeDisposable); + .DisposeWith(CompositeDisposable); // Load custom game settings when game type changes this.WhenAny(x => x.SelectedGame) - .Select(game => settings.ModlistSettings.TryCreate(game.Game)) + .Select(game => _settings.ModlistSettings.TryCreate(game.Game)) .Pairwise() .Subscribe(pair => { // Save old - if (pair.Previous != null) + var (previous, current) = pair; + if (previous != null) { - pair.Previous.GameLocation = this.GameLocation.TargetPath; + previous.GameLocation = GameLocation.TargetPath; } // Load new - this.GameLocation.TargetPath = pair.Current?.GameLocation ?? null; - if (string.IsNullOrWhiteSpace(this.GameLocation.TargetPath)) + GameLocation.TargetPath = current?.GameLocation; + if (string.IsNullOrWhiteSpace(GameLocation.TargetPath)) { - this.SetGameToSteamLocation(); + SetGameToSteamLocation(); } - if (string.IsNullOrWhiteSpace(this.GameLocation.TargetPath)) + if (string.IsNullOrWhiteSpace(GameLocation.TargetPath)) { - this.SetGameToGogLocation(); + SetGameToGogLocation(); } - this.DownloadsLocation.TargetPath = pair.Current?.DownloadLocation ?? null; - if (string.IsNullOrWhiteSpace(this.DownloadsLocation.TargetPath)) + DownloadsLocation.TargetPath = current?.DownloadLocation; + if (string.IsNullOrWhiteSpace(DownloadsLocation.TargetPath)) { - this.DownloadsLocation.TargetPath = VortexCompiler.RetrieveDownloadLocation(this.SelectedGame.Game); + DownloadsLocation.TargetPath = VortexCompiler.RetrieveDownloadLocation(SelectedGame.Game); } - this.StagingLocation.TargetPath = pair.Current?.StagingLocation ?? null; - if (string.IsNullOrWhiteSpace(this.StagingLocation.TargetPath)) + StagingLocation.TargetPath = current?.StagingLocation; + if (string.IsNullOrWhiteSpace(StagingLocation.TargetPath)) { - this.StagingLocation.TargetPath = VortexCompiler.RetrieveStagingLocation(this.SelectedGame.Game); + StagingLocation.TargetPath = VortexCompiler.RetrieveStagingLocation(SelectedGame.Game); } }) - .DisposeWith(this.CompositeDisposable); + .DisposeWith(CompositeDisposable); - // Load custom modlist settings when game type changes - this._ModlistSettings = this.WhenAny(x => x.SelectedGame) + // Load custom ModList settings when game type changes + this._modListSettings = this.WhenAny(x => x.SelectedGame) .Select(game => { - var gameSettings = settings.ModlistSettings.TryCreate(game.Game); + var gameSettings = _settings.ModlistSettings.TryCreate(game.Game); return new ModlistSettingsEditorVM(gameSettings.ModlistSettings); }) // Interject and save old while loading new .Pairwise() .Do(pair => { - pair.Previous?.Save(); - pair.Current?.Init(); + var (previous, current) = pair; + previous?.Save(); + current?.Init(); }) .Select(x => x.Current) // Save to property .ObserveOnGuiThread() - .ToProperty(this, nameof(this.ModlistSettings)); + .ToProperty(this, nameof(ModlistSettings)); // Find game commands - this.FindGameInSteamCommand = ReactiveCommand.Create(SetGameToSteamLocation); - this.FindGameInGogCommand = ReactiveCommand.Create(SetGameToGogLocation); + FindGameInSteamCommand = ReactiveCommand.Create(SetGameToSteamLocation); + FindGameInGogCommand = ReactiveCommand.Create(SetGameToGogLocation); // Add additional criteria to download/staging folders - this.DownloadsLocation.AdditionalError = this.WhenAny(x => x.DownloadsLocation.TargetPath) - .Select(path => - { - if (path == null) return ErrorResponse.Success; - return VortexCompiler.IsValidDownloadsFolder(path); - }); - this.StagingLocation.AdditionalError = this.WhenAny(x => x.StagingLocation.TargetPath) - .Select(path => - { - if (path == null) return ErrorResponse.Success; - return VortexCompiler.IsValidBaseStagingFolder(path); - }); + DownloadsLocation.AdditionalError = this.WhenAny(x => x.DownloadsLocation.TargetPath) + .Select(path => path == null ? ErrorResponse.Success : VortexCompiler.IsValidDownloadsFolder(path)); + StagingLocation.AdditionalError = this.WhenAny(x => x.StagingLocation.TargetPath) + .Select(path => path == null ? ErrorResponse.Success : VortexCompiler.IsValidBaseStagingFolder(path)); } public void Unload() { - settings.LastCompiledGame = this.SelectedGame.Game; - this.ModlistSettings?.Save(); + _settings.LastCompiledGame = SelectedGame.Game; + ModlistSettings?.Save(); } private void SetGameToSteamLocation() { - var steamGame = SteamHandler.Instance.Games.FirstOrDefault(g => g.Game.HasValue && g.Game == this.SelectedGame.Game); - this.GameLocation.TargetPath = steamGame?.InstallDir; + var steamGame = SteamHandler.Instance.Games.FirstOrDefault(g => g.Game.HasValue && g.Game == SelectedGame.Game); + GameLocation.TargetPath = steamGame?.InstallDir; } private void SetGameToGogLocation() { - var gogGame = GOGHandler.Instance.Games.FirstOrDefault(g => g.Game.HasValue && g.Game == this.SelectedGame.Game); - this.GameLocation.TargetPath = gogGame?.Path; + var gogGame = GOGHandler.Instance.Games.FirstOrDefault(g => g.Game.HasValue && g.Game == SelectedGame.Game); + GameLocation.TargetPath = gogGame?.Path; } } } \ No newline at end of file From 1e303a8339b1593108749f1f2eeb5604e480a274 Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 14:35:02 +0100 Subject: [PATCH 07/18] Added RequiredFiles property to GameMetaData --- Wabbajack.Common/GameMetaData.cs | 120 ++++++++++++++++++++++++------- Wabbajack.Common/SteamHandler.cs | 6 +- 2 files changed, 99 insertions(+), 27 deletions(-) diff --git a/Wabbajack.Common/GameMetaData.cs b/Wabbajack.Common/GameMetaData.cs index b0924515..300a9030 100644 --- a/Wabbajack.Common/GameMetaData.cs +++ b/Wabbajack.Common/GameMetaData.cs @@ -55,6 +55,8 @@ namespace Wabbajack.Common public List GOGIDs { get; internal set; } // these are additional folders when a game installs mods outside the game folder public List AdditionalFolders { get; internal set; } + // file to check if the game is present, useful when steamIds and gogIds dont help + public List RequiredFiles { get; internal set; } public string GameLocation { @@ -99,7 +101,11 @@ namespace Wabbajack.Common MO2Name = "Oblivion", MO2ArchiveName = "oblivion", GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Oblivion", - SteamIDs = new List {22330} + SteamIDs = new List {22330}, + RequiredFiles = new List + { + "oblivion.exe" + } } }, @@ -112,7 +118,12 @@ namespace Wabbajack.Common MO2Name = "fallout3", MO2ArchiveName = "fallout3", GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Fallout3", - SteamIDs = new List {22300, 22370} // base game and GotY + SteamIDs = new List {22300, 22370}, // base game and GotY + RequiredFiles = new List + { + "falloutlauncher.exe", + "data\\fallout3.esm" + } } }, { @@ -124,7 +135,11 @@ namespace Wabbajack.Common MO2Name = "New Vegas", MO2ArchiveName = "falloutnv", GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\falloutnv", - SteamIDs = new List {22380} + SteamIDs = new List {22380}, + RequiredFiles = new List + { + "FalloutNV.exe" + } } }, { @@ -136,7 +151,11 @@ namespace Wabbajack.Common MO2Name = "Skyrim", MO2ArchiveName = "skyrim", GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\skyrim", - SteamIDs = new List {72850} + SteamIDs = new List {72850}, + RequiredFiles = new List + { + "tesv.exe" + } } }, { @@ -148,7 +167,11 @@ namespace Wabbajack.Common MO2Name = "Skyrim Special Edition", MO2ArchiveName = "skyrimse", GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Skyrim Special Edition", - SteamIDs = new List {489830} + SteamIDs = new List {489830}, + RequiredFiles = new List + { + "SkyrimSE.exe" + } } }, { @@ -160,7 +183,11 @@ namespace Wabbajack.Common MO2Name = "Fallout 4", MO2ArchiveName = "fallout4", GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Fallout4", - SteamIDs = new List {377160} + SteamIDs = new List {377160}, + RequiredFiles = new List + { + "Fallout4.exe" + } } }, /*{ @@ -183,7 +210,11 @@ namespace Wabbajack.Common MO2Name = "Skyrim VR", MO2ArchiveName = "skyrimse", GameLocationRegistryKey = @"HKEY_LOCAL_MACHINE\SOFTWARE\Bethesda Softworks\Skyrim VR", - SteamIDs = new List {611670} + SteamIDs = new List {611670}, + RequiredFiles = new List + { + "SkyrimVR.exe" + } } }, { @@ -193,20 +224,10 @@ namespace Wabbajack.Common Game = Game.DarkestDungeon, NexusName = "darkestdungeon", SteamIDs = new List {262060}, - GOGIDs = new List{1450711444} - } - }, - { - Game.DivinityOriginalSin2, new GameMetaData - { - SupportedModManager = ModManager.Vortex, - Game = Game.DivinityOriginalSin2, - NexusName = "divinityoriginalsin2", - SteamIDs = new List {435150}, - GOGIDs = new List{1584823040}, - AdditionalFolders = new List + GOGIDs = new List{1450711444}, + RequiredFiles = new List { - "%documents%\\Larian Studios\\Divinity Original Sin 2\\Mods\\", + "_windows\\Darkest.exe" } } }, @@ -221,6 +242,28 @@ namespace Wabbajack.Common AdditionalFolders = new List { "%documents%\\Larian Studios\\Divinity Original Sin 2 Definitive Edition\\Mods\\" + }, + RequiredFiles = new List + { + "DefEd\\bin\\SuppportTool.exe" + } + } + }, + { + Game.DivinityOriginalSin2, new GameMetaData + { + SupportedModManager = ModManager.Vortex, + Game = Game.DivinityOriginalSin2, + NexusName = "divinityoriginalsin2", + SteamIDs = new List {435150}, + GOGIDs = new List{1584823040}, + AdditionalFolders = new List + { + "%documents%\\Larian Studios\\Divinity Original Sin 2\\Mods\\", + }, + RequiredFiles = new List + { + "bin\\SuppportTool.exe" } } }, @@ -231,7 +274,11 @@ namespace Wabbajack.Common Game = Game.Starbound, NexusName = "starbound", SteamIDs = new List{211820}, - GOGIDs = new List{1452598881} + GOGIDs = new List{1452598881}, + RequiredFiles = new List + { + "win64\\starbound.exe" + } } }, { @@ -241,7 +288,11 @@ namespace Wabbajack.Common Game = Game.SWKOTOR, NexusName = "kotor", SteamIDs = new List{32370}, - GOGIDs = new List{1207666283} + GOGIDs = new List{1207666283}, + RequiredFiles = new List + { + "swkotor.exe" + } } }, { @@ -251,7 +302,11 @@ namespace Wabbajack.Common Game = Game.SWKOTOR2, NexusName = "kotor2", SteamIDs = new List{208580}, - GOGIDs = new List{1421404581} + GOGIDs = new List{1421404581}, + RequiredFiles = new List + { + "swkotor2.exe" + } } }, { @@ -261,7 +316,11 @@ namespace Wabbajack.Common Game = Game.Witcher, NexusName = "witcher", SteamIDs = new List{20900}, - GOGIDs = new List{1207658924} + GOGIDs = new List{1207658924}, + RequiredFiles = new List + { + "system\\witcher.exe" + } } }, { @@ -271,7 +330,12 @@ namespace Wabbajack.Common Game = Game.Witcher2, NexusName = "witcher2", SteamIDs = new List{20920}, - GOGIDs = new List{1207658930} + GOGIDs = new List{1207658930}, + RequiredFiles = new List + { + "bin\\witcher2.exe", + "bin\\userContentManager.exe" + } } }, { @@ -281,7 +345,11 @@ namespace Wabbajack.Common Game = Game.Witcher3, NexusName = "witcher3", SteamIDs = new List{292030, 499450}, // normal and GotY - GOGIDs = new List{1207664643, 1495134320, 1207664663, 1640424747} // normal, GotY and both in packages + GOGIDs = new List{1207664643, 1495134320, 1207664663, 1640424747}, // normal, GotY and both in packages + RequiredFiles = new List + { + "bin\\x64\\witcher2.exe" + } } } }; diff --git a/Wabbajack.Common/SteamHandler.cs b/Wabbajack.Common/SteamHandler.cs index 7831602e..0d91a489 100644 --- a/Wabbajack.Common/SteamHandler.cs +++ b/Wabbajack.Common/SteamHandler.cs @@ -105,7 +105,11 @@ namespace Wabbajack.Common }); steamGame.Game = GameRegistry.Games.Values - .FirstOrDefault(g => g.SteamIDs.Contains(steamGame.AppId))?.Game; + .FirstOrDefault(g => + g.SteamIDs.Contains(steamGame.AppId) + && + g.RequiredFiles.TrueForAll(s => File.Exists(Path.Combine(steamGame.InstallDir, s))) + )?.Game; games.Add(steamGame); }); }); From eed53e38df29f000118d15b0361af8eab35e7fd2 Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 14:41:08 +0100 Subject: [PATCH 08/18] Added nosettings argument --- Wabbajack/Settings.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Wabbajack/Settings.cs b/Wabbajack/Settings.cs index 819269c0..0be19afc 100644 --- a/Wabbajack/Settings.cs +++ b/Wabbajack/Settings.cs @@ -32,7 +32,8 @@ namespace Wabbajack public static MainSettings LoadSettings() { - if (!File.Exists(Filename)) return new MainSettings(); + string[] args = Environment.GetCommandLineArgs(); + if (!File.Exists(Filename) || args[1] == "nosettings") return new MainSettings(); return JsonConvert.DeserializeObject(File.ReadAllText(Filename)); } From 76db3cafea4eaa1376cf8e4edad80ad5252ec483 Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 14:44:32 +0100 Subject: [PATCH 09/18] Fixed typo --- Wabbajack.Common/GameMetaData.cs | 4 ++-- Wabbajack/Wabbajack.csproj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Wabbajack.Common/GameMetaData.cs b/Wabbajack.Common/GameMetaData.cs index 300a9030..4d4a470b 100644 --- a/Wabbajack.Common/GameMetaData.cs +++ b/Wabbajack.Common/GameMetaData.cs @@ -245,7 +245,7 @@ namespace Wabbajack.Common }, RequiredFiles = new List { - "DefEd\\bin\\SuppportTool.exe" + "DefEd\\bin\\SupportTool.exe" } } }, @@ -263,7 +263,7 @@ namespace Wabbajack.Common }, RequiredFiles = new List { - "bin\\SuppportTool.exe" + "bin\\SupportTool.exe" } } }, diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index d040cdef..b7a46389 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -17,6 +17,7 @@ false + True publish\ true Disk @@ -31,7 +32,6 @@ 1.0.0.%2a false true - True x64 From ed2a6c559268ec4bbbbcd39c4410010b5e659d39 Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 14:46:10 +0100 Subject: [PATCH 10/18] Updated GOGHandler with new check for RequiredFiles --- Wabbajack.Common/GOGHandler.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Wabbajack.Common/GOGHandler.cs b/Wabbajack.Common/GOGHandler.cs index c85ebcbf..d63826ad 100644 --- a/Wabbajack.Common/GOGHandler.cs +++ b/Wabbajack.Common/GOGHandler.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using Microsoft.Win32; @@ -66,7 +67,9 @@ namespace Wabbajack.Common }; game.Game = GameRegistry.Games.Values - .FirstOrDefault(g => g.GOGIDs != null && g.GOGIDs.Contains(game.GameID))?.Game; + .FirstOrDefault(g => g.GOGIDs != null && g.GOGIDs.Contains(game.GameID) + && + g.RequiredFiles.TrueForAll(s => File.Exists(Path.Combine(game.Path, s))))?.Game; Games.Add(game); } From bfbdc10860cf3e5025eba02c2757e9ace59d815e Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 14:51:28 +0100 Subject: [PATCH 11/18] Fixed Game not being set and vortex file not being ignored --- Wabbajack.Lib/VortexCompiler.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs index 1244591f..1642111d 100644 --- a/Wabbajack.Lib/VortexCompiler.cs +++ b/Wabbajack.Lib/VortexCompiler.cs @@ -33,17 +33,18 @@ namespace Wabbajack.Lib public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder, string stagingFolder) { ModManager = ModManager.Vortex; + Game = game; // TODO: only for testing IgnoreMissingFiles = true; GamePath = gamePath; GameName = GameRegistry.Games[game].NexusName; - this.VortexFolder = vortexFolder; - this.DownloadsFolder = downloadsFolder; - this.StagingFolder = stagingFolder; Queue = new WorkQueue(); VFS = new Context(Queue); + VortexFolder = vortexFolder; + DownloadsFolder = downloadsFolder; + StagingFolder = stagingFolder; ModListOutputFolder = "output_folder"; // TODO: add custom modlist name @@ -413,6 +414,7 @@ namespace Wabbajack.Lib new IncludeVortexDeployment(this), new IncludeRegex(this, "^*\\.meta"), new IgnoreVortex(this), + new IgnoreRegex(this, "^*__vortex_staging_folder$"), Game == Game.DarkestDungeon ? new IncludeRegex(this, "project\\.xml$") : null, From a10a3ff1ff46d813d0edca5e8e47e538f1428ede Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 14:52:00 +0100 Subject: [PATCH 12/18] ModList staging folder will be deleted when finished --- Wabbajack.Lib/VortexCompiler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs index 1642111d..4eac8ec6 100644 --- a/Wabbajack.Lib/VortexCompiler.cs +++ b/Wabbajack.Lib/VortexCompiler.cs @@ -266,7 +266,7 @@ namespace Wabbajack.Lib metadata.ToJSON(ModListOutputFile + ".meta.json"); Utils.Log("Removing ModList staging folder"); - //Directory.Delete(ModListOutputFolder, true); + Directory.Delete(ModListOutputFolder, true); } /*private void GenerateReport() From c22cc20fc8817768b602f835c1993a25edb0c71b Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 15:06:28 +0100 Subject: [PATCH 13/18] Rebase fixes --- Wabbajack.Lib/AInstaller.cs | 17 ++++++++++++----- Wabbajack.Lib/Compiler.cs | 3 +-- Wabbajack.Lib/Installer.cs | 7 +++---- Wabbajack.Lib/VortexCompiler.cs | 5 +++-- Wabbajack.Lib/VortexInstaller.cs | 15 +++------------ 5 files changed, 22 insertions(+), 25 deletions(-) diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 8c566eab..9ba3f525 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -16,7 +16,9 @@ namespace Wabbajack.Lib { public bool IgnoreMissingFiles { get; internal set; } = false; - public Context VFS { get; internal set; } = new Context(); + public StatusUpdateTracker UpdateTracker { get; protected set; } + public WorkQueue Queue { get; protected set; } + public Context VFS { get; internal set; } public string OutputFolder { get; set; } public string DownloadFolder { get; set; } @@ -27,6 +29,11 @@ namespace Wabbajack.Lib public ModList ModList { get; internal set; } public Dictionary HashedArchives { get; set; } + protected AInstaller() + { + Queue = new WorkQueue(); + } + public abstract void Install(); public void Info(string msg) @@ -36,7 +43,7 @@ namespace Wabbajack.Lib public void Status(string msg) { - WorkQueue.Report(msg, 0); + Queue.Report(msg, 0); } public void Error(string msg) @@ -123,7 +130,7 @@ namespace Wabbajack.Lib .ToList(); Info("Installing Archives"); - archives.PMap(a => InstallArchive(a.Archive, a.AbsolutePath, grouped[a.Archive.Hash])); + archives.PMap(Queue,a => InstallArchive(a.Archive, a.AbsolutePath, grouped[a.Archive.Hash])); } private void InstallArchive(Archive archive, string absolutePath, IGrouping grouping) @@ -239,7 +246,7 @@ namespace Wabbajack.Lib } missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State)) - .PMap(archive => + .PMap(Queue, archive => { Info($"Downloading {archive.Name}"); var outputPath = Path.Combine(DownloadFolder, archive.Name); @@ -272,7 +279,7 @@ namespace Wabbajack.Lib { HashedArchives = Directory.EnumerateFiles(DownloadFolder) .Where(e => !e.EndsWith(".sha")) - .PMap(e => (HashArchive(e), e)) + .PMap(Queue, e => (HashArchive(e), e)) .OrderByDescending(e => File.GetLastWriteTime(e.Item2)) .GroupBy(e => e.Item1) .Select(e => e.First()) diff --git a/Wabbajack.Lib/Compiler.cs b/Wabbajack.Lib/Compiler.cs index 7e2627ea..9a8f20b7 100644 --- a/Wabbajack.Lib/Compiler.cs +++ b/Wabbajack.Lib/Compiler.cs @@ -45,8 +45,7 @@ namespace Wabbajack.Lib public Compiler(string mo2_folder) { UpdateTracker = new StatusUpdateTracker(10); - Queue = new WorkQueue(); - VFS = new Context(Queue) {UpdateTracker = UpdateTracker}; + VFS = new Context(Queue) {UpdateTracker = UpdateTracker}; ModManager = ModManager.MO2; MO2Folder = mo2_folder; diff --git a/Wabbajack.Lib/Installer.cs b/Wabbajack.Lib/Installer.cs index f3d86826..8172427f 100644 --- a/Wabbajack.Lib/Installer.cs +++ b/Wabbajack.Lib/Installer.cs @@ -7,6 +7,7 @@ using Wabbajack.Common; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.Validation; +using Wabbajack.VirtualFileSystem; using Directory = Alphaleonis.Win32.Filesystem.Directory; using File = Alphaleonis.Win32.Filesystem.File; using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; @@ -16,12 +17,10 @@ namespace Wabbajack.Lib { public class Installer : AInstaller { - private WorkQueue Queue { get; set; } - public Installer(string archive, ModList mod_list, string output_folder) { - Queue = new WorkQueue(); - VFS = new Context(Queue); + UpdateTracker = new StatusUpdateTracker(10); + VFS = new Context(Queue) {UpdateTracker = UpdateTracker}; ModManager = ModManager.MO2; ModListArchive = archive; OutputFolder = output_folder; diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs index 4eac8ec6..846672be 100644 --- a/Wabbajack.Lib/VortexCompiler.cs +++ b/Wabbajack.Lib/VortexCompiler.cs @@ -32,6 +32,9 @@ namespace Wabbajack.Lib public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder, string stagingFolder) { + UpdateTracker = new StatusUpdateTracker(10); + VFS = new Context(Queue) {UpdateTracker = UpdateTracker}; + ModManager = ModManager.Vortex; Game = game; @@ -40,8 +43,6 @@ namespace Wabbajack.Lib GamePath = gamePath; GameName = GameRegistry.Games[game].NexusName; - Queue = new WorkQueue(); - VFS = new Context(Queue); VortexFolder = vortexFolder; DownloadsFolder = downloadsFolder; StagingFolder = stagingFolder; diff --git a/Wabbajack.Lib/VortexInstaller.cs b/Wabbajack.Lib/VortexInstaller.cs index 8f1b5f78..94f5f949 100644 --- a/Wabbajack.Lib/VortexInstaller.cs +++ b/Wabbajack.Lib/VortexInstaller.cs @@ -1,14 +1,8 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Linq; +using System.Linq; using Wabbajack.Common; -using Wabbajack.Lib.Downloaders; using Wabbajack.VirtualFileSystem; using Directory = Alphaleonis.Win32.Filesystem.Directory; using File = Alphaleonis.Win32.Filesystem.File; -using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib @@ -17,13 +11,10 @@ namespace Wabbajack.Lib { public GameMetaData GameInfo { get; internal set; } - public WorkQueue Queue { get; } - - public bool IgnoreMissingFiles { get; internal set; } - public VortexInstaller(string archive, ModList modList) { - Queue = new WorkQueue(); + UpdateTracker = new StatusUpdateTracker(10); + VFS = new Context(Queue) {UpdateTracker = UpdateTracker}; ModManager = ModManager.Vortex; ModListArchive = archive; ModList = modList; From 24378ac5530f1ef5663182ac21742353e045386d Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 15:30:06 +0100 Subject: [PATCH 14/18] VortexCompiler include Properties --- Wabbajack.Lib/ACompiler.cs | 3 +++ .../CompilationSteps/IncludePropertyFiles.cs | 13 ++++++------- Wabbajack.Lib/Compiler.cs | 10 ---------- Wabbajack.Lib/VortexCompiler.cs | 13 +++++++++---- Wabbajack/View Models/Compilers/VortexCompilerVM.cs | 10 +++++++++- 5 files changed, 27 insertions(+), 22 deletions(-) diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index f7c5ab5c..68de8ddf 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -12,6 +12,9 @@ namespace Wabbajack.Lib { public abstract class ACompiler { + public string ModListName, ModListAuthor, ModListDescription, ModListImage, ModListWebsite, ModListReadme; + public string WabbajackVersion; + public StatusUpdateTracker UpdateTracker { get; protected set; } public WorkQueue Queue { get; protected set; } diff --git a/Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs b/Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs index fa6f3d78..98a992f7 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs @@ -2,43 +2,42 @@ using System.Linq; using Alphaleonis.Win32.Filesystem; using Newtonsoft.Json; +using Wabbajack.Common; namespace Wabbajack.Lib.CompilationSteps { public class IncludePropertyFiles : ACompilationStep { - private readonly Compiler _mo2Compiler; public IncludePropertyFiles(ACompiler compiler) : base(compiler) { - _mo2Compiler = (Compiler) compiler; } public override Directive Run(RawSourceFile source) { var files = new HashSet { - _mo2Compiler.ModListImage, _mo2Compiler.ModListReadme + _compiler.ModListImage, _compiler.ModListReadme }; if (!files.Any(f => source.AbsolutePath.Equals(f))) return null; if (!File.Exists(source.AbsolutePath)) return null; - var isBanner = source.AbsolutePath == _mo2Compiler.ModListImage; + var isBanner = source.AbsolutePath == _compiler.ModListImage; //var isReadme = source.AbsolutePath == ModListReadme; var result = source.EvolveTo(); result.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath)); if (isBanner) { result.Type = PropertyType.Banner; - _mo2Compiler.ModListImage = result.SourceDataID; + _compiler.ModListImage = result.SourceDataID; } else { result.Type = PropertyType.Readme; - _mo2Compiler.ModListReadme = result.SourceDataID; + _compiler.ModListReadme = result.SourceDataID; } return result; - } + } public override IState GetState() { diff --git a/Wabbajack.Lib/Compiler.cs b/Wabbajack.Lib/Compiler.cs index 9a8f20b7..7daa1144 100644 --- a/Wabbajack.Lib/Compiler.cs +++ b/Wabbajack.Lib/Compiler.cs @@ -1,18 +1,12 @@ using CommonMark; using Compression.BSA; using System; -using System.CodeDom; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Linq; -using System.Reactive.Subjects; -using System.Reflection; -using System.Runtime.InteropServices.ComTypes; -using System.Text; -using System.Text.RegularExpressions; using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; @@ -23,7 +17,6 @@ using Wabbajack.Lib.Validation; using Wabbajack.VirtualFileSystem; using Directory = Alphaleonis.Win32.Filesystem.Directory; using File = Alphaleonis.Win32.Filesystem.File; -using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib @@ -37,10 +30,7 @@ namespace Wabbajack.Lib public string MO2Folder; - public string MO2Profile; - public string ModListName, ModListAuthor, ModListDescription, ModListWebsite, ModListImage, ModListReadme; - public string WabbajackVersion; public Compiler(string mo2_folder) { diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs index 846672be..de4648bd 100644 --- a/Wabbajack.Lib/VortexCompiler.cs +++ b/Wabbajack.Lib/VortexCompiler.cs @@ -47,9 +47,6 @@ namespace Wabbajack.Lib DownloadsFolder = downloadsFolder; StagingFolder = stagingFolder; ModListOutputFolder = "output_folder"; - - // TODO: add custom modlist name - ModListOutputFile = $"VORTEX_TEST_MODLIST{ExtensionManager.Extension}"; } public override void Info(string msg) @@ -84,6 +81,8 @@ namespace Wabbajack.Lib public override bool Compile() { + ModListOutputFile = $"{ModListName}{ExtensionManager.Extension}"; + Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}."); Info("Starting pre-compilation steps"); @@ -197,6 +196,12 @@ namespace Wabbajack.Lib ModList = new ModList { + Name = ModListName ?? $"Vortex ModList for {Game.ToString()}", + Author = ModListAuthor ?? "", + Description = ModListDescription ?? "", + Readme = ModListReadme ?? "", + Image = ModListImage ?? "", + Website = ModListWebsite ?? "", Archives = SelectedArchives, ModManager = ModManager.Vortex, Directives = InstallDirectives, @@ -411,7 +416,7 @@ namespace Wabbajack.Lib Utils.Log("Generating compilation stack"); return new List { - //new IncludePropertyFiles(this), + new IncludePropertyFiles(this), new IncludeVortexDeployment(this), new IncludeRegex(this, "^*\\.meta"), new IgnoreVortex(this), diff --git a/Wabbajack/View Models/Compilers/VortexCompilerVM.cs b/Wabbajack/View Models/Compilers/VortexCompilerVM.cs index 57e036a9..f4a2cd9b 100644 --- a/Wabbajack/View Models/Compilers/VortexCompilerVM.cs +++ b/Wabbajack/View Models/Compilers/VortexCompilerVM.cs @@ -90,7 +90,15 @@ namespace Wabbajack GameLocation.TargetPath, VortexCompiler.TypicalVortexFolder(), DownloadsLocation.TargetPath, - StagingLocation.TargetPath); + StagingLocation.TargetPath) + { + ModListName = ModlistSettings.ModListName, + ModListAuthor = ModlistSettings.AuthorText, + ModListDescription = ModlistSettings.Description, + ModListImage = ModlistSettings.ImagePath.TargetPath, + ModListWebsite = ModlistSettings.Website, + ModListReadme = ModlistSettings.ReadMeText.TargetPath + }; } catch (Exception ex) { From b21fdfbd8c5856471b53ad8886d79637946033d6 Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 15:54:04 +0100 Subject: [PATCH 15/18] VortexCompiler will now export MD and HTML files --- Wabbajack.Lib/ReportBuilder.cs | 20 ++++++++++++++------ Wabbajack.Lib/VortexCompiler.cs | 22 +++++++++++++++++++--- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/Wabbajack.Lib/ReportBuilder.cs b/Wabbajack.Lib/ReportBuilder.cs index a87b1f3e..4a5807c5 100644 --- a/Wabbajack.Lib/ReportBuilder.cs +++ b/Wabbajack.Lib/ReportBuilder.cs @@ -44,18 +44,26 @@ namespace Wabbajack.Lib wtr.WriteLine(txt); } - public void Build(Compiler c, ModList lst) + public void Build(ACompiler c, ModList lst) { + Compiler compiler = null; + if (lst.ModManager == ModManager.MO2) + compiler = (Compiler) c; + Text($"### {lst.Name} by {lst.Author} - Installation Summary"); Text($"Build with Wabbajack Version {lst.WabbajackVersion}"); Text(lst.Description); - Text($"#### Website:"); + Text("#### Website:"); NoWrapText($"[{lst.Website}]({lst.Website})"); + Text($"Mod Manager: {lst.ModManager.ToString()}"); - var readme_file = Path.Combine(c.MO2ProfileDir, "readme.md"); - if (File.Exists(readme_file)) - File.ReadAllLines(readme_file) - .Do(NoWrapText); + if (lst.ModManager == ModManager.MO2) + { + var readme_file = Path.Combine(compiler?.MO2ProfileDir, "readme.md"); + if (File.Exists(readme_file)) + File.ReadAllLines(readme_file) + .Do(NoWrapText); + } Text( $"#### Download Summary ({lst.Archives.Count} archives - {lst.Archives.Sum(a => a.Size).ToFileSizeString()})"); diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs index de4648bd..eec1bbae 100644 --- a/Wabbajack.Lib/VortexCompiler.cs +++ b/Wabbajack.Lib/VortexCompiler.cs @@ -1,10 +1,12 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Linq; using System.Security.Cryptography; using System.Text; +using CommonMark; using Microsoft.WindowsAPICodePack.Shell; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; @@ -81,6 +83,8 @@ namespace Wabbajack.Lib public override bool Compile() { + if (string.IsNullOrEmpty(ModListName)) + ModListName = $"Vortex ModList for {Game.ToString()}"; ModListOutputFile = $"{ModListName}{ExtensionManager.Extension}"; Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}."); @@ -196,7 +200,7 @@ namespace Wabbajack.Lib ModList = new ModList { - Name = ModListName ?? $"Vortex ModList for {Game.ToString()}", + Name = ModListName ?? "", Author = ModListAuthor ?? "", Description = ModListDescription ?? "", Readme = ModListReadme ?? "", @@ -208,9 +212,12 @@ namespace Wabbajack.Lib GameType = Game }; + GenerateReport(); ExportModList(); Info("Done Building ModList"); + + ShowReport(); return true; } @@ -275,7 +282,7 @@ namespace Wabbajack.Lib Directory.Delete(ModListOutputFolder, true); } - /*private void GenerateReport() + private void GenerateReport() { string css; using (var cssStream = Utils.GetResourceStream("Wabbajack.Lib.css-min.css")) @@ -292,7 +299,16 @@ namespace Wabbajack.Lib reporter.Build(this, ModList); } } - }*/ + + ModList.ReportHTML = "" + CommonMarkConverter.Convert(File.ReadAllText($"{ModList.Name}.md")); + } + + private void ShowReport() + { + var file = Alphaleonis.Win32.Filesystem.Path.GetTempFileName() + ".html"; + File.WriteAllText(file, ModList.ReportHTML); + Process.Start(file); + } private void CreateMetaFiles() { From ac178ed0c597a1963d71e94bcf20aa84dbd74b2e Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 16:00:33 +0100 Subject: [PATCH 16/18] Moved core Compile functions to ACompiler --- Wabbajack.Lib/ACompiler.cs | 183 ++++++++++++++++++++++++++++++-- Wabbajack.Lib/Compiler.cs | 179 +------------------------------ Wabbajack.Lib/VortexCompiler.cs | 167 ----------------------------- 3 files changed, 177 insertions(+), 352 deletions(-) diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index 68de8ddf..d95b0732 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -1,12 +1,19 @@ using System; using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; using System.Linq; using System.Reactive.Subjects; -using System.Text; -using System.Threading.Tasks; +using CommonMark; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.ModListRegistry; using Wabbajack.VirtualFileSystem; +using Directory = Alphaleonis.Win32.Filesystem.Directory; +using File = Alphaleonis.Win32.Filesystem.File; +using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib { @@ -44,16 +51,176 @@ namespace Wabbajack.Lib public List IndexedArchives = new List(); public Dictionary> IndexedFiles = new Dictionary>(); - public abstract void Info(string msg); - public abstract void Status(string msg); - public abstract void Error(string msg); + public void Info(string msg) + { + Utils.Log(msg); + } - internal abstract string IncludeFile(byte[] data); - internal abstract string IncludeFile(string data); + public void Status(string msg) + { + Queue.Report(msg, 0); + } + + public void Error(string msg) + { + Utils.Log(msg); + throw new Exception(msg); + } + + internal string IncludeFile(byte[] data) + { + var id = Guid.NewGuid().ToString(); + File.WriteAllBytes(Path.Combine(ModListOutputFolder, id), data); + return id; + } + + internal string IncludeFile(string data) + { + var id = Guid.NewGuid().ToString(); + File.WriteAllText(Path.Combine(ModListOutputFolder, id), data); + return id; + } + + public void ExportModList() + { + Utils.Log($"Exporting ModList to : {ModListOutputFile}"); + + //ModList.ToJSON(Path.Combine(ModListOutputFolder, "modlist.json")); + ModList.ToCERAS(Path.Combine(ModListOutputFolder, "modlist"), ref CerasConfig.Config); + + if (File.Exists(ModListOutputFile)) + File.Delete(ModListOutputFile); + + using (var fs = new FileStream(ModListOutputFile, FileMode.Create)) + { + using (var za = new ZipArchive(fs, ZipArchiveMode.Create)) + { + Directory.EnumerateFiles(ModListOutputFolder, "*.*") + .DoProgress("Compressing ModList", + f => + { + var ze = za.CreateEntry(Path.GetFileName(f)); + using (var os = ze.Open()) + using (var ins = File.OpenRead(f)) + { + ins.CopyTo(os); + } + }); + } + } + + Utils.Log("Exporting ModList metadata"); + var metadata = new ModlistMetadata.DownloadMetadata + { + Size = File.GetSize(ModListOutputFile), + Hash = ModListOutputFile.FileHash(), + NumberOfArchives = ModList.Archives.Count, + SizeOfArchives = ModList.Archives.Sum(a => a.Size), + NumberOfInstalledFiles = ModList.Directives.Count, + SizeOfInstalledFiles = ModList.Directives.Sum(a => a.Size) + }; + metadata.ToJSON(ModListOutputFile + ".meta.json"); + + + Utils.Log("Removing ModList staging folder"); + Directory.Delete(ModListOutputFolder, true); + } + + public void ShowReport() + { + //if (!ShowReportWhenFinished) return; + + var file = Path.GetTempFileName() + ".html"; + File.WriteAllText(file, ModList.ReportHTML); + Process.Start(file); + } + + public void GenerateReport() + { + string css; + using (var cssStream = Utils.GetResourceStream("Wabbajack.Lib.css-min.css")) + { + using (var reader = new StreamReader(cssStream)) + { + css = reader.ReadToEnd(); + } + } + + using (var fs = File.OpenWrite($"{ModList.Name}.md")) + { + fs.SetLength(0); + using (var reporter = new ReportBuilder(fs, ModListOutputFolder)) + { + reporter.Build(this, ModList); + } + } + + ModList.ReportHTML = "" + + CommonMarkConverter.Convert(File.ReadAllText($"{ModList.Name}.md")); + } + + public void GatherArchives() + { + Info("Building a list of archives based on the files required"); + + var shas = InstallDirectives.OfType() + .Select(a => a.ArchiveHashPath[0]) + .Distinct(); + + var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified) + .GroupBy(f => f.File.Hash) + .ToDictionary(f => f.Key, f => f.First()); + + SelectedArchives = shas.PMap(Queue, sha => ResolveArchive(sha, archives)); + } + + public Archive ResolveArchive(string sha, IDictionary archives) + { + if (archives.TryGetValue(sha, out var found)) + { + if (found.IniData == null) + Error($"No download metadata found for {found.Name}, please use MO2 to query info or add a .meta file and try again."); + + var result = new Archive + { + State = (AbstractDownloadState) DownloadDispatcher.ResolveArchive(found.IniData) + }; + + if (result.State == null) + Error($"{found.Name} could not be handled by any of the downloaders"); + + result.Name = found.Name; + result.Hash = found.File.Hash; + result.Meta = found.Meta; + result.Size = found.File.Size; + + Info($"Checking link for {found.Name}"); + + if (result.State != null && !result.State.Verify()) + Error( + $"Unable to resolve link for {found.Name}. If this is hosted on the Nexus the file may have been removed."); + + return result; + } + + Error($"No match found for Archive sha: {sha} this shouldn't happen"); + return null; + } public abstract bool Compile(); - public abstract Directive RunStack(IEnumerable stack, RawSourceFile source); + public Directive RunStack(IEnumerable stack, RawSourceFile source) + { + Utils.Status($"Compiling {source.Path}"); + foreach (var step in stack) + { + var result = step.Run(source); + if (result != null) return result; + } + + throw new InvalidDataException("Data fell out of the compilation stack"); + } + public abstract IEnumerable GetStack(); public abstract IEnumerable MakeStack(); diff --git a/Wabbajack.Lib/Compiler.cs b/Wabbajack.Lib/Compiler.cs index 7daa1144..0b66dcc9 100644 --- a/Wabbajack.Lib/Compiler.cs +++ b/Wabbajack.Lib/Compiler.cs @@ -1,17 +1,12 @@ -using CommonMark; -using Compression.BSA; +using Compression.BSA; using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; using System.IO; -using System.IO.Compression; using System.Linq; using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; -using Wabbajack.Lib.Downloaders; -using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.Validation; using Wabbajack.VirtualFileSystem; @@ -76,36 +71,6 @@ namespace Wabbajack.Lib public HashSet SelectedProfiles { get; set; } = new HashSet(); - public override void Info(string msg) - { - Utils.Log(msg); - } - - public override void Status(string msg) - { - Queue.Report(msg, 0); - } - - public override void Error(string msg) - { - Utils.Log(msg); - throw new Exception(msg); - } - - internal override string IncludeFile(byte[] data) - { - var id = Guid.NewGuid().ToString(); - File.WriteAllBytes(Path.Combine(ModListOutputFolder, id), data); - return id; - } - - internal override string IncludeFile(string data) - { - var id = Guid.NewGuid().ToString(); - File.WriteAllText(Path.Combine(ModListOutputFolder, id), data); - return id; - } - public override bool Compile() { UpdateTracker.Reset(); @@ -287,7 +252,7 @@ namespace Wabbajack.Lib ValidateModlist.RunValidation(ModList); GenerateReport(); - ExportModlist(); + ExportModList(); ResetMembers(); @@ -313,87 +278,6 @@ namespace Wabbajack.Lib }); } - private void ExportModlist() - { - Utils.Log($"Exporting Modlist to : {ModListOutputFile}"); - - //ModList.ToJSON(Path.Combine(ModListOutputFolder, "modlist.json")); - ModList.ToCERAS(Path.Combine(ModListOutputFolder, "modlist"), ref CerasConfig.Config); - - if (File.Exists(ModListOutputFile)) - File.Delete(ModListOutputFile); - - using (var fs = new FileStream(ModListOutputFile, FileMode.Create)) - { - using (var za = new ZipArchive(fs, ZipArchiveMode.Create)) - { - Directory.EnumerateFiles(ModListOutputFolder, "*.*") - .DoProgress("Compressing Modlist", - f => - { - var ze = za.CreateEntry(Path.GetFileName(f)); - using (var os = ze.Open()) - using (var ins = File.OpenRead(f)) - { - ins.CopyTo(os); - } - }); - } - } - - Utils.Log("Exporting Modlist metadata"); - var metadata = new ModlistMetadata.DownloadMetadata - { - Size = File.GetSize(ModListOutputFile), - Hash = ModListOutputFile.FileHash(), - NumberOfArchives = ModList.Archives.Count, - SizeOfArchives = ModList.Archives.Sum(a => a.Size), - NumberOfInstalledFiles = ModList.Directives.Count, - SizeOfInstalledFiles = ModList.Directives.Sum(a => a.Size) - }; - metadata.ToJSON(ModListOutputFile + ".meta.json"); - - - Utils.Log("Removing modlist staging folder"); - Directory.Delete(ModListOutputFolder, true); - - - - } - - private void ShowReport() - { - if (!ShowReportWhenFinished) return; - - var file = Path.GetTempFileName() + ".html"; - File.WriteAllText(file, ModList.ReportHTML); - Process.Start(file); - } - - private void GenerateReport() - { - string css = ""; - using (Stream cssStream = Utils.GetResourceStream("Wabbajack.Lib.css-min.css")) - { - using (StreamReader reader = new StreamReader(cssStream)) - { - css = reader.ReadToEnd(); - } - } - - using (var fs = File.OpenWrite($"{ModList.Name}.md")) - { - fs.SetLength(0); - using (var reporter = new ReportBuilder(fs, ModListOutputFolder)) - { - reporter.Build(this, ModList); - } - } - - ModList.ReportHTML = "" - + CommonMarkConverter.Convert(File.ReadAllText($"{ModList.Name}.md")); - } - /// /// Clear references to lists that hold a lot of data. /// @@ -477,65 +361,6 @@ namespace Wabbajack.Lib return null; } - private void GatherArchives() - { - Info("Building a list of archives based on the files required"); - - var shas = InstallDirectives.OfType() - .Select(a => a.ArchiveHashPath[0]) - .Distinct(); - - var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified) - .GroupBy(f => f.File.Hash) - .ToDictionary(f => f.Key, f => f.First()); - - SelectedArchives = shas.PMap(Queue, sha => ResolveArchive(sha, archives)); - } - - private Archive ResolveArchive(string sha, IDictionary archives) - { - if (archives.TryGetValue(sha, out var found)) - { - if (found.IniData == null) - Error($"No download metadata found for {found.Name}, please use MO2 to query info or add a .meta file and try again."); - - var result = new Archive(); - result.State = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(found.IniData); - - if (result.State == null) - Error($"{found.Name} could not be handled by any of the downloaders"); - - result.Name = found.Name; - result.Hash = found.File.Hash; - result.Meta = found.Meta; - result.Size = found.File.Size; - - Info($"Checking link for {found.Name}"); - - if (!result.State.Verify()) - Error( - $"Unable to resolve link for {found.Name}. If this is hosted on the Nexus the file may have been removed."); - - return result; - } - - Error($"No match found for Archive sha: {sha} this shouldn't happen"); - return null; - } - - - public override Directive RunStack(IEnumerable stack, RawSourceFile source) - { - Utils.Status($"Compiling {source.Path}"); - foreach (var step in stack) - { - var result = step.Run(source); - if (result != null) return result; - } - - throw new InvalidDataException("Data fell out of the compilation stack"); - } - public override IEnumerable GetStack() { var user_config = Path.Combine(MO2ProfileDir, "compilation_stack.yml"); diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs index eec1bbae..135fc9a3 100644 --- a/Wabbajack.Lib/VortexCompiler.cs +++ b/Wabbajack.Lib/VortexCompiler.cs @@ -1,17 +1,12 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; -using System.IO.Compression; using System.Linq; using System.Security.Cryptography; using System.Text; -using CommonMark; using Microsoft.WindowsAPICodePack.Shell; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; -using Wabbajack.Lib.Downloaders; -using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.NexusApi; using Wabbajack.VirtualFileSystem; using File = Alphaleonis.Win32.Filesystem.File; @@ -51,36 +46,6 @@ namespace Wabbajack.Lib ModListOutputFolder = "output_folder"; } - public override void Info(string msg) - { - Utils.Log(msg); - } - - public override void Status(string msg) - { - Queue.Report(msg, 0); - } - - public override void Error(string msg) - { - Utils.Log(msg); - throw new Exception(msg); - } - - internal override string IncludeFile(byte[] data) - { - var id = Guid.NewGuid().ToString(); - File.WriteAllBytes(Path.Combine(ModListOutputFolder, id), data); - return id; - } - - internal override string IncludeFile(string data) - { - var id = Guid.NewGuid().ToString(); - File.WriteAllText(Path.Combine(ModListOutputFolder, id), data); - return id; - } - public override bool Compile() { if (string.IsNullOrEmpty(ModListName)) @@ -237,79 +202,6 @@ namespace Wabbajack.Lib }); } - private void ExportModList() - { - Utils.Log($"Exporting ModList to: {ModListOutputFolder}"); - - // using JSON for better debugging - ModList.ToJSON(Path.Combine(ModListOutputFolder, "modlist.json")); - //ModList.ToCERAS(Path.Combine(ModListOutputFolder, "modlist"), ref CerasConfig.Config); - - if(File.Exists(ModListOutputFile)) - File.Delete(ModListOutputFile); - - using (var fs = new FileStream(ModListOutputFile, FileMode.Create)) - { - using (var za = new ZipArchive(fs, ZipArchiveMode.Create)) - { - Directory.EnumerateFiles(ModListOutputFolder, "*.*") - .DoProgress("Compressing ModList", - f => - { - var ze = za.CreateEntry(Path.GetFileName(f)); - using (var os = ze.Open()) - using (var ins = File.OpenRead(f)) - { - ins.CopyTo(os); - } - }); - } - } - - Utils.Log("Exporting ModList metadata"); - var metadata = new ModlistMetadata.DownloadMetadata - { - Size = File.GetSize(ModListOutputFile), - Hash = ModListOutputFile.FileHash(), - NumberOfArchives = ModList.Archives.Count, - SizeOfArchives = ModList.Archives.Sum(a => a.Size), - NumberOfInstalledFiles = ModList.Directives.Count, - SizeOfInstalledFiles = ModList.Directives.Sum(a => a.Size) - }; - metadata.ToJSON(ModListOutputFile + ".meta.json"); - - Utils.Log("Removing ModList staging folder"); - Directory.Delete(ModListOutputFolder, true); - } - - private void GenerateReport() - { - string css; - using (var cssStream = Utils.GetResourceStream("Wabbajack.Lib.css-min.css")) - using (var reader = new StreamReader(cssStream)) - { - css = reader.ReadToEnd(); - } - - using (var fs = File.OpenWrite($"{ModList.Name}.md")) - { - fs.SetLength(0); - using (var reporter = new ReportBuilder(fs, ModListOutputFolder)) - { - reporter.Build(this, ModList); - } - } - - ModList.ReportHTML = "" + CommonMarkConverter.Convert(File.ReadAllText($"{ModList.Name}.md")); - } - - private void ShowReport() - { - var file = Alphaleonis.Win32.Filesystem.Path.GetTempFileName() + ".html"; - File.WriteAllText(file, ModList.ReportHTML); - Process.Start(file); - } - private void CreateMetaFiles() { Utils.Log("Getting Nexus api_key, please click authorize if a browser window appears"); @@ -353,65 +245,6 @@ namespace Wabbajack.Lib }); } - private void GatherArchives() - { - Info("Building a list of archives based on the files required"); - - var shas = InstallDirectives.OfType() - .Select(a => a.ArchiveHashPath[0]) - .Distinct(); - - var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified) - .GroupBy(f => f.File.Hash) - .ToDictionary(f => f.Key, f => f.First()); - - SelectedArchives = shas.PMap(Queue, sha => ResolveArchive(sha, archives)); - } - - private Archive ResolveArchive(string sha, IDictionary archives) - { - if (archives.TryGetValue(sha, out var found)) - { - if(found.IniData == null) - Error($"No download metadata found for {found.Name}, please use MO2 to query info or add a .meta file and try again."); - - var result = new Archive(); - result.State = (AbstractDownloadState) DownloadDispatcher.ResolveArchive(found.IniData); - - if (result.State == null) - Error($"{found.Name} could not be handled by any of the downloaders"); - - result.Name = found.Name; - result.Hash = found.File.Hash; - result.Meta = found.Meta; - result.Size = found.File.Size; - - Info($"Checking link for {found.Name}"); - - if (!result.State.Verify()) - Error( - $"Unable to resolve link for {found.Name}. If this is hosted on the Nexus the file may have been removed."); - - return result; - } - - Error($"No match found for Archive sha: {sha} this shouldn't happen"); - return null; - } - - public override Directive RunStack(IEnumerable stack, RawSourceFile source) - { - Utils.Status($"Compiling {source.Path}"); - foreach (var step in stack) - { - var result = step.Run(source); - if (result != null) return result; - } - - throw new InvalidDataException("Data fell out of the compilation stack"); - - } - public override IEnumerable GetStack() { var s = Consts.TestMode ? DownloadsFolder : VortexFolder; From 7dbe31581f37cb2b6e64660c28f53bfca7f52936 Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 17:26:04 +0100 Subject: [PATCH 17/18] Disabled mods will now be ignored in the VortexCompiler --- .../IgnoreDisabledVortexMods.cs | 41 +++++++++++ .../IncludeVortexDeployment.cs | 11 +-- Wabbajack.Lib/VortexCompiler.cs | 72 ++++++++++++++++++- Wabbajack.Lib/Wabbajack.Lib.csproj | 1 + 4 files changed, 115 insertions(+), 10 deletions(-) create mode 100644 Wabbajack.Lib/CompilationSteps/IgnoreDisabledVortexMods.cs diff --git a/Wabbajack.Lib/CompilationSteps/IgnoreDisabledVortexMods.cs b/Wabbajack.Lib/CompilationSteps/IgnoreDisabledVortexMods.cs new file mode 100644 index 00000000..aa283805 --- /dev/null +++ b/Wabbajack.Lib/CompilationSteps/IgnoreDisabledVortexMods.cs @@ -0,0 +1,41 @@ +using Alphaleonis.Win32.Filesystem; +using Wabbajack.Common; + +namespace Wabbajack.Lib.CompilationSteps +{ + public class IgnoreDisabledVortexMods : ACompilationStep + { + private readonly VortexCompiler _vortexCompiler; + + public IgnoreDisabledVortexMods(ACompiler compiler) : base(compiler) + { + _vortexCompiler = (VortexCompiler) compiler; + } + + public override Directive Run(RawSourceFile source) + { + var b = false; + _vortexCompiler.ActiveArchives.Do(a => + { + if (source.Path.Contains(a)) b = true; + }); + if (b) return null; + var r = source.EvolveTo(); + r.Reason = "Disabled Archive"; + return r; + } + + public override IState GetState() + { + return new State(); + } + + public class State : IState + { + public ICompilationStep CreateStep(ACompiler compiler) + { + return new IgnoreDisabledVortexMods(compiler); + } + } + } +} diff --git a/Wabbajack.Lib/CompilationSteps/IncludeVortexDeployment.cs b/Wabbajack.Lib/CompilationSteps/IncludeVortexDeployment.cs index 774acc50..30ab7d1e 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludeVortexDeployment.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludeVortexDeployment.cs @@ -1,10 +1,4 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Wabbajack.Common; +using System.IO; namespace Wabbajack.Lib.CompilationSteps { @@ -16,8 +10,9 @@ namespace Wabbajack.Lib.CompilationSteps public override Directive Run(RawSourceFile source) { + if (!source.Path.EndsWith("vortex.deployment.msgpack") && - !source.Path.EndsWith("\\vortex.deployment.json")) return null; + !source.Path.EndsWith("\\vortex.deployment.json") && Path.GetExtension(source.Path) != ".meta") return null; var inline = source.EvolveTo(); inline.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath)); return inline; diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs index 135fc9a3..a5bb537d 100644 --- a/Wabbajack.Lib/VortexCompiler.cs +++ b/Wabbajack.Lib/VortexCompiler.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Security.Cryptography; using System.Text; using Microsoft.WindowsAPICodePack.Shell; +using Newtonsoft.Json; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.NexusApi; @@ -15,6 +16,14 @@ namespace Wabbajack.Lib { public class VortexCompiler : ACompiler { + /* vortex creates a vortex.deployment.json file that contains information + about all deployed files, parsing that file, we can get a list of all 'active' + archives so we don't force the user to install all archives found in the downloads folder. + Similar to how IgnoreDisabledMods for MO2 works + */ + public VortexDeployment VortexDeployment; + public List ActiveArchives; + public Game Game { get; } public string GameName { get; } @@ -44,6 +53,8 @@ namespace Wabbajack.Lib DownloadsFolder = downloadsFolder; StagingFolder = stagingFolder; ModListOutputFolder = "output_folder"; + + ActiveArchives = new List(); } public override bool Compile() @@ -54,6 +65,8 @@ namespace Wabbajack.Lib Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}."); + ParseDeploymentFile(); + Info("Starting pre-compilation steps"); CreateMetaFiles(); @@ -186,6 +199,46 @@ namespace Wabbajack.Lib return true; } + private void ParseDeploymentFile() + { + Info("Searching for vortex.deployment.json..."); + + var deploymentFile = ""; + Directory.EnumerateFiles(GamePath, "vortex.deployment.json", SearchOption.AllDirectories) + .Where(File.Exists) + .Do(f => deploymentFile = f); + var currentGame = GameRegistry.Games[Game]; + if (currentGame.AdditionalFolders != null && currentGame.AdditionalFolders.Count != 0) + currentGame.AdditionalFolders.Do(f => Directory.EnumerateFiles(f, "vortex.deployment.json", SearchOption.AllDirectories) + .Where(File.Exists) + .Do(d => deploymentFile = d)); + + if (string.IsNullOrEmpty(deploymentFile)) + { + Info("vortex.deployment.json not found!"); + return; + } + Info("vortex.deployment.json found at "+deploymentFile); + + Info("Parsing vortex.deployment.json..."); + try + { + VortexDeployment = deploymentFile.FromJSON(); + } + catch (JsonSerializationException e) + { + Info("Failed to parse vortex.deployment.json!"); + Utils.LogToFile(e.Message); + Utils.LogToFile(e.StackTrace); + } + + VortexDeployment.files.Do(f => + { + var archive = f.source; + if(!ActiveArchives.Contains(archive)) ActiveArchives.Add(archive); + }); + } + /// /// Some have mods outside their game folder located /// @@ -208,7 +261,7 @@ namespace Wabbajack.Lib var nexusClient = new NexusApiClient(); Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly) - .Where(f => File.Exists(f) && Path.GetExtension(f) != ".meta" && !File.Exists(f+".meta")) + .Where(f => File.Exists(f) && Path.GetExtension(f) != ".meta" && !File.Exists(f+".meta") && ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f))) .Do(f => { Utils.Log($"Trying to create meta file for {Path.GetFileName(f)}"); @@ -266,8 +319,8 @@ namespace Wabbajack.Lib return new List { new IncludePropertyFiles(this), + new IgnoreDisabledVortexMods(this), new IncludeVortexDeployment(this), - new IncludeRegex(this, "^*\\.meta"), new IgnoreVortex(this), new IgnoreRegex(this, "^*__vortex_staging_folder$"), @@ -330,4 +383,19 @@ namespace Wabbajack.Lib return IsValidBaseStagingFolder(Path.GetDirectoryName(path)); } } + + public class VortexDeployment + { + public string instance; + public int version; + public string deploymentMethod; + public List files; + } + + public class VortexFile + { + public string relPath; + public string source; + public string target; + } } diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index b17545b0..8bcb678f 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -86,6 +86,7 @@ + From e7a644ea6fc601b17222ce6c997b2e97a10cf722 Mon Sep 17 00:00:00 2001 From: erri120 Date: Sun, 17 Nov 2019 17:30:05 +0100 Subject: [PATCH 18/18] Removed IgnoreMissingFiles = true --- Wabbajack.Lib/VortexCompiler.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs index a5bb537d..ed8552c4 100644 --- a/Wabbajack.Lib/VortexCompiler.cs +++ b/Wabbajack.Lib/VortexCompiler.cs @@ -44,9 +44,6 @@ namespace Wabbajack.Lib ModManager = ModManager.Vortex; Game = game; - // TODO: only for testing - IgnoreMissingFiles = true; - GamePath = gamePath; GameName = GameRegistry.Games[game].NexusName; VortexFolder = vortexFolder;