From 34e3b1bcfa9c34ef6d949d6e68b9fcc7d56510fb Mon Sep 17 00:00:00 2001 From: erri120 Date: Mon, 24 Feb 2020 16:24:13 +0100 Subject: [PATCH 1/5] Changed the VFSCache location MO2: - uses the hash of the path to ModOrganizer.exe Vortex: - uses the hash of the Staging path --- Wabbajack.Lib/ACompiler.cs | 3 ++- Wabbajack.Lib/MO2Compiler.cs | 4 ++++ Wabbajack.Lib/VortexCompiler.cs | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index 02cf854b..3d5a1936 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -22,7 +22,8 @@ namespace Wabbajack.Lib public bool ReadmeIsWebsite; public string WabbajackVersion; - protected string VFSCacheName => Path.Combine(Consts.LocalAppDataPath, $"vfs_compile_cache_{ModListName?.StringSHA256Hex() ?? "_Unknown_"}.bin"); + public abstract string VFSCacheName { get; } + //protected string VFSCacheName => Path.Combine(Consts.LocalAppDataPath, $"vfs_compile_cache.bin"); /// /// A stream of tuples of ("Update Title", 0.25) which represent the name of the current task /// and the current progress. diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 093bdfef..b3144a07 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -43,6 +43,10 @@ namespace Wabbajack.Lib public override string ModListOutputFile { get; } + public override string VFSCacheName => Path.Combine( + Consts.LocalAppDataPath, + $"vfs_compile_cache-{Path.Combine(MO2Folder ?? "Unknown", "ModOrganizer.exe").StringSHA256Hex()}.bin"); + public MO2Compiler(string mo2Folder, string mo2Profile, string outputFile) { MO2Folder = mo2Folder; diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs index 7bfb3ce1..d625e938 100644 --- a/Wabbajack.Lib/VortexCompiler.cs +++ b/Wabbajack.Lib/VortexCompiler.cs @@ -47,6 +47,10 @@ namespace Wabbajack.Lib private SteamGame _steamGame; private bool _hasSteamWorkshopItems; + public override string VFSCacheName => Path.Combine( + Consts.LocalAppDataPath, + $"vfs_compile_cache-{StagingFolder?.StringSHA256Hex() ?? "Unknown"}.bin"); + public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder, string stagingFolder, string outputFile) { Game = game; From 86e6b478a53921bf6fadd643f51eafa178c6e6cb Mon Sep 17 00:00:00 2001 From: erri120 Date: Mon, 24 Feb 2020 16:29:33 +0100 Subject: [PATCH 2/5] Print assembly version on startup --- Wabbajack/View Models/MainWindowVM.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Wabbajack/View Models/MainWindowVM.cs b/Wabbajack/View Models/MainWindowVM.cs index e98d1963..64b0dae1 100644 --- a/Wabbajack/View Models/MainWindowVM.cs +++ b/Wabbajack/View Models/MainWindowVM.cs @@ -122,9 +122,10 @@ namespace Wabbajack try { - System.Reflection.Assembly assembly = System.Reflection.Assembly.GetExecutingAssembly(); - FileVersionInfo fvi = FileVersionInfo.GetVersionInfo(assembly.Location); + var assembly = Assembly.GetExecutingAssembly(); + var fvi = FileVersionInfo.GetVersionInfo(assembly.Location); VersionDisplay = $"v{fvi.FileVersion}"; + Utils.Log($"Wabbajack Version: {fvi.FileVersion}"); } catch (Exception ex) { From 52784789ab81449c573f597bdf20a137442da451 Mon Sep 17 00:00:00 2001 From: erri120 Date: Mon, 24 Feb 2020 17:50:58 +0100 Subject: [PATCH 3/5] Created CLIUtils with logging --- Wabbajack.CLI/CLIUtils.cs | 21 ++++++ Wabbajack.CLI/Verbs/ChangeDownload.cs | 97 ++++++++++++------------- Wabbajack.CLI/Verbs/DownloadUrl.cs | 2 +- Wabbajack.CLI/Verbs/UpdateModlists.cs | 2 +- Wabbajack.CLI/Verbs/UpdateNexusCache.cs | 2 +- Wabbajack.CLI/Verbs/Validate.cs | 12 +-- 6 files changed, 76 insertions(+), 60 deletions(-) create mode 100644 Wabbajack.CLI/CLIUtils.cs diff --git a/Wabbajack.CLI/CLIUtils.cs b/Wabbajack.CLI/CLIUtils.cs new file mode 100644 index 00000000..7a3f0316 --- /dev/null +++ b/Wabbajack.CLI/CLIUtils.cs @@ -0,0 +1,21 @@ +using System; + +namespace Wabbajack.CLI +{ + internal static class CLIUtils + { + internal static void Log(string msg, bool newLine = true) + { + //TODO: maybe also write to a log file? + if(newLine) + Console.WriteLine(msg); + else + Console.Write(msg); + } + + internal static void LogException(Exception e, string msg) + { + Console.WriteLine($"{msg}\n{e}"); + } + } +} diff --git a/Wabbajack.CLI/Verbs/ChangeDownload.cs b/Wabbajack.CLI/Verbs/ChangeDownload.cs index 774c709b..f50cc8a9 100644 --- a/Wabbajack.CLI/Verbs/ChangeDownload.cs +++ b/Wabbajack.CLI/Verbs/ChangeDownload.cs @@ -41,11 +41,6 @@ namespace Wabbajack.CLI.Verbs [Option("meta", Default = true, HelpText = "Whether to also transfer the meta file for the archive")] public bool IncludeMeta { get; set; } - private static void Log(string msg) - { - Console.WriteLine(msg); - } - private struct TransferFile { public readonly string Input; @@ -64,31 +59,31 @@ namespace Wabbajack.CLI.Verbs { if (!File.Exists(Modlist)) { - Log($"The file {Modlist} does not exist!"); + CLIUtils.Log($"The file {Modlist} does not exist!"); return -1; } if (!Directory.Exists(Input)) { - Log($"The input directory {Input} does not exist!"); + CLIUtils.Log($"The input directory {Input} does not exist!"); return -1; } if (!Directory.Exists(Output)) { - Log($"The output directory {Output} does not exist, it will be created."); + CLIUtils.Log($"The output directory {Output} does not exist, it will be created."); Directory.CreateDirectory(Output); } if (!Modlist.EndsWith(Consts.ModListExtension) && !Modlist.EndsWith("modlist.txt")) { - Log($"The file {Modlist} is not a valid modlist file!"); + CLIUtils.Log($"The file {Modlist} is not a valid modlist file!"); return -1; } if (Copy && Move) { - Log("You can't set both copy and move flags!"); + CLIUtils.Log("You can't set both copy and move flags!"); return -1; } @@ -106,17 +101,17 @@ namespace Wabbajack.CLI.Verbs } catch (Exception e) { - Log($"Error while loading the Modlist!\n{e}"); + CLIUtils.Log($"Error while loading the Modlist!\n{e}"); return 1; } if (modlist == null) { - Log("The Modlist could not be loaded!"); + CLIUtils.Log("The Modlist could not be loaded!"); return 1; } - Log($"Modlist contains {modlist.Archives.Count} archives."); + CLIUtils.Log($"Modlist contains {modlist.Archives.Count} archives."); modlist.Archives.Do(a => { @@ -125,11 +120,11 @@ namespace Wabbajack.CLI.Verbs if (!File.Exists(inputPath)) { - Log($"File {inputPath} does not exist, skipping."); + CLIUtils.Log($"File {inputPath} does not exist, skipping."); return; } - Log($"Adding {inputPath} to the transfer list."); + CLIUtils.Log($"Adding {inputPath} to the transfer list."); list.Add(new TransferFile(inputPath, outputPath)); var metaInputPath = Path.Combine(inputPath, ".meta"); @@ -137,34 +132,34 @@ namespace Wabbajack.CLI.Verbs if (File.Exists(metaInputPath)) { - Log($"Found meta file {metaInputPath}"); + CLIUtils.Log($"Found meta file {metaInputPath}"); if (IncludeMeta) { - Log($"Adding {metaInputPath} to the transfer list."); + CLIUtils.Log($"Adding {metaInputPath} to the transfer list."); list.Add(new TransferFile(metaInputPath, metaOutputPath)); } else { - Log($"Meta file {metaInputPath} will be ignored."); + CLIUtils.Log($"Meta file {metaInputPath} will be ignored."); } } else { - Log($"Found no meta file for {inputPath}"); + CLIUtils.Log($"Found no meta file for {inputPath}"); if (IncludeMeta) { if (string.IsNullOrWhiteSpace(a.Meta)) { - Log($"Meta for {a.Name} is empty, this should not be possible but whatever."); + CLIUtils.Log($"Meta for {a.Name} is empty, this should not be possible but whatever."); return; } - Log("Adding meta from archive info the transfer list"); + CLIUtils.Log("Adding meta from archive info the transfer list"); list.Add(new TransferFile(a.Meta, metaOutputPath, true)); } else { - Log($"Meta will be ignored for {a.Name}"); + CLIUtils.Log($"Meta will be ignored for {a.Name}"); } } }); @@ -173,27 +168,27 @@ namespace Wabbajack.CLI.Verbs { if (!Directory.Exists(Mods)) { - Log($"Mods directory {Mods} does not exist!"); + CLIUtils.Log($"Mods directory {Mods} does not exist!"); return -1; } - Log($"Reading modlist.txt from {Modlist}"); + CLIUtils.Log($"Reading modlist.txt from {Modlist}"); string[] modlist = File.ReadAllLines(Modlist); if (modlist == null || modlist.Length == 0) { - Log($"Provided modlist.txt file at {Modlist} is empty or could not be read!"); + CLIUtils.Log($"Provided modlist.txt file at {Modlist} is empty or could not be read!"); return -1; } var mods = modlist.Where(s => s.StartsWith("+")).Select(s => s.Substring(1)).ToHashSet(); if (mods.Count == 0) { - Log("Counted mods from modlist.txt are 0!"); + CLIUtils.Log("Counted mods from modlist.txt are 0!"); return -1; } - Log($"Found {mods.Count} mods in modlist.txt"); + CLIUtils.Log($"Found {mods.Count} mods in modlist.txt"); var downloads = new HashSet(); @@ -204,14 +199,14 @@ namespace Wabbajack.CLI.Verbs var meta = Path.Combine(d, "meta.ini"); if (!File.Exists(meta)) { - Log($"Mod meta file {meta} does not exist, skipping"); + CLIUtils.Log($"Mod meta file {meta} does not exist, skipping"); return; } string[] ini = File.ReadAllLines(meta); if (ini == null || ini.Length == 0) { - Log($"Mod meta file {meta} could not be read or is empty!"); + CLIUtils.Log($"Mod meta file {meta} could not be read or is empty!"); return; } @@ -219,47 +214,47 @@ namespace Wabbajack.CLI.Verbs .Select(i => i.Replace("installationFile=", "")) .Do(i => { - Log($"Found installationFile {i}"); + CLIUtils.Log($"Found installationFile {i}"); downloads.Add(i); }); }); - Log($"Found {downloads.Count} installationFiles from mod metas."); + CLIUtils.Log($"Found {downloads.Count} installationFiles from mod metas."); Directory.EnumerateFiles(Input, "*", SearchOption.TopDirectoryOnly) .Where(f => downloads.Contains(Path.GetFileNameWithoutExtension(f))) .Do(f => { - Log($"Found archive {f}"); + CLIUtils.Log($"Found archive {f}"); var outputPath = Path.Combine(Output, Path.GetFileName(f)); - Log($"Adding {f} to the transfer list"); + CLIUtils.Log($"Adding {f} to the transfer list"); list.Add(new TransferFile(f, outputPath)); var metaInputPath = Path.Combine(f, ".meta"); if (File.Exists(metaInputPath)) { - Log($"Found meta file for {f} at {metaInputPath}"); + CLIUtils.Log($"Found meta file for {f} at {metaInputPath}"); if (IncludeMeta) { var metaOutputPath = Path.Combine(outputPath, ".meta"); - Log($"Adding {metaInputPath} to the transfer list."); + CLIUtils.Log($"Adding {metaInputPath} to the transfer list."); list.Add(new TransferFile(metaInputPath, metaOutputPath)); } else { - Log("Meta file will be ignored"); + CLIUtils.Log("Meta file will be ignored"); } } else { - Log($"Found no meta file for {f}"); + CLIUtils.Log($"Found no meta file for {f}"); } }); } - Log($"Transfer list contains {list.Count} items"); + CLIUtils.Log($"Transfer list contains {list.Count} items"); var success = 0; var failed = 0; var skipped = 0; @@ -269,24 +264,24 @@ namespace Wabbajack.CLI.Verbs { if (Overwrite) { - Log($"Output file {f.Output} already exists, it will be overwritten"); + CLIUtils.Log($"Output file {f.Output} already exists, it will be overwritten"); if (f.IsMeta || Move) { - Log($"Deleting file at {f.Output}"); + CLIUtils.Log($"Deleting file at {f.Output}"); try { File.Delete(f.Output); } catch (Exception e) { - Log($"Could not delete file {f.Output}!\n{e}"); + CLIUtils.Log($"Could not delete file {f.Output}!\n{e}"); failed++; } } } else { - Log($"Output file {f.Output} already exists, skipping"); + CLIUtils.Log($"Output file {f.Output} already exists, skipping"); skipped++; return; } @@ -294,7 +289,7 @@ namespace Wabbajack.CLI.Verbs if (f.IsMeta) { - Log($"Writing meta data to {f.Output}"); + CLIUtils.Log($"Writing meta data to {f.Output}"); try { File.WriteAllText(f.Output, f.Input, Encoding.UTF8); @@ -302,7 +297,7 @@ namespace Wabbajack.CLI.Verbs } catch (Exception e) { - Log($"Error while writing meta data to {f.Output}!\n{e}"); + CLIUtils.Log($"Error while writing meta data to {f.Output}!\n{e}"); failed++; } } @@ -310,7 +305,7 @@ namespace Wabbajack.CLI.Verbs { if (Copy) { - Log($"Copying file {f.Input} to {f.Output}"); + CLIUtils.Log($"Copying file {f.Input} to {f.Output}"); try { File.Copy(f.Input, f.Output, Overwrite ? CopyOptions.None : CopyOptions.FailIfExists, CopyMoveProgressHandler, null); @@ -318,13 +313,13 @@ namespace Wabbajack.CLI.Verbs } catch (Exception e) { - Log($"Error while copying file {f.Input} to {f.Output}!\n{e}"); + CLIUtils.Log($"Error while copying file {f.Input} to {f.Output}!\n{e}"); failed++; } } else if(Move) { - Log($"Moving file {f.Input} to {f.Output}"); + CLIUtils.Log($"Moving file {f.Input} to {f.Output}"); try { File.Move(f.Input, f.Output, Overwrite ? MoveOptions.ReplaceExisting : MoveOptions.None, CopyMoveProgressHandler, null); @@ -332,16 +327,16 @@ namespace Wabbajack.CLI.Verbs } catch (Exception e) { - Log($"Error while moving file {f.Input} to {f.Output}!\n{e}"); + CLIUtils.Log($"Error while moving file {f.Input} to {f.Output}!\n{e}"); failed++; } } } }); - Log($"Skipped transfers: {skipped}"); - Log($"Failed transfers: {failed}"); - Log($"Successful transfers: {success}"); + CLIUtils.Log($"Skipped transfers: {skipped}"); + CLIUtils.Log($"Failed transfers: {failed}"); + CLIUtils.Log($"Successful transfers: {success}"); return 0; } diff --git a/Wabbajack.CLI/Verbs/DownloadUrl.cs b/Wabbajack.CLI/Verbs/DownloadUrl.cs index 8d2a4dda..02196579 100644 --- a/Wabbajack.CLI/Verbs/DownloadUrl.cs +++ b/Wabbajack.CLI/Verbs/DownloadUrl.cs @@ -24,7 +24,7 @@ namespace Wabbajack.CLI.Verbs var state = DownloadDispatcher.Infer(Url); if (state == null) { - Console.WriteLine($"Could not find download source for URL {Url}"); + CLIUtils.Log($"Could not find download source for URL {Url}"); return 1; } diff --git a/Wabbajack.CLI/Verbs/UpdateModlists.cs b/Wabbajack.CLI/Verbs/UpdateModlists.cs index ed6fac60..bc1922e5 100644 --- a/Wabbajack.CLI/Verbs/UpdateModlists.cs +++ b/Wabbajack.CLI/Verbs/UpdateModlists.cs @@ -10,7 +10,7 @@ namespace Wabbajack.CLI.Verbs { protected override async Task Run() { - Console.WriteLine($"Job ID: {await AuthorAPI.UpdateServerModLists()}"); + CLIUtils.Log($"Job ID: {await AuthorAPI.UpdateServerModLists()}"); return 0; } } diff --git a/Wabbajack.CLI/Verbs/UpdateNexusCache.cs b/Wabbajack.CLI/Verbs/UpdateNexusCache.cs index 6cb2c16d..a293ecbb 100644 --- a/Wabbajack.CLI/Verbs/UpdateNexusCache.cs +++ b/Wabbajack.CLI/Verbs/UpdateNexusCache.cs @@ -11,7 +11,7 @@ namespace Wabbajack.CLI.Verbs { protected override async Task Run() { - Console.WriteLine($"Job ID: {await AuthorAPI.UpdateNexusCache()}"); + CLIUtils.Log($"Job ID: {await AuthorAPI.UpdateNexusCache()}"); return 0; } } diff --git a/Wabbajack.CLI/Verbs/Validate.cs b/Wabbajack.CLI/Verbs/Validate.cs index 8ca8acc4..6c50c817 100644 --- a/Wabbajack.CLI/Verbs/Validate.cs +++ b/Wabbajack.CLI/Verbs/Validate.cs @@ -31,14 +31,14 @@ namespace Wabbajack.CLI.Verbs { if (!File.Exists(Input)) { - Console.WriteLine($"The file {Input} does not exist!"); + CLIUtils.Log($"The file {Input} does not exist!"); return -1; } if (!Input.EndsWith(Consts.ModListExtension)) { - Console.WriteLine($"The file {Input} does not end with {Consts.ModListExtension}!"); + CLIUtils.Log($"The file {Input} does not end with {Consts.ModListExtension}!"); return -1; } @@ -50,13 +50,13 @@ namespace Wabbajack.CLI.Verbs } catch (Exception e) { - Console.WriteLine($"Error while loading the Modlist!\n{e}"); + CLIUtils.Log($"Error while loading the Modlist!\n{e}"); return 1; } if (modlist == null) { - Console.WriteLine($"The Modlist could not be loaded!"); + CLIUtils.Log($"The Modlist could not be loaded!"); return 1; } @@ -69,11 +69,11 @@ namespace Wabbajack.CLI.Verbs } catch (Exception e) { - Console.WriteLine($"Error during Validation!\n{e}"); + CLIUtils.Log($"Error during Validation!\n{e}"); return 1; } - Console.WriteLine("The Modlist passed the Validation"); + CLIUtils.Log("The Modlist passed the Validation"); return 0; } } From 8caa66b6fa4c1ab2d84501ec39b3451bfd2ee2a1 Mon Sep 17 00:00:00 2001 From: erri120 Date: Mon, 24 Feb 2020 18:05:42 +0100 Subject: [PATCH 4/5] Created CLIUtils.Exit --- Wabbajack.CLI/CLIUtils.cs | 6 ++++ Wabbajack.CLI/Verbs/ChangeDownload.cs | 42 +++++++-------------------- Wabbajack.CLI/Verbs/DownloadUrl.cs | 7 ++--- Wabbajack.CLI/Verbs/Validate.cs | 24 +++++---------- 4 files changed, 25 insertions(+), 54 deletions(-) diff --git a/Wabbajack.CLI/CLIUtils.cs b/Wabbajack.CLI/CLIUtils.cs index 7a3f0316..c9a081e8 100644 --- a/Wabbajack.CLI/CLIUtils.cs +++ b/Wabbajack.CLI/CLIUtils.cs @@ -13,6 +13,12 @@ namespace Wabbajack.CLI Console.Write(msg); } + internal static int Exit(string msg, int code) + { + Log(msg); + return code; + } + internal static void LogException(Exception e, string msg) { Console.WriteLine($"{msg}\n{e}"); diff --git a/Wabbajack.CLI/Verbs/ChangeDownload.cs b/Wabbajack.CLI/Verbs/ChangeDownload.cs index f50cc8a9..966dfad9 100644 --- a/Wabbajack.CLI/Verbs/ChangeDownload.cs +++ b/Wabbajack.CLI/Verbs/ChangeDownload.cs @@ -58,16 +58,10 @@ namespace Wabbajack.CLI.Verbs protected override async Task Run() { if (!File.Exists(Modlist)) - { - CLIUtils.Log($"The file {Modlist} does not exist!"); - return -1; - } + return CLIUtils.Exit($"The file {Modlist} does not exist!", -1); if (!Directory.Exists(Input)) - { - CLIUtils.Log($"The input directory {Input} does not exist!"); - return -1; - } + return CLIUtils.Exit($"The input directory {Input} does not exist!", -1); if (!Directory.Exists(Output)) { @@ -76,16 +70,10 @@ namespace Wabbajack.CLI.Verbs } if (!Modlist.EndsWith(Consts.ModListExtension) && !Modlist.EndsWith("modlist.txt")) - { - CLIUtils.Log($"The file {Modlist} is not a valid modlist file!"); - return -1; - } + return CLIUtils.Exit($"The file {Modlist} is not a valid modlist file!", -1); if (Copy && Move) - { - CLIUtils.Log("You can't set both copy and move flags!"); - return -1; - } + return CLIUtils.Exit("You can't set both copy and move flags!", -1); var isModlist = Modlist.EndsWith(Consts.ModListExtension); @@ -101,14 +89,12 @@ namespace Wabbajack.CLI.Verbs } catch (Exception e) { - CLIUtils.Log($"Error while loading the Modlist!\n{e}"); - return 1; + return CLIUtils.Exit($"Error while loading the Modlist!\n{e}", 1); } if (modlist == null) { - CLIUtils.Log("The Modlist could not be loaded!"); - return 1; + return CLIUtils.Exit("The Modlist could not be loaded!", 1); } CLIUtils.Log($"Modlist contains {modlist.Archives.Count} archives."); @@ -167,26 +153,18 @@ namespace Wabbajack.CLI.Verbs else { if (!Directory.Exists(Mods)) - { - CLIUtils.Log($"Mods directory {Mods} does not exist!"); - return -1; - } + return CLIUtils.Exit($"Mods directory {Mods} does not exist!", -1); CLIUtils.Log($"Reading modlist.txt from {Modlist}"); string[] modlist = File.ReadAllLines(Modlist); if (modlist == null || modlist.Length == 0) - { - CLIUtils.Log($"Provided modlist.txt file at {Modlist} is empty or could not be read!"); - return -1; - } + return CLIUtils.Exit($"Provided modlist.txt file at {Modlist} is empty or could not be read!", -1); var mods = modlist.Where(s => s.StartsWith("+")).Select(s => s.Substring(1)).ToHashSet(); + if (mods.Count == 0) - { - CLIUtils.Log("Counted mods from modlist.txt are 0!"); - return -1; - } + return CLIUtils.Exit("Counted mods from modlist.txt are 0!", -1); CLIUtils.Log($"Found {mods.Count} mods in modlist.txt"); diff --git a/Wabbajack.CLI/Verbs/DownloadUrl.cs b/Wabbajack.CLI/Verbs/DownloadUrl.cs index 02196579..ead09310 100644 --- a/Wabbajack.CLI/Verbs/DownloadUrl.cs +++ b/Wabbajack.CLI/Verbs/DownloadUrl.cs @@ -23,11 +23,8 @@ namespace Wabbajack.CLI.Verbs { var state = DownloadDispatcher.Infer(Url); if (state == null) - { - CLIUtils.Log($"Could not find download source for URL {Url}"); - return 1; - } - + return CLIUtils.Exit($"Could not find download source for URL {Url}", 1); + DownloadDispatcher.PrepareAll(new []{state}); using var queue = new WorkQueue(); diff --git a/Wabbajack.CLI/Verbs/Validate.cs b/Wabbajack.CLI/Verbs/Validate.cs index 6c50c817..5a21e0af 100644 --- a/Wabbajack.CLI/Verbs/Validate.cs +++ b/Wabbajack.CLI/Verbs/Validate.cs @@ -30,18 +30,12 @@ namespace Wabbajack.CLI.Verbs protected override async Task Run() { if (!File.Exists(Input)) - { - CLIUtils.Log($"The file {Input} does not exist!"); - return -1; - } + return CLIUtils.Exit($"The file {Input} does not exist!", -1); if (!Input.EndsWith(Consts.ModListExtension)) - { - CLIUtils.Log($"The file {Input} does not end with {Consts.ModListExtension}!"); - return -1; - } - + return CLIUtils.Exit($"The file {Input} does not end with {Consts.ModListExtension}!", -1); + ModList modlist; try @@ -50,14 +44,12 @@ namespace Wabbajack.CLI.Verbs } catch (Exception e) { - CLIUtils.Log($"Error while loading the Modlist!\n{e}"); - return 1; + return CLIUtils.Exit($"Error while loading the Modlist!\n{e}", 1); } if (modlist == null) { - CLIUtils.Log($"The Modlist could not be loaded!"); - return 1; + return CLIUtils.Exit($"The Modlist could not be loaded!", 1); } @@ -69,12 +61,10 @@ namespace Wabbajack.CLI.Verbs } catch (Exception e) { - CLIUtils.Log($"Error during Validation!\n{e}"); - return 1; + return CLIUtils.Exit($"Error during Validation!\n{e}", 1); } - CLIUtils.Log("The Modlist passed the Validation"); - return 0; + return CLIUtils.Exit("The Modlist passed the Validation", 0); } } } From 27025db484cb21dd13b15b51d3fa8840f5c911d0 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 24 Feb 2020 16:18:29 -0700 Subject: [PATCH 5/5] Switch from BSDiff to OctoDiff for patch generation --- CHANGELOG.md | 1 + Wabbajack.Common/OctoDiff.cs | 46 ++++++++++++++++++++++++ Wabbajack.Common/Util/Percent.cs | 5 +++ Wabbajack.Common/Utils.cs | 37 +++++++++++++++++-- Wabbajack.Common/Wabbajack.Common.csproj | 1 + Wabbajack.Lib/AInstaller.cs | 11 +++--- Wabbajack.Lib/MO2Installer.cs | 2 +- Wabbajack.Lib/zEditIntegration.cs | 2 +- Wabbajack.Test/TestUtils.cs | 18 ++++++++-- Wabbajack.Test/UtilsTests.cs | 44 +++++++++++++++++++++++ 10 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 Wabbajack.Common/OctoDiff.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index e5a26620..8e4e0c49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ #### Version - 0.9.22.0 * Server side fixes for CORS support and FTP uploads +* Use OctoDiff instead of BSDiff for better performance during diff generation #### Version - 0.9.21.0 - 2/23/2020 * Fix never ending hash issue diff --git a/Wabbajack.Common/OctoDiff.cs b/Wabbajack.Common/OctoDiff.cs new file mode 100644 index 00000000..132beb0d --- /dev/null +++ b/Wabbajack.Common/OctoDiff.cs @@ -0,0 +1,46 @@ +using System; +using System.IO; +using Octodiff.Core; +using Octodiff.Diagnostics; + +namespace Wabbajack.Common +{ + public class OctoDiff + { + private static ProgressReporter reporter = new ProgressReporter(); + public static void Create(byte[] oldData, byte[] newData, Stream output) + { + using var signature = CreateSignature(oldData); + using var oldStream = new MemoryStream(oldData); + using var newStream = new MemoryStream(newData); + var db = new DeltaBuilder {ProgressReporter = reporter}; + db.BuildDelta(newStream, new SignatureReader(signature, reporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output))); + } + + private static Stream CreateSignature(byte[] oldData) + { + Utils.Status("Creating Patch Signature"); + using var oldDataStream = new MemoryStream(oldData); + var sigStream = new MemoryStream(); + var signatureBuilder = new SignatureBuilder(); + signatureBuilder.Build(oldDataStream, new SignatureWriter(sigStream)); + sigStream.Position = 0; + return sigStream; + } + + private class ProgressReporter : IProgressReporter + { + public void ReportProgress(string operation, long currentPosition, long total) + { + Utils.Status(operation, new Percent(total, currentPosition)); + } + } + + public static void Apply(Stream input, Func openPatchStream, Stream output) + { + using var deltaStream = openPatchStream(); + var deltaApplier = new DeltaApplier(); + deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, reporter), output); + } + } +} diff --git a/Wabbajack.Common/Util/Percent.cs b/Wabbajack.Common/Util/Percent.cs index e9adde5d..bd0bc749 100644 --- a/Wabbajack.Common/Util/Percent.cs +++ b/Wabbajack.Common/Util/Percent.cs @@ -24,6 +24,11 @@ namespace Wabbajack.Common } } + public Percent(long max, long current) : this((double)current / max) + { + + } + public Percent(double d) : this(d, check: true) { diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index ef84d8b8..cab10aa9 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -926,8 +926,19 @@ namespace Wabbajack.Common { if (File.Exists(cacheFile)) { - await using var f = File.OpenRead(cacheFile); - await f.CopyToAsync(output); + RETRY_OPEN: + try + { + await using var f = File.OpenRead(cacheFile); + await f.CopyToAsync(output); + } + catch (IOException) + { + // Race condition with patch caching + await Task.Delay(100); + goto RETRY_OPEN; + } + } else { @@ -936,7 +947,7 @@ namespace Wabbajack.Common await using (var f = File.Open(tmpName, System.IO.FileMode.Create)) { Status("Creating Patch"); - BSDiff.Create(a, b, f); + OctoDiff.Create(a, b, f); } RETRY: @@ -973,6 +984,26 @@ namespace Wabbajack.Common return false; } + public static void ApplyPatch(Stream input, Func openPatchStream, Stream output) + { + using var ps = openPatchStream(); + using var br = new BinaryReader(ps); + var bytes = br.ReadBytes(8); + var str = Encoding.ASCII.GetString(bytes); + switch (str) + { + case "BSDIFF40": + BSDiff.Apply(input, openPatchStream, output); + return; + case "OCTODELT": + OctoDiff.Apply(input, openPatchStream, output); + return; + default: + throw new Exception($"No diff dispatch for: {str}"); + } + + } + /* public static void Warning(string s) { diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 8d20c1b1..ae13b6a0 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -36,6 +36,7 @@ + diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 5c5f8596..b98c4224 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -199,9 +199,10 @@ namespace Wabbajack.Lib onFinish(); // Now patch all the files from this archive - foreach (var toPatch in grouping.OfType()) - using (var patchStream = new MemoryStream()) + await grouping.OfType() + .PMap(queue, async toPatch => { + await using var patchStream = new MemoryStream(); Status($"Patching {Path.GetFileName(toPatch.To)}"); // Read in the patch data @@ -214,16 +215,16 @@ namespace Wabbajack.Lib File.Delete(toFile); // Patch it - using (var outStream = File.Open(toFile, FileMode.Create)) + await using (var outStream = File.Open(toFile, FileMode.Create)) { - BSDiff.Apply(oldData, () => new MemoryStream(patchData), outStream); + Utils.ApplyPatch(oldData, () => new MemoryStream(patchData), outStream); } Status($"Verifying Patch {Path.GetFileName(toPatch.To)}"); var resultSha = toFile.FileHash(); if (resultSha != toPatch.Hash) throw new InvalidDataException($"Invalid Hash for {toPatch.To} after patching"); - } + }); } public async Task DownloadArchives() diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index 24f4a02c..55f97b8d 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -298,7 +298,7 @@ namespace Wabbajack.Lib using (var output = File.Open(toFile, FileMode.Create)) using (var input = File.OpenRead(gameFile)) { - BSDiff.Apply(input, () => new MemoryStream(patchData), output); + Utils.ApplyPatch(input, () => new MemoryStream(patchData), output); } } diff --git a/Wabbajack.Lib/zEditIntegration.cs b/Wabbajack.Lib/zEditIntegration.cs index b1617655..c855e91a 100644 --- a/Wabbajack.Lib/zEditIntegration.cs +++ b/Wabbajack.Lib/zEditIntegration.cs @@ -176,7 +176,7 @@ namespace Wabbajack.Lib var patch_data = installer.LoadBytesFromPath(m.PatchID); using (var fs = File.Open(Path.Combine(installer.OutputFolder, m.To), FileMode.Create)) - BSDiff.Apply(new MemoryStream(src_data), () => new MemoryStream(patch_data), fs); + Utils.ApplyPatch(new MemoryStream(src_data), () => new MemoryStream(patch_data), fs); }); } } diff --git a/Wabbajack.Test/TestUtils.cs b/Wabbajack.Test/TestUtils.cs index 3be92886..e94dfe3c 100644 --- a/Wabbajack.Test/TestUtils.cs +++ b/Wabbajack.Test/TestUtils.cs @@ -15,16 +15,16 @@ namespace Wabbajack.Test { public class TestUtils : IDisposable { + private static Random _rng = new Random(); public TestUtils() { - RNG = new Random(); ID = RandomName(); WorkingDirectory = Path.Combine(Directory.GetCurrentDirectory(), "tmp_data"); } public string WorkingDirectory { get;} public string ID { get; } - public Random RNG { get; } + public Random RNG => _rng; public Game Game { get; set; } @@ -113,6 +113,15 @@ namespace Wabbajack.Test File.WriteAllBytes(full_path, bytes); } + public static byte[] RandomData(int? size = null, int maxSize = 1024) + { + if (size == null) + size = _rng.Next(1, maxSize); + var arr = new byte[(int) size]; + _rng.NextBytes(arr); + return arr; + } + public void Dispose() { var exts = new [] {".md", ".exe"}; @@ -250,5 +259,10 @@ namespace Wabbajack.Test GenerateRandomFileData(full_path, i); return full_path; } + + public static object RandomeOne(params object[] opts) + { + return opts[_rng.Next(0, opts.Length)]; + } } } diff --git a/Wabbajack.Test/UtilsTests.cs b/Wabbajack.Test/UtilsTests.cs index cd913a43..8bde6f9c 100644 --- a/Wabbajack.Test/UtilsTests.cs +++ b/Wabbajack.Test/UtilsTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -11,6 +12,7 @@ namespace Wabbajack.Test [TestClass] public class UtilsTests { + [TestMethod] public void IsInPathTests() { @@ -22,5 +24,47 @@ namespace Wabbajack.Test Assert.IsTrue("c:\\foo\\bar.exe".IsInPath("c:\\foo\\")); Assert.IsTrue("c:\\foo\\bar\\".IsInPath("c:\\foo\\")); } + + + [TestMethod] + [DataTestMethod] + [DynamicData(nameof(PatchData), DynamicDataSourceType.Method)] + public async Task DiffCreateAndApply(byte[] src, byte[] dest, DiffMethod method) + { + await using var ms = new MemoryStream(); + switch (method) + { + case DiffMethod.Default: + await Utils.CreatePatch(src, dest, ms); + break; + case DiffMethod.BSDiff: + BSDiff.Create(src, dest, ms); + break; + case DiffMethod.OctoDiff: + OctoDiff.Create(src, dest, ms); + break; + default: + throw new ArgumentOutOfRangeException(nameof(method), method, null); + } + + ms.Position = 0; + var patch = ms.ToArray(); + await using var resultStream = new MemoryStream(); + Utils.ApplyPatch(new MemoryStream(src), () => new MemoryStream(patch), resultStream); + CollectionAssert.AreEqual(dest, resultStream.ToArray()); + } + + + public enum DiffMethod + { + Default, + BSDiff, + OctoDiff + } + public static IEnumerable PatchData() + { + var maxSize = 1024 * 1024 * 8; + return Enumerable.Range(0, 10).Select(x => new[] {TestUtils.RandomData(maxSize:maxSize), TestUtils.RandomData(maxSize:maxSize), TestUtils.RandomeOne(DiffMethod.Default, DiffMethod.OctoDiff, DiffMethod.BSDiff)}); + } } }