From d0accf01e827b00e16cfaee75aebfde31907e476 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 22 Apr 2020 14:58:50 -0600 Subject: [PATCH] Fixes for issues found in alpha4 --- CHANGELOG.md | 21 +++++ Wabbajack.Common/Utils.cs | 30 ++++--- Wabbajack.Lib/ABatchProcessor.cs | 24 +++--- .../CompilationSteps/IncludePatches.cs | 14 ++-- Wabbajack.Test/SanityTests.cs | 79 +++++++++++++++++++ Wabbajack.Test/TestUtils.cs | 9 +++ 6 files changed, 145 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4b55a20..346964c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ ### Changelog +#### Version - 2.0.0.0 - ? +* Reworked all internal routines to use Relative/Absolute path values instead of strings +* Reworked all internal routines to use Hash values instead of strings +* Reworked all internal routines to use Game values instead of strings +* Vortex support has been removed, it wasn't well tested, and wasn't used by enough people to justify further support +* Modlists are no longer saved in a binary format, everything uses Json +* Json type names are now a bit more human friendly +* All server-side code that used MongoDB now uses SQL (unifying the database) +* All Nexus validation code has been reworked to leverage RSS feeds for faster response times to updates +* All non-Nexus validation code has been reworked for better performance +* Feeds are now validated on demand, this is possible due to having a SQL backend and improved Nexus support +* Jobs in the job queue no long clobber each other so much +* BSA routines are now mostly async +* During installation, only the bare minimum number of files are extracted from a 7zip +* During indexing/extraction BSA files are not extracted, instead they are opened and files are read on-demand +* File extraction is now mostly async +* Modlists now only support website readmes (file readmes weren't used much and were a pain to read) +* Modlists now require a machine-readable version field +* Added support for games installed via the Bethesda Launcher +* Cache disk benchmarking results to save startup time of compilation/install + #### Version - 1.1.5.0 - 4/6/2020 * Included LOOT configs are no longer Base64 encoded * Reworked Wabbajack-cli diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 160b4875..1124c3a3 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -891,7 +891,7 @@ namespace Wabbajack.Common Log(s); } - private static async Task TestDiskSpeedInner(WorkQueue queue, string path) + private static async Task TestDiskSpeedInner(WorkQueue queue, AbsolutePath path) { var startTime = DateTime.Now; var seconds = 2; @@ -900,11 +900,11 @@ namespace Wabbajack.Common { var random = new Random(); - var file = Path.Combine(path, $"size_test{idx}.bin"); + var file = path.Combine($"size_test{idx}.bin"); long size = 0; byte[] buffer = new byte[1024 * 8]; random.NextBytes(buffer); - using (var fs = File.Open(file, System.IO.FileMode.Create)) + using (var fs = file.Create()) { while (DateTime.Now < startTime + new TimeSpan(0, 0, seconds)) { @@ -914,19 +914,29 @@ namespace Wabbajack.Common size += buffer.Length; } } - File.Delete(file); + file.Delete(); return size; }); return results.Sum() / seconds; } - private static Dictionary _cachedDiskSpeeds = new Dictionary(); - public static async Task TestDiskSpeed(WorkQueue queue, string path) + public static async Task TestDiskSpeed(WorkQueue queue, AbsolutePath path) { - if (_cachedDiskSpeeds.TryGetValue(path, out long speed)) - return speed; - speed = await TestDiskSpeedInner(queue, path); - _cachedDiskSpeeds[path] = speed; + var benchmarkFile = path.Combine("disk_benchmark.bin"); + if (benchmarkFile.Exists) + { + try + { + return benchmarkFile.FromJson(); + } + catch (Exception ex) + { + // ignored + } + } + var speed = await TestDiskSpeedInner(queue, path); + speed.ToJson(benchmarkFile); + return speed; } diff --git a/Wabbajack.Lib/ABatchProcessor.cs b/Wabbajack.Lib/ABatchProcessor.cs index 12a0aab8..b2734334 100644 --- a/Wabbajack.Lib/ABatchProcessor.cs +++ b/Wabbajack.Lib/ABatchProcessor.cs @@ -76,7 +76,7 @@ namespace Wabbajack.Lib var memory = Utils.GetMemoryStatus(); // Assume roughly 2GB of ram needed to extract each 7zip archive, and then leave 2GB for the OS. If calculation is lower or equal to 1 GB, use 1GB var based_on_memory = Math.Max((memory.ullTotalPhys - (2 * GB)) / (2 * GB), 1); - var scratch_size = await RecommendQueueSize(Directory.GetCurrentDirectory()); + var scratch_size = await RecommendQueueSize(AbsolutePath.EntryPoint); var result = Math.Min((int)based_on_memory, (int)scratch_size); Utils.Log($"Recommending a queue size of {result} based on disk performance, number of cores, and {((long)memory.ullTotalPhys).ToFileSizeString()} of system RAM"); return result; @@ -90,21 +90,17 @@ namespace Wabbajack.Lib /// /// /// Recommended maximum number of threads to use - public static async Task RecommendQueueSize(string folder) + public static async Task RecommendQueueSize(AbsolutePath folder) { - if (!Directory.Exists(folder)) - Directory.CreateDirectory(folder); + using var queue = new WorkQueue(); + + Utils.Log($"Benchmarking {folder}"); + var raw_speed = await Utils.TestDiskSpeed(queue, folder); + Utils.Log($"{raw_speed.ToFileSizeString()}/sec for {folder}"); + int speed = (int)(raw_speed / 1024 / 1024); - using (var queue = new WorkQueue()) - { - Utils.Log($"Benchmarking {folder}"); - var raw_speed = await Utils.TestDiskSpeed(queue, folder); - Utils.Log($"{raw_speed.ToFileSizeString()}/sec for {folder}"); - int speed = (int)(raw_speed / 1024 / 1024); - - // Less than 100MB/sec, stick with two threads. - return speed < 100 ? 2 : Math.Min(Environment.ProcessorCount, speed / 100 * 2); - } + // Less than 100MB/sec, stick with two threads. + return speed < 100 ? 2 : Math.Min(Environment.ProcessorCount, speed / 100 * 2); } /// diff --git a/Wabbajack.Lib/CompilationSteps/IncludePatches.cs b/Wabbajack.Lib/CompilationSteps/IncludePatches.cs index b36152d7..3a68c53d 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludePatches.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludePatches.cs @@ -41,15 +41,13 @@ namespace Wabbajack.Lib.CompilationSteps _indexed.TryGetValue(nameWithoutExt, out choices); dynamic? modIni = null; - if (source.AbsolutePath.InFolder(_mo2Compiler.MO2ModsFolder)) + + if (_bsa == null && source.File.IsNative && source.AbsolutePath.InFolder(_mo2Compiler.MO2ModsFolder)) + ((MO2Compiler)_compiler).ModInis.TryGetValue(ModForFile(source.AbsolutePath), out modIni); + else if (_bsa != null) { - if (_bsa == null) - ((MO2Compiler)_compiler).ModInis.TryGetValue(ModForFile(source.AbsolutePath), out modIni); - else - { - var bsaPath = _bsa.FullPath.Paths.Last().RelativeTo(((MO2Compiler)_compiler).MO2Folder); - ((MO2Compiler)_compiler).ModInis.TryGetValue(ModForFile(bsaPath), out modIni); - } + var bsaPath = _bsa.FullPath.Base; + ((MO2Compiler)_compiler).ModInis.TryGetValue(ModForFile(bsaPath), out modIni); } var installationFile = (string?)modIni?.General?.installationFile; diff --git a/Wabbajack.Test/SanityTests.cs b/Wabbajack.Test/SanityTests.cs index fe503611..7f3ccada 100644 --- a/Wabbajack.Test/SanityTests.cs +++ b/Wabbajack.Test/SanityTests.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using System.Threading.Tasks; +using Compression.BSA; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.CompilationSteps.CompilationErrors; @@ -260,6 +261,84 @@ namespace Wabbajack.Test Assert.NotNull(directive); Assert.IsAssignableFrom(directive); } + + [Fact] + public async Task CanPatchFilesSourcedFromBSAs() + { + var profile = utils.AddProfile(); + var mod = utils.AddMod(); + var file = utils.AddModFile(mod, @"baz.bin", 10); + + await utils.Configure(); + + + using var tempFile = new TempFile(); + var bsaState = new BSAStateObject + { + Magic = "BSA\0", Version = 0x69, ArchiveFlags = 0x107, FileFlags = 0x0, + }; + + await using (var bsa = bsaState.MakeBuilder(1024 * 1024)) + { + await bsa.AddFile(new BSAFileStateObject + { + Path = (RelativePath)@"foo\bar\baz.bin", Index = 0, FlipCompression = false + }, new MemoryStream(utils.RandomData())); + await bsa.Build(tempFile.Path); + } + + var archive = utils.AddManualDownload( + new Dictionary { { "/stuff/files.bsa", await tempFile.Path.ReadAllBytesAsync() } }); + + await CompileAndInstall(profile); + utils.VerifyInstalledFile(mod, @"baz.bin"); + + } + + [Fact] + public async Task CanRecreateBSAsFromFilesSourcedInOtherBSAs() + { + var profile = utils.AddProfile(); + var mod = utils.AddMod(); + var file = utils.AddModFile(mod, @"baz.bsa", 10); + + await utils.Configure(); + + + var bsaState = new BSAStateObject + { + Magic = "BSA\0", Version = 0x69, ArchiveFlags = 0x107, FileFlags = 0x0, + }; + + // Create the download + using var tempFile = new TempFile(); + await using (var bsa = bsaState.MakeBuilder(1024 * 1024)) + { + await bsa.AddFile(new BSAFileStateObject + { + Path = (RelativePath)@"foo\bar\baz.bin", Index = 0, FlipCompression = false + }, new MemoryStream(utils.RandomData())); + await bsa.Build(tempFile.Path); + } + var archive = utils.AddManualDownload( + new Dictionary { { "/stuff/baz.bsa", await tempFile.Path.ReadAllBytesAsync() } }); + + + // Create the result + await using (var bsa = bsaState.MakeBuilder(1024 * 1024)) + { + await bsa.AddFile(new BSAFileStateObject + { + Path = (RelativePath)@"foo\bar\baz.bin", Index = 0, FlipCompression = false + }, new MemoryStream(utils.RandomData())); + await bsa.Build(file); + } + + + await CompileAndInstall(profile); + utils.VerifyInstalledFile(mod, @"baz.bsa"); + + } [Fact] public async Task NoMatchIncludeIncludesNonMatchingFiles() diff --git a/Wabbajack.Test/TestUtils.cs b/Wabbajack.Test/TestUtils.cs index 0081c560..b7175090 100644 --- a/Wabbajack.Test/TestUtils.cs +++ b/Wabbajack.Test/TestUtils.cs @@ -142,6 +142,15 @@ namespace Wabbajack.Test return Guid.NewGuid().ToString(); } + public byte[] RandomData(int size = 0) + { + if (size == 0) + size = _rng.Next(256); + var data = new byte[size]; + _rng.NextBytes(data); + return data; + } + public string AddManualDownload(Dictionary contents) { var name = RandomName() + ".zip";