Merge pull request #739 from wabbajack-tools/path-issues-in-alpha4

Fixes for issues found in alpha4

CHANGELOG.md

@@ -1,5 +1,26 @@
 ### Changelog
 
+#### Version - 2.0.0.0 - ?
+* Reworked all internal routines to use Relative/Absolute path values instead of strings
+* Reworked all internal routines to use Hash values instead of strings
+* Reworked all internal routines to use Game values instead of strings
+* Vortex support has been removed, it wasn't well tested, and wasn't used by enough people to justify further support
+* Modlists are no longer saved in a binary format, everything uses Json
+* Json type names are now a bit more human friendly
+* All server-side code that used MongoDB now uses SQL (unifying the database)
+* All Nexus validation code has been reworked to leverage RSS feeds for faster response times to updates
+* All non-Nexus validation code has been reworked for better performance
+* Feeds are now validated on demand, this is possible due to having a SQL backend and improved Nexus support
+* Jobs in the job queue no long clobber each other so much
+* BSA routines are now mostly async
+* During installation, only the bare minimum number of files are extracted from a 7zip
+* During indexing/extraction BSA files are not extracted, instead they are opened and files are read on-demand
+* File extraction is now mostly async
+* Modlists now only support website readmes (file readmes weren't used much and were a pain to read)
+* Modlists now require a machine-readable version field 
+* Added support for games installed via the Bethesda Launcher
+* Cache disk benchmarking results to save startup time of compilation/install
+
 #### Version - 1.1.5.0 - 4/6/2020
 * Included LOOT configs are no longer Base64 encoded
 * Reworked Wabbajack-cli

Wabbajack.Common/Utils.cs

@@ -891,7 +891,7 @@ namespace Wabbajack.Common
             Log(s);
         }
 
-        private static async Task<long> TestDiskSpeedInner(WorkQueue queue, string path)
+        private static async Task<long> TestDiskSpeedInner(WorkQueue queue, AbsolutePath path)
         {
             var startTime = DateTime.Now;
             var seconds = 2;
@@ -900,11 +900,11 @@ namespace Wabbajack.Common
                 {
                     var random = new Random();
 
-                    var file = Path.Combine(path, $"size_test{idx}.bin");
+                    var file = path.Combine($"size_test{idx}.bin");
                     long size = 0;
                     byte[] buffer = new byte[1024 * 8];
                     random.NextBytes(buffer);
-                    using (var fs = File.Open(file, System.IO.FileMode.Create))
+                    using (var fs = file.Create())
                     {
                         while (DateTime.Now < startTime + new TimeSpan(0, 0, seconds))
                         {
@@ -914,19 +914,29 @@ namespace Wabbajack.Common
                             size += buffer.Length;
                         }
                     }
-                    File.Delete(file);
+                    file.Delete();
                     return size;
                 });
             return results.Sum() / seconds;
         }
 
-        private static Dictionary<string, long> _cachedDiskSpeeds = new Dictionary<string, long>();
-        public static async Task<long> TestDiskSpeed(WorkQueue queue, string path)
+        public static async Task<long> TestDiskSpeed(WorkQueue queue, AbsolutePath path)
         {
-            if (_cachedDiskSpeeds.TryGetValue(path, out long speed))
-                return speed;
-            speed = await TestDiskSpeedInner(queue, path);
-            _cachedDiskSpeeds[path] = speed;
+            var benchmarkFile = path.Combine("disk_benchmark.bin");
+            if (benchmarkFile.Exists)
+            {
+                try
+                {
+                    return benchmarkFile.FromJson<long>();
+                }
+                catch (Exception ex)
+                {
+                    // ignored
+                }
+            }
+            var speed = await TestDiskSpeedInner(queue, path);
+            speed.ToJson(benchmarkFile);
+           
             return speed;
         }
 

Wabbajack.Lib/ABatchProcessor.cs

@@ -76,7 +76,7 @@ namespace Wabbajack.Lib
             var memory = Utils.GetMemoryStatus();
             // Assume roughly 2GB of ram needed to extract each 7zip archive, and then leave 2GB for the OS. If calculation is lower or equal to 1 GB, use 1GB
             var based_on_memory = Math.Max((memory.ullTotalPhys - (2 * GB)) / (2 * GB), 1);
-            var scratch_size = await RecommendQueueSize(Directory.GetCurrentDirectory());
+            var scratch_size = await RecommendQueueSize(AbsolutePath.EntryPoint);
             var result = Math.Min((int)based_on_memory, (int)scratch_size);
             Utils.Log($"Recommending a queue size of {result} based on disk performance, number of cores, and {((long)memory.ullTotalPhys).ToFileSizeString()} of system RAM");
             return result;
@@ -90,21 +90,17 @@ namespace Wabbajack.Lib
         /// </summary>
         /// <param name="folder"></param>
         /// <returns>Recommended maximum number of threads to use</returns>
-        public static async Task<int> RecommendQueueSize(string folder)
+        public static async Task<int> RecommendQueueSize(AbsolutePath folder)
         {
-            if (!Directory.Exists(folder))
-                Directory.CreateDirectory(folder);
+            using var queue = new WorkQueue();
+            
+            Utils.Log($"Benchmarking {folder}");
+            var raw_speed = await Utils.TestDiskSpeed(queue, folder);
+            Utils.Log($"{raw_speed.ToFileSizeString()}/sec for {folder}");
+            int speed = (int)(raw_speed / 1024 / 1024);
 
-            using (var queue = new WorkQueue())
-            {
-                Utils.Log($"Benchmarking {folder}");
-                var raw_speed = await Utils.TestDiskSpeed(queue, folder);
-                Utils.Log($"{raw_speed.ToFileSizeString()}/sec for {folder}");
-                int speed = (int)(raw_speed / 1024 / 1024);
-
-                // Less than 100MB/sec, stick with two threads.
-                return speed < 100 ? 2 : Math.Min(Environment.ProcessorCount, speed / 100 * 2);
-            }
+            // Less than 100MB/sec, stick with two threads.
+            return speed < 100 ? 2 : Math.Min(Environment.ProcessorCount, speed / 100 * 2);
         }
 
         /// <summary>

Wabbajack.Lib/CompilationSteps/IncludePatches.cs

@@ -41,15 +41,13 @@ namespace Wabbajack.Lib.CompilationSteps
                 _indexed.TryGetValue(nameWithoutExt, out choices);
 
             dynamic? modIni = null;
-            if (source.AbsolutePath.InFolder(_mo2Compiler.MO2ModsFolder))
+            
+            if (_bsa == null && source.File.IsNative && source.AbsolutePath.InFolder(_mo2Compiler.MO2ModsFolder))
+                ((MO2Compiler)_compiler).ModInis.TryGetValue(ModForFile(source.AbsolutePath), out modIni);
+            else if (_bsa != null)
             {
-                if (_bsa == null)
-                    ((MO2Compiler)_compiler).ModInis.TryGetValue(ModForFile(source.AbsolutePath), out modIni);
-                else
-                {
-                    var bsaPath = _bsa.FullPath.Paths.Last().RelativeTo(((MO2Compiler)_compiler).MO2Folder);
-                    ((MO2Compiler)_compiler).ModInis.TryGetValue(ModForFile(bsaPath), out modIni);
-                }
+                var bsaPath = _bsa.FullPath.Base;
+                ((MO2Compiler)_compiler).ModInis.TryGetValue(ModForFile(bsaPath), out modIni);
             }
 
             var installationFile = (string?)modIni?.General?.installationFile;

Wabbajack.Test/SanityTests.cs

@@ -2,6 +2,7 @@
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
+using Compression.BSA;
 using Wabbajack.Common;
 using Wabbajack.Lib;
 using Wabbajack.Lib.CompilationSteps.CompilationErrors;
@@ -260,6 +261,84 @@ namespace Wabbajack.Test
             Assert.NotNull(directive);
             Assert.IsAssignableFrom<PatchedFromArchive>(directive);
         }
+
+        [Fact]
+        public async Task CanPatchFilesSourcedFromBSAs()
+        {
+            var profile = utils.AddProfile();
+            var mod = utils.AddMod();
+            var file = utils.AddModFile(mod, @"baz.bin", 10);
+            
+            await utils.Configure();
+
+            
+            using var tempFile = new TempFile();
+            var bsaState = new BSAStateObject
+            {
+                Magic = "BSA\0", Version = 0x69, ArchiveFlags = 0x107, FileFlags = 0x0,
+            };
+
+            await using (var bsa = bsaState.MakeBuilder(1024 * 1024))
+            {
+                await bsa.AddFile(new BSAFileStateObject
+                {
+                    Path = (RelativePath)@"foo\bar\baz.bin", Index = 0, FlipCompression = false
+                }, new MemoryStream(utils.RandomData()));
+                await bsa.Build(tempFile.Path);
+            }
+            
+            var archive = utils.AddManualDownload(
+                new Dictionary<string, byte[]> { { "/stuff/files.bsa", await tempFile.Path.ReadAllBytesAsync() } });
+            
+            await CompileAndInstall(profile);
+            utils.VerifyInstalledFile(mod, @"baz.bin");
+            
+        }
+        
+        [Fact]
+        public async Task CanRecreateBSAsFromFilesSourcedInOtherBSAs()
+        {
+            var profile = utils.AddProfile();
+            var mod = utils.AddMod();
+            var file = utils.AddModFile(mod, @"baz.bsa", 10);
+            
+            await utils.Configure();
+
+            
+            var bsaState = new BSAStateObject
+            {
+                Magic = "BSA\0", Version = 0x69, ArchiveFlags = 0x107, FileFlags = 0x0,
+            };
+
+            // Create the download
+            using var tempFile = new TempFile();
+            await using (var bsa = bsaState.MakeBuilder(1024 * 1024))
+            {
+                await bsa.AddFile(new BSAFileStateObject
+                {
+                    Path = (RelativePath)@"foo\bar\baz.bin", Index = 0, FlipCompression = false
+                }, new MemoryStream(utils.RandomData()));
+                await bsa.Build(tempFile.Path);
+            }
+            var archive = utils.AddManualDownload(
+                new Dictionary<string, byte[]> { { "/stuff/baz.bsa", await tempFile.Path.ReadAllBytesAsync() } });
+
+            
+            // Create the result
+            await using (var bsa = bsaState.MakeBuilder(1024 * 1024))
+            {
+                await bsa.AddFile(new BSAFileStateObject
+                {
+                    Path = (RelativePath)@"foo\bar\baz.bin", Index = 0, FlipCompression = false
+                }, new MemoryStream(utils.RandomData()));
+                await bsa.Build(file);
+            }
+
+            
+            await CompileAndInstall(profile);
+            utils.VerifyInstalledFile(mod, @"baz.bsa");
+            
+        }
         
         [Fact]
         public async Task NoMatchIncludeIncludesNonMatchingFiles() 

Wabbajack.Test/TestUtils.cs

@@ -142,6 +142,15 @@ namespace Wabbajack.Test
             return Guid.NewGuid().ToString();
         }
 
+        public byte[] RandomData(int size = 0)
+        {
+            if (size == 0)
+                size = _rng.Next(256);
+            var data = new byte[size];
+            _rng.NextBytes(data);
+            return data;
+        }
+
         public string AddManualDownload(Dictionary<string, byte[]> contents)
         {
             var name = RandomName() + ".zip";