Merge pull request #1024 from Noggog/bsa-optimization

Bsa Optimizations

Compression.BSA/BA2Builder.cs

@@ -8,6 +8,7 @@ using System.Threading.Tasks;
 using ICSharpCode.SharpZipLib.Zip.Compression;
 using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
 using Wabbajack.Common;
+#nullable disable
 
 namespace Compression.BSA
 {

Compression.BSA/BA2Reader.cs

@@ -8,6 +8,7 @@ using ICSharpCode.SharpZipLib.Zip.Compression;
 using Wabbajack.Common;
 using Wabbajack.Common.Serialization.Json;
 using File = Alphaleonis.Win32.Filesystem.File;
+#nullable disable
 
 namespace Compression.BSA
 {

Compression.BSA/BSA/ArchiveFlags.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Compression.BSA
+{
+    [Flags]
+    public enum ArchiveFlags : uint
+    {
+        HasFolderNames = 0x1,
+        HasFileNames = 0x2,
+        Compressed = 0x4,
+        Unk4 = 0x8,
+        Unk5 = 0x10,
+        Unk6 = 0x20,
+        XBox360Archive = 0x40,
+        Unk8 = 0x80,
+        HasFileNameBlobs = 0x100,
+        Unk10 = 0x200,
+        Unk11 = 0x400
+    }
+
+}

Compression.BSA/BSA/Builder/BSABuilder.cs

@@ -10,20 +10,18 @@ using K4os.Compression.LZ4.Streams;
 using Wabbajack.Common;
 using File = Alphaleonis.Win32.Filesystem.File;
 using Path = Alphaleonis.Win32.Filesystem.Path;
+#nullable disable
 
 namespace Compression.BSA
 {
     public class BSABuilder : IBSABuilder
     {
-        internal uint _archiveFlags;
-        internal uint _fileFlags;
         internal byte[] _fileId;
 
         private List<FileEntry> _files = new List<FileEntry>();
         internal List<FolderRecordBuilder> _folders = new List<FolderRecordBuilder>();
         internal uint _offset;
         internal uint _totalFileNameLength;
-        internal uint _version;
         internal DiskSlabAllocator _slab;
 
         public static async Task<BSABuilder> Create(long size)
@@ -39,32 +37,20 @@ namespace Compression.BSA
 
         public static async Task<BSABuilder> Create(BSAStateObject bsaStateObject, long size)
         {
-            var self = await Create(size);
-            self._version = bsaStateObject.Version;
-            self._fileFlags = bsaStateObject.FileFlags;
-            self._archiveFlags = bsaStateObject.ArchiveFlags;
+            var self = await Create(size).ConfigureAwait(false);
+            self.HeaderType = (VersionType)bsaStateObject.Version;
+            self.FileFlags = (FileFlags)bsaStateObject.FileFlags;
+            self.ArchiveFlags = (ArchiveFlags)bsaStateObject.ArchiveFlags;
             return self;
         }
 
         public IEnumerable<FileEntry> Files => _files;
 
-        public ArchiveFlags ArchiveFlags
-        {
-            get => (ArchiveFlags) _archiveFlags;
-            set => _archiveFlags = (uint) value;
-        }
+        public ArchiveFlags ArchiveFlags { get; set; }
 
-        public FileFlags FileFlags
-        {
-            get => (FileFlags) _archiveFlags;
-            set => _archiveFlags = (uint) value;
-        }
+        public FileFlags FileFlags { get; set; }
 
-        public VersionType HeaderType
-        {
-            get => (VersionType) _version;
-            set => _version = (uint) value;
-        }
+        public VersionType HeaderType { get; set; }
 
         public IEnumerable<RelativePath> FolderNames
         {
@@ -74,13 +60,13 @@ namespace Compression.BSA
             }
         }
 
-        public bool HasFolderNames => (_archiveFlags & 0x1) > 0;
+        public bool HasFolderNames => ArchiveFlags.HasFlag(ArchiveFlags.HasFileNames);
 
-        public bool HasFileNames => (_archiveFlags & 0x2) > 0;
+        public bool HasFileNames => ArchiveFlags.HasFlag(ArchiveFlags.HasFileNames);
 
-        public bool CompressedByDefault => (_archiveFlags & 0x4) > 0;
+        public bool CompressedByDefault => ArchiveFlags.HasFlag(ArchiveFlags.Compressed);
 
-        public bool HasNameBlobs => (_archiveFlags & 0x100) > 0;
+        public bool HasNameBlobs => ArchiveFlags.HasFlag(ArchiveFlags.HasFileNameBlobs);
 
         public async ValueTask DisposeAsync()
         {
@@ -105,9 +91,9 @@ namespace Compression.BSA
             await using var wtr = new BinaryWriter(fs);
             
             wtr.Write(_fileId);
-            wtr.Write(_version);
+            wtr.Write((uint)HeaderType);
             wtr.Write(_offset);
-            wtr.Write(_archiveFlags);
+            wtr.Write((uint)ArchiveFlags);
             var folders = FolderNames.ToList();
             wtr.Write((uint) folders.Count);
             wtr.Write((uint) _files.Count);
@@ -115,7 +101,7 @@ namespace Compression.BSA
             var s = _files.Select(f => f._pathBytes.Count()).Sum();
             _totalFileNameLength = (uint) _files.Select(f => f._nameBytes.Count()).Sum();
             wtr.Write(_totalFileNameLength); // totalFileNameLength
-            wtr.Write(_fileFlags);
+            wtr.Write((uint)FileFlags);
 
             foreach (var folder in _folders) folder.WriteFolderRecord(wtr);
 

Compression.BSA/BSA/FileFlags.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Compression.BSA
+{
+    [Flags]
+    public enum FileFlags : uint
+    {
+        Meshes = 0x1,
+        Textures = 0x2,
+        Menus = 0x4,
+        Sounds = 0x8,
+        Voices = 0x10,
+        Shaders = 0x20,
+        Trees = 0x40,
+        Fonts = 0x80,
+        Miscellaneous = 0x100
+    }
+}

Compression.BSA/BSA/Reader/BSAFileStateObject.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Wabbajack.Common.Serialization.Json;
+using File = Alphaleonis.Win32.Filesystem.File;
+
+namespace Compression.BSA
+{
+    [JsonName("BSAFileState")]
+    public class BSAFileStateObject : FileStateObject
+    {
+        public bool FlipCompression { get; set; }
+
+        public BSAFileStateObject() { }
+
+        public BSAFileStateObject(FileRecord fileRecord)
+        {
+            FlipCompression = fileRecord.FlipCompression;
+            Path = fileRecord.Path;
+            Index = fileRecord._index;
+        }
+    }
+}

Compression.BSA/BSA/Reader/BSAReader.cs

@@ -0,0 +1,174 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.Common.Serialization.Json;
+using File = Alphaleonis.Win32.Filesystem.File;
+
+namespace Compression.BSA
+{
+    public class BSAReader : IBSAReader
+    {
+        public const int HeaderLength = 0x24;
+
+        internal uint _fileCount;
+        internal AbsolutePath _fileName;
+        internal uint _folderCount;
+        internal uint _folderRecordOffset;
+        private Lazy<FolderRecord[]> _folders = null!;
+        private Lazy<Dictionary<string, FolderRecord>> _foldersByName = null!;
+        internal string _magic = string.Empty;
+        internal uint _totalFileNameLength;
+        internal uint _totalFolderNameLength;
+
+        public VersionType HeaderType { get; private set; }
+
+        public ArchiveFlags ArchiveFlags { get; private set; }
+
+        public FileFlags FileFlags { get; private set; }
+
+        public IEnumerable<IFile> Files => _folders.Value.SelectMany(f => f.Files);
+
+        public IEnumerable<IFolder> Folders => _folders.Value;
+
+        public ArchiveStateObject State => new BSAStateObject(this);
+
+        public bool HasFolderNames => ArchiveFlags.HasFlag(ArchiveFlags.HasFolderNames);
+
+        public bool HasFileNames => ArchiveFlags.HasFlag(ArchiveFlags.HasFileNames);
+
+        public bool CompressedByDefault => ArchiveFlags.HasFlag(ArchiveFlags.Compressed);
+
+        public bool Bit9Set => ArchiveFlags.HasFlag(ArchiveFlags.HasFileNameBlobs);
+
+        public bool HasNameBlobs
+        {
+            get
+            {
+                if (HeaderType == VersionType.FO3 || HeaderType == VersionType.SSE) return Bit9Set;
+                return false;
+            }
+        }
+
+        public void Dump(Action<string> print)
+        {
+            print($"File Name: {_fileName}");
+            print($"File Count: {_fileCount}");
+            print($"Magic: {_magic}");
+
+            foreach (var file in Files)
+            {
+                print("\n");
+                file.Dump(print);
+            }
+        }
+
+        public static async ValueTask<BSAReader> LoadAsync(AbsolutePath filename)
+        {
+            using var stream = await filename.OpenRead().ConfigureAwait(false);
+            using var br = new BinaryReader(stream);
+            var bsa = new BSAReader { _fileName = filename };
+            bsa.LoadHeaders(br);
+            return bsa;
+        }
+
+        public static BSAReader Load(AbsolutePath filename)
+        {
+            var bsa = new BSAReader { _fileName = filename };
+            using var rdr = bsa.GetStream();
+            bsa.LoadHeaders(rdr);
+            return bsa;
+        }
+
+        internal BinaryReader GetStream()
+        {
+            return new BinaryReader(File.Open(_fileName.ToString(), FileMode.Open, FileAccess.Read, FileShare.Read));
+        }
+
+        private void LoadHeaders(BinaryReader rdr)
+        {
+            var fourcc = Encoding.ASCII.GetString(rdr.ReadBytes(4));
+
+            if (fourcc != "BSA\0")
+                throw new InvalidDataException("Archive is not a BSA");
+
+            _magic = fourcc;
+            HeaderType = (VersionType)rdr.ReadUInt32();
+            _folderRecordOffset = rdr.ReadUInt32();
+            ArchiveFlags = (ArchiveFlags)rdr.ReadUInt32();
+            _folderCount = rdr.ReadUInt32();
+            _fileCount = rdr.ReadUInt32();
+            _totalFolderNameLength = rdr.ReadUInt32();
+            _totalFileNameLength = rdr.ReadUInt32();
+            FileFlags = (FileFlags)rdr.ReadUInt32();
+
+            _folders = new Lazy<FolderRecord[]>(
+                isThreadSafe: true,
+                valueFactory: () => LoadFolderRecords());
+            _foldersByName = new Lazy<Dictionary<string, FolderRecord>>(
+                isThreadSafe: true,
+                valueFactory: GetFolderDictionary);
+        }
+
+        private FolderRecord[] LoadFolderRecords()
+        {
+            using var rdr = GetStream();
+            rdr.BaseStream.Position = _folderRecordOffset;
+            var folderHeaderLength = FolderRecord.HeaderLength(HeaderType);
+            ReadOnlyMemorySlice<byte> folderHeaderData = rdr.ReadBytes(checked((int)(folderHeaderLength * _folderCount)));
+
+            var ret = new FolderRecord[_folderCount];
+            for (var idx = 0; idx < _folderCount; idx += 1)
+                ret[idx] = new FolderRecord(this, folderHeaderData.Slice(idx * folderHeaderLength, folderHeaderLength), idx);
+
+            // Slice off appropriate file header data per folder
+            int fileCountTally = 0;
+            foreach (var folder in ret)
+            {
+                folder.ProcessFileRecordHeadersBlock(rdr, fileCountTally);
+                fileCountTally = checked((int)(fileCountTally + folder.FileCount));
+            }
+
+            if (HasFileNames)
+            {
+                var filenameBlock = new FileNameBlock(this, rdr.BaseStream.Position);
+                foreach (var folder in ret)
+                {
+                    folder.FileNameBlock = filenameBlock;
+                }
+            }
+
+            return ret;
+        }
+
+        private Dictionary<string, FolderRecord> GetFolderDictionary()
+        {
+            if (!HasFolderNames)
+            {
+                throw new ArgumentException("Cannot get folders by name if the BSA does not have folder names.");
+            }
+            var ret = new Dictionary<string, FolderRecord>();
+            foreach (var folder in _folders.Value)
+            {
+                ret.Add(folder.Name!, folder);
+            }
+            return ret;
+        }
+
+        public bool TryGetFolder(string path, [MaybeNullWhen(false)] out IFolder folder)
+        {
+            if (!HasFolderNames
+                || !_foldersByName.Value.TryGetValue(path, out var folderRec))
+            {
+                folder = default;
+                return false;
+            }
+            folder = folderRec;
+            return true;
+        }
+    }
+}

Compression.BSA/BSA/Reader/BSAStateObject.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading.Tasks;
+using Wabbajack.Common.Serialization.Json;
+using File = Alphaleonis.Win32.Filesystem.File;
+
+namespace Compression.BSA
+{
+    [JsonName("BSAState")]
+    public class BSAStateObject : ArchiveStateObject
+    {
+        public string Magic { get; set; } = string.Empty;
+        public uint Version { get; set; }
+        public uint ArchiveFlags { get; set; }
+        public uint FileFlags { get; set; }
+
+        public BSAStateObject()
+        {
+        }
+
+        public BSAStateObject(BSAReader bsaReader)
+        {
+            Magic = bsaReader._magic;
+            Version = (uint)bsaReader.HeaderType;
+            ArchiveFlags = (uint)bsaReader.ArchiveFlags;
+            FileFlags = (uint)bsaReader.FileFlags;
+        }
+
+        public override async Task<IBSABuilder> MakeBuilder(long size)
+        {
+            return await BSABuilder.Create(this, size).ConfigureAwait(false);
+        }
+    }
+}

Compression.BSA/BSA/Reader/FileNameBlock.cs

@@ -0,0 +1,40 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using Wabbajack.Common;
+
+namespace Compression.BSA
+{
+    internal class FileNameBlock
+    {
+        public readonly Lazy<ReadOnlyMemorySlice<byte>[]> Names;
+
+        public FileNameBlock(BSAReader bsa, long position)
+        {
+            Names = new Lazy<ReadOnlyMemorySlice<byte>[]>(
+                mode: System.Threading.LazyThreadSafetyMode.ExecutionAndPublication,
+                valueFactory: () =>
+                {
+                    using var stream = bsa.GetStream();
+                    stream.BaseStream.Position = position;
+                    ReadOnlyMemorySlice<byte> data = stream.ReadBytes(checked((int)bsa._totalFileNameLength));
+                    ReadOnlyMemorySlice<byte>[] names = new ReadOnlyMemorySlice<byte>[bsa._fileCount];
+                    for (int i = 0; i < bsa._fileCount; i++)
+                    {
+                        var index = data.Span.IndexOf(default(byte));
+                        if (index == -1)
+                        {
+                            throw new InvalidDataException("Did not end all of its strings in null bytes");
+                        }
+                        names[i] = data.Slice(0, index + 1);
+                        var str = names[i].ReadStringTerm(bsa.HeaderType);
+                        data = data.Slice(index + 1);
+                    }
+                    // Data doesn't seem to need to be fully consumed.
+                    // Official BSAs have overflow of zeros
+                    return names;
+                });
+        }
+    }
+}

Compression.BSA/BSA/Reader/FileRecord.cs

@@ -0,0 +1,167 @@
+using System;
+using System.Buffers.Binary;
+using System.Collections.Generic;
+using System.IO;
+using System.Runtime.Versioning;
+using System.Text;
+using System.Threading.Tasks;
+using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
+using K4os.Compression.LZ4.Streams;
+using Wabbajack.Common;
+using File = Alphaleonis.Win32.Filesystem.File;
+
+namespace Compression.BSA
+{
+    public class FileRecord : IFile
+    {
+        public const int HeaderLength = 0x10;
+
+        private readonly ReadOnlyMemorySlice<byte> _headerData;
+        internal readonly int _index;
+        internal readonly int _overallIndex;
+        internal readonly FileNameBlock _nameBlock;
+        internal readonly Lazy<string> _name;
+        internal Lazy<(uint Size, uint OnDisk, uint Original)> _size;
+
+        public ulong Hash => BinaryPrimitives.ReadUInt64LittleEndian(_headerData);
+        protected uint RawSize => BinaryPrimitives.ReadUInt32LittleEndian(_headerData.Slice(0x8));
+        public uint Offset => BinaryPrimitives.ReadUInt32LittleEndian(_headerData.Slice(0xC));
+        public string Name => _name.Value;
+        public uint Size => _size.Value.Size;
+
+        public bool FlipCompression => (RawSize & (0x1 << 30)) > 0;
+
+        internal FolderRecord Folder { get; }
+        internal BSAReader BSA => Folder.BSA;
+
+        internal FileRecord(
+            FolderRecord folderRecord, 
+            ReadOnlyMemorySlice<byte> data,
+            int index,
+            int overallIndex,
+            FileNameBlock nameBlock)
+        {
+            _index = index;
+            _overallIndex = overallIndex;
+            _headerData = data;
+            _nameBlock = nameBlock;
+            Folder = folderRecord;
+            _name = new Lazy<string>(GetName, System.Threading.LazyThreadSafetyMode.PublicationOnly);
+
+            // Will be replaced if CopyDataTo is called before value is created
+            _size = new Lazy<(uint Size, uint OnDisk, uint Original)>(
+                mode: System.Threading.LazyThreadSafetyMode.ExecutionAndPublication,
+                valueFactory: () =>
+                {
+                    using var rdr = BSA.GetStream();
+                    rdr.BaseStream.Position = Offset;
+                    return ReadSize(rdr);
+                });
+        }
+
+        public RelativePath Path => new RelativePath(string.IsNullOrEmpty(Folder.Name) ? Name : Folder.Name + "\\" + Name, skipValidation: true);
+
+        public bool Compressed
+        {
+            get
+            {
+                if (FlipCompression) return !BSA.CompressedByDefault;
+                return BSA.CompressedByDefault;
+            }
+        }
+
+        public FileStateObject State => new BSAFileStateObject(this);
+
+        public async ValueTask CopyDataTo(Stream output)
+        {
+            await using var in_file = await BSA._fileName.OpenRead().ConfigureAwait(false);
+            using var rdr = new BinaryReader(in_file);
+            rdr.BaseStream.Position = Offset;
+
+            (uint Size, uint OnDisk, uint Original) size = ReadSize(rdr);
+            if (!_size.IsValueCreated)
+            {
+                _size = new Lazy<(uint Size, uint OnDisk, uint Original)>(value: size);
+            }
+
+            if (BSA.HeaderType == VersionType.SSE)
+            {
+                if (Compressed)
+                {
+                    using var r = LZ4Stream.Decode(rdr.BaseStream);
+                    await r.CopyToLimitAsync(output, size.Original).ConfigureAwait(false);
+                }
+                else
+                {
+                    await rdr.BaseStream.CopyToLimitAsync(output, size.OnDisk).ConfigureAwait(false);
+                }
+            }
+            else
+            {
+                if (Compressed)
+                {
+                    await using var z = new InflaterInputStream(rdr.BaseStream);
+                    await z.CopyToLimitAsync(output, size.Original).ConfigureAwait(false);
+                }
+                else
+                    await rdr.BaseStream.CopyToLimitAsync(output, size.OnDisk).ConfigureAwait(false);
+            }
+        }
+
+        private string GetName()
+        {
+            var names = _nameBlock.Names.Value;
+            return names[_overallIndex].ReadStringTerm(BSA.HeaderType);
+        }
+
+        private (uint Size, uint OnDisk, uint Original) ReadSize(BinaryReader rdr)
+        {
+            uint size = RawSize;
+            if (FlipCompression)
+                size = size ^ (0x1 << 30);
+
+            if (Compressed)
+                size -= 4;
+
+            byte nameBlobOffset;
+            if (BSA.HasNameBlobs)
+            {
+                nameBlobOffset = rdr.ReadByte();
+                // Just skip, not using
+                rdr.BaseStream.Position += nameBlobOffset;
+            }
+            else
+            {
+                nameBlobOffset = 0;
+            }
+
+            uint originalSize;
+            if (Compressed)
+            {
+                originalSize = rdr.ReadUInt32();
+            }
+            else
+            {
+                originalSize = 0;
+            }
+
+            uint onDiskSize = size - nameBlobOffset;
+            if (Compressed)
+            {
+                return (Size: originalSize, OnDisk: onDiskSize, Original: originalSize);
+            }
+            else
+            {
+                return (Size: onDiskSize, OnDisk: onDiskSize, Original: originalSize);
+            }
+        }
+
+        public void Dump(Action<string> print)
+        {
+            print($"Name: {Name}");
+            print($"Offset: {Offset}");
+            print($"Raw Size: {RawSize}");
+            print($"Index: {_index}");
+        }
+    }
+}

Compression.BSA/BSA/Reader/FolderRecord.cs

@@ -0,0 +1,89 @@
+using System;
+using System.Buffers.Binary;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using NativeImport;
+using Wabbajack.Common;
+using File = Alphaleonis.Win32.Filesystem.File;
+
+namespace Compression.BSA
+{
+    public class FolderRecord : IFolder
+    {
+        internal readonly BSAReader BSA;
+        private readonly ReadOnlyMemorySlice<byte> _data;
+        internal Lazy<FileRecord[]> _files = null!;
+        private int _prevFileCount;
+        internal FileNameBlock FileNameBlock = null!;
+        internal int Index { get; }
+        public string? Name { get; private set; }
+
+        public IEnumerable<IFile> Files => _files.Value;
+
+        internal FolderRecord(BSAReader bsa, ReadOnlyMemorySlice<byte> data, int index)
+        {
+            BSA = bsa;
+            _data = data;
+            Index = index;
+        }
+
+        private bool IsLongform => BSA.HeaderType == VersionType.SSE;
+
+        public ulong Hash => BinaryPrimitives.ReadUInt64LittleEndian(_data);
+
+        public int FileCount => checked((int)BinaryPrimitives.ReadUInt32LittleEndian(_data.Slice(0x8)));
+
+        public uint Unknown => IsLongform ?
+            BinaryPrimitives.ReadUInt32LittleEndian(_data.Slice(0xC)) : 
+            0;
+
+        public ulong Offset => IsLongform ?
+            BinaryPrimitives.ReadUInt64LittleEndian(_data.Slice(0x10)) : 
+            BinaryPrimitives.ReadUInt32LittleEndian(_data.Slice(0xC));
+
+        public static int HeaderLength(VersionType version)
+        {
+            return version switch
+            {
+                VersionType.SSE => 0x18,
+                _ => 0x10,
+            };
+        }
+
+        internal void ProcessFileRecordHeadersBlock(BinaryReader rdr, int fileCountTally)
+        {
+            _prevFileCount = fileCountTally;
+            var totalFileLen = checked((int)(FileCount * FileRecord.HeaderLength));
+
+            ReadOnlyMemorySlice<byte> data;
+            if (BSA.HasFolderNames)
+            {
+                var len = rdr.ReadByte();
+                data = rdr.ReadBytes(len + totalFileLen);
+                Name = data.Slice(0, len).ReadStringTerm(BSA.HeaderType);
+                data = data.Slice(len);
+            }
+            else
+            {
+                data = rdr.ReadBytes(totalFileLen);
+            }
+
+            _files = new Lazy<FileRecord[]>(
+                isThreadSafe: true,
+                valueFactory: () => ParseFileRecords(data));
+        }
+
+        private FileRecord[] ParseFileRecords(ReadOnlyMemorySlice<byte> data)
+        {
+            var fileCount = FileCount;
+            var ret = new FileRecord[fileCount];
+            for (var idx = 0; idx < fileCount; idx += 1)
+            {
+                var fileData = data.Slice(idx * FileRecord.HeaderLength, FileRecord.HeaderLength);
+                ret[idx] = new FileRecord(this, fileData, idx, idx + _prevFileCount, FileNameBlock);
+            }
+            return ret;
+        }
+    }
+}

Compression.BSA/BSA/VersionType.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Compression.BSA
+{
+    public enum VersionType : uint
+    {
+        TES4 = 0x67,
+        FO3 = 0x68, // FO3, FNV, TES5
+        SSE = 0x69,
+        FO4 = 0x01,
+        TES3 = 0xFF // Not a real Bethesda version number
+    }
+
+}

Compression.BSA/BSAReader.cs

@@ -1,376 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Text;
-using System.Threading.Tasks;
-using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
-using K4os.Compression.LZ4.Streams;
-using Wabbajack.Common;
-using Wabbajack.Common.Serialization.Json;
-using File = Alphaleonis.Win32.Filesystem.File;
-
-namespace Compression.BSA
-{
-    public enum VersionType : uint
-    {
-        TES4 = 0x67,
-        FO3 = 0x68, // FO3, FNV, TES5
-        SSE = 0x69,
-        FO4 = 0x01,
-        TES3 = 0xFF // Not a real Bethesda version number
-    }
-
-    [Flags]
-    public enum ArchiveFlags : uint
-    {
-        HasFolderNames = 0x1,
-        HasFileNames = 0x2,
-        Compressed = 0x4,
-        Unk4 = 0x8,
-        Unk5 = 0x10,
-        Unk6 = 0x20,
-        XBox360Archive = 0x40,
-        Unk8 = 0x80,
-        HasFileNameBlobs = 0x100,
-        Unk10 = 0x200,
-        Unk11 = 0x400
-    }
-
-    [Flags]
-    public enum FileFlags : uint
-    {
-        Meshes = 0x1,
-        Textures = 0x2,
-        Menus = 0x4,
-        Sounds = 0x8,
-        Voices = 0x10,
-        Shaders = 0x20,
-        Trees = 0x40,
-        Fonts = 0x80,
-        Miscellaneous = 0x100
-    }
-
-    public class BSAReader : IBSAReader
-    {
-        internal uint _archiveFlags;
-        internal uint _fileCount;
-        internal uint _fileFlags;
-        internal AbsolutePath _fileName;
-        internal uint _folderCount;
-        internal uint _folderRecordOffset;
-        private List<FolderRecord> _folders;
-        internal string _magic;
-        internal uint _totalFileNameLength;
-        internal uint _totalFolderNameLength;
-        internal uint _version;
-        
-        public void Dump(Action<string> print)
-        {
-            print($"File Name: {_fileName}");
-            print($"File Count: {_fileCount}");
-            print($"Magic: {_magic}");
-
-            foreach (var file in Files)
-            {
-                print("\n");
-                file.Dump(print);
-            }
-        }
-
-        public static async ValueTask<BSAReader> LoadAsync(AbsolutePath filename)
-        {
-            using var stream = await filename.OpenRead();
-            using var br = new BinaryReader(stream);
-            var bsa = new BSAReader { _fileName = filename };
-            bsa.LoadHeaders(br);
-            return bsa;
-        }
-
-        public static BSAReader Load(AbsolutePath filename)
-        {
-            using var stream = File.Open(filename.ToString(), FileMode.Open, FileAccess.Read, FileShare.Read);
-            using var br = new BinaryReader(stream);
-            var bsa = new BSAReader { _fileName = filename };
-            bsa.LoadHeaders(br);
-            return bsa;
-        }
-
-        public IEnumerable<IFile> Files
-        {
-            get
-            {
-                foreach (var folder in _folders)
-                foreach (var file in folder._files)
-                    yield return file;
-            }
-        }
-
-        public ArchiveStateObject State => new BSAStateObject(this);
-
-        public VersionType HeaderType => (VersionType) _version;
-
-        public ArchiveFlags ArchiveFlags => (ArchiveFlags) _archiveFlags;
-
-        public FileFlags FileFlags => (FileFlags)_fileFlags;
-
-
-        public bool HasFolderNames => (_archiveFlags & 0x1) > 0;
-
-        public bool HasFileNames => (_archiveFlags & 0x2) > 0;
-
-        public bool CompressedByDefault => (_archiveFlags & 0x4) > 0;
-
-        public bool Bit9Set => (_archiveFlags & 0x100) > 0;
-
-        public bool HasNameBlobs
-        {
-            get
-            {
-                if (HeaderType == VersionType.FO3 || HeaderType == VersionType.SSE) return (_archiveFlags & 0x100) > 0;
-                return false;
-            }
-        }
-
-        private void LoadHeaders(BinaryReader rdr)
-        {
-            var fourcc = Encoding.ASCII.GetString(rdr.ReadBytes(4));
-
-            if (fourcc != "BSA\0")
-                throw new InvalidDataException("Archive is not a BSA");
-
-            _magic = fourcc;
-            _version = rdr.ReadUInt32();
-            _folderRecordOffset = rdr.ReadUInt32();
-            _archiveFlags = rdr.ReadUInt32();
-            _folderCount = rdr.ReadUInt32();
-            _fileCount = rdr.ReadUInt32();
-            _totalFolderNameLength = rdr.ReadUInt32();
-            _totalFileNameLength = rdr.ReadUInt32();
-            _fileFlags = rdr.ReadUInt32();
-
-            LoadFolderRecords(rdr);
-        }
-
-        private void LoadFolderRecords(BinaryReader rdr)
-        {
-            _folders = new List<FolderRecord>();
-            for (var idx = 0; idx < _folderCount; idx += 1)
-                _folders.Add(new FolderRecord(this, rdr));
-
-            foreach (var folder in _folders)
-                folder.LoadFileRecordBlock(this, rdr);
-
-            foreach (var folder in _folders)
-            foreach (var file in folder._files)
-                file.LoadFileRecord(this, folder, file, rdr);
-        }
-    }
-
-    [JsonName("BSAState")]
-    public class BSAStateObject : ArchiveStateObject
-    {
-        public BSAStateObject() { }
-        public BSAStateObject(BSAReader bsaReader)
-        {
-            Magic = bsaReader._magic;
-            Version = bsaReader._version;
-            ArchiveFlags = bsaReader._archiveFlags;
-            FileFlags = bsaReader._fileFlags;
-
-        }
-
-        public override async Task<IBSABuilder> MakeBuilder(long size)
-        {
-            return await BSABuilder.Create(this, size);
-        }
-
-        public string Magic { get; set; }
-        public uint Version { get; set; }
-        public uint ArchiveFlags { get; set; }
-        public uint FileFlags { get; set; }
-    }
-
-    public class FolderRecord
-    {
-        private readonly uint _fileCount;
-        internal List<FileRecord> _files;
-        private ulong _offset;
-        private uint _unk;
-
-        internal FolderRecord(BSAReader bsa, BinaryReader src)
-        {
-            Hash = src.ReadUInt64();
-            _fileCount = src.ReadUInt32();
-            if (bsa.HeaderType == VersionType.SSE)
-            {
-                _unk = src.ReadUInt32();
-                _offset = src.ReadUInt64();
-            }
-            else
-            {
-                _offset = src.ReadUInt32();
-            }
-        }
-
-        public string Name { get; private set; }
-
-        public ulong Hash { get; }
-
-        internal void LoadFileRecordBlock(BSAReader bsa, BinaryReader src)
-        {
-            if (bsa.HasFolderNames) Name = src.ReadStringLen(bsa.HeaderType);
-
-            _files = new List<FileRecord>();
-            for (var idx = 0; idx < _fileCount; idx += 1)
-                _files.Add(new FileRecord(bsa, this, src, idx));
-        }
-    }
-
-    public class FileRecord : IFile
-    {
-        private readonly BSAReader _bsa;
-        private readonly long _dataOffset;
-        private readonly uint _dataSize;
-        private string _name;
-        private readonly string _nameBlob;
-        private readonly uint _offset;
-        private readonly uint _onDiskSize;
-        private readonly uint _originalSize;
-        private readonly uint _size;
-        internal readonly int _index;
-        
-        
-        public void Dump(Action<string> print)
-        {
-            print($"Name: {_name}");
-            print($"Offset: {_offset}");
-            print($"On Disk Size: {_onDiskSize}");
-            print($"Original Size: {_originalSize}");
-            print($"Size: {_size}");
-            print($"Index: {_index}");
-        }
-
-
-        public FileRecord(BSAReader bsa, FolderRecord folderRecord, BinaryReader src, int index)
-        {
-            _index = index;
-            _bsa = bsa;
-            Hash = src.ReadUInt64();
-            var size = src.ReadUInt32();
-            FlipCompression = (size & (0x1 << 30)) > 0;
-
-            if (FlipCompression)
-                _size = size ^ (0x1 << 30);
-            else
-                _size = size;
-
-            if (Compressed)
-                _size -= 4;
-
-            _offset = src.ReadUInt32();
-            Folder = folderRecord;
-
-            var old_pos = src.BaseStream.Position;
-
-            src.BaseStream.Position = _offset;
-
-            if (bsa.HasNameBlobs)
-                _nameBlob = src.ReadStringLenNoTerm(bsa.HeaderType);
-
-
-            if (Compressed)
-                _originalSize = src.ReadUInt32();
-
-            _onDiskSize = (uint) (_size - (_nameBlob == null ? 0 : _nameBlob.Length + 1));
-
-            if (Compressed)
-            {
-                _dataSize = _originalSize;
-                _onDiskSize -= 4;
-            }
-            else
-            {
-                _dataSize = _onDiskSize;
-            }
-
-            _dataOffset = src.BaseStream.Position;
-
-            src.BaseStream.Position = old_pos;
-        }
-
-        public RelativePath Path
-        {
-            get
-            {
-                return string.IsNullOrEmpty(Folder.Name) ? new RelativePath(_name) : new RelativePath(Folder.Name + "\\" + _name);
-            }
-        }
-
-        public bool Compressed
-        {
-            get
-            {
-                if (FlipCompression) return !_bsa.CompressedByDefault;
-                return _bsa.CompressedByDefault;
-            }
-        }
-
-        public uint Size => _dataSize;
-        public FileStateObject State => new BSAFileStateObject(this);
-
-        public ulong Hash { get; }
-
-        public FolderRecord Folder { get; }
-
-        public bool FlipCompression { get; }
-
-        internal void LoadFileRecord(BSAReader bsaReader, FolderRecord folder, FileRecord file, BinaryReader rdr)
-        {
-            _name = rdr.ReadStringTerm(_bsa.HeaderType);
-        }
-
-        public async ValueTask CopyDataTo(Stream output)
-        {
-            await using var in_file = await _bsa._fileName.OpenRead();
-            using var rdr = new BinaryReader(in_file);
-            rdr.BaseStream.Position = _dataOffset;
-
-            if (_bsa.HeaderType == VersionType.SSE)
-            {
-                if (Compressed)
-                {
-                    using var r = LZ4Stream.Decode(rdr.BaseStream);
-                    await r.CopyToLimitAsync(output, (int) _originalSize);
-                }
-                else
-                {
-                    await rdr.BaseStream.CopyToLimitAsync(output, (int) _onDiskSize);
-                }
-            }
-            else
-            {
-                if (Compressed)
-                {
-                    await using var z = new InflaterInputStream(rdr.BaseStream);
-                    await z.CopyToLimitAsync(output, (int) _originalSize);
-                }
-                else
-                    await rdr.BaseStream.CopyToLimitAsync(output, (int) _onDiskSize);
-            }
-        }
-    }
-
-    [JsonName("BSAFileState")]
-    public class BSAFileStateObject : FileStateObject
-    {
-        public BSAFileStateObject() { }
-        public BSAFileStateObject(FileRecord fileRecord)
-        {
-            FlipCompression = fileRecord.FlipCompression;
-            Path = fileRecord.Path;
-            Index = fileRecord._index;
-        }
-
-        public bool FlipCompression { get; set; }
-    }
-}

Compression.BSA/Compression.BSA.csproj

@@ -6,7 +6,10 @@
       <Platforms>x64</Platforms>
       <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
       <Version>3.0</Version>
+      <Nullable>enable</Nullable>
+      <WarningsAsErrors>nullable</WarningsAsErrors>
       <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
+      <AllowedOutputExtensionsInPackageBuildOutputFolder>$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb</AllowedOutputExtensionsInPackageBuildOutputFolder>
     </PropertyGroup>
     <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
       <DocumentationFile>Compression.BSA.xml</DocumentationFile>

Compression.BSA/Interfaces/IBSABuilder.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Compression.BSA
+{
+    public interface IBSABuilder : IAsyncDisposable
+    {
+        Task AddFile(FileStateObject state, Stream src);
+        Task Build(AbsolutePath filename);
+    }
+}

Compression.BSA/Interfaces/IBSAReader.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Compression.BSA
+{
+    public interface IBSAReader
+    {
+        /// <summary>
+        /// The files defined by the archive
+        /// </summary>
+        IEnumerable<IFile> Files { get; }
+
+        ArchiveStateObject State { get; }
+
+        void Dump(Action<string> print);
+    }
+}

Compression.BSA/Interfaces/IFile.cs

@@ -1,43 +1,12 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Text;
 using System.Threading.Tasks;
 using Wabbajack.Common;
 
 namespace Compression.BSA
 {
-    public interface IBSAReader
-    {
-        /// <summary>
-        /// The files defined by the archive
-        /// </summary>
-        IEnumerable<IFile> Files { get; }
-
-        ArchiveStateObject State { get; }
-
-        void Dump(Action<string> print);
-    }
-
-    public interface IBSABuilder : IAsyncDisposable
-    {
-        Task AddFile(FileStateObject state, Stream src);
-        Task Build(AbsolutePath filename);
-    }
-
-    public class ArchiveStateObject
-    {
-        public virtual async Task<IBSABuilder> MakeBuilder(long size)
-        {
-            throw new NotImplementedException();
-        }
-    }
-
-    public class FileStateObject
-    {
-        public int Index { get; set; }
-        public RelativePath Path { get; set; }
-    }
-    
     public interface IFile
     {
         /// <summary>

Compression.BSA/Interfaces/IFolder.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Compression.BSA
+{
+    public interface IFolder
+    {
+        string? Name { get; }
+        IEnumerable<IFile> Files { get; }
+        int FileCount { get; }
+    }
+}

Compression.BSA/State/ArchiveStateObject.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Compression.BSA
+{
+    public abstract class ArchiveStateObject
+    {
+        public abstract Task<IBSABuilder> MakeBuilder(long size);
+    }
+}

Compression.BSA/State/FileStateObject.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Wabbajack.Common;
+
+namespace Compression.BSA
+{
+    public abstract class FileStateObject
+    {
+        public int Index { get; set; }
+        public RelativePath Path { get; set; }
+    }
+}

Compression.BSA/TES3Reader.cs

@@ -5,6 +5,7 @@ using System.Text;
 using System.Threading.Tasks;
 using Wabbajack.Common;
 using Wabbajack.Common.Serialization.Json;
+#nullable disable
 
 namespace Compression.BSA
 {

Compression.BSA/Utils.cs

@@ -30,9 +30,7 @@ namespace Compression.BSA
         public static string ReadStringLen(this BinaryReader rdr, VersionType version)
         {
             var len = rdr.ReadByte();
-            if (len == 0)
-                //rdr.ReadByte();
-                return "";
+            if (len == 0) return string.Empty;
 
             var bytes = rdr.ReadBytes(len - 1);
             rdr.ReadByte();
@@ -61,6 +59,18 @@ namespace Compression.BSA
             return GetEncoding(version).GetString(acc.ToArray());
         }
 
+        public static string ReadStringLenTerm(this ReadOnlyMemorySlice<byte> bytes, VersionType version)
+        {
+            if (bytes.Length <= 1) return string.Empty;
+            return GetEncoding(version).GetString(bytes.Slice(1, bytes[0]));
+        }
+
+        public static string ReadStringTerm(this ReadOnlyMemorySlice<byte> bytes, VersionType version)
+        {
+            if (bytes.Length <= 1) return string.Empty;
+            return GetEncoding(version).GetString(bytes[0..^1]);
+        }
+
         /// <summary>
         ///     Returns bytes for a \0 terminated string
         /// </summary>

Wabbajack.Common.Test/WorkQueueTests.cs

@@ -6,7 +6,6 @@ using System.Reactive.Linq;
 using System.Reactive.Subjects;
 using System.Threading;
 using System.Threading.Tasks;
-using Splat;
 using Wabbajack;
 using Wabbajack.Common;
 using Xunit;

Wabbajack.Common/Error States/ErrorResponse.cs

@@ -1,5 +1,4 @@
 using System;
-using DynamicData.Kernel;
 
 namespace Wabbajack
 {

Wabbajack.Common/Extensions/StreamExt.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace Wabbajack.Common
+{
+    public static class StreamExt
+    {
+        public static long Remaining(this Stream stream)
+        {
+            return stream.Length - stream.Position;
+        }
+    }
+}

Wabbajack.Common/MemorySlice.cs

@@ -0,0 +1,218 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Text;
+
+namespace Wabbajack.Common
+{
+    public struct MemorySlice<T> : IEnumerable<T>
+    {
+        private T[] _arr;
+        private int _startPos;
+        private int _length;
+        public int Length => _length;
+        public int StartPosition => _startPos;
+
+        [DebuggerStepThrough]
+        public MemorySlice(T[] arr)
+        {
+            this._arr = arr;
+            this._startPos = 0;
+            this._length = arr.Length;
+        }
+
+        [DebuggerStepThrough]
+        public MemorySlice(T[] arr, int startPos, int length)
+        {
+            this._arr = arr;
+            this._startPos = startPos;
+            this._length = length;
+        }
+
+        public Span<T> Span => _arr.AsSpan(start: _startPos, length: _length);
+
+        public T this[int index]
+        {
+            get => _arr[index + _startPos];
+            set => _arr[index + _startPos] = value;
+        }
+
+        [DebuggerStepThrough]
+        public MemorySlice<T> Slice(int start)
+        {
+            var startPos = _startPos + start;
+            if (startPos < 0)
+            {
+                throw new ArgumentOutOfRangeException();
+            }
+            return new MemorySlice<T>()
+            {
+                _arr = _arr,
+                _startPos = startPos,
+                _length = _length - start
+            };
+        }
+
+        [DebuggerStepThrough]
+        public MemorySlice<T> Slice(int start, int length)
+        {
+            var startPos = _startPos + start;
+            if (startPos < 0)
+            {
+                throw new ArgumentOutOfRangeException();
+            }
+            if (startPos + length > _arr.Length)
+            {
+                throw new ArgumentOutOfRangeException();
+            }
+            return new MemorySlice<T>()
+            {
+                _arr = _arr,
+                _startPos = startPos,
+                _length = length
+            };
+        }
+
+        public IEnumerator<T> GetEnumerator()
+        {
+            for (int i = 0; i < _length; i++)
+            {
+                yield return this._arr[i + _startPos];
+            }
+        }
+
+        IEnumerator IEnumerable.GetEnumerator() => this.GetEnumerator();
+
+        public static implicit operator ReadOnlyMemorySlice<T>(MemorySlice<T> mem)
+        {
+            return new ReadOnlyMemorySlice<T>(
+                mem._arr,
+                mem._startPos,
+                mem._length);
+        }
+
+        public static implicit operator ReadOnlySpan<T>(MemorySlice<T> mem)
+        {
+            return mem.Span;
+        }
+
+        public static implicit operator Span<T>(MemorySlice<T> mem)
+        {
+            return mem.Span;
+        }
+
+        public static implicit operator MemorySlice<T>(T[] mem)
+        {
+            return new MemorySlice<T>(mem);
+        }
+
+        public static implicit operator MemorySlice<T>?(T[]? mem)
+        {
+            if (mem == null) return null;
+            return new MemorySlice<T>(mem);
+        }
+    }
+
+    public struct ReadOnlyMemorySlice<T> : IEnumerable<T>
+    {
+        private T[] _arr;
+        private int _startPos;
+        private int _length;
+        public int Length => _length;
+        public int StartPosition => _startPos;
+
+        [DebuggerStepThrough]
+        public ReadOnlyMemorySlice(T[] arr)
+        {
+            this._arr = arr;
+            this._startPos = 0;
+            this._length = arr.Length;
+        }
+
+        [DebuggerStepThrough]
+        public ReadOnlyMemorySlice(T[] arr, int startPos, int length)
+        {
+            this._arr = arr;
+            this._startPos = startPos;
+            this._length = length;
+        }
+
+        public ReadOnlySpan<T> Span => _arr.AsSpan(start: _startPos, length: _length);
+
+        public T this[int index] => _arr[index + _startPos];
+
+        [DebuggerStepThrough]
+        public ReadOnlyMemorySlice<T> Slice(int start)
+        {
+            var startPos = _startPos + start;
+            if (startPos < 0)
+            {
+                throw new ArgumentOutOfRangeException();
+            }
+            return new ReadOnlyMemorySlice<T>()
+            {
+                _arr = _arr,
+                _startPos = _startPos + start,
+                _length = _length - start
+            };
+        }
+
+        [DebuggerStepThrough]
+        public ReadOnlyMemorySlice<T> Slice(int start, int length)
+        {
+            var startPos = _startPos + start;
+            if (startPos < 0)
+            {
+                throw new ArgumentOutOfRangeException();
+            }
+            if (startPos + length > _arr.Length)
+            {
+                throw new ArgumentOutOfRangeException();
+            }
+            return new ReadOnlyMemorySlice<T>()
+            {
+                _arr = _arr,
+                _startPos = _startPos + start,
+                _length = length
+            };
+        }
+
+        public IEnumerator<T> GetEnumerator()
+        {
+            for (int i = 0; i < _length; i++)
+            {
+                yield return this._arr[i + _startPos];
+            }
+        }
+
+        IEnumerator IEnumerable.GetEnumerator() => this.GetEnumerator();
+
+        public static implicit operator ReadOnlySpan<T>(ReadOnlyMemorySlice<T> mem)
+        {
+            return mem.Span;
+        }
+
+        public static implicit operator ReadOnlyMemorySlice<T>?(T[]? mem)
+        {
+            if (mem == null) return null;
+            return new ReadOnlyMemorySlice<T>(mem);
+        }
+
+        public static implicit operator ReadOnlyMemorySlice<T>(T[] mem)
+        {
+            return new ReadOnlyMemorySlice<T>(mem);
+        }
+    }
+
+    public static class MemorySliceExt
+    {
+        public static bool Equal<T>(ReadOnlyMemorySlice<T>? lhs, ReadOnlyMemorySlice<T>? rhs)
+            where T : IEquatable<T>
+        {
+            if (lhs == null && rhs == null) return true;
+            if (lhs == null || rhs == null) return false;
+            return MemoryExtensions.SequenceEqual(lhs.Value.Span, rhs.Value.Span);
+        }
+    }
+}

Wabbajack.Common/Paths/AbsolutePath.cs

@@ -397,14 +397,14 @@ namespace Wabbajack.Common
         {
             var path = _path;
             return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () =>
-                File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite));
+                File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, bufferSize: 1048576, useAsync: true));
         }
 
         public ValueTask<FileStream> WriteShared()
         {
             var path = _path;
             return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () =>
-                File.Open(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite));
+                File.Open(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite, bufferSize: 1048576, useAsync: true));
         }
 
         public async Task CopyDirectoryToAsync(AbsolutePath destination)

Wabbajack.Common/Paths/RelativePath.cs

@@ -13,7 +13,7 @@ namespace Wabbajack.Common
         private readonly string? _nullable_path;
         private string _path => _nullable_path ?? string.Empty;
 
-        public RelativePath(string path)
+        public RelativePath(string path, bool skipValidation = false)
         {
             if (string.IsNullOrWhiteSpace(path))
             {
@@ -28,7 +28,10 @@ namespace Wabbajack.Common
             }
 
             _nullable_path = trimmed;
-            Validate();
+            if (!skipValidation)
+            {
+                Validate();
+            }
         }
 
         public override string ToString()

Wabbajack.Common/Utils.cs

@@ -7,6 +7,7 @@ using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
+using System.Reactive.Concurrency;
 using System.Reactive.Linq;
 using System.Reactive.Subjects;
 using System.Reflection;
@@ -23,7 +24,6 @@ using IniParser.Model.Configuration;
 using IniParser.Parser;
 using Microsoft.Win32;
 using Newtonsoft.Json;
-using ReactiveUI;
 using RocksDbSharp;
 using Wabbajack.Common.StatusFeed;
 using Wabbajack.Common.StatusFeed.Errors;
@@ -107,7 +107,7 @@ namespace Wabbajack.Common
             AppLocalEvents = Observable.Merge(Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Changed += h, h => watcher.Changed -= h).Select(e => (FileEventType.Changed, e.EventArgs)),
                                                 Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Created += h, h => watcher.Created -= h).Select(e => (FileEventType.Created, e.EventArgs)),
                                                 Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Deleted += h, h => watcher.Deleted -= h).Select(e => (FileEventType.Deleted, e.EventArgs)))
-                                       .ObserveOn(RxApp.TaskpoolScheduler);
+                                       .ObserveOn(Scheduler.Default);
             watcher.EnableRaisingEvents = true;
             InitPatches();
         }
@@ -1098,13 +1098,6 @@ namespace Wabbajack.Common
             return bytes;
         }
 
-        public static async Task CopyFileAsync(string src, string dest)
-        {
-            await using var s = File.OpenRead(src);
-            await using var d = File.Create(dest);
-            await s.CopyToAsync(d);
-        }
-
         public static string ToNormalString(this SecureString value)
         {
             var valuePtr = IntPtr.Zero;

Wabbajack.Common/Wabbajack.Common.csproj

@@ -15,6 +15,7 @@
         <PackageIconUrl>https://www.wabbajack.org/favicon.ico</PackageIconUrl>
         <RepositoryUrl>https://github.com/wabbajack-tools/wabbajack</RepositoryUrl>
         <Version>3.0</Version>
+        <AllowedOutputExtensionsInPackageBuildOutputFolder>$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb</AllowedOutputExtensionsInPackageBuildOutputFolder>
     </PropertyGroup>
     <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
       <DocumentationFile>Wabbajack.Common.xml</DocumentationFile>
@@ -50,12 +51,12 @@
         <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0-preview.6.20305.6" />
         <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
         <PackageReference Include="Octodiff" Version="1.2.1" />
-        <PackageReference Include="ReactiveUI" Version="11.5.17" />
         <PackageReference Include="RocksDbNative" Version="6.2.2" />
         <PackageReference Include="RocksDbSharp" Version="6.2.2" />
         <PackageReference Include="SharpZipLib" Version="1.2.0" />
         <PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
         <PackageReference Include="System.Net.Http" Version="4.3.4" />
+        <PackageReference Include="System.Reactive" Version="4.4.1" />
         <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="5.0.0-preview.6.20305.6" />
         <PackageReference Include="System.Security.Principal.Windows" Version="5.0.0-preview.6.20305.6" />
         <PackageReference Include="YamlDotNet" Version="8.1.2" />

Wabbajack.Common/WorkQueue.cs

@@ -8,7 +8,6 @@ using System.Reactive.Subjects;
 using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
-using DynamicData;
 using Wabbajack.Common.StatusFeed;
 
 [assembly: InternalsVisibleTo("Wabbajack.Test")]
@@ -26,6 +25,7 @@ namespace Wabbajack.Common
         public static bool WorkerThread => AsyncLocalCurrentQueue.Value != null;
         public bool IsWorkerThread => WorkerThread;
         internal static readonly AsyncLocal<WorkQueue?> AsyncLocalCurrentQueue = new AsyncLocal<WorkQueue?>();
+        public static WorkQueue? AsyncLocalQueue => AsyncLocalCurrentQueue.Value;
 
         private readonly Subject<CPUStatus> _Status = new Subject<CPUStatus>();
         public IObservable<CPUStatus> Status => _Status;
@@ -69,15 +69,14 @@ namespace Wabbajack.Common
         public WorkQueue(IObservable<int>? numThreads)
         {
             // Hook onto the number of active threads subject, and subscribe to it for changes
-            _activeNumThreadsObservable
+            _disposables.Add(_activeNumThreadsObservable
                 // Select the latest driving observable
                 .Select(x => x ?? Observable.Return(Environment.ProcessorCount))
                 .Switch()
                 .DistinctUntilChanged()
                 // Add new threads if it increases
                 .SelectTask(AddNewThreadsIfNeeded)
-                .Subscribe()
-                .DisposeWith(_disposables);
+                .Subscribe());
             // Set the incoming driving observable to be active
             SetActiveThreadsObservable(numThreads);
         }

Wabbajack.Lib/Interventions/AUserIntervention.cs

@@ -5,8 +5,9 @@ using System.Text;
 using System.Threading.Tasks;
 using System.Windows.Input;
 using ReactiveUI;
+using Wabbajack.Common;
 
-namespace Wabbajack.Common
+namespace Wabbajack.Lib
 {
     public abstract class AUserIntervention : ReactiveObject, IUserIntervention
     {
@@ -17,7 +18,7 @@ namespace Wabbajack.Common
         private bool _handled;
         public bool Handled { get => _handled; set => this.RaiseAndSetIfChanged(ref _handled, value); }
 
-        public int CpuID { get; } = WorkQueue.AsyncLocalCurrentQueue.Value?.CpuId ?? WorkQueue.UnassignedCpuId;
+        public int CpuID { get; } = WorkQueue.AsyncLocalQueue?.CpuId ?? WorkQueue.UnassignedCpuId;
 
         public abstract void Cancel();
         public ICommand CancelCommand { get; }

Wabbajack.Lib/Interventions/ConfirmationIntervention.cs

@@ -6,7 +6,7 @@ using System.Threading.Tasks;
 using System.Windows.Input;
 using ReactiveUI;
 
-namespace Wabbajack.Common
+namespace Wabbajack.Lib
 {
     public abstract class ConfirmationIntervention : AUserIntervention
     {

Wabbajack.Lib/Interventions/IUserIntervention.cs

@@ -6,7 +6,7 @@ using System.Threading.Tasks;
 using ReactiveUI;
 using Wabbajack.Common.StatusFeed;
 
-namespace Wabbajack.Common
+namespace Wabbajack.Lib
 {
     /// <summary>
     /// Defines a message that requires user interaction. The user must perform some action

Wabbajack.Test/RestartingDownloadsTests.cs

@@ -7,6 +7,7 @@ using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
 using Wabbajack.Common.StatusFeed;
+using Wabbajack.Lib;
 using Wabbajack.Lib.Downloaders;
 using Xunit;
 using Xunit.Abstractions;

Wabbajack.sln

@@ -57,6 +57,7 @@ Global
 		{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Debug|x64.ActiveCfg = Debug|x64
 		{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Debug|x64.Build.0 = Debug|x64
 		{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|Any CPU.ActiveCfg = Release|x64
+		{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|Any CPU.Build.0 = Release|x64
 		{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|x64.ActiveCfg = Release|x64
 		{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|x64.Build.0 = Release|x64
 		{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Debug|Any CPU.ActiveCfg = Debug|x64
@@ -64,6 +65,7 @@ Global
 		{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Debug|x64.ActiveCfg = Debug|x64
 		{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Debug|x64.Build.0 = Debug|x64
 		{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|Any CPU.ActiveCfg = Release|x64
+		{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|Any CPU.Build.0 = Release|x64
 		{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|x64.ActiveCfg = Release|x64
 		{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|x64.Build.0 = Release|x64
 		{0A820830-A298-497D-85E0-E9A89EFEF5FE}.Debug|Any CPU.ActiveCfg = Debug|x64
@@ -99,6 +101,7 @@ Global
 		{89281BA1-67C8-48D2-9D6E-0F5CC85AD8C9}.Debug|x64.ActiveCfg = Debug|x64
 		{89281BA1-67C8-48D2-9D6E-0F5CC85AD8C9}.Debug|x64.Build.0 = Debug|x64
 		{89281BA1-67C8-48D2-9D6E-0F5CC85AD8C9}.Release|Any CPU.ActiveCfg = Release|x64
+		{89281BA1-67C8-48D2-9D6E-0F5CC85AD8C9}.Release|Any CPU.Build.0 = Release|x64
 		{89281BA1-67C8-48D2-9D6E-0F5CC85AD8C9}.Release|x64.ActiveCfg = Release|x64
 		{89281BA1-67C8-48D2-9D6E-0F5CC85AD8C9}.Release|x64.Build.0 = Release|x64
 		{F72C17EC-0881-4455-8B0E-E1CC4FFD642E}.Debug|Any CPU.ActiveCfg = Debug|x64
@@ -120,6 +123,7 @@ Global
 		{685D8BB1-D178-4D2C-85C7-C54A36FB7454}.Debug|x64.ActiveCfg = Debug|x64
 		{685D8BB1-D178-4D2C-85C7-C54A36FB7454}.Debug|x64.Build.0 = Debug|x64
 		{685D8BB1-D178-4D2C-85C7-C54A36FB7454}.Release|Any CPU.ActiveCfg = Release|x64
+		{685D8BB1-D178-4D2C-85C7-C54A36FB7454}.Release|Any CPU.Build.0 = Release|x64
 		{685D8BB1-D178-4D2C-85C7-C54A36FB7454}.Release|x64.ActiveCfg = Release|x64
 		{685D8BB1-D178-4D2C-85C7-C54A36FB7454}.Release|x64.Build.0 = Release|x64
 		{D6856DBF-C959-4867-A8A8-343DA2D2715E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU

Wabbajack/Views/Compilers/CompilerView.xaml

@@ -2,7 +2,7 @@
     x:Class="Wabbajack.CompilerView"
     xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
     xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-    xmlns:common="clr-namespace:Wabbajack.Common;assembly=Wabbajack.Common"
+    xmlns:lib="clr-namespace:Wabbajack.Lib;assembly=Wabbajack.Lib"
     xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
     xmlns:icon="http://metro.mahapps.com/winfx/xaml/iconpacks"
     xmlns:local="clr-namespace:Wabbajack"
@@ -200,7 +200,7 @@
                 ViewModel="{Binding}" />
             <local:AttentionBorder x:Name="UserInterventionsControl" Grid.Column="2">
                 <Grid>
-                    <local:ConfirmationInterventionView DataContext="{Binding ActiveGlobalUserIntervention}" Visibility="{Binding ActiveGlobalUserIntervention, Converter={StaticResource IsTypeVisibilityConverter}, ConverterParameter={x:Type common:ConfirmationIntervention}}" />
+                    <local:ConfirmationInterventionView DataContext="{Binding ActiveGlobalUserIntervention}" Visibility="{Binding ActiveGlobalUserIntervention, Converter={StaticResource IsTypeVisibilityConverter}, ConverterParameter={x:Type lib:ConfirmationIntervention}}" />
                 </Grid>
             </local:AttentionBorder>
             <local:CompilationCompleteView Grid.Column="2"

Wabbajack/Views/Installers/InstallationView.xaml

@@ -304,7 +304,7 @@
                 x:Name="UserInterventionsControl"
                 Content="{Binding ActiveGlobalUserIntervention}">
                 <local:AttentionBorder.Resources>
-                    <DataTemplate DataType="{x:Type common:ConfirmationIntervention}">
+                    <DataTemplate DataType="{x:Type lib:ConfirmationIntervention}">
                         <local:ConfirmationInterventionView ViewModel="{Binding}" />
                     </DataTemplate>
                     <DataTemplate DataType="{x:Type local:ConfirmUpdateOfExistingInstallVM}">

Wabbajack/Views/Interventions/ConfirmationInterventionView.xaml

@@ -2,14 +2,14 @@
     x:Class="Wabbajack.ConfirmationInterventionView"
     xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
     xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-    xmlns:common="clr-namespace:Wabbajack.Common;assembly=Wabbajack.Common"
     xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
     xmlns:local="clr-namespace:Wabbajack"
     xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+    xmlns:lib="clr-namespace:Wabbajack.Lib;assembly=Wabbajack.Lib"
     xmlns:rxui="http://reactiveui.net"
     d:DesignHeight="450"
     d:DesignWidth="800"
-    x:TypeArguments="common:ConfirmationIntervention"
+    x:TypeArguments="lib:ConfirmationIntervention"
     mc:Ignorable="d">
     <Grid Margin="10">
         <Grid.ColumnDefinitions>

Wabbajack/Views/Interventions/ConfirmationInterventionView.xaml.cs

@@ -14,7 +14,7 @@ using System.Windows.Media.Imaging;
 using System.Windows.Navigation;
 using System.Windows.Shapes;
 using ReactiveUI;
-using Wabbajack.Common;
+using Wabbajack.Lib;
 
 namespace Wabbajack
 {