diff --git a/Compression.BSA/BA2Builder.cs b/Compression.BSA/BA2Builder.cs index 2e366a80..f9a2e346 100644 --- a/Compression.BSA/BA2Builder.cs +++ b/Compression.BSA/BA2Builder.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using ICSharpCode.SharpZipLib.Zip.Compression; using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using Wabbajack.Common; +#nullable disable namespace Compression.BSA { diff --git a/Compression.BSA/BA2Reader.cs b/Compression.BSA/BA2Reader.cs index 32e5639e..d6c6137b 100644 --- a/Compression.BSA/BA2Reader.cs +++ b/Compression.BSA/BA2Reader.cs @@ -8,6 +8,7 @@ using ICSharpCode.SharpZipLib.Zip.Compression; using Wabbajack.Common; using Wabbajack.Common.Serialization.Json; using File = Alphaleonis.Win32.Filesystem.File; +#nullable disable namespace Compression.BSA { diff --git a/Compression.BSA/BSA/ArchiveFlags.cs b/Compression.BSA/BSA/ArchiveFlags.cs new file mode 100644 index 00000000..59c01a1a --- /dev/null +++ b/Compression.BSA/BSA/ArchiveFlags.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Compression.BSA +{ + [Flags] + public enum ArchiveFlags : uint + { + HasFolderNames = 0x1, + HasFileNames = 0x2, + Compressed = 0x4, + Unk4 = 0x8, + Unk5 = 0x10, + Unk6 = 0x20, + XBox360Archive = 0x40, + Unk8 = 0x80, + HasFileNameBlobs = 0x100, + Unk10 = 0x200, + Unk11 = 0x400 + } + +} diff --git a/Compression.BSA/BSABuilder.cs b/Compression.BSA/BSA/Builder/BSABuilder.cs similarity index 90% rename from Compression.BSA/BSABuilder.cs rename to Compression.BSA/BSA/Builder/BSABuilder.cs index cf56468d..c4c84eea 100644 --- a/Compression.BSA/BSABuilder.cs +++ b/Compression.BSA/BSA/Builder/BSABuilder.cs @@ -10,20 +10,18 @@ using K4os.Compression.LZ4.Streams; using Wabbajack.Common; using File = Alphaleonis.Win32.Filesystem.File; using Path = Alphaleonis.Win32.Filesystem.Path; +#nullable disable namespace Compression.BSA { public class BSABuilder : IBSABuilder { - internal uint _archiveFlags; - internal uint _fileFlags; internal byte[] _fileId; private List _files = new List(); internal List _folders = new List(); internal uint _offset; internal uint _totalFileNameLength; - internal uint _version; internal DiskSlabAllocator _slab; public static async Task Create(long size) @@ -39,32 +37,20 @@ namespace Compression.BSA public static async Task Create(BSAStateObject bsaStateObject, long size) { - var self = await Create(size); - self._version = bsaStateObject.Version; - self._fileFlags = bsaStateObject.FileFlags; - self._archiveFlags = bsaStateObject.ArchiveFlags; + var self = await Create(size).ConfigureAwait(false); + self.HeaderType = (VersionType)bsaStateObject.Version; + self.FileFlags = (FileFlags)bsaStateObject.FileFlags; + self.ArchiveFlags = (ArchiveFlags)bsaStateObject.ArchiveFlags; return self; } public IEnumerable Files => _files; - public ArchiveFlags ArchiveFlags - { - get => (ArchiveFlags) _archiveFlags; - set => _archiveFlags = (uint) value; - } + public ArchiveFlags ArchiveFlags { get; set; } - public FileFlags FileFlags - { - get => (FileFlags) _archiveFlags; - set => _archiveFlags = (uint) value; - } + public FileFlags FileFlags { get; set; } - public VersionType HeaderType - { - get => (VersionType) _version; - set => _version = (uint) value; - } + public VersionType HeaderType { get; set; } public IEnumerable FolderNames { @@ -74,13 +60,13 @@ namespace Compression.BSA } } - public bool HasFolderNames => (_archiveFlags & 0x1) > 0; + public bool HasFolderNames => ArchiveFlags.HasFlag(ArchiveFlags.HasFileNames); - public bool HasFileNames => (_archiveFlags & 0x2) > 0; + public bool HasFileNames => ArchiveFlags.HasFlag(ArchiveFlags.HasFileNames); - public bool CompressedByDefault => (_archiveFlags & 0x4) > 0; + public bool CompressedByDefault => ArchiveFlags.HasFlag(ArchiveFlags.Compressed); - public bool HasNameBlobs => (_archiveFlags & 0x100) > 0; + public bool HasNameBlobs => ArchiveFlags.HasFlag(ArchiveFlags.HasFileNameBlobs); public async ValueTask DisposeAsync() { @@ -105,9 +91,9 @@ namespace Compression.BSA await using var wtr = new BinaryWriter(fs); wtr.Write(_fileId); - wtr.Write(_version); + wtr.Write((uint)HeaderType); wtr.Write(_offset); - wtr.Write(_archiveFlags); + wtr.Write((uint)ArchiveFlags); var folders = FolderNames.ToList(); wtr.Write((uint) folders.Count); wtr.Write((uint) _files.Count); @@ -115,7 +101,7 @@ namespace Compression.BSA var s = _files.Select(f => f._pathBytes.Count()).Sum(); _totalFileNameLength = (uint) _files.Select(f => f._nameBytes.Count()).Sum(); wtr.Write(_totalFileNameLength); // totalFileNameLength - wtr.Write(_fileFlags); + wtr.Write((uint)FileFlags); foreach (var folder in _folders) folder.WriteFolderRecord(wtr); diff --git a/Compression.BSA/BSA/FileFlags.cs b/Compression.BSA/BSA/FileFlags.cs new file mode 100644 index 00000000..9612eb6a --- /dev/null +++ b/Compression.BSA/BSA/FileFlags.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Compression.BSA +{ + [Flags] + public enum FileFlags : uint + { + Meshes = 0x1, + Textures = 0x2, + Menus = 0x4, + Sounds = 0x8, + Voices = 0x10, + Shaders = 0x20, + Trees = 0x40, + Fonts = 0x80, + Miscellaneous = 0x100 + } +} diff --git a/Compression.BSA/BSA/Reader/BSAFileStateObject.cs b/Compression.BSA/BSA/Reader/BSAFileStateObject.cs new file mode 100644 index 00000000..cdc258cc --- /dev/null +++ b/Compression.BSA/BSA/Reader/BSAFileStateObject.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Wabbajack.Common.Serialization.Json; +using File = Alphaleonis.Win32.Filesystem.File; + +namespace Compression.BSA +{ + [JsonName("BSAFileState")] + public class BSAFileStateObject : FileStateObject + { + public bool FlipCompression { get; set; } + + public BSAFileStateObject() { } + + public BSAFileStateObject(FileRecord fileRecord) + { + FlipCompression = fileRecord.FlipCompression; + Path = fileRecord.Path; + Index = fileRecord._index; + } + } +} diff --git a/Compression.BSA/BSA/Reader/BSAReader.cs b/Compression.BSA/BSA/Reader/BSAReader.cs new file mode 100644 index 00000000..57f23cf5 --- /dev/null +++ b/Compression.BSA/BSA/Reader/BSAReader.cs @@ -0,0 +1,174 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Common.Serialization.Json; +using File = Alphaleonis.Win32.Filesystem.File; + +namespace Compression.BSA +{ + public class BSAReader : IBSAReader + { + public const int HeaderLength = 0x24; + + internal uint _fileCount; + internal AbsolutePath _fileName; + internal uint _folderCount; + internal uint _folderRecordOffset; + private Lazy _folders = null!; + private Lazy> _foldersByName = null!; + internal string _magic = string.Empty; + internal uint _totalFileNameLength; + internal uint _totalFolderNameLength; + + public VersionType HeaderType { get; private set; } + + public ArchiveFlags ArchiveFlags { get; private set; } + + public FileFlags FileFlags { get; private set; } + + public IEnumerable Files => _folders.Value.SelectMany(f => f.Files); + + public IEnumerable Folders => _folders.Value; + + public ArchiveStateObject State => new BSAStateObject(this); + + public bool HasFolderNames => ArchiveFlags.HasFlag(ArchiveFlags.HasFolderNames); + + public bool HasFileNames => ArchiveFlags.HasFlag(ArchiveFlags.HasFileNames); + + public bool CompressedByDefault => ArchiveFlags.HasFlag(ArchiveFlags.Compressed); + + public bool Bit9Set => ArchiveFlags.HasFlag(ArchiveFlags.HasFileNameBlobs); + + public bool HasNameBlobs + { + get + { + if (HeaderType == VersionType.FO3 || HeaderType == VersionType.SSE) return Bit9Set; + return false; + } + } + + public void Dump(Action print) + { + print($"File Name: {_fileName}"); + print($"File Count: {_fileCount}"); + print($"Magic: {_magic}"); + + foreach (var file in Files) + { + print("\n"); + file.Dump(print); + } + } + + public static async ValueTask LoadAsync(AbsolutePath filename) + { + using var stream = await filename.OpenRead().ConfigureAwait(false); + using var br = new BinaryReader(stream); + var bsa = new BSAReader { _fileName = filename }; + bsa.LoadHeaders(br); + return bsa; + } + + public static BSAReader Load(AbsolutePath filename) + { + var bsa = new BSAReader { _fileName = filename }; + using var rdr = bsa.GetStream(); + bsa.LoadHeaders(rdr); + return bsa; + } + + internal BinaryReader GetStream() + { + return new BinaryReader(File.Open(_fileName.ToString(), FileMode.Open, FileAccess.Read, FileShare.Read)); + } + + private void LoadHeaders(BinaryReader rdr) + { + var fourcc = Encoding.ASCII.GetString(rdr.ReadBytes(4)); + + if (fourcc != "BSA\0") + throw new InvalidDataException("Archive is not a BSA"); + + _magic = fourcc; + HeaderType = (VersionType)rdr.ReadUInt32(); + _folderRecordOffset = rdr.ReadUInt32(); + ArchiveFlags = (ArchiveFlags)rdr.ReadUInt32(); + _folderCount = rdr.ReadUInt32(); + _fileCount = rdr.ReadUInt32(); + _totalFolderNameLength = rdr.ReadUInt32(); + _totalFileNameLength = rdr.ReadUInt32(); + FileFlags = (FileFlags)rdr.ReadUInt32(); + + _folders = new Lazy( + isThreadSafe: true, + valueFactory: () => LoadFolderRecords()); + _foldersByName = new Lazy>( + isThreadSafe: true, + valueFactory: GetFolderDictionary); + } + + private FolderRecord[] LoadFolderRecords() + { + using var rdr = GetStream(); + rdr.BaseStream.Position = _folderRecordOffset; + var folderHeaderLength = FolderRecord.HeaderLength(HeaderType); + ReadOnlyMemorySlice folderHeaderData = rdr.ReadBytes(checked((int)(folderHeaderLength * _folderCount))); + + var ret = new FolderRecord[_folderCount]; + for (var idx = 0; idx < _folderCount; idx += 1) + ret[idx] = new FolderRecord(this, folderHeaderData.Slice(idx * folderHeaderLength, folderHeaderLength), idx); + + // Slice off appropriate file header data per folder + int fileCountTally = 0; + foreach (var folder in ret) + { + folder.ProcessFileRecordHeadersBlock(rdr, fileCountTally); + fileCountTally = checked((int)(fileCountTally + folder.FileCount)); + } + + if (HasFileNames) + { + var filenameBlock = new FileNameBlock(this, rdr.BaseStream.Position); + foreach (var folder in ret) + { + folder.FileNameBlock = filenameBlock; + } + } + + return ret; + } + + private Dictionary GetFolderDictionary() + { + if (!HasFolderNames) + { + throw new ArgumentException("Cannot get folders by name if the BSA does not have folder names."); + } + var ret = new Dictionary(); + foreach (var folder in _folders.Value) + { + ret.Add(folder.Name!, folder); + } + return ret; + } + + public bool TryGetFolder(string path, [MaybeNullWhen(false)] out IFolder folder) + { + if (!HasFolderNames + || !_foldersByName.Value.TryGetValue(path, out var folderRec)) + { + folder = default; + return false; + } + folder = folderRec; + return true; + } + } +} diff --git a/Compression.BSA/BSA/Reader/BSAStateObject.cs b/Compression.BSA/BSA/Reader/BSAStateObject.cs new file mode 100644 index 00000000..2d1852dc --- /dev/null +++ b/Compression.BSA/BSA/Reader/BSAStateObject.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common.Serialization.Json; +using File = Alphaleonis.Win32.Filesystem.File; + +namespace Compression.BSA +{ + [JsonName("BSAState")] + public class BSAStateObject : ArchiveStateObject + { + public string Magic { get; set; } = string.Empty; + public uint Version { get; set; } + public uint ArchiveFlags { get; set; } + public uint FileFlags { get; set; } + + public BSAStateObject() + { + } + + public BSAStateObject(BSAReader bsaReader) + { + Magic = bsaReader._magic; + Version = (uint)bsaReader.HeaderType; + ArchiveFlags = (uint)bsaReader.ArchiveFlags; + FileFlags = (uint)bsaReader.FileFlags; + } + + public override async Task MakeBuilder(long size) + { + return await BSABuilder.Create(this, size).ConfigureAwait(false); + } + } +} diff --git a/Compression.BSA/BSA/Reader/FileNameBlock.cs b/Compression.BSA/BSA/Reader/FileNameBlock.cs new file mode 100644 index 00000000..b8644234 --- /dev/null +++ b/Compression.BSA/BSA/Reader/FileNameBlock.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Wabbajack.Common; + +namespace Compression.BSA +{ + internal class FileNameBlock + { + public readonly Lazy[]> Names; + + public FileNameBlock(BSAReader bsa, long position) + { + Names = new Lazy[]>( + mode: System.Threading.LazyThreadSafetyMode.ExecutionAndPublication, + valueFactory: () => + { + using var stream = bsa.GetStream(); + stream.BaseStream.Position = position; + ReadOnlyMemorySlice data = stream.ReadBytes(checked((int)bsa._totalFileNameLength)); + ReadOnlyMemorySlice[] names = new ReadOnlyMemorySlice[bsa._fileCount]; + for (int i = 0; i < bsa._fileCount; i++) + { + var index = data.Span.IndexOf(default(byte)); + if (index == -1) + { + throw new InvalidDataException("Did not end all of its strings in null bytes"); + } + names[i] = data.Slice(0, index + 1); + var str = names[i].ReadStringTerm(bsa.HeaderType); + data = data.Slice(index + 1); + } + // Data doesn't seem to need to be fully consumed. + // Official BSAs have overflow of zeros + return names; + }); + } + } +} diff --git a/Compression.BSA/BSA/Reader/FileRecord.cs b/Compression.BSA/BSA/Reader/FileRecord.cs new file mode 100644 index 00000000..d6e2b7bf --- /dev/null +++ b/Compression.BSA/BSA/Reader/FileRecord.cs @@ -0,0 +1,167 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.IO; +using System.Runtime.Versioning; +using System.Text; +using System.Threading.Tasks; +using ICSharpCode.SharpZipLib.Zip.Compression.Streams; +using K4os.Compression.LZ4.Streams; +using Wabbajack.Common; +using File = Alphaleonis.Win32.Filesystem.File; + +namespace Compression.BSA +{ + public class FileRecord : IFile + { + public const int HeaderLength = 0x10; + + private readonly ReadOnlyMemorySlice _headerData; + internal readonly int _index; + internal readonly int _overallIndex; + internal readonly FileNameBlock _nameBlock; + internal readonly Lazy _name; + internal Lazy<(uint Size, uint OnDisk, uint Original)> _size; + + public ulong Hash => BinaryPrimitives.ReadUInt64LittleEndian(_headerData); + protected uint RawSize => BinaryPrimitives.ReadUInt32LittleEndian(_headerData.Slice(0x8)); + public uint Offset => BinaryPrimitives.ReadUInt32LittleEndian(_headerData.Slice(0xC)); + public string Name => _name.Value; + public uint Size => _size.Value.Size; + + public bool FlipCompression => (RawSize & (0x1 << 30)) > 0; + + internal FolderRecord Folder { get; } + internal BSAReader BSA => Folder.BSA; + + internal FileRecord( + FolderRecord folderRecord, + ReadOnlyMemorySlice data, + int index, + int overallIndex, + FileNameBlock nameBlock) + { + _index = index; + _overallIndex = overallIndex; + _headerData = data; + _nameBlock = nameBlock; + Folder = folderRecord; + _name = new Lazy(GetName, System.Threading.LazyThreadSafetyMode.PublicationOnly); + + // Will be replaced if CopyDataTo is called before value is created + _size = new Lazy<(uint Size, uint OnDisk, uint Original)>( + mode: System.Threading.LazyThreadSafetyMode.ExecutionAndPublication, + valueFactory: () => + { + using var rdr = BSA.GetStream(); + rdr.BaseStream.Position = Offset; + return ReadSize(rdr); + }); + } + + public RelativePath Path => new RelativePath(string.IsNullOrEmpty(Folder.Name) ? Name : Folder.Name + "\\" + Name, skipValidation: true); + + public bool Compressed + { + get + { + if (FlipCompression) return !BSA.CompressedByDefault; + return BSA.CompressedByDefault; + } + } + + public FileStateObject State => new BSAFileStateObject(this); + + public async ValueTask CopyDataTo(Stream output) + { + await using var in_file = await BSA._fileName.OpenRead().ConfigureAwait(false); + using var rdr = new BinaryReader(in_file); + rdr.BaseStream.Position = Offset; + + (uint Size, uint OnDisk, uint Original) size = ReadSize(rdr); + if (!_size.IsValueCreated) + { + _size = new Lazy<(uint Size, uint OnDisk, uint Original)>(value: size); + } + + if (BSA.HeaderType == VersionType.SSE) + { + if (Compressed) + { + using var r = LZ4Stream.Decode(rdr.BaseStream); + await r.CopyToLimitAsync(output, size.Original).ConfigureAwait(false); + } + else + { + await rdr.BaseStream.CopyToLimitAsync(output, size.OnDisk).ConfigureAwait(false); + } + } + else + { + if (Compressed) + { + await using var z = new InflaterInputStream(rdr.BaseStream); + await z.CopyToLimitAsync(output, size.Original).ConfigureAwait(false); + } + else + await rdr.BaseStream.CopyToLimitAsync(output, size.OnDisk).ConfigureAwait(false); + } + } + + private string GetName() + { + var names = _nameBlock.Names.Value; + return names[_overallIndex].ReadStringTerm(BSA.HeaderType); + } + + private (uint Size, uint OnDisk, uint Original) ReadSize(BinaryReader rdr) + { + uint size = RawSize; + if (FlipCompression) + size = size ^ (0x1 << 30); + + if (Compressed) + size -= 4; + + byte nameBlobOffset; + if (BSA.HasNameBlobs) + { + nameBlobOffset = rdr.ReadByte(); + // Just skip, not using + rdr.BaseStream.Position += nameBlobOffset; + } + else + { + nameBlobOffset = 0; + } + + uint originalSize; + if (Compressed) + { + originalSize = rdr.ReadUInt32(); + } + else + { + originalSize = 0; + } + + uint onDiskSize = size - nameBlobOffset; + if (Compressed) + { + return (Size: originalSize, OnDisk: onDiskSize, Original: originalSize); + } + else + { + return (Size: onDiskSize, OnDisk: onDiskSize, Original: originalSize); + } + } + + public void Dump(Action print) + { + print($"Name: {Name}"); + print($"Offset: {Offset}"); + print($"Raw Size: {RawSize}"); + print($"Index: {_index}"); + } + } +} diff --git a/Compression.BSA/BSA/Reader/FolderRecord.cs b/Compression.BSA/BSA/Reader/FolderRecord.cs new file mode 100644 index 00000000..ce9bb18d --- /dev/null +++ b/Compression.BSA/BSA/Reader/FolderRecord.cs @@ -0,0 +1,89 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.IO; +using System.Text; +using NativeImport; +using Wabbajack.Common; +using File = Alphaleonis.Win32.Filesystem.File; + +namespace Compression.BSA +{ + public class FolderRecord : IFolder + { + internal readonly BSAReader BSA; + private readonly ReadOnlyMemorySlice _data; + internal Lazy _files = null!; + private int _prevFileCount; + internal FileNameBlock FileNameBlock = null!; + internal int Index { get; } + public string? Name { get; private set; } + + public IEnumerable Files => _files.Value; + + internal FolderRecord(BSAReader bsa, ReadOnlyMemorySlice data, int index) + { + BSA = bsa; + _data = data; + Index = index; + } + + private bool IsLongform => BSA.HeaderType == VersionType.SSE; + + public ulong Hash => BinaryPrimitives.ReadUInt64LittleEndian(_data); + + public int FileCount => checked((int)BinaryPrimitives.ReadUInt32LittleEndian(_data.Slice(0x8))); + + public uint Unknown => IsLongform ? + BinaryPrimitives.ReadUInt32LittleEndian(_data.Slice(0xC)) : + 0; + + public ulong Offset => IsLongform ? + BinaryPrimitives.ReadUInt64LittleEndian(_data.Slice(0x10)) : + BinaryPrimitives.ReadUInt32LittleEndian(_data.Slice(0xC)); + + public static int HeaderLength(VersionType version) + { + return version switch + { + VersionType.SSE => 0x18, + _ => 0x10, + }; + } + + internal void ProcessFileRecordHeadersBlock(BinaryReader rdr, int fileCountTally) + { + _prevFileCount = fileCountTally; + var totalFileLen = checked((int)(FileCount * FileRecord.HeaderLength)); + + ReadOnlyMemorySlice data; + if (BSA.HasFolderNames) + { + var len = rdr.ReadByte(); + data = rdr.ReadBytes(len + totalFileLen); + Name = data.Slice(0, len).ReadStringTerm(BSA.HeaderType); + data = data.Slice(len); + } + else + { + data = rdr.ReadBytes(totalFileLen); + } + + _files = new Lazy( + isThreadSafe: true, + valueFactory: () => ParseFileRecords(data)); + } + + private FileRecord[] ParseFileRecords(ReadOnlyMemorySlice data) + { + var fileCount = FileCount; + var ret = new FileRecord[fileCount]; + for (var idx = 0; idx < fileCount; idx += 1) + { + var fileData = data.Slice(idx * FileRecord.HeaderLength, FileRecord.HeaderLength); + ret[idx] = new FileRecord(this, fileData, idx, idx + _prevFileCount, FileNameBlock); + } + return ret; + } + } +} diff --git a/Compression.BSA/BSA/VersionType.cs b/Compression.BSA/BSA/VersionType.cs new file mode 100644 index 00000000..f0e718c7 --- /dev/null +++ b/Compression.BSA/BSA/VersionType.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Compression.BSA +{ + public enum VersionType : uint + { + TES4 = 0x67, + FO3 = 0x68, // FO3, FNV, TES5 + SSE = 0x69, + FO4 = 0x01, + TES3 = 0xFF // Not a real Bethesda version number + } + +} diff --git a/Compression.BSA/BSAReader.cs b/Compression.BSA/BSAReader.cs deleted file mode 100644 index f580555a..00000000 --- a/Compression.BSA/BSAReader.cs +++ /dev/null @@ -1,376 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; -using System.Threading.Tasks; -using ICSharpCode.SharpZipLib.Zip.Compression.Streams; -using K4os.Compression.LZ4.Streams; -using Wabbajack.Common; -using Wabbajack.Common.Serialization.Json; -using File = Alphaleonis.Win32.Filesystem.File; - -namespace Compression.BSA -{ - public enum VersionType : uint - { - TES4 = 0x67, - FO3 = 0x68, // FO3, FNV, TES5 - SSE = 0x69, - FO4 = 0x01, - TES3 = 0xFF // Not a real Bethesda version number - } - - [Flags] - public enum ArchiveFlags : uint - { - HasFolderNames = 0x1, - HasFileNames = 0x2, - Compressed = 0x4, - Unk4 = 0x8, - Unk5 = 0x10, - Unk6 = 0x20, - XBox360Archive = 0x40, - Unk8 = 0x80, - HasFileNameBlobs = 0x100, - Unk10 = 0x200, - Unk11 = 0x400 - } - - [Flags] - public enum FileFlags : uint - { - Meshes = 0x1, - Textures = 0x2, - Menus = 0x4, - Sounds = 0x8, - Voices = 0x10, - Shaders = 0x20, - Trees = 0x40, - Fonts = 0x80, - Miscellaneous = 0x100 - } - - public class BSAReader : IBSAReader - { - internal uint _archiveFlags; - internal uint _fileCount; - internal uint _fileFlags; - internal AbsolutePath _fileName; - internal uint _folderCount; - internal uint _folderRecordOffset; - private List _folders; - internal string _magic; - internal uint _totalFileNameLength; - internal uint _totalFolderNameLength; - internal uint _version; - - public void Dump(Action print) - { - print($"File Name: {_fileName}"); - print($"File Count: {_fileCount}"); - print($"Magic: {_magic}"); - - foreach (var file in Files) - { - print("\n"); - file.Dump(print); - } - } - - public static async ValueTask LoadAsync(AbsolutePath filename) - { - using var stream = await filename.OpenRead(); - using var br = new BinaryReader(stream); - var bsa = new BSAReader { _fileName = filename }; - bsa.LoadHeaders(br); - return bsa; - } - - public static BSAReader Load(AbsolutePath filename) - { - using var stream = File.Open(filename.ToString(), FileMode.Open, FileAccess.Read, FileShare.Read); - using var br = new BinaryReader(stream); - var bsa = new BSAReader { _fileName = filename }; - bsa.LoadHeaders(br); - return bsa; - } - - public IEnumerable Files - { - get - { - foreach (var folder in _folders) - foreach (var file in folder._files) - yield return file; - } - } - - public ArchiveStateObject State => new BSAStateObject(this); - - public VersionType HeaderType => (VersionType) _version; - - public ArchiveFlags ArchiveFlags => (ArchiveFlags) _archiveFlags; - - public FileFlags FileFlags => (FileFlags)_fileFlags; - - - public bool HasFolderNames => (_archiveFlags & 0x1) > 0; - - public bool HasFileNames => (_archiveFlags & 0x2) > 0; - - public bool CompressedByDefault => (_archiveFlags & 0x4) > 0; - - public bool Bit9Set => (_archiveFlags & 0x100) > 0; - - public bool HasNameBlobs - { - get - { - if (HeaderType == VersionType.FO3 || HeaderType == VersionType.SSE) return (_archiveFlags & 0x100) > 0; - return false; - } - } - - private void LoadHeaders(BinaryReader rdr) - { - var fourcc = Encoding.ASCII.GetString(rdr.ReadBytes(4)); - - if (fourcc != "BSA\0") - throw new InvalidDataException("Archive is not a BSA"); - - _magic = fourcc; - _version = rdr.ReadUInt32(); - _folderRecordOffset = rdr.ReadUInt32(); - _archiveFlags = rdr.ReadUInt32(); - _folderCount = rdr.ReadUInt32(); - _fileCount = rdr.ReadUInt32(); - _totalFolderNameLength = rdr.ReadUInt32(); - _totalFileNameLength = rdr.ReadUInt32(); - _fileFlags = rdr.ReadUInt32(); - - LoadFolderRecords(rdr); - } - - private void LoadFolderRecords(BinaryReader rdr) - { - _folders = new List(); - for (var idx = 0; idx < _folderCount; idx += 1) - _folders.Add(new FolderRecord(this, rdr)); - - foreach (var folder in _folders) - folder.LoadFileRecordBlock(this, rdr); - - foreach (var folder in _folders) - foreach (var file in folder._files) - file.LoadFileRecord(this, folder, file, rdr); - } - } - - [JsonName("BSAState")] - public class BSAStateObject : ArchiveStateObject - { - public BSAStateObject() { } - public BSAStateObject(BSAReader bsaReader) - { - Magic = bsaReader._magic; - Version = bsaReader._version; - ArchiveFlags = bsaReader._archiveFlags; - FileFlags = bsaReader._fileFlags; - - } - - public override async Task MakeBuilder(long size) - { - return await BSABuilder.Create(this, size); - } - - public string Magic { get; set; } - public uint Version { get; set; } - public uint ArchiveFlags { get; set; } - public uint FileFlags { get; set; } - } - - public class FolderRecord - { - private readonly uint _fileCount; - internal List _files; - private ulong _offset; - private uint _unk; - - internal FolderRecord(BSAReader bsa, BinaryReader src) - { - Hash = src.ReadUInt64(); - _fileCount = src.ReadUInt32(); - if (bsa.HeaderType == VersionType.SSE) - { - _unk = src.ReadUInt32(); - _offset = src.ReadUInt64(); - } - else - { - _offset = src.ReadUInt32(); - } - } - - public string Name { get; private set; } - - public ulong Hash { get; } - - internal void LoadFileRecordBlock(BSAReader bsa, BinaryReader src) - { - if (bsa.HasFolderNames) Name = src.ReadStringLen(bsa.HeaderType); - - _files = new List(); - for (var idx = 0; idx < _fileCount; idx += 1) - _files.Add(new FileRecord(bsa, this, src, idx)); - } - } - - public class FileRecord : IFile - { - private readonly BSAReader _bsa; - private readonly long _dataOffset; - private readonly uint _dataSize; - private string _name; - private readonly string _nameBlob; - private readonly uint _offset; - private readonly uint _onDiskSize; - private readonly uint _originalSize; - private readonly uint _size; - internal readonly int _index; - - - public void Dump(Action print) - { - print($"Name: {_name}"); - print($"Offset: {_offset}"); - print($"On Disk Size: {_onDiskSize}"); - print($"Original Size: {_originalSize}"); - print($"Size: {_size}"); - print($"Index: {_index}"); - } - - - public FileRecord(BSAReader bsa, FolderRecord folderRecord, BinaryReader src, int index) - { - _index = index; - _bsa = bsa; - Hash = src.ReadUInt64(); - var size = src.ReadUInt32(); - FlipCompression = (size & (0x1 << 30)) > 0; - - if (FlipCompression) - _size = size ^ (0x1 << 30); - else - _size = size; - - if (Compressed) - _size -= 4; - - _offset = src.ReadUInt32(); - Folder = folderRecord; - - var old_pos = src.BaseStream.Position; - - src.BaseStream.Position = _offset; - - if (bsa.HasNameBlobs) - _nameBlob = src.ReadStringLenNoTerm(bsa.HeaderType); - - - if (Compressed) - _originalSize = src.ReadUInt32(); - - _onDiskSize = (uint) (_size - (_nameBlob == null ? 0 : _nameBlob.Length + 1)); - - if (Compressed) - { - _dataSize = _originalSize; - _onDiskSize -= 4; - } - else - { - _dataSize = _onDiskSize; - } - - _dataOffset = src.BaseStream.Position; - - src.BaseStream.Position = old_pos; - } - - public RelativePath Path - { - get - { - return string.IsNullOrEmpty(Folder.Name) ? new RelativePath(_name) : new RelativePath(Folder.Name + "\\" + _name); - } - } - - public bool Compressed - { - get - { - if (FlipCompression) return !_bsa.CompressedByDefault; - return _bsa.CompressedByDefault; - } - } - - public uint Size => _dataSize; - public FileStateObject State => new BSAFileStateObject(this); - - public ulong Hash { get; } - - public FolderRecord Folder { get; } - - public bool FlipCompression { get; } - - internal void LoadFileRecord(BSAReader bsaReader, FolderRecord folder, FileRecord file, BinaryReader rdr) - { - _name = rdr.ReadStringTerm(_bsa.HeaderType); - } - - public async ValueTask CopyDataTo(Stream output) - { - await using var in_file = await _bsa._fileName.OpenRead(); - using var rdr = new BinaryReader(in_file); - rdr.BaseStream.Position = _dataOffset; - - if (_bsa.HeaderType == VersionType.SSE) - { - if (Compressed) - { - using var r = LZ4Stream.Decode(rdr.BaseStream); - await r.CopyToLimitAsync(output, (int) _originalSize); - } - else - { - await rdr.BaseStream.CopyToLimitAsync(output, (int) _onDiskSize); - } - } - else - { - if (Compressed) - { - await using var z = new InflaterInputStream(rdr.BaseStream); - await z.CopyToLimitAsync(output, (int) _originalSize); - } - else - await rdr.BaseStream.CopyToLimitAsync(output, (int) _onDiskSize); - } - } - } - - [JsonName("BSAFileState")] - public class BSAFileStateObject : FileStateObject - { - public BSAFileStateObject() { } - public BSAFileStateObject(FileRecord fileRecord) - { - FlipCompression = fileRecord.FlipCompression; - Path = fileRecord.Path; - Index = fileRecord._index; - } - - public bool FlipCompression { get; set; } - } -} diff --git a/Compression.BSA/Compression.BSA.csproj b/Compression.BSA/Compression.BSA.csproj index 800e278d..4517d254 100644 --- a/Compression.BSA/Compression.BSA.csproj +++ b/Compression.BSA/Compression.BSA.csproj @@ -6,7 +6,10 @@ x64 win10-x64 3.0 + enable + nullable true + $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb Compression.BSA.xml diff --git a/Compression.BSA/Interfaces/IBSABuilder.cs b/Compression.BSA/Interfaces/IBSABuilder.cs new file mode 100644 index 00000000..a157a52b --- /dev/null +++ b/Compression.BSA/Interfaces/IBSABuilder.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Compression.BSA +{ + public interface IBSABuilder : IAsyncDisposable + { + Task AddFile(FileStateObject state, Stream src); + Task Build(AbsolutePath filename); + } +} diff --git a/Compression.BSA/Interfaces/IBSAReader.cs b/Compression.BSA/Interfaces/IBSAReader.cs new file mode 100644 index 00000000..6eba5e52 --- /dev/null +++ b/Compression.BSA/Interfaces/IBSAReader.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Compression.BSA +{ + public interface IBSAReader + { + /// + /// The files defined by the archive + /// + IEnumerable Files { get; } + + ArchiveStateObject State { get; } + + void Dump(Action print); + } +} diff --git a/Compression.BSA/IBSAReader.cs b/Compression.BSA/Interfaces/IFile.cs similarity index 54% rename from Compression.BSA/IBSAReader.cs rename to Compression.BSA/Interfaces/IFile.cs index 316a8a11..fe8054b5 100644 --- a/Compression.BSA/IBSAReader.cs +++ b/Compression.BSA/Interfaces/IFile.cs @@ -1,43 +1,12 @@ using System; using System.Collections.Generic; using System.IO; +using System.Text; using System.Threading.Tasks; using Wabbajack.Common; namespace Compression.BSA { - public interface IBSAReader - { - /// - /// The files defined by the archive - /// - IEnumerable Files { get; } - - ArchiveStateObject State { get; } - - void Dump(Action print); - } - - public interface IBSABuilder : IAsyncDisposable - { - Task AddFile(FileStateObject state, Stream src); - Task Build(AbsolutePath filename); - } - - public class ArchiveStateObject - { - public virtual async Task MakeBuilder(long size) - { - throw new NotImplementedException(); - } - } - - public class FileStateObject - { - public int Index { get; set; } - public RelativePath Path { get; set; } - } - public interface IFile { /// diff --git a/Compression.BSA/Interfaces/IFolder.cs b/Compression.BSA/Interfaces/IFolder.cs new file mode 100644 index 00000000..0b2794a3 --- /dev/null +++ b/Compression.BSA/Interfaces/IFolder.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Compression.BSA +{ + public interface IFolder + { + string? Name { get; } + IEnumerable Files { get; } + int FileCount { get; } + } +} diff --git a/Compression.BSA/State/ArchiveStateObject.cs b/Compression.BSA/State/ArchiveStateObject.cs new file mode 100644 index 00000000..e9aabd74 --- /dev/null +++ b/Compression.BSA/State/ArchiveStateObject.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +namespace Compression.BSA +{ + public abstract class ArchiveStateObject + { + public abstract Task MakeBuilder(long size); + } +} diff --git a/Compression.BSA/State/FileStateObject.cs b/Compression.BSA/State/FileStateObject.cs new file mode 100644 index 00000000..967c627a --- /dev/null +++ b/Compression.BSA/State/FileStateObject.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Wabbajack.Common; + +namespace Compression.BSA +{ + public abstract class FileStateObject + { + public int Index { get; set; } + public RelativePath Path { get; set; } + } +} diff --git a/Compression.BSA/TES3Reader.cs b/Compression.BSA/TES3Reader.cs index b57d38cf..0ebe34f6 100644 --- a/Compression.BSA/TES3Reader.cs +++ b/Compression.BSA/TES3Reader.cs @@ -5,6 +5,7 @@ using System.Text; using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Common.Serialization.Json; +#nullable disable namespace Compression.BSA { diff --git a/Compression.BSA/Utils.cs b/Compression.BSA/Utils.cs index 32893261..a718ff16 100644 --- a/Compression.BSA/Utils.cs +++ b/Compression.BSA/Utils.cs @@ -30,9 +30,7 @@ namespace Compression.BSA public static string ReadStringLen(this BinaryReader rdr, VersionType version) { var len = rdr.ReadByte(); - if (len == 0) - //rdr.ReadByte(); - return ""; + if (len == 0) return string.Empty; var bytes = rdr.ReadBytes(len - 1); rdr.ReadByte(); @@ -61,6 +59,18 @@ namespace Compression.BSA return GetEncoding(version).GetString(acc.ToArray()); } + public static string ReadStringLenTerm(this ReadOnlyMemorySlice bytes, VersionType version) + { + if (bytes.Length <= 1) return string.Empty; + return GetEncoding(version).GetString(bytes.Slice(1, bytes[0])); + } + + public static string ReadStringTerm(this ReadOnlyMemorySlice bytes, VersionType version) + { + if (bytes.Length <= 1) return string.Empty; + return GetEncoding(version).GetString(bytes[0..^1]); + } + /// /// Returns bytes for a \0 terminated string /// diff --git a/Wabbajack.Common.Test/WorkQueueTests.cs b/Wabbajack.Common.Test/WorkQueueTests.cs index ee5be9ef..5cf5924f 100644 --- a/Wabbajack.Common.Test/WorkQueueTests.cs +++ b/Wabbajack.Common.Test/WorkQueueTests.cs @@ -6,7 +6,6 @@ using System.Reactive.Linq; using System.Reactive.Subjects; using System.Threading; using System.Threading.Tasks; -using Splat; using Wabbajack; using Wabbajack.Common; using Xunit; diff --git a/Wabbajack.Common/Error States/ErrorResponse.cs b/Wabbajack.Common/Error States/ErrorResponse.cs index e06f4ed7..2aedfe1b 100644 --- a/Wabbajack.Common/Error States/ErrorResponse.cs +++ b/Wabbajack.Common/Error States/ErrorResponse.cs @@ -1,5 +1,4 @@ using System; -using DynamicData.Kernel; namespace Wabbajack { diff --git a/Wabbajack.Common/Extensions/StreamExt.cs b/Wabbajack.Common/Extensions/StreamExt.cs new file mode 100644 index 00000000..dbb1b612 --- /dev/null +++ b/Wabbajack.Common/Extensions/StreamExt.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Wabbajack.Common +{ + public static class StreamExt + { + public static long Remaining(this Stream stream) + { + return stream.Length - stream.Position; + } + } +} diff --git a/Wabbajack.Common/MemorySlice.cs b/Wabbajack.Common/MemorySlice.cs new file mode 100644 index 00000000..42071d53 --- /dev/null +++ b/Wabbajack.Common/MemorySlice.cs @@ -0,0 +1,218 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; + +namespace Wabbajack.Common +{ + public struct MemorySlice : IEnumerable + { + private T[] _arr; + private int _startPos; + private int _length; + public int Length => _length; + public int StartPosition => _startPos; + + [DebuggerStepThrough] + public MemorySlice(T[] arr) + { + this._arr = arr; + this._startPos = 0; + this._length = arr.Length; + } + + [DebuggerStepThrough] + public MemorySlice(T[] arr, int startPos, int length) + { + this._arr = arr; + this._startPos = startPos; + this._length = length; + } + + public Span Span => _arr.AsSpan(start: _startPos, length: _length); + + public T this[int index] + { + get => _arr[index + _startPos]; + set => _arr[index + _startPos] = value; + } + + [DebuggerStepThrough] + public MemorySlice Slice(int start) + { + var startPos = _startPos + start; + if (startPos < 0) + { + throw new ArgumentOutOfRangeException(); + } + return new MemorySlice() + { + _arr = _arr, + _startPos = startPos, + _length = _length - start + }; + } + + [DebuggerStepThrough] + public MemorySlice Slice(int start, int length) + { + var startPos = _startPos + start; + if (startPos < 0) + { + throw new ArgumentOutOfRangeException(); + } + if (startPos + length > _arr.Length) + { + throw new ArgumentOutOfRangeException(); + } + return new MemorySlice() + { + _arr = _arr, + _startPos = startPos, + _length = length + }; + } + + public IEnumerator GetEnumerator() + { + for (int i = 0; i < _length; i++) + { + yield return this._arr[i + _startPos]; + } + } + + IEnumerator IEnumerable.GetEnumerator() => this.GetEnumerator(); + + public static implicit operator ReadOnlyMemorySlice(MemorySlice mem) + { + return new ReadOnlyMemorySlice( + mem._arr, + mem._startPos, + mem._length); + } + + public static implicit operator ReadOnlySpan(MemorySlice mem) + { + return mem.Span; + } + + public static implicit operator Span(MemorySlice mem) + { + return mem.Span; + } + + public static implicit operator MemorySlice(T[] mem) + { + return new MemorySlice(mem); + } + + public static implicit operator MemorySlice?(T[]? mem) + { + if (mem == null) return null; + return new MemorySlice(mem); + } + } + + public struct ReadOnlyMemorySlice : IEnumerable + { + private T[] _arr; + private int _startPos; + private int _length; + public int Length => _length; + public int StartPosition => _startPos; + + [DebuggerStepThrough] + public ReadOnlyMemorySlice(T[] arr) + { + this._arr = arr; + this._startPos = 0; + this._length = arr.Length; + } + + [DebuggerStepThrough] + public ReadOnlyMemorySlice(T[] arr, int startPos, int length) + { + this._arr = arr; + this._startPos = startPos; + this._length = length; + } + + public ReadOnlySpan Span => _arr.AsSpan(start: _startPos, length: _length); + + public T this[int index] => _arr[index + _startPos]; + + [DebuggerStepThrough] + public ReadOnlyMemorySlice Slice(int start) + { + var startPos = _startPos + start; + if (startPos < 0) + { + throw new ArgumentOutOfRangeException(); + } + return new ReadOnlyMemorySlice() + { + _arr = _arr, + _startPos = _startPos + start, + _length = _length - start + }; + } + + [DebuggerStepThrough] + public ReadOnlyMemorySlice Slice(int start, int length) + { + var startPos = _startPos + start; + if (startPos < 0) + { + throw new ArgumentOutOfRangeException(); + } + if (startPos + length > _arr.Length) + { + throw new ArgumentOutOfRangeException(); + } + return new ReadOnlyMemorySlice() + { + _arr = _arr, + _startPos = _startPos + start, + _length = length + }; + } + + public IEnumerator GetEnumerator() + { + for (int i = 0; i < _length; i++) + { + yield return this._arr[i + _startPos]; + } + } + + IEnumerator IEnumerable.GetEnumerator() => this.GetEnumerator(); + + public static implicit operator ReadOnlySpan(ReadOnlyMemorySlice mem) + { + return mem.Span; + } + + public static implicit operator ReadOnlyMemorySlice?(T[]? mem) + { + if (mem == null) return null; + return new ReadOnlyMemorySlice(mem); + } + + public static implicit operator ReadOnlyMemorySlice(T[] mem) + { + return new ReadOnlyMemorySlice(mem); + } + } + + public static class MemorySliceExt + { + public static bool Equal(ReadOnlyMemorySlice? lhs, ReadOnlyMemorySlice? rhs) + where T : IEquatable + { + if (lhs == null && rhs == null) return true; + if (lhs == null || rhs == null) return false; + return MemoryExtensions.SequenceEqual(lhs.Value.Span, rhs.Value.Span); + } + } +} diff --git a/Wabbajack.Common/Paths/AbsolutePath.cs b/Wabbajack.Common/Paths/AbsolutePath.cs index af7e2f55..4c9f0e3d 100644 --- a/Wabbajack.Common/Paths/AbsolutePath.cs +++ b/Wabbajack.Common/Paths/AbsolutePath.cs @@ -397,14 +397,14 @@ namespace Wabbajack.Common { var path = _path; return CircuitBreaker.WithAutoRetryAsync(async () => - File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)); + File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, bufferSize: 1048576, useAsync: true)); } public ValueTask WriteShared() { var path = _path; return CircuitBreaker.WithAutoRetryAsync(async () => - File.Open(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)); + File.Open(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite, bufferSize: 1048576, useAsync: true)); } public async Task CopyDirectoryToAsync(AbsolutePath destination) diff --git a/Wabbajack.Common/Paths/RelativePath.cs b/Wabbajack.Common/Paths/RelativePath.cs index d4c9ba6f..21aa1238 100644 --- a/Wabbajack.Common/Paths/RelativePath.cs +++ b/Wabbajack.Common/Paths/RelativePath.cs @@ -13,7 +13,7 @@ namespace Wabbajack.Common private readonly string? _nullable_path; private string _path => _nullable_path ?? string.Empty; - public RelativePath(string path) + public RelativePath(string path, bool skipValidation = false) { if (string.IsNullOrWhiteSpace(path)) { @@ -28,7 +28,10 @@ namespace Wabbajack.Common } _nullable_path = trimmed; - Validate(); + if (!skipValidation) + { + Validate(); + } } public override string ToString() diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 06394170..09e1a7ec 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -7,6 +7,7 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Net.Http; +using System.Reactive.Concurrency; using System.Reactive.Linq; using System.Reactive.Subjects; using System.Reflection; @@ -23,7 +24,6 @@ using IniParser.Model.Configuration; using IniParser.Parser; using Microsoft.Win32; using Newtonsoft.Json; -using ReactiveUI; using RocksDbSharp; using Wabbajack.Common.StatusFeed; using Wabbajack.Common.StatusFeed.Errors; @@ -107,7 +107,7 @@ namespace Wabbajack.Common AppLocalEvents = Observable.Merge(Observable.FromEventPattern(h => watcher.Changed += h, h => watcher.Changed -= h).Select(e => (FileEventType.Changed, e.EventArgs)), Observable.FromEventPattern(h => watcher.Created += h, h => watcher.Created -= h).Select(e => (FileEventType.Created, e.EventArgs)), Observable.FromEventPattern(h => watcher.Deleted += h, h => watcher.Deleted -= h).Select(e => (FileEventType.Deleted, e.EventArgs))) - .ObserveOn(RxApp.TaskpoolScheduler); + .ObserveOn(Scheduler.Default); watcher.EnableRaisingEvents = true; InitPatches(); } @@ -1098,13 +1098,6 @@ namespace Wabbajack.Common return bytes; } - public static async Task CopyFileAsync(string src, string dest) - { - await using var s = File.OpenRead(src); - await using var d = File.Create(dest); - await s.CopyToAsync(d); - } - public static string ToNormalString(this SecureString value) { var valuePtr = IntPtr.Zero; diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index decd2ff4..99c9a70d 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -15,6 +15,7 @@ https://www.wabbajack.org/favicon.ico https://github.com/wabbajack-tools/wabbajack 3.0 + $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb Wabbajack.Common.xml @@ -50,12 +51,12 @@ - + diff --git a/Wabbajack.Common/WorkQueue.cs b/Wabbajack.Common/WorkQueue.cs index f935b5e7..540bd989 100644 --- a/Wabbajack.Common/WorkQueue.cs +++ b/Wabbajack.Common/WorkQueue.cs @@ -8,7 +8,6 @@ using System.Reactive.Subjects; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; -using DynamicData; using Wabbajack.Common.StatusFeed; [assembly: InternalsVisibleTo("Wabbajack.Test")] @@ -26,6 +25,7 @@ namespace Wabbajack.Common public static bool WorkerThread => AsyncLocalCurrentQueue.Value != null; public bool IsWorkerThread => WorkerThread; internal static readonly AsyncLocal AsyncLocalCurrentQueue = new AsyncLocal(); + public static WorkQueue? AsyncLocalQueue => AsyncLocalCurrentQueue.Value; private readonly Subject _Status = new Subject(); public IObservable Status => _Status; @@ -69,15 +69,14 @@ namespace Wabbajack.Common public WorkQueue(IObservable? numThreads) { // Hook onto the number of active threads subject, and subscribe to it for changes - _activeNumThreadsObservable + _disposables.Add(_activeNumThreadsObservable // Select the latest driving observable .Select(x => x ?? Observable.Return(Environment.ProcessorCount)) .Switch() .DistinctUntilChanged() // Add new threads if it increases .SelectTask(AddNewThreadsIfNeeded) - .Subscribe() - .DisposeWith(_disposables); + .Subscribe()); // Set the incoming driving observable to be active SetActiveThreadsObservable(numThreads); } diff --git a/Wabbajack.Common/StatusFeed/Interventions/AUserIntervention.cs b/Wabbajack.Lib/Interventions/AUserIntervention.cs similarity index 84% rename from Wabbajack.Common/StatusFeed/Interventions/AUserIntervention.cs rename to Wabbajack.Lib/Interventions/AUserIntervention.cs index 879d4feb..16dfe691 100644 --- a/Wabbajack.Common/StatusFeed/Interventions/AUserIntervention.cs +++ b/Wabbajack.Lib/Interventions/AUserIntervention.cs @@ -5,8 +5,9 @@ using System.Text; using System.Threading.Tasks; using System.Windows.Input; using ReactiveUI; +using Wabbajack.Common; -namespace Wabbajack.Common +namespace Wabbajack.Lib { public abstract class AUserIntervention : ReactiveObject, IUserIntervention { @@ -17,7 +18,7 @@ namespace Wabbajack.Common private bool _handled; public bool Handled { get => _handled; set => this.RaiseAndSetIfChanged(ref _handled, value); } - public int CpuID { get; } = WorkQueue.AsyncLocalCurrentQueue.Value?.CpuId ?? WorkQueue.UnassignedCpuId; + public int CpuID { get; } = WorkQueue.AsyncLocalQueue?.CpuId ?? WorkQueue.UnassignedCpuId; public abstract void Cancel(); public ICommand CancelCommand { get; } diff --git a/Wabbajack.Common/StatusFeed/Interventions/ConfirmationIntervention.cs b/Wabbajack.Lib/Interventions/ConfirmationIntervention.cs similarity index 97% rename from Wabbajack.Common/StatusFeed/Interventions/ConfirmationIntervention.cs rename to Wabbajack.Lib/Interventions/ConfirmationIntervention.cs index de097c6e..2a3a585b 100644 --- a/Wabbajack.Common/StatusFeed/Interventions/ConfirmationIntervention.cs +++ b/Wabbajack.Lib/Interventions/ConfirmationIntervention.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; using System.Windows.Input; using ReactiveUI; -namespace Wabbajack.Common +namespace Wabbajack.Lib { public abstract class ConfirmationIntervention : AUserIntervention { diff --git a/Wabbajack.Common/StatusFeed/Interventions/IUserIntervention.cs b/Wabbajack.Lib/Interventions/IUserIntervention.cs similarity index 97% rename from Wabbajack.Common/StatusFeed/Interventions/IUserIntervention.cs rename to Wabbajack.Lib/Interventions/IUserIntervention.cs index 1f2ce864..183703f5 100644 --- a/Wabbajack.Common/StatusFeed/Interventions/IUserIntervention.cs +++ b/Wabbajack.Lib/Interventions/IUserIntervention.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; using ReactiveUI; using Wabbajack.Common.StatusFeed; -namespace Wabbajack.Common +namespace Wabbajack.Lib { /// /// Defines a message that requires user interaction. The user must perform some action diff --git a/Wabbajack.Test/RestartingDownloadsTests.cs b/Wabbajack.Test/RestartingDownloadsTests.cs index 3bf8b88f..dc46f0f0 100644 --- a/Wabbajack.Test/RestartingDownloadsTests.cs +++ b/Wabbajack.Test/RestartingDownloadsTests.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; using Wabbajack.Common.StatusFeed; +using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Xunit; using Xunit.Abstractions; diff --git a/Wabbajack.sln b/Wabbajack.sln index 745caef4..bcd18b18 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -57,6 +57,7 @@ Global {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Debug|x64.ActiveCfg = Debug|x64 {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Debug|x64.Build.0 = Debug|x64 {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|Any CPU.ActiveCfg = Release|x64 + {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|Any CPU.Build.0 = Release|x64 {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|x64.ActiveCfg = Release|x64 {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|x64.Build.0 = Release|x64 {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Debug|Any CPU.ActiveCfg = Debug|x64 @@ -64,6 +65,7 @@ Global {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Debug|x64.ActiveCfg = Debug|x64 {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Debug|x64.Build.0 = Debug|x64 {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|Any CPU.ActiveCfg = Release|x64 + {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|Any CPU.Build.0 = Release|x64 {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|x64.ActiveCfg = Release|x64 {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|x64.Build.0 = Release|x64 {0A820830-A298-497D-85E0-E9A89EFEF5FE}.Debug|Any CPU.ActiveCfg = Debug|x64 @@ -99,6 +101,7 @@ Global {89281BA1-67C8-48D2-9D6E-0F5CC85AD8C9}.Debug|x64.ActiveCfg = Debug|x64 {89281BA1-67C8-48D2-9D6E-0F5CC85AD8C9}.Debug|x64.Build.0 = Debug|x64 {89281BA1-67C8-48D2-9D6E-0F5CC85AD8C9}.Release|Any CPU.ActiveCfg = Release|x64 + {89281BA1-67C8-48D2-9D6E-0F5CC85AD8C9}.Release|Any CPU.Build.0 = Release|x64 {89281BA1-67C8-48D2-9D6E-0F5CC85AD8C9}.Release|x64.ActiveCfg = Release|x64 {89281BA1-67C8-48D2-9D6E-0F5CC85AD8C9}.Release|x64.Build.0 = Release|x64 {F72C17EC-0881-4455-8B0E-E1CC4FFD642E}.Debug|Any CPU.ActiveCfg = Debug|x64 @@ -120,6 +123,7 @@ Global {685D8BB1-D178-4D2C-85C7-C54A36FB7454}.Debug|x64.ActiveCfg = Debug|x64 {685D8BB1-D178-4D2C-85C7-C54A36FB7454}.Debug|x64.Build.0 = Debug|x64 {685D8BB1-D178-4D2C-85C7-C54A36FB7454}.Release|Any CPU.ActiveCfg = Release|x64 + {685D8BB1-D178-4D2C-85C7-C54A36FB7454}.Release|Any CPU.Build.0 = Release|x64 {685D8BB1-D178-4D2C-85C7-C54A36FB7454}.Release|x64.ActiveCfg = Release|x64 {685D8BB1-D178-4D2C-85C7-C54A36FB7454}.Release|x64.Build.0 = Release|x64 {D6856DBF-C959-4867-A8A8-343DA2D2715E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU diff --git a/Wabbajack/Views/Compilers/CompilerView.xaml b/Wabbajack/Views/Compilers/CompilerView.xaml index 95ff7a67..4cdd8a1a 100644 --- a/Wabbajack/Views/Compilers/CompilerView.xaml +++ b/Wabbajack/Views/Compilers/CompilerView.xaml @@ -2,7 +2,7 @@ x:Class="Wabbajack.CompilerView" xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" - xmlns:common="clr-namespace:Wabbajack.Common;assembly=Wabbajack.Common" + xmlns:lib="clr-namespace:Wabbajack.Lib;assembly=Wabbajack.Lib" xmlns:d="http://schemas.microsoft.com/expression/blend/2008" xmlns:icon="http://metro.mahapps.com/winfx/xaml/iconpacks" xmlns:local="clr-namespace:Wabbajack" @@ -200,7 +200,7 @@ ViewModel="{Binding}" /> - + - + diff --git a/Wabbajack/Views/Interventions/ConfirmationInterventionView.xaml b/Wabbajack/Views/Interventions/ConfirmationInterventionView.xaml index 4914fd74..c7e40cd9 100644 --- a/Wabbajack/Views/Interventions/ConfirmationInterventionView.xaml +++ b/Wabbajack/Views/Interventions/ConfirmationInterventionView.xaml @@ -2,14 +2,14 @@ x:Class="Wabbajack.ConfirmationInterventionView" xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" - xmlns:common="clr-namespace:Wabbajack.Common;assembly=Wabbajack.Common" xmlns:d="http://schemas.microsoft.com/expression/blend/2008" xmlns:local="clr-namespace:Wabbajack" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" + xmlns:lib="clr-namespace:Wabbajack.Lib;assembly=Wabbajack.Lib" xmlns:rxui="http://reactiveui.net" d:DesignHeight="450" d:DesignWidth="800" - x:TypeArguments="common:ConfirmationIntervention" + x:TypeArguments="lib:ConfirmationIntervention" mc:Ignorable="d"> diff --git a/Wabbajack/Views/Interventions/ConfirmationInterventionView.xaml.cs b/Wabbajack/Views/Interventions/ConfirmationInterventionView.xaml.cs index 05c6222f..91f9d783 100644 --- a/Wabbajack/Views/Interventions/ConfirmationInterventionView.xaml.cs +++ b/Wabbajack/Views/Interventions/ConfirmationInterventionView.xaml.cs @@ -14,7 +14,7 @@ using System.Windows.Media.Imaging; using System.Windows.Navigation; using System.Windows.Shapes; using ReactiveUI; -using Wabbajack.Common; +using Wabbajack.Lib; namespace Wabbajack {