From 099021d8905f9e777d6db1c1d398c315682903f7 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 28 Jul 2019 17:04:23 -0600 Subject: [PATCH] first commit of new BSA reader/writer --- BSA.Tools/BSAFile.cs | 372 ++++++++++++++++++ Compression.BSA.Test/App.config | 6 + .../Compression.BSA.Test.csproj | 59 +++ Compression.BSA.Test/Program.cs | 56 +++ .../Properties/AssemblyInfo.cs | 36 ++ Compression.BSA/BSABuilder.cs | 312 +++++++++++++++ Compression.BSA/BSAReader.cs | 330 ++++++++++++++++ Compression.BSA/Compression.BSA.csproj | 76 ++++ Compression.BSA/Properties/AssemblyInfo.cs | 36 ++ Compression.BSA/Utils.cs | 112 ++++++ Compression.BSA/packages.config | 11 + Wabbajack.Common/Data.cs | 2 + Wabbajack.sln | 12 + Wabbajack/Compiler.cs | 24 +- Wabbajack/Installer.cs | 44 ++- Wabbajack/MainWindow.xaml.cs | 1 + Wabbajack/Wabbajack.csproj | 3 + Wabbajack/packages.config | 1 + 18 files changed, 1489 insertions(+), 4 deletions(-) create mode 100644 BSA.Tools/BSAFile.cs create mode 100644 Compression.BSA.Test/App.config create mode 100644 Compression.BSA.Test/Compression.BSA.Test.csproj create mode 100644 Compression.BSA.Test/Program.cs create mode 100644 Compression.BSA.Test/Properties/AssemblyInfo.cs create mode 100644 Compression.BSA/BSABuilder.cs create mode 100644 Compression.BSA/BSAReader.cs create mode 100644 Compression.BSA/Compression.BSA.csproj create mode 100644 Compression.BSA/Properties/AssemblyInfo.cs create mode 100644 Compression.BSA/Utils.cs create mode 100644 Compression.BSA/packages.config diff --git a/BSA.Tools/BSAFile.cs b/BSA.Tools/BSAFile.cs new file mode 100644 index 00000000..9c726299 --- /dev/null +++ b/BSA.Tools/BSAFile.cs @@ -0,0 +1,372 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; +using static BSA.Tools.libbsarch; + +namespace BSA.Tools +{ + // Represents a BSA archive on disk (in READ mode) + public class BSAFile : IDisposable + { + private static Mutex GlobalLock = new Mutex(false); + protected unsafe libbsarch.bsa_archive_t* _archive; + + public UInt32 Version + { + get + { + unsafe + { + return libbsarch.bsa_version_get(_archive); + } + } + } + + public bsa_archive_type_t Type + { + get + { + unsafe + { + return libbsarch.bsa_archive_type_get(_archive); + } + } + } + + public UInt32 FileCount + { + get + { + unsafe + { + return libbsarch.bsa_file_count_get(_archive); + } + } + } + + public UInt32 ArchiveFlags + { + get + { + unsafe + { + return libbsarch.bsa_archive_flags_get(_archive); + } + } + set + { + unsafe + { + libbsarch.bsa_archive_flags_set(_archive, value); + } + } + } + + public UInt32 FileFlags + { + get + { + unsafe + { + return libbsarch.bsa_file_flags_get(_archive); + } + } + set + { + unsafe + { + libbsarch.bsa_file_flags_set(_archive, value); + } + } + } + + public bool Compress + { + get + { + unsafe + { + return libbsarch.bsa_compress_get(_archive); + } + } + set + { + unsafe + { + libbsarch.bsa_compress_set(_archive, value); + } + } + } + + public bool ShareData + { + get + { + unsafe + { + return libbsarch.bsa_share_data_get(_archive); + } + } + set + { + unsafe + { + libbsarch.bsa_share_data_set(_archive, value); + } + } + } + + + public void Save() + { + unsafe + { + check_err(libbsarch.bsa_save(_archive)); + } + } + + private IEnumerable _entries = null; + public IEnumerable Entries { + get + { + if (_entries != null) + return _entries; + + return GetAndCacheEntries(); + } + + + } + + private IEnumerable GetAndCacheEntries() + { + var entries = new List(); + unsafe + { + foreach (var filename in GetFileNames()) + { + entries.Add(new ArchiveEntry(this, _archive, filename)); + } + } + _entries = entries; + return entries; + } + + public BSAFile() + { + GlobalLock.WaitOne(); + unsafe + { + _archive = libbsarch.bsa_create(); + } + } + + public void Create(string filename, bsa_archive_type_t type, EntryList entries) + { + unsafe + { + check_err(libbsarch.bsa_create_archive(_archive, filename, type, entries._list)); + } + } + + public BSAFile(string filename) + { + GlobalLock.WaitOne(); + unsafe + { + _archive = libbsarch.bsa_create(); + check_err(libbsarch.bsa_load_from_file(_archive, filename)); + } + } + + public void AddFile(string filename, byte[] data) + { + unsafe + { + var ptr = Marshal.AllocHGlobal(data.Length); + Marshal.Copy(data, 0, ptr, data.Length); + libbsarch.bsa_add_file_from_memory(_archive, filename, (UInt32)data.Length, (byte*)ptr); + Marshal.FreeHGlobal(ptr); + } + } + + public void Dispose() + { + unsafe + { + check_err(libbsarch.bsa_free(_archive)); + } + GlobalLock.ReleaseMutex(); + } + + public static void check_err(libbsarch.bsa_result_message_t bsa_result_message_t) + { + if (bsa_result_message_t.code != 0) + { + unsafe + { + int i = 0; + for (i = 0; i < 1024 * 2; i += 2) + if (bsa_result_message_t.text[i] == 0) break; + + var msg = new String((sbyte*)bsa_result_message_t.text, 0, i, Encoding.Unicode); + throw new Exception(msg); + } + } + } + + public IEnumerable GetFileNames() + { + List filenames = new List(); + unsafe + { + check_err(libbsarch.bsa_iterate_files(_archive, (archive, filename, file, folder, context) => + { + lock (filenames) + { + filenames.Add(filename); + } + return false; + }, null)); + } + return filenames; + } + } + + public class ArchiveEntry + { + private BSAFile _archive; + private unsafe libbsarch.bsa_archive_t* _archivep; + private string _filename; + + public string Filename { + get + { + return _filename; + } + } + + public unsafe ArchiveEntry(BSAFile archive, libbsarch.bsa_archive_t* archivep, string filename) + { + _archive = archive; + _archivep = archivep; + _filename = filename; + } + + public FileData GetFileData() + { + unsafe + { + var result = libbsarch.bsa_extract_file_data_by_filename(_archivep, _filename); + BSAFile.check_err(result.message); + return new FileData(_archive, _archivep, result.buffer); + } + } + + public void ExtractTo(Stream stream) + { + using (var data = GetFileData()) + { + data.WriteTo(stream); + } + } + + public void ExtractTo(string filename) + { + unsafe + { + libbsarch.bsa_extract_file(_archivep, _filename, filename); + } + } + } + + public class FileData : IDisposable + { + private BSAFile archive; + private unsafe libbsarch.bsa_archive_t* archivep; + private libbsarch.bsa_result_buffer_t result; + + public unsafe FileData(BSAFile archive, libbsarch.bsa_archive_t* archivep, libbsarch.bsa_result_buffer_t result) + { + this.archive = archive; + this.archivep = archivep; + this.result = result; + } + + public void WriteTo(Stream stream) + { + var memory = ToByteArray(); + stream.Write(memory, 0, (int)result.size); + } + + public byte[] ToByteArray() + { + unsafe + { + byte[] memory = new byte[result.size]; + Marshal.Copy((IntPtr)result.data, memory, 0, (int)result.size); + return memory; + } + } + + public void Dispose() + { + unsafe + { + BSAFile.check_err(libbsarch.bsa_file_data_free(archivep, result)); + } + } + } + + public class EntryList : IDisposable + { + public unsafe bsa_entry_list_t* _list; + + public EntryList() + { + unsafe + { + _list = libbsarch.bsa_entry_list_create(); + } + } + + public UInt32 Count + { + get + { + lock (this) + { + unsafe + { + return libbsarch.bsa_entry_list_count(_list); + } + } + } + } + + public void Add(string entry) + { + lock(this) + { + unsafe + { + libbsarch.bsa_entry_list_add(_list, entry); + } + } + } + + public void Dispose() + { + lock (this) + { + unsafe + { + libbsarch.bsa_entry_list_free(_list); + } + } + } + } +} diff --git a/Compression.BSA.Test/App.config b/Compression.BSA.Test/App.config new file mode 100644 index 00000000..56efbc7b --- /dev/null +++ b/Compression.BSA.Test/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Compression.BSA.Test/Compression.BSA.Test.csproj b/Compression.BSA.Test/Compression.BSA.Test.csproj new file mode 100644 index 00000000..9bfa7e3b --- /dev/null +++ b/Compression.BSA.Test/Compression.BSA.Test.csproj @@ -0,0 +1,59 @@ + + + + + Debug + AnyCPU + {BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9} + Exe + Compression.BSA.Test + Compression.BSA.Test + v4.7.2 + 512 + true + true + + + x64 + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + {ff5d892f-8ff4-44fc-8f7f-cd58f307ad1b} + Compression.BSA + + + + \ No newline at end of file diff --git a/Compression.BSA.Test/Program.cs b/Compression.BSA.Test/Program.cs new file mode 100644 index 00000000..f0a36f66 --- /dev/null +++ b/Compression.BSA.Test/Program.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Compression.BSA.Test +{ + class Program + { + const string TestDir = "c:\\Mod Organizer 2\\mods"; + static void Main(string[] args) + { + foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories)) + { + Console.WriteLine($"From {bsa}"); + using (var a = new BSAReader(bsa)) + { + + Parallel.ForEach(a.Files, file => + { + var abs_name = Path.Combine("c:\\tmp\\out", file.Path); + + if (!Directory.Exists(Path.GetDirectoryName(abs_name))) + Directory.CreateDirectory(Path.GetDirectoryName(abs_name)); + + using (var fs = File.OpenWrite(abs_name)) + file.CopyDataTo(fs); + }); + + using (var w = new BSABuilder()) + { + w.ArchiveFlags = a.ArchiveFlags; + w.FileFlags = a.FileFlags; + w.HeaderType = a.HeaderType; + + foreach (var file in a.Files) + { + var abs_path = Path.Combine("c:\\tmp\\out", file.Path); + using (var str = File.OpenRead(abs_path)) + w.AddFile(file.Path, str); + + } + + w.RegenFolderRecords(); + + w.Build("c:\\tmp\\built.bsa"); + + } + break; + } + } + } + } +} diff --git a/Compression.BSA.Test/Properties/AssemblyInfo.cs b/Compression.BSA.Test/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..e2c26383 --- /dev/null +++ b/Compression.BSA.Test/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Compression.BSA.Test")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Compression.BSA.Test")] +[assembly: AssemblyCopyright("Copyright © 2019")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("ba2cfea1-072b-42d6-822a-8c6d0e3ae5d9")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/Compression.BSA/BSABuilder.cs b/Compression.BSA/BSABuilder.cs new file mode 100644 index 00000000..759616f8 --- /dev/null +++ b/Compression.BSA/BSABuilder.cs @@ -0,0 +1,312 @@ +using K4os.Compression.LZ4.Streams; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Compression.BSA +{ + public class BSABuilder : IDisposable + { + internal byte[] _fileId; + internal uint _version; + internal uint _offset; + internal uint _archiveFlags; + internal uint _folderCount; + internal uint _fileCount; + internal uint _totalFolderNameLength; + internal uint _totalFileNameLength; + internal uint _fileFlags; + + private List _files = new List(); + internal List _folders = new List(); + + public BSABuilder() + { + _fileId = Encoding.ASCII.GetBytes("BSA\0"); + _offset = 0x24; + } + + public ArchiveFlags ArchiveFlags + { + get + { + return (ArchiveFlags)_archiveFlags; + } + set + { + _archiveFlags = (uint)value; + } + } + + public FileFlags FileFlags + { + get + { + return (FileFlags)_archiveFlags; + } + set + { + _archiveFlags = (uint)value; + } + } + + public VersionType HeaderType + { + get + { + return (VersionType)_version; + } + set + { + _version = (uint)value; + } + } + + public void AddFile(string path, Stream src, bool flipCompression = false) + { + FileEntry r = new FileEntry(this, path, src, flipCompression); + + lock (this) + { + _files.Add(r); + } + } + + public IEnumerable FolderNames + { + get + { + return _files.Select(f => Path.GetDirectoryName(f.Path)) + .ToHashSet(); + } + } + + public bool HasFolderNames + { + get + { + return (_archiveFlags & 0x1) > 0; + } + } + + public bool HasFileNames + { + get + { + return (_archiveFlags & 0x2) > 0; + } + } + + public bool CompressedByDefault + { + get + { + return (_archiveFlags & 0x4) > 0; + } + } + + public void Build(string outputName) + { + if (File.Exists(outputName)) File.Delete(outputName); + + using (var fs = File.OpenWrite(outputName)) + using (var wtr = new BinaryWriter(fs)) + { + wtr.Write(_fileId); + wtr.Write(_version); + wtr.Write(_offset); + wtr.Write(_archiveFlags); + var folders = FolderNames.ToList(); + wtr.Write((uint)folders.Count); + wtr.Write((uint)_files.Count); + wtr.Write((uint)_folders.Select(f => f._nameBytes.Count() - 1).Sum()); // totalFolderNameLength + var s = _files.Select(f => f._pathBytes.Count()).Sum(); + _totalFileNameLength = (uint)_files.Select(f => f._nameBytes.Count()).Sum(); + wtr.Write(_totalFileNameLength); // totalFileNameLength + wtr.Write(_fileFlags); + + uint idx = 0; + foreach (var folder in _folders) + { + folder.WriteFolderRecord(wtr, idx); + idx += 1; + } + + } + } + + public void RegenFolderRecords() + { + _folders = _files.GroupBy(f => Path.GetDirectoryName(f.Path).ToLowerInvariant()) + .Select(f => new FolderRecordBuilder(this, f.Key, f.ToList())) + .OrderBy(f => f._hash) + .ToList(); + } + + public void Dispose() + { + + } + } + + public class FolderRecordBuilder + { + internal IEnumerable _files; + internal BSABuilder _bsa; + internal ulong _hash; + internal uint _fileCount; + internal byte[] _nameBytes; + internal uint _recordSize; + internal ulong _offset; + + public ulong SelfSize + { + get + { + if (_bsa.HeaderType == VersionType.SSE) + { + return sizeof(ulong) + sizeof(uint) + sizeof(uint) + sizeof(ulong); + } + else + { + return sizeof(ulong) + sizeof(uint) + sizeof(uint); + } + } + } + + public ulong FileRecordSize + { + get + { + ulong size = 0; + if (_bsa.HasFolderNames) + size += (ulong)_nameBytes.Length; + size += (ulong)_files.Select(f => sizeof(ulong) + sizeof(uint) + sizeof(uint)).Sum(); + return size; + } + } + + public FolderRecordBuilder(BSABuilder bsa, string folderName, IEnumerable files) + { + _files = files; + _bsa = bsa; + _hash = folderName.GetBSAHash(); + _fileCount = (uint)files.Count(); + _nameBytes = folderName.ToBZString(); + _recordSize = sizeof(ulong) + sizeof(uint) + sizeof(uint); + } + + public void WriteFolderRecord(BinaryWriter wtr, uint idx) + { + _offset = (ulong)wtr.BaseStream.Position; + _offset += (ulong)_bsa._folders.Skip((int)idx).Select(f => (long)f.SelfSize).Sum(); + _offset += _bsa._totalFileNameLength; + _offset += (ulong)_bsa._folders.Take((int)idx).Select(f => (long)f.FileRecordSize).Sum(); + + var sp = wtr.BaseStream.Position; + wtr.Write(_hash); + wtr.Write(_fileCount); + if (_bsa.HeaderType == VersionType.SSE) + { + wtr.Write((uint)0); // unk + wtr.Write((ulong)_offset); // offset + } + } + + public void WriteFileRecordBlocks(BinaryWriter wtr) + { + if (_bsa.HasFolderNames) + { + wtr.Write(_nameBytes); + foreach (var file in _files) + file.WriteFileRecord(wtr); + } + } + } + + public class FileEntry + { + internal BSABuilder _bsa; + internal string _path; + internal string _filenameSource; + internal Stream _bytesSource; + internal bool _flipCompression; + + internal ulong _hash; + internal byte[] _nameBytes; + internal byte[] _pathBytes; + internal byte[] _rawData; + internal int _originalSize; + + public FileEntry(BSABuilder bsa, string path, Stream src, bool flipCompression) + { + _bsa = bsa; + _path = path.ToLowerInvariant(); + _hash = _path.GetBSAHash(); + _nameBytes = System.IO.Path.GetFileName(_path).ToTermString(); + _pathBytes = _path.ToTermString(); + _flipCompression = flipCompression; + + var ms = new MemoryStream(); + src.CopyTo(ms); + _rawData = ms.ToArray(); + _originalSize = _rawData.Length; + + if (Compressed) + CompressData(); + + } + + private void CompressData() + { + if (_bsa.HeaderType == VersionType.SSE) + { + var r = new MemoryStream(); + var w = LZ4Stream.Encode(r); + (new MemoryStream(_rawData)).CopyTo(w); + _rawData = r.ToArray(); + } + } + + public bool Compressed + { + get + { + if (_flipCompression) + return !_bsa.CompressedByDefault; + else + return _bsa.CompressedByDefault; + } + } + + + + public string Path + { + get + { + return _path; + } + } + + public bool FlipCompression + { + get + { + return _flipCompression; + } + set + { + _flipCompression = value; + } + } + + internal void WriteFileRecord(BinaryWriter wtr) + { + wtr.Write(_hash); + } + } +} diff --git a/Compression.BSA/BSAReader.cs b/Compression.BSA/BSAReader.cs new file mode 100644 index 00000000..fd831ed2 --- /dev/null +++ b/Compression.BSA/BSAReader.cs @@ -0,0 +1,330 @@ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using K4os.Compression.LZ4.Streams; + +namespace Compression.BSA +{ + public enum VersionType : uint + { + TES4 = 0x67, + FO3 = 0x68, + SSE = 0x69, + FO4 = 0x01 + }; + + [Flags] + public enum ArchiveFlags : uint + { + HasFolderNames = 0x1, + HasFileNames = 0x2, + Compressed = 0x4, + Unk4 = 0x8, + Unk5 = 0x10, + Unk6 = 0x20, + XBox360Archive = 0x40, + Unk8 = 0x80, + HasFileNameBlobs = 0x100, + Unk10 = 0x200, + Unk11 = 0x400 + } + + [Flags] + public enum FileFlags : uint + { + Meshes = 0x1, + Textures = 0x2, + Menus = 0x4, + Sounds = 0x8, + Voices = 0x10, + Shaders = 0x20, + Trees = 0x40, + Fonts = 0x80, + Miscellaneous = 0x100 + } + + public class BSAReader : IDisposable + { + private Stream _stream; + private BinaryReader _rdr; + private string _magic; + private uint _version; + private uint _folderRecordOffset; + private uint _archiveFlags; + private uint _folderCount; + private uint _fileCount; + private uint _totalFolderNameLength; + private uint _totalFileNameLength; + private uint _fileFlags; + private List _folders; + internal string _fileName; + + public IEnumerable Files + { + get + { + foreach (var folder in _folders) + foreach (var file in folder._files) + yield return file; + } + } + + public VersionType HeaderType + { + get + { + return (VersionType)_version; + } + } + + public ArchiveFlags ArchiveFlags + { + get + { + return (ArchiveFlags)_archiveFlags; + } + } + + public FileFlags FileFlags + { + get + { + return (FileFlags)_archiveFlags; + } + } + + + public bool HasFolderNames + { + get + { + return (_archiveFlags & 0x1) > 0; + } + } + + public bool HasFileNames + { + get + { + return (_archiveFlags & 0x2) > 0; + } + } + + public bool CompressedByDefault + { + get + { + return (_archiveFlags & 0x4) > 0; + } + } + + public bool Bit9Set + { + get + { + return (_archiveFlags & 0x100) > 0; + } + } + + public bool HasNameBlobs + { + get + { + if (HeaderType == VersionType.FO3 || HeaderType == VersionType.SSE) + { + return (_archiveFlags & 0x100) > 0; + } + return false; + } + } + + public BSAReader(string filename) : this(File.OpenRead(filename)) + { + _fileName = filename; + + } + + public BSAReader(Stream stream) + { + _stream = stream; + _rdr = new BinaryReader(_stream); + LoadHeaders(); + } + + public void Dispose() + { + _stream.Close(); + } + + private void LoadHeaders() + { + var fourcc = Encoding.ASCII.GetString(_rdr.ReadBytes(4)); + + if (fourcc != "BSA\0") + throw new InvalidDataException("Archive is not a BSA"); + + _magic = fourcc; + _version = _rdr.ReadUInt32(); + _folderRecordOffset = _rdr.ReadUInt32(); + _archiveFlags = _rdr.ReadUInt32(); + _folderCount = _rdr.ReadUInt32(); + _fileCount = _rdr.ReadUInt32(); + _totalFolderNameLength = _rdr.ReadUInt32(); + _totalFileNameLength = _rdr.ReadUInt32(); + _fileFlags = _rdr.ReadUInt32(); + + LoadFolderRecords(); + } + + private void LoadFolderRecords() + { + _folders = new List(); + for (int idx = 0; idx < _folderCount; idx += 1) + _folders.Add(new FolderRecord(this, _rdr)); + + foreach (var folder in _folders) + folder.LoadFileRecordBlock(this, _rdr); + + foreach (var folder in _folders) + foreach (var file in folder._files) + file.LoadFileRecord(this, folder, file, _rdr); + + } + } + + public class FolderRecord + { + private ulong _nameHash; + private uint _fileCount; + private uint _unk; + private ulong _offset; + internal List _files; + + internal FolderRecord(BSAReader bsa, BinaryReader src) + { + _nameHash = src.ReadUInt64(); + _fileCount = src.ReadUInt32(); + if (bsa.HeaderType == VersionType.SSE) + { + _unk = src.ReadUInt32(); + _offset = src.ReadUInt64(); + } + else + { + _offset = src.ReadUInt32(); + } + } + + public string Name { get; private set; } + + internal void LoadFileRecordBlock(BSAReader bsa, BinaryReader src) + { + if (bsa.HasFolderNames) + { + Name = src.ReadStringLen(); + } + + _files = new List(); + for (int idx = 0; idx < _fileCount; idx += 1) + { + _files.Add(new FileRecord(bsa, this, src)); + } + } + + } + + public class FileRecord + { + private BSAReader _bsa; + private ulong _hash; + private bool _compressedFlag; + private int _size; + private int _offset; + private FolderRecord _folder; + private string _name; + private uint _originalSize; + + public FileRecord(BSAReader bsa, FolderRecord folderRecord, BinaryReader src) + { + _bsa = bsa; + _hash = src.ReadUInt64(); + var size = src.ReadInt32(); + _compressedFlag = (size & (0x1 << 30)) > 0; + + if (_compressedFlag) + _size = size ^ (0x1 << 30); + else + _size = size; + + _offset = src.ReadInt32(); + + _folder = folderRecord; + } + + internal void LoadFileRecord(BSAReader bsaReader, FolderRecord folder, FileRecord file, BinaryReader rdr) + { + _name = rdr.ReadStringTerm(); + } + + public string Path + { + get + { + return _folder.Name + "\\" + _name; + } + } + + public bool Compressed + { + get + { + if (_compressedFlag) return !_bsa.CompressedByDefault; + return _bsa.CompressedByDefault; + } + } + + public int Size + { + get + { + if (Compressed) return (int)_originalSize; + return _size; + } + } + + public void CopyDataTo(Stream output) + { + using (var in_file = File.OpenRead(_bsa._fileName)) + using (var rdr = new BinaryReader(in_file)) + { + rdr.BaseStream.Position = _offset; + if (Compressed) + { + string _name; + int file_size = _size; + if (_bsa.HasNameBlobs) + { + var name_size = rdr.ReadByte(); + file_size -= name_size + 1; + rdr.BaseStream.Position = _offset + 1 + name_size; + } + + var original_size = rdr.ReadUInt32(); + if (_bsa.HeaderType == VersionType.SSE) + { + var r = LZ4Stream.Decode(rdr.BaseStream); + r.CopyTo(output); + } + else + { + + } + } + } + } + + + } +} diff --git a/Compression.BSA/Compression.BSA.csproj b/Compression.BSA/Compression.BSA.csproj new file mode 100644 index 00000000..1af8efe3 --- /dev/null +++ b/Compression.BSA/Compression.BSA.csproj @@ -0,0 +1,76 @@ + + + + + Debug + AnyCPU + {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B} + Library + Properties + Compression.BSA + Compression.BSA + v4.7.2 + 512 + true + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + x64 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\packages\K4os.Compression.LZ4.1.1.11\lib\net46\K4os.Compression.LZ4.dll + + + ..\packages\K4os.Compression.LZ4.Streams.1.1.11\lib\net46\K4os.Compression.LZ4.Streams.dll + + + ..\packages\K4os.Hash.xxHash.1.0.6\lib\net46\K4os.Hash.xxHash.dll + + + + ..\packages\System.Buffers.4.4.0\lib\netstandard2.0\System.Buffers.dll + + + + ..\packages\System.Memory.4.5.3\lib\netstandard2.0\System.Memory.dll + + + + ..\packages\System.Numerics.Vectors.4.4.0\lib\net46\System.Numerics.Vectors.dll + + + ..\packages\System.Runtime.CompilerServices.Unsafe.4.5.2\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Compression.BSA/Properties/AssemblyInfo.cs b/Compression.BSA/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..5d2029be --- /dev/null +++ b/Compression.BSA/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Compression.BSA")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Compression.BSA")] +[assembly: AssemblyCopyright("Copyright © 2019")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("ff5d892f-8ff4-44fc-8f7f-cd58f307ad1b")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/Compression.BSA/Utils.cs b/Compression.BSA/Utils.cs new file mode 100644 index 00000000..64f7eee3 --- /dev/null +++ b/Compression.BSA/Utils.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Compression.BSA +{ + public static class Utils + { + private static Encoding Windows1251 = Encoding.GetEncoding(1251); + public static string ReadStringLen(this BinaryReader rdr) + { + var len = rdr.ReadByte(); + var bytes = rdr.ReadBytes(len - 1); + rdr.ReadByte(); + return Windows1251.GetString(bytes); + } + + public static string ReadStringTerm(this BinaryReader rdr) + { + List acc = new List(); + while (true) + { + var c = rdr.ReadByte(); + + if (c == '\0') break; + + acc.Add(c); + } + return Windows1251.GetString(acc.ToArray()); + } + + + /// + /// Returns bytes for a \0 terminated string + /// + /// + /// + public static byte[] ToBZString(this string val) + { + var b = Windows1251.GetBytes(val); + var b2 = new byte[b.Length + 2]; + b.CopyTo(b2, 1); + b[0] = (byte)b.Length; + return b2; + } + + /// + /// Returns bytes for a \0 terminated string prefixed by a length + /// + /// + /// + public static byte[] ToTermString(this string val) + { + var b = Windows1251.GetBytes(val); + var b2 = new byte[b.Length + 1]; + b.CopyTo(b2, 0); + b[0] = (byte)b.Length; + return b2; + } + + public static ulong GetBSAHash(this string name) + { + name = name.Replace('/', '\\'); + return GetBSAHash(Path.ChangeExtension(name, null), Path.GetExtension(name)); + } + + private static ulong GetBSAHash(string name, string ext) + { + name = name.ToLowerInvariant(); + ext = ext.ToLowerInvariant(); + var hashBytes = new byte[] + { + (byte)(name.Length == 0 ? '\0' : name[name.Length - 1]), + (byte)(name.Length < 3 ? '\0' : name[name.Length - 2]), + (byte)name.Length, + (byte)name[0] + }; + var hash1 = BitConverter.ToUInt32(hashBytes, 0); + switch (ext) + { + case ".kf": + hash1 |= 0x80; + break; + case ".nif": + hash1 |= 0x8000; + break; + case ".dds": + hash1 |= 0x8080; + break; + case ".wav": + hash1 |= 0x80000000; + break; + } + + uint hash2 = 0; + for (var i = 1; i < name.Length - 2; i++) + { + hash2 = hash2 * 0x1003f + (byte)name[i]; + } + + uint hash3 = 0; + for (var i = 0; i < ext.Length; i++) + { + hash3 = hash3 * 0x1003f + (byte)ext[i]; + } + + return (((ulong)(hash2 + hash3)) << 32) + hash1; + } + } +} + diff --git a/Compression.BSA/packages.config b/Compression.BSA/packages.config new file mode 100644 index 00000000..7ad1ad47 --- /dev/null +++ b/Compression.BSA/packages.config @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/Wabbajack.Common/Data.cs b/Wabbajack.Common/Data.cs index 7b8b2785..c50f39a6 100644 --- a/Wabbajack.Common/Data.cs +++ b/Wabbajack.Common/Data.cs @@ -104,9 +104,11 @@ namespace Wabbajack.Common public string IsCompressed; public uint Version; public Int32 Type; + public bool ShareData; public uint FileFlags { get; set; } public bool Compress { get; set; } + public uint ArchiveFlags { get; set; } } public class PatchedFromArchive : FromArchive diff --git a/Wabbajack.sln b/Wabbajack.sln index 97d8040c..bf44ceec 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -11,6 +11,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SevenZipExtractor", "SevenZ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack", "Wabbajack\Wabbajack.csproj", "{33602679-8484-40C7-A10C-774DFF5D8314}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Compression.BSA", "Compression.BSA\Compression.BSA.csproj", "{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Compression.BSA.Test", "Compression.BSA.Test\Compression.BSA.Test.csproj", "{BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -33,6 +37,14 @@ Global {33602679-8484-40C7-A10C-774DFF5D8314}.Debug|Any CPU.Build.0 = Debug|Any CPU {33602679-8484-40C7-A10C-774DFF5D8314}.Release|Any CPU.ActiveCfg = Release|Any CPU {33602679-8484-40C7-A10C-774DFF5D8314}.Release|Any CPU.Build.0 = Release|Any CPU + {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|Any CPU.Build.0 = Release|Any CPU + {BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Wabbajack/Compiler.cs b/Wabbajack/Compiler.cs index f748b9f3..f17d26d8 100644 --- a/Wabbajack/Compiler.cs +++ b/Wabbajack/Compiler.cs @@ -1,6 +1,7 @@ using BSA.Tools; using Newtonsoft.Json; using SevenZipExtractor; +using SharpCompress.Archives; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -250,6 +251,21 @@ namespace Wabbajack }, false); } + /* + using (var a = ArchiveFactory.Open(archive.AbsolutePath)) + { + foreach (var entry in a.Entries) + { + var path = entry.Key.Replace("/", "\\"); + if (!paths.Contains(path)) continue; + var result = new MemoryStream(); + streams.Add(path, result); + Info("Extracting {0}", path); + using (var stream = entry.OpenEntryStream()) + stream.CopyTo(result); + } + }*/ + var extracted = streams.ToDictionary(k => k.Key, v => v.Value.ToArray()); // Now Create the patches Status("Building Patches for {0}", archive.Name); @@ -486,12 +502,14 @@ namespace Wabbajack { directive = new CreateBSA() { - To = source.Path, + To = source.Path, TempID = id, Version = bsa.Version, Type = (int)bsa.Type, FileFlags = bsa.FileFlags, - Compress = bsa.Compress + ArchiveFlags = bsa.ArchiveFlags, + Compress = bsa.Compress, + ShareData = bsa.ShareData }; }; @@ -528,7 +546,7 @@ namespace Wabbajack { var disabled_mods = File.ReadAllLines(Path.Combine(MO2ProfileDir, "modlist.txt")) .Where(line => line.StartsWith("-") && !line.EndsWith("_separator")) - .Select(line => Path.Combine("mods", line.Substring(1))) + .Select(line => Path.Combine("mods", line.Substring(1)) + "\\") .ToList(); return source => { diff --git a/Wabbajack/Installer.cs b/Wabbajack/Installer.cs index e45bf5b3..4816b1f5 100644 --- a/Wabbajack/Installer.cs +++ b/Wabbajack/Installer.cs @@ -1,4 +1,5 @@ -using SevenZipExtractor; +using BSA.Tools; +using SevenZipExtractor; using System; using System.Collections.Generic; using System.IO; @@ -8,6 +9,7 @@ using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using Wabbajack.Common; +using static BSA.Tools.libbsarch; namespace Wabbajack { @@ -81,10 +83,50 @@ namespace Wabbajack BuildFolderStructure(); InstallArchives(); InstallIncludedFiles(); + BuildBSAs(); Info("Installation complete! You may exit the program."); } + private void BuildBSAs() + { + var bsas = ModList.Directives.OfType().ToList(); + Info("Building {0} bsa files"); + + bsas.Do(bsa => + { + Status($"Building {bsa.To}"); + var source_dir = Path.Combine(Outputfolder, Consts.BSACreationDir, bsa.TempID); + using (var entries = new EntryList()) + { + var source_files = Directory.EnumerateFiles(source_dir, "*", SearchOption.AllDirectories) + .Select(e => e.Substring(source_dir.Length + 1)) + .ToList(); + + source_files.Do(name => entries.Add(name)); + + using (var a = new BSAFile()) + { + + a.Create(Path.Combine(Outputfolder, bsa.To), (bsa_archive_type_t)bsa.Type, entries); + a.FileFlags = bsa.FileFlags; + a.ArchiveFlags = bsa.ArchiveFlags; + a.ShareData = bsa.ShareData; + + + source_files.Do(e => + { + a.AddFile(e, File.ReadAllBytes(Path.Combine(source_dir, e))); + }); + + a.Save(); + + } + } + }); + + } + private void InstallIncludedFiles() { Info("Writing inline files"); diff --git a/Wabbajack/MainWindow.xaml.cs b/Wabbajack/MainWindow.xaml.cs index 62b6cd47..cf8a4250 100644 --- a/Wabbajack/MainWindow.xaml.cs +++ b/Wabbajack/MainWindow.xaml.cs @@ -39,6 +39,7 @@ namespace Wabbajack compiler.ModList.ToJSON("C:\\tmp\\modpack.json"); var modlist = compiler.ModList; + var create = modlist.Directives.OfType().ToList(); compiler = null; var installer = new Installer(modlist, "c:\\tmp\\install\\", msg => context.LogMsg(msg)); installer.Install(); diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 67d9de78..d48b3207 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -44,6 +44,9 @@ ..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll + + ..\packages\SharpCompress.0.23.0\lib\net45\SharpCompress.dll + diff --git a/Wabbajack/packages.config b/Wabbajack/packages.config index e0797ff5..fa75f9fb 100644 --- a/Wabbajack/packages.config +++ b/Wabbajack/packages.config @@ -3,5 +3,6 @@ + \ No newline at end of file