Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

first commit of new BSA reader/writer

Browse Source
This commit is contained in:
Timothy Baldridge 2019-07-28 17:04:23 -06:00
parent e63e3be3b7
commit 099021d890
18 changed files with 1489 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

372
BSA.Tools/BSAFile.cs Normal file
View File

@ -0,0 +1,372 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using static BSA.Tools.libbsarch;
namespace BSA.Tools
{
// Represents a BSA archive on disk (in READ mode)
public class BSAFile : IDisposable
{
private static Mutex GlobalLock = new Mutex(false);
protected unsafe libbsarch.bsa_archive_t* _archive;
public UInt32 Version
{
get
{
unsafe
{
return libbsarch.bsa_version_get(_archive);
}
}
}
public bsa_archive_type_t Type
{
get
{
unsafe
{
return libbsarch.bsa_archive_type_get(_archive);
}
}
}
public UInt32 FileCount
{
get
{
unsafe
{
return libbsarch.bsa_file_count_get(_archive);
}
}
}
public UInt32 ArchiveFlags
{
get
{
unsafe
{
return libbsarch.bsa_archive_flags_get(_archive);
}
}
set
{
unsafe
{
libbsarch.bsa_archive_flags_set(_archive, value);
}
}
}
public UInt32 FileFlags
{
get
{
unsafe
{
return libbsarch.bsa_file_flags_get(_archive);
}
}
set
{
unsafe
{
libbsarch.bsa_file_flags_set(_archive, value);
}
}
}
public bool Compress
{
get
{
unsafe
{
return libbsarch.bsa_compress_get(_archive);
}
}
set
{
unsafe
{
libbsarch.bsa_compress_set(_archive, value);
}
}
}
public bool ShareData
{
get
{
unsafe
{
return libbsarch.bsa_share_data_get(_archive);
}
}
set
{
unsafe
{
libbsarch.bsa_share_data_set(_archive, value);
}
}
}
public void Save()
{
unsafe
{
check_err(libbsarch.bsa_save(_archive));
}
}
private IEnumerable<ArchiveEntry> _entries = null;
public IEnumerable<ArchiveEntry> Entries {
get
{
if (_entries != null)
return _entries;
return GetAndCacheEntries();
}
}
private IEnumerable<ArchiveEntry> GetAndCacheEntries()
{
var entries = new List<ArchiveEntry>();
unsafe
{
foreach (var filename in GetFileNames())
{
entries.Add(new ArchiveEntry(this, _archive, filename));
}
}
_entries = entries;
return entries;
}
public BSAFile()
{
GlobalLock.WaitOne();
unsafe
{
_archive = libbsarch.bsa_create();
}
}
public void Create(string filename, bsa_archive_type_t type, EntryList entries)
{
unsafe
{
check_err(libbsarch.bsa_create_archive(_archive, filename, type, entries._list));
}
}
public BSAFile(string filename)
{
GlobalLock.WaitOne();
unsafe
{
_archive = libbsarch.bsa_create();
check_err(libbsarch.bsa_load_from_file(_archive, filename));
}
}
public void AddFile(string filename, byte[] data)
{
unsafe
{
var ptr = Marshal.AllocHGlobal(data.Length);
Marshal.Copy(data, 0, ptr, data.Length);
libbsarch.bsa_add_file_from_memory(_archive, filename, (UInt32)data.Length, (byte*)ptr);
Marshal.FreeHGlobal(ptr);
}
}
public void Dispose()
{
unsafe
{
check_err(libbsarch.bsa_free(_archive));
}
GlobalLock.ReleaseMutex();
}
public static void check_err(libbsarch.bsa_result_message_t bsa_result_message_t)
{
if (bsa_result_message_t.code != 0)
{
unsafe
{
int i = 0;
for (i = 0; i < 1024 * 2; i += 2)
if (bsa_result_message_t.text[i] == 0) break;
var msg = new String((sbyte*)bsa_result_message_t.text, 0, i, Encoding.Unicode);
throw new Exception(msg);
}
}
}
public IEnumerable<string> GetFileNames()
{
List<string> filenames = new List<string>();
unsafe
{
check_err(libbsarch.bsa_iterate_files(_archive, (archive, filename, file, folder, context) =>
{
lock (filenames)
{
filenames.Add(filename);
}
return false;
}, null));
}
return filenames;
}
}
public class ArchiveEntry
{
private BSAFile _archive;
private unsafe libbsarch.bsa_archive_t* _archivep;
private string _filename;
public string Filename {
get
{
return _filename;
}
}
public unsafe ArchiveEntry(BSAFile archive, libbsarch.bsa_archive_t* archivep, string filename)
{
_archive = archive;
_archivep = archivep;
_filename = filename;
}
public FileData GetFileData()
{
unsafe
{
var result = libbsarch.bsa_extract_file_data_by_filename(_archivep, _filename);
BSAFile.check_err(result.message);
return new FileData(_archive, _archivep, result.buffer);
}
}
public void ExtractTo(Stream stream)
{
using (var data = GetFileData())
{
data.WriteTo(stream);
}
}
public void ExtractTo(string filename)
{
unsafe
{
libbsarch.bsa_extract_file(_archivep, _filename, filename);
}
}
}
public class FileData : IDisposable
{
private BSAFile archive;
private unsafe libbsarch.bsa_archive_t* archivep;
private libbsarch.bsa_result_buffer_t result;
public unsafe FileData(BSAFile archive, libbsarch.bsa_archive_t* archivep, libbsarch.bsa_result_buffer_t result)
{
this.archive = archive;
this.archivep = archivep;
this.result = result;
}
public void WriteTo(Stream stream)
{
var memory = ToByteArray();
stream.Write(memory, 0, (int)result.size);
}
public byte[] ToByteArray()
{
unsafe
{
byte[] memory = new byte[result.size];
Marshal.Copy((IntPtr)result.data, memory, 0, (int)result.size);
return memory;
}
}
public void Dispose()
{
unsafe
{
BSAFile.check_err(libbsarch.bsa_file_data_free(archivep, result));
}
}
}
public class EntryList : IDisposable
{
public unsafe bsa_entry_list_t* _list;
public EntryList()
{
unsafe
{
_list = libbsarch.bsa_entry_list_create();
}
}
public UInt32 Count
{
get
{
lock (this)
{
unsafe
{
return libbsarch.bsa_entry_list_count(_list);
}
}
}
}
public void Add(string entry)
{
lock(this)
{
unsafe
{
libbsarch.bsa_entry_list_add(_list, entry);
}
}
}
public void Dispose()
{
lock (this)
{
unsafe
{
libbsarch.bsa_entry_list_free(_list);
}
}
}
}
}

6
Compression.BSA.Test/App.config Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2" />
</startup>
</configuration>

59
Compression.BSA.Test/Compression.BSA.Test.csproj Normal file
View File

@ -0,0 +1,59 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}</ProjectGuid>
<OutputType>Exe</OutputType>
<RootNamespace>Compression.BSA.Test</RootNamespace>
<AssemblyName>Compression.BSA.Test</AssemblyName>
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<Deterministic>true</Deterministic>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>x64</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Compression.BSA\Compression.BSA.csproj">
<Project>{ff5d892f-8ff4-44fc-8f7f-cd58f307ad1b}</Project>
<Name>Compression.BSA</Name>
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>

56
Compression.BSA.Test/Program.cs Normal file
View File

@ -0,0 +1,56 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Compression.BSA.Test
{
class Program
{
const string TestDir = "c:\\Mod Organizer 2\\mods";
static void Main(string[] args)
{
foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories))
{
Console.WriteLine($"From {bsa}");
using (var a = new BSAReader(bsa))
{
Parallel.ForEach(a.Files, file =>
{
var abs_name = Path.Combine("c:\\tmp\\out", file.Path);
if (!Directory.Exists(Path.GetDirectoryName(abs_name)))
Directory.CreateDirectory(Path.GetDirectoryName(abs_name));
using (var fs = File.OpenWrite(abs_name))
file.CopyDataTo(fs);
});
using (var w = new BSABuilder())
{
w.ArchiveFlags = a.ArchiveFlags;
w.FileFlags = a.FileFlags;
w.HeaderType = a.HeaderType;
foreach (var file in a.Files)
{
var abs_path = Path.Combine("c:\\tmp\\out", file.Path);
using (var str = File.OpenRead(abs_path))
w.AddFile(file.Path, str);
}
w.RegenFolderRecords();
w.Build("c:\\tmp\\built.bsa");
}
break;
}
}
}
}
}

36
Compression.BSA.Test/Properties/AssemblyInfo.cs Normal file
View File

@ -0,0 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Compression.BSA.Test")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("Compression.BSA.Test")]
[assembly: AssemblyCopyright("Copyright © 2019")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("ba2cfea1-072b-42d6-822a-8c6d0e3ae5d9")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

312
Compression.BSA/BSABuilder.cs Normal file
View File

@ -0,0 +1,312 @@
using K4os.Compression.LZ4.Streams;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Compression.BSA
{
public class BSABuilder : IDisposable
{
internal byte[] _fileId;
internal uint _version;
internal uint _offset;
internal uint _archiveFlags;
internal uint _folderCount;
internal uint _fileCount;
internal uint _totalFolderNameLength;
internal uint _totalFileNameLength;
internal uint _fileFlags;
private List<FileEntry> _files = new List<FileEntry>();
internal List<FolderRecordBuilder> _folders = new List<FolderRecordBuilder>();
public BSABuilder()
{
_fileId = Encoding.ASCII.GetBytes("BSA\0");
_offset = 0x24;
}
public ArchiveFlags ArchiveFlags
{
get
{
return (ArchiveFlags)_archiveFlags;
}
set
{
_archiveFlags = (uint)value;
}
}
public FileFlags FileFlags
{
get
{
return (FileFlags)_archiveFlags;
}
set
{
_archiveFlags = (uint)value;
}
}
public VersionType HeaderType
{
get
{
return (VersionType)_version;
}
set
{
_version = (uint)value;
}
}
public void AddFile(string path, Stream src, bool flipCompression = false)
{
FileEntry r = new FileEntry(this, path, src, flipCompression);
lock (this)
{
_files.Add(r);
}
}
public IEnumerable<string> FolderNames
{
get
{
return _files.Select(f => Path.GetDirectoryName(f.Path))
.ToHashSet();
}
}
public bool HasFolderNames
{
get
{
return (_archiveFlags & 0x1) > 0;
}
}
public bool HasFileNames
{
get
{
return (_archiveFlags & 0x2) > 0;
}
}
public bool CompressedByDefault
{
get
{
return (_archiveFlags & 0x4) > 0;
}
}
public void Build(string outputName)
{
if (File.Exists(outputName)) File.Delete(outputName);
using (var fs = File.OpenWrite(outputName))
using (var wtr = new BinaryWriter(fs))
{
wtr.Write(_fileId);
wtr.Write(_version);
wtr.Write(_offset);
wtr.Write(_archiveFlags);
var folders = FolderNames.ToList();
wtr.Write((uint)folders.Count);
wtr.Write((uint)_files.Count);
wtr.Write((uint)_folders.Select(f => f._nameBytes.Count() - 1).Sum()); // totalFolderNameLength
var s = _files.Select(f => f._pathBytes.Count()).Sum();
_totalFileNameLength = (uint)_files.Select(f => f._nameBytes.Count()).Sum();
wtr.Write(_totalFileNameLength); // totalFileNameLength
wtr.Write(_fileFlags);
uint idx = 0;
foreach (var folder in _folders)
{
folder.WriteFolderRecord(wtr, idx);
idx += 1;
}
}
}
public void RegenFolderRecords()
{
_folders = _files.GroupBy(f => Path.GetDirectoryName(f.Path).ToLowerInvariant())
.Select(f => new FolderRecordBuilder(this, f.Key, f.ToList()))
.OrderBy(f => f._hash)
.ToList();
}
public void Dispose()
{
}
}
public class FolderRecordBuilder
{
internal IEnumerable<FileEntry> _files;
internal BSABuilder _bsa;
internal ulong _hash;
internal uint _fileCount;
internal byte[] _nameBytes;
internal uint _recordSize;
internal ulong _offset;
public ulong SelfSize
{
get
{
if (_bsa.HeaderType == VersionType.SSE)
{
return sizeof(ulong) + sizeof(uint) + sizeof(uint) + sizeof(ulong);
}
else
{
return sizeof(ulong) + sizeof(uint) + sizeof(uint);
}
}
}
public ulong FileRecordSize
{
get
{
ulong size = 0;
if (_bsa.HasFolderNames)
size += (ulong)_nameBytes.Length;
size += (ulong)_files.Select(f => sizeof(ulong) + sizeof(uint) + sizeof(uint)).Sum();
return size;
}
}
public FolderRecordBuilder(BSABuilder bsa, string folderName, IEnumerable<FileEntry> files)
{
_files = files;
_bsa = bsa;
_hash = folderName.GetBSAHash();
_fileCount = (uint)files.Count();
_nameBytes = folderName.ToBZString();
_recordSize = sizeof(ulong) + sizeof(uint) + sizeof(uint);
}
public void WriteFolderRecord(BinaryWriter wtr, uint idx)
{
_offset = (ulong)wtr.BaseStream.Position;
_offset += (ulong)_bsa._folders.Skip((int)idx).Select(f => (long)f.SelfSize).Sum();
_offset += _bsa._totalFileNameLength;
_offset += (ulong)_bsa._folders.Take((int)idx).Select(f => (long)f.FileRecordSize).Sum();
var sp = wtr.BaseStream.Position;
wtr.Write(_hash);
wtr.Write(_fileCount);
if (_bsa.HeaderType == VersionType.SSE)
{
wtr.Write((uint)0); // unk
wtr.Write((ulong)_offset); // offset
}
}
public void WriteFileRecordBlocks(BinaryWriter wtr)
{
if (_bsa.HasFolderNames)
{
wtr.Write(_nameBytes);
foreach (var file in _files)
file.WriteFileRecord(wtr);
}
}
}
public class FileEntry
{
internal BSABuilder _bsa;
internal string _path;
internal string _filenameSource;
internal Stream _bytesSource;
internal bool _flipCompression;
internal ulong _hash;
internal byte[] _nameBytes;
internal byte[] _pathBytes;
internal byte[] _rawData;
internal int _originalSize;
public FileEntry(BSABuilder bsa, string path, Stream src, bool flipCompression)
{
_bsa = bsa;
_path = path.ToLowerInvariant();
_hash = _path.GetBSAHash();
_nameBytes = System.IO.Path.GetFileName(_path).ToTermString();
_pathBytes = _path.ToTermString();
_flipCompression = flipCompression;
var ms = new MemoryStream();
src.CopyTo(ms);
_rawData = ms.ToArray();
_originalSize = _rawData.Length;
if (Compressed)
CompressData();
}
private void CompressData()
{
if (_bsa.HeaderType == VersionType.SSE)
{
var r = new MemoryStream();
var w = LZ4Stream.Encode(r);
(new MemoryStream(_rawData)).CopyTo(w);
_rawData = r.ToArray();
}
}
public bool Compressed
{
get
{
if (_flipCompression)
return !_bsa.CompressedByDefault;
else
return _bsa.CompressedByDefault;
}
}
public string Path
{
get
{
return _path;
}
}
public bool FlipCompression
{
get
{
return _flipCompression;
}
set
{
_flipCompression = value;
}
}
internal void WriteFileRecord(BinaryWriter wtr)
{
wtr.Write(_hash);
}
}
}

330
Compression.BSA/BSAReader.cs Normal file
View File

@ -0,0 +1,330 @@

using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using K4os.Compression.LZ4.Streams;
namespace Compression.BSA
{
public enum VersionType : uint
{
TES4 = 0x67,
FO3 = 0x68,
SSE = 0x69,
FO4 = 0x01
};
[Flags]
public enum ArchiveFlags : uint
{
HasFolderNames = 0x1,
HasFileNames = 0x2,
Compressed = 0x4,
Unk4 = 0x8,
Unk5 = 0x10,
Unk6 = 0x20,
XBox360Archive = 0x40,
Unk8 = 0x80,
HasFileNameBlobs = 0x100,
Unk10 = 0x200,
Unk11 = 0x400
}
[Flags]
public enum FileFlags : uint
{
Meshes = 0x1,
Textures = 0x2,
Menus = 0x4,
Sounds = 0x8,
Voices = 0x10,
Shaders = 0x20,
Trees = 0x40,
Fonts = 0x80,
Miscellaneous = 0x100
}
public class BSAReader : IDisposable
{
private Stream _stream;
private BinaryReader _rdr;
private string _magic;
private uint _version;
private uint _folderRecordOffset;
private uint _archiveFlags;
private uint _folderCount;
private uint _fileCount;
private uint _totalFolderNameLength;
private uint _totalFileNameLength;
private uint _fileFlags;
private List<FolderRecord> _folders;
internal string _fileName;
public IEnumerable<FileRecord> Files
{
get
{
foreach (var folder in _folders)
foreach (var file in folder._files)
yield return file;
}
}
public VersionType HeaderType
{
get
{
return (VersionType)_version;
}
}
public ArchiveFlags ArchiveFlags
{
get
{
return (ArchiveFlags)_archiveFlags;
}
}
public FileFlags FileFlags
{
get
{
return (FileFlags)_archiveFlags;
}
}
public bool HasFolderNames
{
get
{
return (_archiveFlags & 0x1) > 0;
}
}
public bool HasFileNames
{
get
{
return (_archiveFlags & 0x2) > 0;
}
}
public bool CompressedByDefault
{
get
{
return (_archiveFlags & 0x4) > 0;
}
}
public bool Bit9Set
{
get
{
return (_archiveFlags & 0x100) > 0;
}
}
public bool HasNameBlobs
{
get
{
if (HeaderType == VersionType.FO3 || HeaderType == VersionType.SSE)
{
return (_archiveFlags & 0x100) > 0;
}
return false;
}
}
public BSAReader(string filename) : this(File.OpenRead(filename))
{
_fileName = filename;
}
public BSAReader(Stream stream)
{
_stream = stream;
_rdr = new BinaryReader(_stream);
LoadHeaders();
}
public void Dispose()
{
_stream.Close();
}
private void LoadHeaders()
{
var fourcc = Encoding.ASCII.GetString(_rdr.ReadBytes(4));
if (fourcc != "BSA\0")
throw new InvalidDataException("Archive is not a BSA");
_magic = fourcc;
_version = _rdr.ReadUInt32();
_folderRecordOffset = _rdr.ReadUInt32();
_archiveFlags = _rdr.ReadUInt32();
_folderCount = _rdr.ReadUInt32();
_fileCount = _rdr.ReadUInt32();
_totalFolderNameLength = _rdr.ReadUInt32();
_totalFileNameLength = _rdr.ReadUInt32();
_fileFlags = _rdr.ReadUInt32();
LoadFolderRecords();
}
private void LoadFolderRecords()
{
_folders = new List<FolderRecord>();
for (int idx = 0; idx < _folderCount; idx += 1)
_folders.Add(new FolderRecord(this, _rdr));
foreach (var folder in _folders)
folder.LoadFileRecordBlock(this, _rdr);
foreach (var folder in _folders)
foreach (var file in folder._files)
file.LoadFileRecord(this, folder, file, _rdr);
}
}
public class FolderRecord
{
private ulong _nameHash;
private uint _fileCount;
private uint _unk;
private ulong _offset;
internal List<FileRecord> _files;
internal FolderRecord(BSAReader bsa, BinaryReader src)
{
_nameHash = src.ReadUInt64();
_fileCount = src.ReadUInt32();
if (bsa.HeaderType == VersionType.SSE)
{
_unk = src.ReadUInt32();
_offset = src.ReadUInt64();
}
else
{
_offset = src.ReadUInt32();
}
}
public string Name { get; private set; }
internal void LoadFileRecordBlock(BSAReader bsa, BinaryReader src)
{
if (bsa.HasFolderNames)
{
Name = src.ReadStringLen();
}
_files = new List<FileRecord>();
for (int idx = 0; idx < _fileCount; idx += 1)
{
_files.Add(new FileRecord(bsa, this, src));
}
}
}
public class FileRecord
{
private BSAReader _bsa;
private ulong _hash;
private bool _compressedFlag;
private int _size;
private int _offset;
private FolderRecord _folder;
private string _name;
private uint _originalSize;
public FileRecord(BSAReader bsa, FolderRecord folderRecord, BinaryReader src)
{
_bsa = bsa;
_hash = src.ReadUInt64();
var size = src.ReadInt32();
_compressedFlag = (size & (0x1 << 30)) > 0;
if (_compressedFlag)
_size = size ^ (0x1 << 30);
else
_size = size;
_offset = src.ReadInt32();
_folder = folderRecord;
}
internal void LoadFileRecord(BSAReader bsaReader, FolderRecord folder, FileRecord file, BinaryReader rdr)
{
_name = rdr.ReadStringTerm();
}
public string Path
{
get
{
return _folder.Name + "\\" + _name;
}
}
public bool Compressed
{
get
{
if (_compressedFlag) return !_bsa.CompressedByDefault;
return _bsa.CompressedByDefault;
}
}
public int Size
{
get
{
if (Compressed) return (int)_originalSize;
return _size;
}
}
public void CopyDataTo(Stream output)
{
using (var in_file = File.OpenRead(_bsa._fileName))
using (var rdr = new BinaryReader(in_file))
{
rdr.BaseStream.Position = _offset;
if (Compressed)
{
string _name;
int file_size = _size;
if (_bsa.HasNameBlobs)
{
var name_size = rdr.ReadByte();
file_size -= name_size + 1;
rdr.BaseStream.Position = _offset + 1 + name_size;
}
var original_size = rdr.ReadUInt32();
if (_bsa.HeaderType == VersionType.SSE)
{
var r = LZ4Stream.Decode(rdr.BaseStream);
r.CopyTo(output);
}
else
{
}
}
}
}
}
}

76
Compression.BSA/Compression.BSA.csproj Normal file
View File

@ -0,0 +1,76 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Compression.BSA</RootNamespace>
<AssemblyName>Compression.BSA</AssemblyName>
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<Deterministic>true</Deterministic>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<PlatformTarget>x64</PlatformTarget>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="K4os.Compression.LZ4, Version=1.1.11.0, Culture=neutral, PublicKeyToken=2186fa9121ef231d, processorArchitecture=MSIL">
<HintPath>..\packages\K4os.Compression.LZ4.1.1.11\lib\net46\K4os.Compression.LZ4.dll</HintPath>
</Reference>
<Reference Include="K4os.Compression.LZ4.Streams, Version=1.1.11.0, Culture=neutral, PublicKeyToken=2186fa9121ef231d, processorArchitecture=MSIL">
<HintPath>..\packages\K4os.Compression.LZ4.Streams.1.1.11\lib\net46\K4os.Compression.LZ4.Streams.dll</HintPath>
</Reference>
<Reference Include="K4os.Hash.xxHash, Version=1.0.6.0, Culture=neutral, PublicKeyToken=32cd54395057cec3, processorArchitecture=MSIL">
<HintPath>..\packages\K4os.Hash.xxHash.1.0.6\lib\net46\K4os.Hash.xxHash.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Buffers, Version=4.0.2.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
<HintPath>..\packages\System.Buffers.4.4.0\lib\netstandard2.0\System.Buffers.dll</HintPath>
</Reference>
<Reference Include="System.Core" />
<Reference Include="System.Memory, Version=4.0.1.1, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
<HintPath>..\packages\System.Memory.4.5.3\lib\netstandard2.0\System.Memory.dll</HintPath>
</Reference>
<Reference Include="System.Numerics" />
<Reference Include="System.Numerics.Vectors, Version=4.1.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Numerics.Vectors.4.4.0\lib\net46\System.Numerics.Vectors.dll</HintPath>
</Reference>
<Reference Include="System.Runtime.CompilerServices.Unsafe, Version=4.0.4.1, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Runtime.CompilerServices.Unsafe.4.5.2\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll</HintPath>
</Reference>
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="BSABuilder.cs" />
<Compile Include="BSAReader.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Utils.cs" />
</ItemGroup>
<ItemGroup>
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>

36
Compression.BSA/Properties/AssemblyInfo.cs Normal file
View File

@ -0,0 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Compression.BSA")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("Compression.BSA")]
[assembly: AssemblyCopyright("Copyright © 2019")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("ff5d892f-8ff4-44fc-8f7f-cd58f307ad1b")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

112
Compression.BSA/Utils.cs Normal file
View File

@ -0,0 +1,112 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
namespace Compression.BSA
{
public static class Utils
{
private static Encoding Windows1251 = Encoding.GetEncoding(1251);
public static string ReadStringLen(this BinaryReader rdr)
{
var len = rdr.ReadByte();
var bytes = rdr.ReadBytes(len - 1);
rdr.ReadByte();
return Windows1251.GetString(bytes);
}
public static string ReadStringTerm(this BinaryReader rdr)
{
List<byte> acc = new List<byte>();
while (true)
{
var c = rdr.ReadByte();
if (c == '\0') break;
acc.Add(c);
}
return Windows1251.GetString(acc.ToArray());
}
/// <summary>
/// Returns bytes for a \0 terminated string
/// </summary>
/// <param name="val"></param>
/// <returns></returns>
public static byte[] ToBZString(this string val)
{
var b = Windows1251.GetBytes(val);
var b2 = new byte[b.Length + 2];
b.CopyTo(b2, 1);
b[0] = (byte)b.Length;
return b2;
}
/// <summary>
/// Returns bytes for a \0 terminated string prefixed by a length
/// </summary>
/// <param name="val"></param>
/// <returns></returns>
public static byte[] ToTermString(this string val)
{
var b = Windows1251.GetBytes(val);
var b2 = new byte[b.Length + 1];
b.CopyTo(b2, 0);
b[0] = (byte)b.Length;
return b2;
}
public static ulong GetBSAHash(this string name)
{
name = name.Replace('/', '\\');
return GetBSAHash(Path.ChangeExtension(name, null), Path.GetExtension(name));
}
private static ulong GetBSAHash(string name, string ext)
{
name = name.ToLowerInvariant();
ext = ext.ToLowerInvariant();
var hashBytes = new byte[]
{
(byte)(name.Length == 0 ? '\0' : name[name.Length - 1]),
(byte)(name.Length < 3 ? '\0' : name[name.Length - 2]),
(byte)name.Length,
(byte)name[0]
};
var hash1 = BitConverter.ToUInt32(hashBytes, 0);
switch (ext)
{
case ".kf":
hash1 |= 0x80;
break;
case ".nif":
hash1 |= 0x8000;
break;
case ".dds":
hash1 |= 0x8080;
break;
case ".wav":
hash1 |= 0x80000000;
break;
}
uint hash2 = 0;
for (var i = 1; i < name.Length - 2; i++)
{
hash2 = hash2 * 0x1003f + (byte)name[i];
}
uint hash3 = 0;
for (var i = 0; i < ext.Length; i++)
{
hash3 = hash3 * 0x1003f + (byte)ext[i];
}
return (((ulong)(hash2 + hash3)) << 32) + hash1;
}
}
}

11
Compression.BSA/packages.config Normal file
View File

@ -0,0 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="IonKiwi.lz4.net" version="1.0.12" targetFramework="net472" />
<package id="K4os.Compression.LZ4" version="1.1.11" targetFramework="net472" />
<package id="K4os.Compression.LZ4.Streams" version="1.1.11" targetFramework="net472" />
<package id="K4os.Hash.xxHash" version="1.0.6" targetFramework="net472" />
<package id="System.Buffers" version="4.4.0" targetFramework="net472" />
<package id="System.Memory" version="4.5.3" targetFramework="net472" />
<package id="System.Numerics.Vectors" version="4.4.0" targetFramework="net472" />
<package id="System.Runtime.CompilerServices.Unsafe" version="4.5.2" targetFramework="net472" />
</packages>

2
Wabbajack.Common/Data.cs
View File

@ -104,9 +104,11 @@ namespace Wabbajack.Common
public string IsCompressed;
public uint Version;
public Int32 Type;
public bool ShareData;
public uint FileFlags { get; set; }
public bool Compress { get; set; }
public uint ArchiveFlags { get; set; }
}
public class PatchedFromArchive : FromArchive

12
Wabbajack.sln
View File

@ -11,6 +11,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SevenZipExtractor", "SevenZ
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack", "Wabbajack\Wabbajack.csproj", "{33602679-8484-40C7-A10C-774DFF5D8314}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Compression.BSA", "Compression.BSA\Compression.BSA.csproj", "{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Compression.BSA.Test", "Compression.BSA.Test\Compression.BSA.Test.csproj", "{BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -33,6 +37,14 @@ Global
{33602679-8484-40C7-A10C-774DFF5D8314}.Debug|Any CPU.Build.0 = Debug|Any CPU
{33602679-8484-40C7-A10C-774DFF5D8314}.Release|Any CPU.ActiveCfg = Release|Any CPU
{33602679-8484-40C7-A10C-774DFF5D8314}.Release|Any CPU.Build.0 = Release|Any CPU
{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{FF5D892F-8FF4-44FC-8F7F-CD58F307AD1B}.Release|Any CPU.Build.0 = Release|Any CPU
{BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BA2CFEA1-072B-42D6-822A-8C6D0E3AE5D9}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

24
Wabbajack/Compiler.cs
View File

@ -1,6 +1,7 @@
using BSA.Tools;
using Newtonsoft.Json;
using SevenZipExtractor;
using SharpCompress.Archives;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
@ -250,6 +251,21 @@ namespace Wabbajack
}, false);
}
/*
using (var a = ArchiveFactory.Open(archive.AbsolutePath))
{
foreach (var entry in a.Entries)
{
var path = entry.Key.Replace("/", "\\");
if (!paths.Contains(path)) continue;
var result = new MemoryStream();
streams.Add(path, result);
Info("Extracting {0}", path);
using (var stream = entry.OpenEntryStream())
stream.CopyTo(result);
}
}*/
var extracted = streams.ToDictionary(k => k.Key, v => v.Value.ToArray());
// Now Create the patches
Status("Building Patches for {0}", archive.Name);
@ -486,12 +502,14 @@ namespace Wabbajack
{
directive = new CreateBSA()
{
To = source.Path,
To = source.Path,
TempID = id,
Version = bsa.Version,
Type = (int)bsa.Type,
FileFlags = bsa.FileFlags,
Compress = bsa.Compress
ArchiveFlags = bsa.ArchiveFlags,
Compress = bsa.Compress,
ShareData = bsa.ShareData
};
};
@ -528,7 +546,7 @@ namespace Wabbajack
{
var disabled_mods = File.ReadAllLines(Path.Combine(MO2ProfileDir, "modlist.txt"))
.Where(line => line.StartsWith("-") && !line.EndsWith("_separator"))
.Select(line => Path.Combine("mods", line.Substring(1)))
.Select(line => Path.Combine("mods", line.Substring(1)) + "\\")
.ToList();
return source =>
{

44
Wabbajack/Installer.cs
View File

@ -1,4 +1,5 @@
using SevenZipExtractor;
using BSA.Tools;
using SevenZipExtractor;
using System;
using System.Collections.Generic;
using System.IO;
@ -8,6 +9,7 @@ using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Wabbajack.Common;
using static BSA.Tools.libbsarch;
namespace Wabbajack
{
@ -81,10 +83,50 @@ namespace Wabbajack
BuildFolderStructure();
InstallArchives();
InstallIncludedFiles();
BuildBSAs();
Info("Installation complete! You may exit the program.");
}
private void BuildBSAs()
{
var bsas = ModList.Directives.OfType<CreateBSA>().ToList();
Info("Building {0} bsa files");
bsas.Do(bsa =>
{
Status($"Building {bsa.To}");
var source_dir = Path.Combine(Outputfolder, Consts.BSACreationDir, bsa.TempID);
using (var entries = new EntryList())
{
var source_files = Directory.EnumerateFiles(source_dir, "*", SearchOption.AllDirectories)
.Select(e => e.Substring(source_dir.Length + 1))
.ToList();
source_files.Do(name => entries.Add(name));
using (var a = new BSAFile())
{
a.Create(Path.Combine(Outputfolder, bsa.To), (bsa_archive_type_t)bsa.Type, entries);
a.FileFlags = bsa.FileFlags;
a.ArchiveFlags = bsa.ArchiveFlags;
a.ShareData = bsa.ShareData;
source_files.Do(e =>
{
a.AddFile(e, File.ReadAllBytes(Path.Combine(source_dir, e)));
});
a.Save();
}
}
});
}
private void InstallIncludedFiles()
{
Info("Writing inline files");

1
Wabbajack/MainWindow.xaml.cs
View File

@ -39,6 +39,7 @@ namespace Wabbajack
compiler.ModList.ToJSON("C:\\tmp\\modpack.json");
var modlist = compiler.ModList;
var create = modlist.Directives.OfType<CreateBSA>().ToList();
compiler = null;
var installer = new Installer(modlist, "c:\\tmp\\install\\", msg => context.LogMsg(msg));
installer.Install();

3
Wabbajack/Wabbajack.csproj
View File

@ -44,6 +44,9 @@
<Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="SharpCompress, Version=0.23.0.0, Culture=neutral, PublicKeyToken=afb0a02973931d96, processorArchitecture=MSIL">
<HintPath>..\packages\SharpCompress.0.23.0\lib\net45\SharpCompress.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Data" />
<Reference Include="System.Web" />

1
Wabbajack/packages.config
View File

@ -3,5 +3,6 @@
<package id="Costura.Fody" version="4.0.0" targetFramework="net472" />
<package id="Fody" version="5.1.1" targetFramework="net472" developmentDependency="true" />
<package id="Newtonsoft.Json" version="12.0.2" targetFramework="net472" />
<package id="SharpCompress" version="0.23.0" targetFramework="net472" />
<package id="WebSocketSharpFork" version="1.0.4.0" targetFramework="net472" />
</packages>