Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

BSA tools should now support TES4, FO3, FNV, Skyrim and SSE

Browse Source
This commit is contained in:
Timothy Baldridge 2019-08-21 22:27:24 -06:00
parent e2817ef949
commit 0c74967dd8
7 changed files with 138 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Compression.BSA.Test/Compression.BSA.Test.csproj
View File

@ -53,6 +53,9 @@
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<ItemGroup>
<Reference Include="ICSharpCode.SharpZipLib, Version=1.2.0.246, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
<HintPath>..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
@ -68,6 +71,7 @@
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Compression.BSA\Compression.BSA.csproj">

40
Compression.BSA.Test/Program.cs
View File

@ -1,4 +1,5 @@
using System;
using ICSharpCode.SharpZipLib;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@ -9,11 +10,11 @@ namespace Compression.BSA.Test
{
class Program
{
const string TestDir = "d:\\Personal YASHEd\\mods";
const string TestDir = "d:\\MO2 Instances\\Mod Organizer 2 - LE";
const string TempDir = "c:\\tmp\\out";
static void Main(string[] args)
{
foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories).Skip(3))
foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories).Skip(2))
{
Console.WriteLine($"From {bsa}");
Console.WriteLine("Cleaning Output Dir");
@ -23,20 +24,25 @@ namespace Compression.BSA.Test
}
Directory.CreateDirectory(TempDir);
Console.WriteLine($"Reading {bsa}");
using (var a = new BSAReader(bsa))
{
Parallel.ForEach(a.Files, file =>
{
var abs_name = Path.Combine("c:\\tmp\\out", file.Path);
var abs_name = Path.Combine(TempDir, file.Path);
if (!Directory.Exists(Path.GetDirectoryName(abs_name)))
Directory.CreateDirectory(Path.GetDirectoryName(abs_name));
using (var fs = File.OpenWrite(abs_name))
file.CopyDataTo(fs);
Equal((long)file.Size, new FileInfo(abs_name).Length);
});
Console.WriteLine($"Building {bsa}");
using (var w = new BSABuilder())
{
w.ArchiveFlags = a.ArchiveFlags;
@ -47,7 +53,9 @@ namespace Compression.BSA.Test
{
var abs_path = Path.Combine("c:\\tmp\\out", file.Path);
using (var str = File.OpenRead(abs_path))
w.AddFile(file.Path, str);
{
var entry = w.AddFile(file.Path, str, file.FlipCompression);
}
});
@ -57,13 +65,20 @@ namespace Compression.BSA.Test
Equal(a.Files.Count(), w.Files.Count());
Equal(a.Files.Select(f => f.Path).ToHashSet(), w.Files.Select(f => f.Path).ToHashSet());
/*foreach (var pair in Enumerable.Zip(a.Files, w.Files, (ai, bi) => (ai, bi)))
{
Console.WriteLine($"{pair.ai.Path}, {pair.ai.Hash}, {pair.bi.Path}, {pair.bi.Hash}");
}*/
foreach (var pair in Enumerable.Zip(a.Files, w.Files, (ai, bi) => (ai, bi)))
{
Equal(pair.ai.Path, pair.bi.Path);
Equal(pair.ai.Hash, pair.bi.Hash);
}
}
Console.WriteLine($"Verifying {bsa}");
using (var b = new BSAReader("c:\\tmp\\built.bsa"))
{
Console.WriteLine($"Performing A/B tests on {bsa}");
@ -84,7 +99,6 @@ namespace Compression.BSA.Test
}
}
break;
}
}
}
@ -110,6 +124,20 @@ namespace Compression.BSA.Test
throw new InvalidDataException($"{a} != {b}");
}
public static void Equal(long a, long b)
{
if (a == b) return;
throw new InvalidDataException($"{a} != {b}");
}
public static void Equal(ulong a, ulong b)
{
if (a == b) return;
throw new InvalidDataException($"{a} != {b}");
}
public static void Equal(int a, int b)
{
if (a == b) return;

47
Compression.BSA/BSABuilder.cs
View File

@ -1,4 +1,5 @@
using K4os.Compression.LZ4;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using K4os.Compression.LZ4;
using K4os.Compression.LZ4.Streams;
using System;
using System.Collections.Generic;
@ -73,7 +74,7 @@ namespace Compression.BSA
}
}
public void AddFile(string path, Stream src, bool flipCompression = false)
public FileEntry AddFile(string path, Stream src, bool flipCompression = false)
{
FileEntry r = new FileEntry(this, path, src, flipCompression);
@ -81,6 +82,7 @@ namespace Compression.BSA
{
_files.Add(r);
}
return r;
}
public IEnumerable<string> FolderNames
@ -176,11 +178,13 @@ namespace Compression.BSA
public void RegenFolderRecords()
{
_folders = _files.GroupBy(f => Path.GetDirectoryName(f.Path).ToLowerInvariant())
_folders = _files.GroupBy(f => Path.GetDirectoryName(f.Path.ToLowerInvariant()))
.Select(f => new FolderRecordBuilder(this, f.Key, f.ToList()))
.OrderBy(f => f._hash)
.ToList();
var lnk = _files.Where(f => f.Path.EndsWith(".lnk")).FirstOrDefault();
foreach (var folder in _folders)
foreach (var file in folder._files)
file._folder = folder;
@ -279,6 +283,14 @@ namespace Compression.BSA
wtr.Write((uint)0); // unk
wtr.Write((ulong)_offset); // offset
}
else if (_bsa.HeaderType == VersionType.FO3 || _bsa.HeaderType == VersionType.TES4)
{
wtr.Write((uint)_offset);
}
else
{
throw new NotImplementedException($"Cannot write to BSAs of type {_bsa.HeaderType}");
}
}
}
@ -331,7 +343,18 @@ namespace Compression.BSA
(new MemoryStream(_rawData)).CopyTo(w);
_rawData = r.ToArray();
}
else if (_bsa.HeaderType == VersionType.FO3 || _bsa.HeaderType == VersionType.TES4)
{
var r = new MemoryStream();
using (var w = new DeflaterOutputStream(r))
(new MemoryStream(_rawData)).CopyTo(w);
_rawData = r.ToArray();
}
else
{
throw new NotImplementedException($"Can't compress data for {_bsa.HeaderType} BSAs.");
}
}
@ -360,10 +383,6 @@ namespace Compression.BSA
{
return _flipCompression;
}
set
{
_flipCompression = value;
}
}
public ulong Hash { get
@ -409,21 +428,19 @@ namespace Compression.BSA
wtr.Write((uint)offset);
wtr.BaseStream.Position = offset;
if (_bsa.HasNameBlobs)
{
wtr.Write(_pathBSBytes);
}
if (Compressed)
{
if (_bsa.HasNameBlobs)
{
wtr.Write(_pathBSBytes);
}
wtr.Write((uint)_originalSize);
wtr.Write(_rawData);
}
else
{
if (_bsa.HasNameBlobs)
{
wtr.Write(_pathBSBytes);
}
wtr.Write(_rawData);
}
}

86
Compression.BSA/BSAReader.cs
View File

@ -3,6 +3,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using K4os.Compression.LZ4.Streams;
namespace Compression.BSA
@ -10,7 +11,7 @@ namespace Compression.BSA
public enum VersionType : uint
{
TES4 = 0x67,
FO3 = 0x68,
FO3 = 0x68, // FO3, FNV, TES5
SSE = 0x69,
FO4 = 0x01
};
@ -251,7 +252,11 @@ namespace Compression.BSA
private int _offset;
private FolderRecord _folder;
private string _name;
private uint? _originalSize;
private uint _originalSize;
private uint _dataSize;
private uint _onDiskSize;
private string _nameBlob;
private long _dataOffset;
public FileRecord(BSAReader bsa, FolderRecord folderRecord, BinaryReader src)
{
@ -265,9 +270,36 @@ namespace Compression.BSA
else
_size = size;
if (Compressed)
_size -= 4;
_offset = src.ReadInt32();
_folder = folderRecord;
var old_pos = src.BaseStream.Position;
src.BaseStream.Position = _offset;
if (bsa.HasNameBlobs)
_nameBlob = src.ReadStringLenNoTerm();
if (Compressed)
_originalSize = src.ReadUInt32();
_onDiskSize = (uint)(_size - (_nameBlob == null ? 0 : _nameBlob.Length + 1));
if (Compressed)
{
_dataSize = _originalSize;
_onDiskSize -= 4;
}
else
_dataSize = _onDiskSize;
_dataOffset = src.BaseStream.Position;
src.BaseStream.Position = old_pos;
}
internal void LoadFileRecord(BSAReader bsaReader, FolderRecord folder, FileRecord file, BinaryReader rdr)
@ -296,13 +328,7 @@ namespace Compression.BSA
{
get
{
if (Compressed)
{
if (_originalSize == null)
LoadOriginalSize();
return (int)_originalSize;
}
return _size;
return (int)_dataSize;
}
}
@ -341,46 +367,42 @@ namespace Compression.BSA
}
}
public bool FlipCompression { get => _compressedFlag; }
public void CopyDataTo(Stream output)
{
using (var in_file = File.OpenRead(_bsa._fileName))
using (var rdr = new BinaryReader(in_file))
{
rdr.BaseStream.Position = _offset;
if (Compressed)
{
string _name;
int file_size = _size;
if (_bsa.HasNameBlobs)
{
var name_size = rdr.ReadByte();
file_size -= name_size + 1;
rdr.BaseStream.Position = _offset + 1 + name_size;
}
rdr.BaseStream.Position = _dataOffset;
var original_size = rdr.ReadUInt32();
file_size -= 4;
if (_bsa.HeaderType == VersionType.SSE)
if (_bsa.HeaderType == VersionType.SSE)
{
if (Compressed)
{
var r = LZ4Stream.Decode(rdr.BaseStream);
r.CopyTo(output);
r.CopyToLimit(output, (int)_onDiskSize);
}
else
{
throw new NotImplementedException("Compressed Skyrim LE archives not yet implemented");
rdr.BaseStream.CopyToLimit(output, (int)_onDiskSize);
}
}
else
{
string _name;
int file_size = _size;
if (_bsa.HasNameBlobs)
if (Compressed)
{
var name_size = rdr.ReadByte();
file_size -= name_size + 1;
rdr.BaseStream.Position = _offset + 1 + name_size;
using (var z = new InflaterInputStream(rdr.BaseStream))
z.CopyToLimit(output, (int)_originalSize);
}
else
{
rdr.BaseStream.CopyToLimit(output, (int)_onDiskSize);
}
rdr.BaseStream.CopyToLimit(output, file_size);
}
}
}

3
Compression.BSA/Compression.BSA.csproj
View File

@ -50,6 +50,9 @@
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<ItemGroup>
<Reference Include="ICSharpCode.SharpZipLib, Version=1.2.0.246, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
<HintPath>..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll</HintPath>
</Reference>
<Reference Include="K4os.Compression.LZ4, Version=1.1.11.0, Culture=neutral, PublicKeyToken=2186fa9121ef231d, processorArchitecture=MSIL">
<HintPath>..\packages\K4os.Compression.LZ4.1.1.11\lib\net46\K4os.Compression.LZ4.dll</HintPath>
</Reference>

12
Compression.BSA/Utils.cs
View File

@ -7,7 +7,7 @@ namespace Compression.BSA
{
internal static class Utils
{
private static Encoding Windows1251 = Encoding.GetEncoding(1251);
private static Encoding Windows1251 = Encoding.UTF7;// Encoding.GetEncoding(1251);
public static string ReadStringLen(this BinaryReader rdr)
{
@ -17,6 +17,13 @@ namespace Compression.BSA
return Windows1251.GetString(bytes);
}
public static string ReadStringLenNoTerm(this BinaryReader rdr)
{
var len = rdr.ReadByte();
var bytes = rdr.ReadBytes(len);
return Windows1251.GetString(bytes);
}
public static string ReadStringTerm(this BinaryReader rdr)
{
List<byte> acc = new List<byte>();
@ -53,7 +60,7 @@ namespace Compression.BSA
/// <returns></returns>
public static byte[] ToBSString(this string val)
{
var b = Windows1251.GetBytes(val);
var b = Encoding.ASCII.GetBytes(val);
var b2 = new byte[b.Length + 1];
b.CopyTo(b2, 1);
b2[0] = (byte)b.Length;
@ -134,6 +141,7 @@ namespace Compression.BSA
tw.Write(buff, 0, read);
limit -= read;
}
tw.Flush();
}
}
}

1
Compression.BSA/packages.config
View File

@ -3,6 +3,7 @@
<package id="K4os.Compression.LZ4" version="1.1.11" targetFramework="net472" />
<package id="K4os.Compression.LZ4.Streams" version="1.1.11" targetFramework="net472" />
<package id="K4os.Hash.xxHash" version="1.0.6" targetFramework="net472" />
<package id="SharpZipLib" version="1.2.0" targetFramework="net472" />
<package id="System.Buffers" version="4.4.0" targetFramework="net472" />
<package id="System.Memory" version="4.5.3" targetFramework="net472" />
<package id="System.Numerics.Vectors" version="4.4.0" targetFramework="net472" />