diff --git a/Compression.BSA.Test/Compression.BSA.Test.csproj b/Compression.BSA.Test/Compression.BSA.Test.csproj
index 30627566..9590988e 100644
--- a/Compression.BSA.Test/Compression.BSA.Test.csproj
+++ b/Compression.BSA.Test/Compression.BSA.Test.csproj
@@ -53,6 +53,9 @@
true
+
+ ..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll
+
@@ -68,6 +71,7 @@
+
diff --git a/Compression.BSA.Test/Program.cs b/Compression.BSA.Test/Program.cs
index ad638ff4..2a66e658 100644
--- a/Compression.BSA.Test/Program.cs
+++ b/Compression.BSA.Test/Program.cs
@@ -1,4 +1,5 @@
-using System;
+using ICSharpCode.SharpZipLib;
+using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@@ -9,11 +10,11 @@ namespace Compression.BSA.Test
{
class Program
{
- const string TestDir = "d:\\Personal YASHEd\\mods";
+ const string TestDir = "d:\\MO2 Instances\\Mod Organizer 2 - LE";
const string TempDir = "c:\\tmp\\out";
static void Main(string[] args)
{
- foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories).Skip(3))
+ foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories).Skip(2))
{
Console.WriteLine($"From {bsa}");
Console.WriteLine("Cleaning Output Dir");
@@ -23,20 +24,25 @@ namespace Compression.BSA.Test
}
Directory.CreateDirectory(TempDir);
+ Console.WriteLine($"Reading {bsa}");
using (var a = new BSAReader(bsa))
{
Parallel.ForEach(a.Files, file =>
{
- var abs_name = Path.Combine("c:\\tmp\\out", file.Path);
+ var abs_name = Path.Combine(TempDir, file.Path);
if (!Directory.Exists(Path.GetDirectoryName(abs_name)))
Directory.CreateDirectory(Path.GetDirectoryName(abs_name));
using (var fs = File.OpenWrite(abs_name))
file.CopyDataTo(fs);
+ Equal((long)file.Size, new FileInfo(abs_name).Length);
});
+
+ Console.WriteLine($"Building {bsa}");
+
using (var w = new BSABuilder())
{
w.ArchiveFlags = a.ArchiveFlags;
@@ -47,7 +53,9 @@ namespace Compression.BSA.Test
{
var abs_path = Path.Combine("c:\\tmp\\out", file.Path);
using (var str = File.OpenRead(abs_path))
- w.AddFile(file.Path, str);
+ {
+ var entry = w.AddFile(file.Path, str, file.FlipCompression);
+ }
});
@@ -57,13 +65,20 @@ namespace Compression.BSA.Test
Equal(a.Files.Count(), w.Files.Count());
Equal(a.Files.Select(f => f.Path).ToHashSet(), w.Files.Select(f => f.Path).ToHashSet());
+ /*foreach (var pair in Enumerable.Zip(a.Files, w.Files, (ai, bi) => (ai, bi)))
+ {
+ Console.WriteLine($"{pair.ai.Path}, {pair.ai.Hash}, {pair.bi.Path}, {pair.bi.Hash}");
+ }*/
+
foreach (var pair in Enumerable.Zip(a.Files, w.Files, (ai, bi) => (ai, bi)))
{
Equal(pair.ai.Path, pair.bi.Path);
+ Equal(pair.ai.Hash, pair.bi.Hash);
}
}
+ Console.WriteLine($"Verifying {bsa}");
using (var b = new BSAReader("c:\\tmp\\built.bsa"))
{
Console.WriteLine($"Performing A/B tests on {bsa}");
@@ -84,7 +99,6 @@ namespace Compression.BSA.Test
}
}
- break;
}
}
}
@@ -110,6 +124,20 @@ namespace Compression.BSA.Test
throw new InvalidDataException($"{a} != {b}");
}
+ public static void Equal(long a, long b)
+ {
+ if (a == b) return;
+
+ throw new InvalidDataException($"{a} != {b}");
+ }
+
+ public static void Equal(ulong a, ulong b)
+ {
+ if (a == b) return;
+
+ throw new InvalidDataException($"{a} != {b}");
+ }
+
public static void Equal(int a, int b)
{
if (a == b) return;
diff --git a/Compression.BSA/BSABuilder.cs b/Compression.BSA/BSABuilder.cs
index d8460b65..e89093e8 100644
--- a/Compression.BSA/BSABuilder.cs
+++ b/Compression.BSA/BSABuilder.cs
@@ -1,4 +1,5 @@
-using K4os.Compression.LZ4;
+using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
+using K4os.Compression.LZ4;
using K4os.Compression.LZ4.Streams;
using System;
using System.Collections.Generic;
@@ -73,7 +74,7 @@ namespace Compression.BSA
}
}
- public void AddFile(string path, Stream src, bool flipCompression = false)
+ public FileEntry AddFile(string path, Stream src, bool flipCompression = false)
{
FileEntry r = new FileEntry(this, path, src, flipCompression);
@@ -81,6 +82,7 @@ namespace Compression.BSA
{
_files.Add(r);
}
+ return r;
}
public IEnumerable FolderNames
@@ -176,11 +178,13 @@ namespace Compression.BSA
public void RegenFolderRecords()
{
- _folders = _files.GroupBy(f => Path.GetDirectoryName(f.Path).ToLowerInvariant())
+ _folders = _files.GroupBy(f => Path.GetDirectoryName(f.Path.ToLowerInvariant()))
.Select(f => new FolderRecordBuilder(this, f.Key, f.ToList()))
.OrderBy(f => f._hash)
.ToList();
+ var lnk = _files.Where(f => f.Path.EndsWith(".lnk")).FirstOrDefault();
+
foreach (var folder in _folders)
foreach (var file in folder._files)
file._folder = folder;
@@ -279,6 +283,14 @@ namespace Compression.BSA
wtr.Write((uint)0); // unk
wtr.Write((ulong)_offset); // offset
}
+ else if (_bsa.HeaderType == VersionType.FO3 || _bsa.HeaderType == VersionType.TES4)
+ {
+ wtr.Write((uint)_offset);
+ }
+ else
+ {
+ throw new NotImplementedException($"Cannot write to BSAs of type {_bsa.HeaderType}");
+ }
}
}
@@ -331,7 +343,18 @@ namespace Compression.BSA
(new MemoryStream(_rawData)).CopyTo(w);
_rawData = r.ToArray();
+ }
+ else if (_bsa.HeaderType == VersionType.FO3 || _bsa.HeaderType == VersionType.TES4)
+ {
+ var r = new MemoryStream();
+ using (var w = new DeflaterOutputStream(r))
+ (new MemoryStream(_rawData)).CopyTo(w);
+ _rawData = r.ToArray();
+ }
+ else
+ {
+ throw new NotImplementedException($"Can't compress data for {_bsa.HeaderType} BSAs.");
}
}
@@ -360,10 +383,6 @@ namespace Compression.BSA
{
return _flipCompression;
}
- set
- {
- _flipCompression = value;
- }
}
public ulong Hash { get
@@ -409,21 +428,19 @@ namespace Compression.BSA
wtr.Write((uint)offset);
wtr.BaseStream.Position = offset;
+ if (_bsa.HasNameBlobs)
+ {
+ wtr.Write(_pathBSBytes);
+ }
+
if (Compressed)
{
- if (_bsa.HasNameBlobs)
- {
- wtr.Write(_pathBSBytes);
- }
+
wtr.Write((uint)_originalSize);
wtr.Write(_rawData);
}
else
{
- if (_bsa.HasNameBlobs)
- {
- wtr.Write(_pathBSBytes);
- }
wtr.Write(_rawData);
}
}
diff --git a/Compression.BSA/BSAReader.cs b/Compression.BSA/BSAReader.cs
index 865b72be..06415f28 100644
--- a/Compression.BSA/BSAReader.cs
+++ b/Compression.BSA/BSAReader.cs
@@ -3,6 +3,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
+using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using K4os.Compression.LZ4.Streams;
namespace Compression.BSA
@@ -10,7 +11,7 @@ namespace Compression.BSA
public enum VersionType : uint
{
TES4 = 0x67,
- FO3 = 0x68,
+ FO3 = 0x68, // FO3, FNV, TES5
SSE = 0x69,
FO4 = 0x01
};
@@ -251,7 +252,11 @@ namespace Compression.BSA
private int _offset;
private FolderRecord _folder;
private string _name;
- private uint? _originalSize;
+ private uint _originalSize;
+ private uint _dataSize;
+ private uint _onDiskSize;
+ private string _nameBlob;
+ private long _dataOffset;
public FileRecord(BSAReader bsa, FolderRecord folderRecord, BinaryReader src)
{
@@ -265,9 +270,36 @@ namespace Compression.BSA
else
_size = size;
+ if (Compressed)
+ _size -= 4;
+
_offset = src.ReadInt32();
_folder = folderRecord;
+
+ var old_pos = src.BaseStream.Position;
+
+ src.BaseStream.Position = _offset;
+
+ if (bsa.HasNameBlobs)
+ _nameBlob = src.ReadStringLenNoTerm();
+
+ if (Compressed)
+ _originalSize = src.ReadUInt32();
+
+ _onDiskSize = (uint)(_size - (_nameBlob == null ? 0 : _nameBlob.Length + 1));
+
+ if (Compressed)
+ {
+ _dataSize = _originalSize;
+ _onDiskSize -= 4;
+ }
+ else
+ _dataSize = _onDiskSize;
+
+ _dataOffset = src.BaseStream.Position;
+
+ src.BaseStream.Position = old_pos;
}
internal void LoadFileRecord(BSAReader bsaReader, FolderRecord folder, FileRecord file, BinaryReader rdr)
@@ -296,13 +328,7 @@ namespace Compression.BSA
{
get
{
- if (Compressed)
- {
- if (_originalSize == null)
- LoadOriginalSize();
- return (int)_originalSize;
- }
- return _size;
+ return (int)_dataSize;
}
}
@@ -341,46 +367,42 @@ namespace Compression.BSA
}
}
+ public bool FlipCompression { get => _compressedFlag; }
+
public void CopyDataTo(Stream output)
{
using (var in_file = File.OpenRead(_bsa._fileName))
using (var rdr = new BinaryReader(in_file))
{
- rdr.BaseStream.Position = _offset;
- if (Compressed)
- {
- string _name;
- int file_size = _size;
- if (_bsa.HasNameBlobs)
- {
- var name_size = rdr.ReadByte();
- file_size -= name_size + 1;
- rdr.BaseStream.Position = _offset + 1 + name_size;
- }
+ rdr.BaseStream.Position = _dataOffset;
- var original_size = rdr.ReadUInt32();
- file_size -= 4;
- if (_bsa.HeaderType == VersionType.SSE)
+ if (_bsa.HeaderType == VersionType.SSE)
+ {
+
+ if (Compressed)
{
var r = LZ4Stream.Decode(rdr.BaseStream);
- r.CopyTo(output);
+ r.CopyToLimit(output, (int)_onDiskSize);
+
}
else
{
- throw new NotImplementedException("Compressed Skyrim LE archives not yet implemented");
+ rdr.BaseStream.CopyToLimit(output, (int)_onDiskSize);
}
}
else
{
- string _name;
- int file_size = _size;
- if (_bsa.HasNameBlobs)
+
+ if (Compressed)
{
- var name_size = rdr.ReadByte();
- file_size -= name_size + 1;
- rdr.BaseStream.Position = _offset + 1 + name_size;
+ using (var z = new InflaterInputStream(rdr.BaseStream))
+ z.CopyToLimit(output, (int)_originalSize);
+
+ }
+ else
+ {
+ rdr.BaseStream.CopyToLimit(output, (int)_onDiskSize);
}
- rdr.BaseStream.CopyToLimit(output, file_size);
}
}
}
diff --git a/Compression.BSA/Compression.BSA.csproj b/Compression.BSA/Compression.BSA.csproj
index 69c3e49d..b174d7b9 100644
--- a/Compression.BSA/Compression.BSA.csproj
+++ b/Compression.BSA/Compression.BSA.csproj
@@ -50,6 +50,9 @@
MinimumRecommendedRules.ruleset
+
+ ..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll
+
..\packages\K4os.Compression.LZ4.1.1.11\lib\net46\K4os.Compression.LZ4.dll
diff --git a/Compression.BSA/Utils.cs b/Compression.BSA/Utils.cs
index 31117c22..57f12c80 100644
--- a/Compression.BSA/Utils.cs
+++ b/Compression.BSA/Utils.cs
@@ -7,7 +7,7 @@ namespace Compression.BSA
{
internal static class Utils
{
- private static Encoding Windows1251 = Encoding.GetEncoding(1251);
+ private static Encoding Windows1251 = Encoding.UTF7;// Encoding.GetEncoding(1251);
public static string ReadStringLen(this BinaryReader rdr)
{
@@ -17,6 +17,13 @@ namespace Compression.BSA
return Windows1251.GetString(bytes);
}
+ public static string ReadStringLenNoTerm(this BinaryReader rdr)
+ {
+ var len = rdr.ReadByte();
+ var bytes = rdr.ReadBytes(len);
+ return Windows1251.GetString(bytes);
+ }
+
public static string ReadStringTerm(this BinaryReader rdr)
{
List acc = new List();
@@ -53,7 +60,7 @@ namespace Compression.BSA
///
public static byte[] ToBSString(this string val)
{
- var b = Windows1251.GetBytes(val);
+ var b = Encoding.ASCII.GetBytes(val);
var b2 = new byte[b.Length + 1];
b.CopyTo(b2, 1);
b2[0] = (byte)b.Length;
@@ -134,6 +141,7 @@ namespace Compression.BSA
tw.Write(buff, 0, read);
limit -= read;
}
+ tw.Flush();
}
}
}
diff --git a/Compression.BSA/packages.config b/Compression.BSA/packages.config
index 0fb520a3..c59cc70f 100644
--- a/Compression.BSA/packages.config
+++ b/Compression.BSA/packages.config
@@ -3,6 +3,7 @@
+