diff --git a/Compression.BSA.Test/Compression.BSA.Test.csproj b/Compression.BSA.Test/Compression.BSA.Test.csproj index 30627566..9590988e 100644 --- a/Compression.BSA.Test/Compression.BSA.Test.csproj +++ b/Compression.BSA.Test/Compression.BSA.Test.csproj @@ -53,6 +53,9 @@ true + + ..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll + @@ -68,6 +71,7 @@ + diff --git a/Compression.BSA.Test/Program.cs b/Compression.BSA.Test/Program.cs index ad638ff4..2a66e658 100644 --- a/Compression.BSA.Test/Program.cs +++ b/Compression.BSA.Test/Program.cs @@ -1,4 +1,5 @@ -using System; +using ICSharpCode.SharpZipLib; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -9,11 +10,11 @@ namespace Compression.BSA.Test { class Program { - const string TestDir = "d:\\Personal YASHEd\\mods"; + const string TestDir = "d:\\MO2 Instances\\Mod Organizer 2 - LE"; const string TempDir = "c:\\tmp\\out"; static void Main(string[] args) { - foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories).Skip(3)) + foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories).Skip(2)) { Console.WriteLine($"From {bsa}"); Console.WriteLine("Cleaning Output Dir"); @@ -23,20 +24,25 @@ namespace Compression.BSA.Test } Directory.CreateDirectory(TempDir); + Console.WriteLine($"Reading {bsa}"); using (var a = new BSAReader(bsa)) { Parallel.ForEach(a.Files, file => { - var abs_name = Path.Combine("c:\\tmp\\out", file.Path); + var abs_name = Path.Combine(TempDir, file.Path); if (!Directory.Exists(Path.GetDirectoryName(abs_name))) Directory.CreateDirectory(Path.GetDirectoryName(abs_name)); using (var fs = File.OpenWrite(abs_name)) file.CopyDataTo(fs); + Equal((long)file.Size, new FileInfo(abs_name).Length); }); + + Console.WriteLine($"Building {bsa}"); + using (var w = new BSABuilder()) { w.ArchiveFlags = a.ArchiveFlags; @@ -47,7 +53,9 @@ namespace Compression.BSA.Test { var abs_path = Path.Combine("c:\\tmp\\out", file.Path); using (var str = File.OpenRead(abs_path)) - w.AddFile(file.Path, str); + { + var entry = w.AddFile(file.Path, str, file.FlipCompression); + } }); @@ -57,13 +65,20 @@ namespace Compression.BSA.Test Equal(a.Files.Count(), w.Files.Count()); Equal(a.Files.Select(f => f.Path).ToHashSet(), w.Files.Select(f => f.Path).ToHashSet()); + /*foreach (var pair in Enumerable.Zip(a.Files, w.Files, (ai, bi) => (ai, bi))) + { + Console.WriteLine($"{pair.ai.Path}, {pair.ai.Hash}, {pair.bi.Path}, {pair.bi.Hash}"); + }*/ + foreach (var pair in Enumerable.Zip(a.Files, w.Files, (ai, bi) => (ai, bi))) { Equal(pair.ai.Path, pair.bi.Path); + Equal(pair.ai.Hash, pair.bi.Hash); } } + Console.WriteLine($"Verifying {bsa}"); using (var b = new BSAReader("c:\\tmp\\built.bsa")) { Console.WriteLine($"Performing A/B tests on {bsa}"); @@ -84,7 +99,6 @@ namespace Compression.BSA.Test } } - break; } } } @@ -110,6 +124,20 @@ namespace Compression.BSA.Test throw new InvalidDataException($"{a} != {b}"); } + public static void Equal(long a, long b) + { + if (a == b) return; + + throw new InvalidDataException($"{a} != {b}"); + } + + public static void Equal(ulong a, ulong b) + { + if (a == b) return; + + throw new InvalidDataException($"{a} != {b}"); + } + public static void Equal(int a, int b) { if (a == b) return; diff --git a/Compression.BSA/BSABuilder.cs b/Compression.BSA/BSABuilder.cs index d8460b65..e89093e8 100644 --- a/Compression.BSA/BSABuilder.cs +++ b/Compression.BSA/BSABuilder.cs @@ -1,4 +1,5 @@ -using K4os.Compression.LZ4; +using ICSharpCode.SharpZipLib.Zip.Compression.Streams; +using K4os.Compression.LZ4; using K4os.Compression.LZ4.Streams; using System; using System.Collections.Generic; @@ -73,7 +74,7 @@ namespace Compression.BSA } } - public void AddFile(string path, Stream src, bool flipCompression = false) + public FileEntry AddFile(string path, Stream src, bool flipCompression = false) { FileEntry r = new FileEntry(this, path, src, flipCompression); @@ -81,6 +82,7 @@ namespace Compression.BSA { _files.Add(r); } + return r; } public IEnumerable FolderNames @@ -176,11 +178,13 @@ namespace Compression.BSA public void RegenFolderRecords() { - _folders = _files.GroupBy(f => Path.GetDirectoryName(f.Path).ToLowerInvariant()) + _folders = _files.GroupBy(f => Path.GetDirectoryName(f.Path.ToLowerInvariant())) .Select(f => new FolderRecordBuilder(this, f.Key, f.ToList())) .OrderBy(f => f._hash) .ToList(); + var lnk = _files.Where(f => f.Path.EndsWith(".lnk")).FirstOrDefault(); + foreach (var folder in _folders) foreach (var file in folder._files) file._folder = folder; @@ -279,6 +283,14 @@ namespace Compression.BSA wtr.Write((uint)0); // unk wtr.Write((ulong)_offset); // offset } + else if (_bsa.HeaderType == VersionType.FO3 || _bsa.HeaderType == VersionType.TES4) + { + wtr.Write((uint)_offset); + } + else + { + throw new NotImplementedException($"Cannot write to BSAs of type {_bsa.HeaderType}"); + } } } @@ -331,7 +343,18 @@ namespace Compression.BSA (new MemoryStream(_rawData)).CopyTo(w); _rawData = r.ToArray(); + } + else if (_bsa.HeaderType == VersionType.FO3 || _bsa.HeaderType == VersionType.TES4) + { + var r = new MemoryStream(); + using (var w = new DeflaterOutputStream(r)) + (new MemoryStream(_rawData)).CopyTo(w); + _rawData = r.ToArray(); + } + else + { + throw new NotImplementedException($"Can't compress data for {_bsa.HeaderType} BSAs."); } } @@ -360,10 +383,6 @@ namespace Compression.BSA { return _flipCompression; } - set - { - _flipCompression = value; - } } public ulong Hash { get @@ -409,21 +428,19 @@ namespace Compression.BSA wtr.Write((uint)offset); wtr.BaseStream.Position = offset; + if (_bsa.HasNameBlobs) + { + wtr.Write(_pathBSBytes); + } + if (Compressed) { - if (_bsa.HasNameBlobs) - { - wtr.Write(_pathBSBytes); - } + wtr.Write((uint)_originalSize); wtr.Write(_rawData); } else { - if (_bsa.HasNameBlobs) - { - wtr.Write(_pathBSBytes); - } wtr.Write(_rawData); } } diff --git a/Compression.BSA/BSAReader.cs b/Compression.BSA/BSAReader.cs index 865b72be..06415f28 100644 --- a/Compression.BSA/BSAReader.cs +++ b/Compression.BSA/BSAReader.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Text; +using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using K4os.Compression.LZ4.Streams; namespace Compression.BSA @@ -10,7 +11,7 @@ namespace Compression.BSA public enum VersionType : uint { TES4 = 0x67, - FO3 = 0x68, + FO3 = 0x68, // FO3, FNV, TES5 SSE = 0x69, FO4 = 0x01 }; @@ -251,7 +252,11 @@ namespace Compression.BSA private int _offset; private FolderRecord _folder; private string _name; - private uint? _originalSize; + private uint _originalSize; + private uint _dataSize; + private uint _onDiskSize; + private string _nameBlob; + private long _dataOffset; public FileRecord(BSAReader bsa, FolderRecord folderRecord, BinaryReader src) { @@ -265,9 +270,36 @@ namespace Compression.BSA else _size = size; + if (Compressed) + _size -= 4; + _offset = src.ReadInt32(); _folder = folderRecord; + + var old_pos = src.BaseStream.Position; + + src.BaseStream.Position = _offset; + + if (bsa.HasNameBlobs) + _nameBlob = src.ReadStringLenNoTerm(); + + if (Compressed) + _originalSize = src.ReadUInt32(); + + _onDiskSize = (uint)(_size - (_nameBlob == null ? 0 : _nameBlob.Length + 1)); + + if (Compressed) + { + _dataSize = _originalSize; + _onDiskSize -= 4; + } + else + _dataSize = _onDiskSize; + + _dataOffset = src.BaseStream.Position; + + src.BaseStream.Position = old_pos; } internal void LoadFileRecord(BSAReader bsaReader, FolderRecord folder, FileRecord file, BinaryReader rdr) @@ -296,13 +328,7 @@ namespace Compression.BSA { get { - if (Compressed) - { - if (_originalSize == null) - LoadOriginalSize(); - return (int)_originalSize; - } - return _size; + return (int)_dataSize; } } @@ -341,46 +367,42 @@ namespace Compression.BSA } } + public bool FlipCompression { get => _compressedFlag; } + public void CopyDataTo(Stream output) { using (var in_file = File.OpenRead(_bsa._fileName)) using (var rdr = new BinaryReader(in_file)) { - rdr.BaseStream.Position = _offset; - if (Compressed) - { - string _name; - int file_size = _size; - if (_bsa.HasNameBlobs) - { - var name_size = rdr.ReadByte(); - file_size -= name_size + 1; - rdr.BaseStream.Position = _offset + 1 + name_size; - } + rdr.BaseStream.Position = _dataOffset; - var original_size = rdr.ReadUInt32(); - file_size -= 4; - if (_bsa.HeaderType == VersionType.SSE) + if (_bsa.HeaderType == VersionType.SSE) + { + + if (Compressed) { var r = LZ4Stream.Decode(rdr.BaseStream); - r.CopyTo(output); + r.CopyToLimit(output, (int)_onDiskSize); + } else { - throw new NotImplementedException("Compressed Skyrim LE archives not yet implemented"); + rdr.BaseStream.CopyToLimit(output, (int)_onDiskSize); } } else { - string _name; - int file_size = _size; - if (_bsa.HasNameBlobs) + + if (Compressed) { - var name_size = rdr.ReadByte(); - file_size -= name_size + 1; - rdr.BaseStream.Position = _offset + 1 + name_size; + using (var z = new InflaterInputStream(rdr.BaseStream)) + z.CopyToLimit(output, (int)_originalSize); + + } + else + { + rdr.BaseStream.CopyToLimit(output, (int)_onDiskSize); } - rdr.BaseStream.CopyToLimit(output, file_size); } } } diff --git a/Compression.BSA/Compression.BSA.csproj b/Compression.BSA/Compression.BSA.csproj index 69c3e49d..b174d7b9 100644 --- a/Compression.BSA/Compression.BSA.csproj +++ b/Compression.BSA/Compression.BSA.csproj @@ -50,6 +50,9 @@ MinimumRecommendedRules.ruleset + + ..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll + ..\packages\K4os.Compression.LZ4.1.1.11\lib\net46\K4os.Compression.LZ4.dll diff --git a/Compression.BSA/Utils.cs b/Compression.BSA/Utils.cs index 31117c22..57f12c80 100644 --- a/Compression.BSA/Utils.cs +++ b/Compression.BSA/Utils.cs @@ -7,7 +7,7 @@ namespace Compression.BSA { internal static class Utils { - private static Encoding Windows1251 = Encoding.GetEncoding(1251); + private static Encoding Windows1251 = Encoding.UTF7;// Encoding.GetEncoding(1251); public static string ReadStringLen(this BinaryReader rdr) { @@ -17,6 +17,13 @@ namespace Compression.BSA return Windows1251.GetString(bytes); } + public static string ReadStringLenNoTerm(this BinaryReader rdr) + { + var len = rdr.ReadByte(); + var bytes = rdr.ReadBytes(len); + return Windows1251.GetString(bytes); + } + public static string ReadStringTerm(this BinaryReader rdr) { List acc = new List(); @@ -53,7 +60,7 @@ namespace Compression.BSA /// public static byte[] ToBSString(this string val) { - var b = Windows1251.GetBytes(val); + var b = Encoding.ASCII.GetBytes(val); var b2 = new byte[b.Length + 1]; b.CopyTo(b2, 1); b2[0] = (byte)b.Length; @@ -134,6 +141,7 @@ namespace Compression.BSA tw.Write(buff, 0, read); limit -= read; } + tw.Flush(); } } } diff --git a/Compression.BSA/packages.config b/Compression.BSA/packages.config index 0fb520a3..c59cc70f 100644 --- a/Compression.BSA/packages.config +++ b/Compression.BSA/packages.config @@ -3,6 +3,7 @@ +