BSA tools should now support TES4, FO3, FNV, Skyrim and SSE

Compression.BSA.Test/Compression.BSA.Test.csproj

@@ -53,6 +53,9 @@
     <Prefer32Bit>true</Prefer32Bit>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="ICSharpCode.SharpZipLib, Version=1.2.0.246, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
+      <HintPath>..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll</HintPath>
+    </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />
     <Reference Include="System.Xml.Linq" />
@@ -68,6 +71,7 @@
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config" />
+    <None Include="packages.config" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Compression.BSA\Compression.BSA.csproj">

Compression.BSA.Test/Program.cs

@@ -1,4 +1,5 @@
-using System;
+using ICSharpCode.SharpZipLib;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -9,11 +10,11 @@ namespace Compression.BSA.Test
 {
     class Program
     {
-        const string TestDir = "d:\\Personal YASHEd\\mods";
+        const string TestDir = "d:\\MO2 Instances\\Mod Organizer 2 - LE";
         const string TempDir = "c:\\tmp\\out";
         static void Main(string[] args)
         {
-            foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories).Skip(3))
+            foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.bsa", SearchOption.AllDirectories).Skip(2))
             {
                 Console.WriteLine($"From {bsa}");
                 Console.WriteLine("Cleaning Output Dir");
@@ -23,20 +24,25 @@ namespace Compression.BSA.Test
                 }
                 Directory.CreateDirectory(TempDir);
 
+                Console.WriteLine($"Reading {bsa}");
                 using (var a = new BSAReader(bsa))
                 {
 
                     Parallel.ForEach(a.Files, file =>
                     {
-                        var abs_name = Path.Combine("c:\\tmp\\out", file.Path);
+                        var abs_name = Path.Combine(TempDir, file.Path);
 
                         if (!Directory.Exists(Path.GetDirectoryName(abs_name)))
                             Directory.CreateDirectory(Path.GetDirectoryName(abs_name));
 
                         using (var fs = File.OpenWrite(abs_name))
                             file.CopyDataTo(fs);
+                        Equal((long)file.Size, new FileInfo(abs_name).Length);
                     });
 
+
+                    Console.WriteLine($"Building {bsa}");
+
                     using (var w = new BSABuilder())
                     {
                         w.ArchiveFlags = a.ArchiveFlags;
@@ -47,7 +53,9 @@ namespace Compression.BSA.Test
                         {
                             var abs_path = Path.Combine("c:\\tmp\\out", file.Path);
                             using (var str = File.OpenRead(abs_path))
-                                w.AddFile(file.Path, str);
+                            {
+                                var entry = w.AddFile(file.Path, str, file.FlipCompression);
+                            }
 
                         });
                         
@@ -57,13 +65,20 @@ namespace Compression.BSA.Test
                         Equal(a.Files.Count(), w.Files.Count());
                         Equal(a.Files.Select(f => f.Path).ToHashSet(), w.Files.Select(f => f.Path).ToHashSet());
 
+                        /*foreach (var pair in Enumerable.Zip(a.Files, w.Files, (ai, bi) => (ai, bi)))
+                        {
+                            Console.WriteLine($"{pair.ai.Path},  {pair.ai.Hash},  {pair.bi.Path}, {pair.bi.Hash}");
+                        }*/
+
                         foreach (var pair in Enumerable.Zip(a.Files, w.Files, (ai, bi) => (ai, bi)))
                         {
                             Equal(pair.ai.Path, pair.bi.Path);
+                            Equal(pair.ai.Hash, pair.bi.Hash);
                         }
 
                     }
 
+                    Console.WriteLine($"Verifying {bsa}");
                     using (var b = new BSAReader("c:\\tmp\\built.bsa"))
                     {
                         Console.WriteLine($"Performing A/B tests on {bsa}");
@@ -84,7 +99,6 @@ namespace Compression.BSA.Test
 
                         }
                     }
-                    break;
                 }
             }
         }
@@ -110,6 +124,20 @@ namespace Compression.BSA.Test
             throw new InvalidDataException($"{a} != {b}");
         }
 
+        public static void Equal(long a, long b)
+        {
+            if (a == b) return;
+
+            throw new InvalidDataException($"{a} != {b}");
+        }
+
+        public static void Equal(ulong a, ulong b)
+        {
+            if (a == b) return;
+
+            throw new InvalidDataException($"{a} != {b}");
+        }
+
         public static void Equal(int a, int b)
         {
             if (a == b) return;

Compression.BSA/BSABuilder.cs

@@ -1,4 +1,5 @@
-using K4os.Compression.LZ4;
+using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
+using K4os.Compression.LZ4;
 using K4os.Compression.LZ4.Streams;
 using System;
 using System.Collections.Generic;
@@ -73,7 +74,7 @@ namespace Compression.BSA
             }
         }
 
-        public void AddFile(string path, Stream src, bool flipCompression = false)
+        public FileEntry AddFile(string path, Stream src, bool flipCompression = false)
         {
             FileEntry r = new FileEntry(this, path, src, flipCompression);
 
@@ -81,6 +82,7 @@ namespace Compression.BSA
             {
                 _files.Add(r);
             }
+            return r;
         }
 
         public IEnumerable<string> FolderNames
@@ -176,11 +178,13 @@ namespace Compression.BSA
 
         public void RegenFolderRecords()
         {
-            _folders = _files.GroupBy(f => Path.GetDirectoryName(f.Path).ToLowerInvariant())
+            _folders = _files.GroupBy(f => Path.GetDirectoryName(f.Path.ToLowerInvariant()))
                              .Select(f => new FolderRecordBuilder(this, f.Key, f.ToList()))
                              .OrderBy(f => f._hash)
                              .ToList();
 
+            var lnk = _files.Where(f => f.Path.EndsWith(".lnk")).FirstOrDefault();
+
             foreach (var folder in _folders)
                 foreach (var file in folder._files)
                     file._folder = folder;
@@ -279,6 +283,14 @@ namespace Compression.BSA
                 wtr.Write((uint)0); // unk
                 wtr.Write((ulong)_offset); // offset
             }
+            else if (_bsa.HeaderType == VersionType.FO3 || _bsa.HeaderType == VersionType.TES4)
+            {
+                wtr.Write((uint)_offset);
+            }
+            else
+            {
+                throw new NotImplementedException($"Cannot write to BSAs of type {_bsa.HeaderType}");
+            }
         }
 
     }
@@ -331,7 +343,18 @@ namespace Compression.BSA
                     (new MemoryStream(_rawData)).CopyTo(w);
 
                 _rawData = r.ToArray();
+            }
+            else if (_bsa.HeaderType == VersionType.FO3 || _bsa.HeaderType == VersionType.TES4)
+            {
+                var r = new MemoryStream();
+                using (var w = new DeflaterOutputStream(r))
+                    (new MemoryStream(_rawData)).CopyTo(w);
 
+                _rawData = r.ToArray();
+            }
+            else
+            {
+                throw new NotImplementedException($"Can't compress data for {_bsa.HeaderType} BSAs.");
             }
         }
 
@@ -360,10 +383,6 @@ namespace Compression.BSA
             {
                 return _flipCompression;
             }
-            set
-            {
-                _flipCompression = value;
-            }
         }
 
         public ulong Hash { get
@@ -409,21 +428,19 @@ namespace Compression.BSA
             wtr.Write((uint)offset);
             wtr.BaseStream.Position = offset;
 
+            if (_bsa.HasNameBlobs)
+            {
+                wtr.Write(_pathBSBytes);
+            }
+
             if (Compressed)
             {
-                if (_bsa.HasNameBlobs)
+
-                {
-                    wtr.Write(_pathBSBytes);
-                }
                 wtr.Write((uint)_originalSize);
                 wtr.Write(_rawData);
             }
             else
             {
-                if (_bsa.HasNameBlobs)
-                {
-                    wtr.Write(_pathBSBytes);
-                }
                 wtr.Write(_rawData);
             }
         }

Compression.BSA/BSAReader.cs

@@ -3,6 +3,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Text;
+using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
 using K4os.Compression.LZ4.Streams;
 
 namespace Compression.BSA
@@ -10,7 +11,7 @@ namespace Compression.BSA
     public enum VersionType : uint
     {
         TES4 = 0x67,
-        FO3 = 0x68,
+        FO3 = 0x68, // FO3, FNV, TES5
         SSE = 0x69,
         FO4 = 0x01
     };
@@ -251,7 +252,11 @@ namespace Compression.BSA
         private int _offset;
         private FolderRecord _folder;
         private string _name;
-        private uint? _originalSize;
+        private uint _originalSize;
+        private uint _dataSize;
+        private uint _onDiskSize;
+        private string _nameBlob;
+        private long _dataOffset;
 
         public FileRecord(BSAReader bsa, FolderRecord folderRecord, BinaryReader src)
         {
@@ -265,9 +270,36 @@ namespace Compression.BSA
             else
                 _size = size;
 
+            if (Compressed)
+                _size -= 4;
+
             _offset = src.ReadInt32();
 
             _folder = folderRecord;
+
+            var old_pos = src.BaseStream.Position;
+
+            src.BaseStream.Position = _offset;
+
+            if (bsa.HasNameBlobs)
+                _nameBlob = src.ReadStringLenNoTerm();
+
+            if (Compressed)
+                _originalSize = src.ReadUInt32();
+
+            _onDiskSize = (uint)(_size - (_nameBlob == null ? 0 : _nameBlob.Length + 1));
+
+            if (Compressed)
+            {
+                _dataSize = _originalSize;
+                _onDiskSize -= 4;
+            }
+            else
+                _dataSize = _onDiskSize;
+
+            _dataOffset = src.BaseStream.Position;
+
+            src.BaseStream.Position = old_pos;
         }
 
         internal void LoadFileRecord(BSAReader bsaReader, FolderRecord folder, FileRecord file, BinaryReader rdr)
@@ -296,13 +328,7 @@ namespace Compression.BSA
         {
             get
             {
-                if (Compressed)
+                return (int)_dataSize;
-                {
-                    if (_originalSize == null)
-                        LoadOriginalSize();
-                    return (int)_originalSize;
-                }
-                return _size;
             }
         }
 
@@ -341,46 +367,42 @@ namespace Compression.BSA
             } 
         }
 
+        public bool FlipCompression { get => _compressedFlag; }
+
         public void CopyDataTo(Stream output)
         {
             using (var in_file = File.OpenRead(_bsa._fileName))
             using (var rdr = new BinaryReader(in_file))
             {
-                rdr.BaseStream.Position = _offset;
+                rdr.BaseStream.Position = _dataOffset;
-                if (Compressed)
-                {
-                    string _name;
-                    int file_size = _size;
-                    if (_bsa.HasNameBlobs)
-                    {
-                        var name_size = rdr.ReadByte();
-                        file_size -= name_size + 1;
-                        rdr.BaseStream.Position = _offset + 1 + name_size;
-                    }
 
-                    var original_size = rdr.ReadUInt32();
+                if (_bsa.HeaderType == VersionType.SSE)
-                    file_size -= 4;
+                {
-                    if (_bsa.HeaderType == VersionType.SSE)
+
+                    if (Compressed)
                     {
                         var r = LZ4Stream.Decode(rdr.BaseStream);
-                        r.CopyTo(output);
+                        r.CopyToLimit(output, (int)_onDiskSize);
+
                     }
                     else
                     {
-                        throw new NotImplementedException("Compressed Skyrim LE archives not yet implemented");
+                        rdr.BaseStream.CopyToLimit(output, (int)_onDiskSize);
                     }
                 }
                 else
                 {
-                    string _name;
+
-                    int file_size = _size;
+                    if (Compressed)
-                    if (_bsa.HasNameBlobs)
                     {
-                        var name_size = rdr.ReadByte();
+                        using (var z = new InflaterInputStream(rdr.BaseStream))
-                        file_size -= name_size + 1;
+                            z.CopyToLimit(output, (int)_originalSize);
-                        rdr.BaseStream.Position = _offset + 1 + name_size;
+
+                    }
+                    else
+                    {
+                        rdr.BaseStream.CopyToLimit(output, (int)_onDiskSize);
                     }
-                    rdr.BaseStream.CopyToLimit(output, file_size);
                 }
             }
         }

Compression.BSA/Compression.BSA.csproj

@@ -50,6 +50,9 @@
     <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="ICSharpCode.SharpZipLib, Version=1.2.0.246, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
+      <HintPath>..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll</HintPath>
+    </Reference>
     <Reference Include="K4os.Compression.LZ4, Version=1.1.11.0, Culture=neutral, PublicKeyToken=2186fa9121ef231d, processorArchitecture=MSIL">
       <HintPath>..\packages\K4os.Compression.LZ4.1.1.11\lib\net46\K4os.Compression.LZ4.dll</HintPath>
     </Reference>

Compression.BSA/Utils.cs

@@ -7,7 +7,7 @@ namespace Compression.BSA
 {
     internal static class Utils
     {
-        private static Encoding Windows1251 = Encoding.GetEncoding(1251);
+        private static Encoding Windows1251 = Encoding.UTF7;// Encoding.GetEncoding(1251);
 
         public static string ReadStringLen(this BinaryReader rdr)
         {
@@ -17,6 +17,13 @@ namespace Compression.BSA
             return Windows1251.GetString(bytes);
         }
 
+        public static string ReadStringLenNoTerm(this BinaryReader rdr)
+        {
+            var len = rdr.ReadByte();
+            var bytes = rdr.ReadBytes(len);
+            return Windows1251.GetString(bytes);
+        }
+
         public static string ReadStringTerm(this BinaryReader rdr)
         {
             List<byte> acc = new List<byte>();
@@ -53,7 +60,7 @@ namespace Compression.BSA
         /// <returns></returns>
         public static byte[] ToBSString(this string val)
         {
-            var b = Windows1251.GetBytes(val);
+            var b = Encoding.ASCII.GetBytes(val);
             var b2 = new byte[b.Length + 1];
             b.CopyTo(b2, 1);
             b2[0] = (byte)b.Length;
@@ -134,6 +141,7 @@ namespace Compression.BSA
                 tw.Write(buff, 0, read);
                 limit -= read;
             }
+            tw.Flush();
         }
     }
 }

Compression.BSA/packages.config

@@ -3,6 +3,7 @@
   <package id="K4os.Compression.LZ4" version="1.1.11" targetFramework="net472" />
   <package id="K4os.Compression.LZ4.Streams" version="1.1.11" targetFramework="net472" />
   <package id="K4os.Hash.xxHash" version="1.0.6" targetFramework="net472" />
+  <package id="SharpZipLib" version="1.2.0" targetFramework="net472" />
   <package id="System.Buffers" version="4.4.0" targetFramework="net472" />
   <package id="System.Memory" version="4.5.3" targetFramework="net472" />
   <package id="System.Numerics.Vectors" version="4.4.0" targetFramework="net472" />