Merge pull request #556 from wabbajack-tools/morrowind-bsas

Add support for extracting/building TES3 .bsa files

CHANGELOG.md

@@ -5,6 +5,7 @@
 * Block popups in the in-app browser (#535)
 * Don't print API keys in logs (#533)
 * Store xxHash caches in binary format (#530)
+* Added support for Morrowind BSA creation/unpacking
 
 
 #### Version - 0.9.19.0

Compression.BSA.Test/BSATests.cs

@@ -45,7 +45,8 @@ namespace Compression.BSA.Test
                 (Game.Skyrim, 3863), // SkyUI
                 (Game.Skyrim, 51473), // iNeed
                 //(Game.Fallout4, 22223) // 10mm SMG
-                (Game.Fallout4, 4472) // True Storms
+                (Game.Fallout4, 4472), // True Storms
+                (Game.Morrowind, 44537) // Morrowind TAMRIEL_DATA
             };
 
             await Task.WhenAll(modIDs.Select(async (info) =>

Compression.BSA/BSADispatch.cs

@@ -13,6 +13,8 @@ namespace Compression.BSA
                 fourcc = Encoding.ASCII.GetString(new BinaryReader(file).ReadBytes(4));
             }
 
+            if (fourcc == TES3Reader.TES3_MAGIC)
+                return new TES3Reader(filename);
             if (fourcc == "BSA\0")
                 return new BSAReader(filename);
             if (fourcc == "BTDX")

Compression.BSA/BSAReader.cs

@@ -13,7 +13,8 @@ namespace Compression.BSA
         TES4 = 0x67,
         FO3 = 0x68, // FO3, FNV, TES5
         SSE = 0x69,
-        FO4 = 0x01
+        FO4 = 0x01,
+        TES3 = 0xFF // Not a real Bethesda version number
     }
 
     [Flags]

Compression.BSA/TES3Builder.cs

@@ -0,0 +1,77 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using File = Alphaleonis.Win32.Filesystem.File;
+
+namespace Compression.BSA
+{
+    public class TES3Builder : IBSABuilder
+    {
+        private TES3ArchiveState _state;
+        private (TES3FileState state, byte[] data)[] _files;
+
+        public TES3Builder(TES3ArchiveState state)
+        {
+            _state = state;
+            _files = new (TES3FileState state, byte[] data)[_state.FileCount];
+        }
+
+        public void AddFile(FileStateObject state, Stream src)
+        {
+            using var br = new BinaryReader(src);
+            var cstate = (TES3FileState)state;
+            _files[state.Index] = (cstate, br.ReadBytes((int)cstate.Size));
+        }
+
+        public void Build(string filename)
+        {
+            using var fs = File.Create(filename);
+            using var bw = new BinaryWriter(fs);
+            
+            bw.Write(_state.VersionNumber);
+            bw.Write(_state.HashOffset);
+            bw.Write(_state.FileCount);
+
+            foreach (var (state, _) in _files)
+            {
+                bw.Write(state.Size);
+                bw.Write(state.Offset);
+            }
+
+            foreach (var (state, _) in _files)
+            {
+                bw.Write(state.NameOffset);
+            }
+
+            var orgPos = bw.BaseStream.Position;
+            
+            foreach (var (state, _) in _files)
+            {
+                if (bw.BaseStream.Position != orgPos + state.NameOffset)
+                    throw new InvalidDataException("Offsets don't match when writing TES3 BSA");
+                bw.Write(Encoding.ASCII.GetBytes(state.Path));
+                bw.Write((byte)0);
+            }
+
+            bw.BaseStream.Position = _state.HashOffset + 12;
+            foreach (var (state, _) in _files)
+            {
+                bw.Write(state.Hash1);
+                bw.Write(state.Hash2);
+            }
+            
+            if (bw.BaseStream.Position != _state.DataOffset)
+                throw new InvalidDataException("Data offset doesn't match when writing TES3 BSA");
+
+            foreach (var (state, data) in _files)
+            {
+                bw.BaseStream.Position = _state.DataOffset + state.Offset;
+                bw.Write(data);
+            }
+        }
+
+        public void Dispose()
+        {
+        }
+    }
+}

Compression.BSA/TES3Reader.cs

@@ -0,0 +1,139 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace Compression.BSA
+{
+    public class TES3Reader : IBSAReader
+    {
+        public static string TES3_MAGIC = Encoding.ASCII.GetString(new byte[] {0, 1, 0, 0});
+        private uint _versionNumber;
+        private uint _hashTableOffset;
+        private uint _fileCount;
+        private TES3FileEntry[] _files;
+        internal long _dataOffset;
+        internal string _filename;
+
+        public TES3Reader(string filename)
+        {
+            _filename = filename;
+            using var fs = File.OpenRead(filename);
+            using var br = new BinaryReader(fs);
+            _versionNumber = br.ReadUInt32();
+            _hashTableOffset = br.ReadUInt32();
+            _fileCount = br.ReadUInt32();
+
+            _files = new TES3FileEntry[_fileCount];
+            for (int i = 0; i < _fileCount; i++)
+            {
+                var file = new TES3FileEntry {
+                    Index = i,
+                    Archive = this, 
+                    Size = br.ReadUInt32(), 
+                    Offset = br.ReadUInt32()
+                    
+                };
+                _files[i] = file;
+            }
+
+            for (int i = 0; i < _fileCount; i++)
+            {
+                _files[i].NameOffset = br.ReadUInt32();
+            }
+
+            var origPos = br.BaseStream.Position;
+            for (int i = 0; i < _fileCount; i++)
+            {
+                br.BaseStream.Position = origPos + _files[i].NameOffset;
+                _files[i].Path = br.ReadStringTerm(VersionType.TES3);
+            }
+
+            br.BaseStream.Position = _hashTableOffset + 12;
+            for (int i = 0; i < _fileCount; i++)
+            {
+                _files[i].Hash1 = br.ReadUInt32();
+                _files[i].Hash2 = br.ReadUInt32();
+            }
+
+            _dataOffset = br.BaseStream.Position;
+        }
+
+        public void Dispose()
+        {
+        }
+
+        public IEnumerable<IFile> Files => _files;
+        public ArchiveStateObject State
+        {
+            get
+            {
+                return new TES3ArchiveState
+                {
+                    FileCount = _fileCount,
+                    DataOffset = _dataOffset,
+                    HashOffset = _hashTableOffset,
+                    VersionNumber = _versionNumber,
+                    
+                };
+            }
+        }
+    }
+
+    public class TES3ArchiveState : ArchiveStateObject
+    {
+        public uint FileCount { get; set; }
+        public long DataOffset { get; set; }
+        public uint HashOffset { get; set; }
+        public uint VersionNumber { get; set; }
+
+        public override IBSABuilder MakeBuilder()
+        {
+            return new TES3Builder(this);
+        }
+    }
+
+    public class TES3FileEntry : IFile
+    {
+        public string Path { get; set;  }
+        public uint Size { get; set; }
+        public FileStateObject State =>
+            new TES3FileState
+            {
+                Index = Index,
+                Path = Path,
+                Size = Size,
+                Offset = Offset,
+                NameOffset = NameOffset,
+                Hash1 = Hash1,
+                Hash2 = Hash2
+            };
+
+        public void CopyDataTo(Stream output)
+        {
+            using var fs = File.OpenRead(Archive._filename);
+            fs.Position = Archive._dataOffset + Offset;
+            fs.CopyToLimit(output, (int)Size);
+        }
+
+        public uint Offset { get; set; }
+        public uint NameOffset { get; set; }
+        public uint Hash1 { get; set; }
+        public uint Hash2 { get; set; }
+        public TES3Reader Archive { get; set; }
+        public int Index { get; set; }
+
+    }
+    
+    
+    public class TES3FileState : FileStateObject
+    {
+        public uint Offset { get; set; }
+        public uint NameOffset { get; set; }
+        public uint Hash1 { get; set; }
+        public uint Hash2 { get; set; }
+        public uint Size { get; set; }
+    }
+
+
+}

Compression.BSA/Utils.cs

@@ -18,6 +18,8 @@ namespace Compression.BSA
         
         private static Encoding GetEncoding(VersionType version)
         {
+            if (version == VersionType.TES3)
+                return Encoding.ASCII;
             if (version == VersionType.SSE)
                 return Windows1252;
             return Encoding.UTF7;

Wabbajack.Lib/CerasConfig.cs

@@ -30,7 +30,7 @@ namespace Wabbajack.Lib
                     typeof(PropertyFile), typeof(SteamMeta), typeof(SteamWorkshopDownloader), typeof(SteamWorkshopDownloader.State),
                     typeof(LoversLabDownloader.State), typeof(GameFileSourceDownloader.State), typeof(VectorPlexusDownloader.State),
                     typeof(DeadlyStreamDownloader.State), typeof(AFKModsDownloader.State), typeof(TESAllianceDownloader.State),
-                    typeof(BethesdaNetDownloader.State)
+                    typeof(TES3ArchiveState), typeof(TES3FileState), typeof(BethesdaNetDownloader.State)
                 },
             };
             Config.VersionTolerance.Mode = VersionToleranceMode.Standard;