using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using K4os.Compression.LZ4; using K4os.Compression.LZ4.Streams; using Wabbajack.Common; using Wabbajack.Compression.BSA.Interfaces; using Wabbajack.Compression.BSA.TES3Archive; using Wabbajack.DTOs.BSA.ArchiveStates; using Wabbajack.DTOs.BSA.FileStates; using Wabbajack.Paths; using Wabbajack.Paths.IO; namespace Wabbajack.Compression.BSA.TES5Archive; public class Builder : IBuilder { internal byte[] _fileId; private List _files = new(); internal List _folders = new(); internal uint _offset; internal DiskSlabAllocator _slab; internal uint _totalFileNameLength; public IEnumerable Files => _files; public ArchiveFlags ArchiveFlags { get; set; } public FileFlags FileFlags { get; set; } public VersionType HeaderType { get; set; } public IEnumerable FolderNames { get { return _files.Select(f => f.Path.Level == 1 ? default : f.Path.Parent).Distinct(); } } public bool HasFolderNames => ArchiveFlags.HasFlag(ArchiveFlags.HasFileNames); public bool HasFileNames => ArchiveFlags.HasFlag(ArchiveFlags.HasFileNames); public bool CompressedByDefault => ArchiveFlags.HasFlag(ArchiveFlags.Compressed); public bool HasNameBlobs => ArchiveFlags.HasFlag(ArchiveFlags.HasFileNameBlobs); public async ValueTask AddFile(AFile state, Stream src, CancellationToken token) { var bsaState = (BSAFile) state; var r = await FileEntry.Create(this, bsaState.Path, src, bsaState.FlipCompression, token); lock (this) { _files.Add(r); } } public async ValueTask Build(Stream fs, CancellationToken token) { RegenFolderRecords(); await using var wtr = new BinaryWriter(fs, Encoding.Default, true); wtr.Write(_fileId); wtr.Write((uint) HeaderType); wtr.Write(_offset); wtr.Write((uint) ArchiveFlags); var folders = FolderNames.ToList(); wtr.Write((uint) folders.Count); wtr.Write((uint) _files.Count); wtr.Write((uint) _folders.Select(f => f._nameBytes.Length - 1).Sum()); // totalFolderNameLength var s = _files.Select(f => f._pathBytes.Length).Sum(); _totalFileNameLength = (uint) _files.Select(f => f._nameBytes.Length).Sum(); wtr.Write(_totalFileNameLength); // totalFileNameLength wtr.Write((uint) FileFlags); foreach (var folder in _folders) folder.WriteFolderRecord(wtr); foreach (var folder in _folders) { if (HasFolderNames) wtr.Write(folder._nameBytes); foreach (var file in folder._files) file.WriteFileRecord(wtr); } foreach (var file in _files) await wtr.BaseStream.WriteAsync(file._nameBytes, token); foreach (var file in _files) await file.WriteData(wtr, token); } public static Builder Create(TemporaryFileManager tempGenerator) { var self = new Builder { _fileId = Encoding.ASCII.GetBytes("BSA\0"), _offset = 0x24, _slab = new DiskSlabAllocator(tempGenerator) }; return self; } public static Builder Create(BSAState bsaStateObject, TemporaryFileManager tempGenerator) { var self = Create(tempGenerator); self.HeaderType = (VersionType) bsaStateObject.Version; self.FileFlags = (FileFlags) bsaStateObject.FileFlags; self.ArchiveFlags = (ArchiveFlags) bsaStateObject.ArchiveFlags; return self; } public async ValueTask DisposeAsync() { await _slab.DisposeAsync(); } public void RegenFolderRecords() { _folders = _files.GroupBy(f => f.Path.Level == 1 ? default : f.Path.Parent) .Select(f => new FolderRecordBuilder(this, f.Key, f.ToList())) .OrderBy(f => f._hash) .ToList(); foreach (var folder in _folders) foreach (var file in folder._files) file._folder = folder; _files = (from folder in _folders from file in folder._files orderby folder._hash, file._hash select file).ToList(); } } public class FolderRecordBuilder { internal Builder _bsa; internal uint _fileCount; internal IEnumerable _files; internal ulong _hash; internal byte[] _nameBytes; internal ulong _offset; internal uint _recordSize; public FolderRecordBuilder(Builder bsa, RelativePath folderName, IEnumerable files) { _files = files.OrderBy(f => f._hash); Name = folderName; _bsa = bsa; // Folders don't have extensions, so let's make sure we cut it out _hash = Name.GetFolderBSAHash(); _fileCount = (uint) files.Count(); _nameBytes = folderName.ToBZString(_bsa.HeaderType); _recordSize = sizeof(ulong) + sizeof(uint) + sizeof(uint); } public ulong Hash => _hash; public RelativePath Name { get; } public ulong SelfSize { get { if (_bsa.HeaderType == VersionType.SSE) return sizeof(ulong) + sizeof(uint) + sizeof(uint) + sizeof(ulong); return sizeof(ulong) + sizeof(uint) + sizeof(uint); } } public ulong FileRecordSize { get { ulong size = 0; if (_bsa.HasFolderNames) size += (ulong) _nameBytes.Length; size += (ulong) _files.Select(f => sizeof(ulong) + sizeof(uint) + sizeof(uint)).Sum(); return size; } } public void WriteFolderRecord(BinaryWriter wtr) { var idx = _bsa._folders.IndexOf(this); _offset = (ulong) wtr.BaseStream.Position; _offset += (ulong) _bsa._folders.Skip(idx).Select(f => (long) f.SelfSize).Sum(); _offset += _bsa._totalFileNameLength; _offset += (ulong) _bsa._folders.Take(idx).Select(f => (long) f.FileRecordSize).Sum(); var sp = wtr.BaseStream.Position; wtr.Write(_hash); wtr.Write(_fileCount); if (_bsa.HeaderType == VersionType.SSE) { wtr.Write((uint) 0); // unk wtr.Write(_offset); // offset } else if (_bsa.HeaderType is VersionType.FO3 or VersionType.TES4) { wtr.Write((uint) _offset); } else { throw new NotImplementedException($"Cannot write to BSAs of type {_bsa.HeaderType}"); } } } public class FileEntry { internal Builder _bsa; internal bool _flipCompression; internal FolderRecordBuilder _folder; internal ulong _hash; internal string _name; internal byte[] _nameBytes; private long _offsetOffset; internal int _originalSize; internal RelativePath _path; private byte[] _pathBSBytes; internal byte[] _pathBytes; private Stream _srcData; public bool Compressed { get { if (_flipCompression) return !_bsa.CompressedByDefault; return _bsa.CompressedByDefault; } } public RelativePath Path => _path; public bool FlipCompression => _flipCompression; public ulong Hash => _hash; public FolderRecordBuilder Folder => _folder; public static async Task Create(Builder bsa, RelativePath path, Stream src, bool flipCompression, CancellationToken token) { var entry = new FileEntry(); entry._bsa = bsa; entry._path = path; entry._name = (string) entry._path.FileName; entry._hash = entry._name.GetBSAHash(); entry._nameBytes = entry._name.ToTermString(bsa.HeaderType); entry._pathBytes = entry._path.ToTermString(bsa.HeaderType); entry._pathBSBytes = entry._path.ToBSString(); entry._flipCompression = flipCompression; entry._srcData = src; entry._originalSize = (int) entry._srcData.Length; if (entry.Compressed) await entry.CompressData(token); return entry; } private async Task CompressData(CancellationToken token) { switch (_bsa.HeaderType) { case VersionType.SSE: { var r = new MemoryStream(); await using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L12_MAX}, true)) { await _srcData.CopyToWithStatusAsync(_srcData.Length, w, token); } await _srcData.DisposeAsync(); _srcData = _bsa._slab.Allocate(r.Length); r.Position = 0; await r.CopyToWithStatusAsync(r.Length, _srcData, token); _srcData.Position = 0; break; } case VersionType.FO3: case VersionType.TES4: { var r = new MemoryStream(); using (var w = new DeflaterOutputStream(r)) { w.IsStreamOwner = false; await _srcData.CopyToWithStatusAsync(_srcData.Length, w, token); } await _srcData.DisposeAsync(); _srcData = _bsa._slab.Allocate(r.Length); r.Position = 0; await r.CopyToWithStatusAsync(r.Length, _srcData, token); _srcData.Position = 0; break; } default: throw new NotImplementedException($"Can't compress data for {_bsa.HeaderType} BSAs."); } } internal void WriteFileRecord(BinaryWriter wtr) { wtr.Write(_hash); var size = _srcData.Length; if (_bsa.HasNameBlobs) size += _pathBSBytes.Length; if (Compressed) size += 4; if (_flipCompression) wtr.Write((uint) size | (0x1 << 30)); else wtr.Write((uint) size); _offsetOffset = wtr.BaseStream.Position; wtr.Write(0xDEADBEEF); } internal async Task WriteData(BinaryWriter wtr, CancellationToken token) { var offset = (uint) wtr.BaseStream.Position; wtr.BaseStream.Position = _offsetOffset; wtr.Write(offset); wtr.BaseStream.Position = offset; if (_bsa.HasNameBlobs) wtr.Write(_pathBSBytes); if (Compressed) { wtr.Write((uint) _originalSize); _srcData.Position = 0; await _srcData.CopyToLimitAsync(wtr.BaseStream, (int) _srcData.Length, token); await _srcData.DisposeAsync(); } else { _srcData.Position = 0; await _srcData.CopyToLimitAsync(wtr.BaseStream, (int) _srcData.Length, token); await _srcData.DisposeAsync(); } } }