Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

can write .ba2 general files, and they read back without issue

Browse Source
This commit is contained in:
Timothy Baldridge 2019-10-07 16:13:38 -06:00
parent af05894ae3
commit 9f3ee6a5cc
8 changed files with 300 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Compression.BSA.Test/Compression.BSA.Test.csproj
View File

@ -56,6 +56,9 @@
<Reference Include="ICSharpCode.SharpZipLib, Version=1.2.0.246, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
<HintPath>..\packages\SharpZipLib.1.2.0\lib\net45\ICSharpCode.SharpZipLib.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />

53
Compression.BSA.Test/Program.cs
View File

@ -3,17 +3,19 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Newtonsoft.Json;
namespace Compression.BSA.Test
{
internal class Program
{
private const string TestDir = @"D:\MO2 Instances\F4EE";
//private const string TestDir = @"D:\Steam\steamapps\common\Fallout 4";
private const string TempDir = @"c:\tmp\out\f4ee";
private static void Main(string[] args)
{
foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.ba2", SearchOption.AllDirectories).Skip(0))
foreach (var bsa in Directory.EnumerateFiles(TestDir, "*.ba2", SearchOption.AllDirectories).Skip(0).Take(1))
{
Console.WriteLine($"From {bsa}");
Console.WriteLine("Cleaning Output Dir");
@ -41,44 +43,36 @@ namespace Compression.BSA.Test
});
/*
Console.WriteLine($"Building {bsa}");
using (var w = new BSABuilder())
using (var w = ViaJson(a.State).MakeBuilder())
{
w.ArchiveFlags = a.ArchiveFlags;
w.FileFlags = a.FileFlags;
w.HeaderType = a.HeaderType;
Parallel.ForEach(a.Files, file =>
{
var abs_path = Path.Combine("c:\\tmp\\out", file.Path);
var abs_path = Path.Combine(TempDir, file.Path);
using (var str = File.OpenRead(abs_path))
{
var entry = w.AddFile(file.Path, str, file.FlipCompression);
w.AddFile(ViaJson(file.State), str);
}
});
w.Build("c:\\tmp\\tmp.bsa");
// Sanity Checks
Equal(a.Files.Count(), w.Files.Count());
Equal(a.Files.Select(f => f.Path).ToHashSet(), w.Files.Select(f => f.Path).ToHashSet());
foreach (var pair in a.Files.Zip(w.Files, (ai, bi) => (ai, bi)))
{
Equal(pair.ai.Path, pair.bi.Path);
Equal(pair.ai.Hash, pair.bi.Hash);
}
}
Console.WriteLine($"Verifying {bsa}");
using (var b = new BSAReader("c:\\tmp\\tmp.bsa"))
using (var b = BSADispatch.OpenRead("c:\\tmp\\tmp.bsa"))
{
Console.WriteLine($"Performing A/B tests on {bsa}");
Equal((uint) a.ArchiveFlags, (uint) b.ArchiveFlags);
Equal((uint) a.FileFlags, (uint) b.FileFlags);
Equal(JsonConvert.SerializeObject(a.State), JsonConvert.SerializeObject(b.State));
//Equal((uint) a.ArchiveFlags, (uint) b.ArchiveFlags);
//Equal((uint) a.FileFlags, (uint) b.FileFlags);
// Check same number of files
Equal(a.Files.Count(), b.Files.Count());
@ -86,17 +80,28 @@ namespace Compression.BSA.Test
foreach (var pair in a.Files.Zip(b.Files, (ai, bi) => (ai, bi)))
{
idx++;
Equal(JsonConvert.SerializeObject(pair.ai.State),
JsonConvert.SerializeObject(pair.bi.State));
//Console.WriteLine($" - {pair.ai.Path}");
Equal(pair.ai.Path, pair.bi.Path);
Equal(pair.ai.Compressed, pair.bi.Compressed);
//Equal(pair.ai.Compressed, pair.bi.Compressed);
Equal(pair.ai.Size, pair.bi.Size);
Equal(pair.ai.GetData(), pair.bi.GetData());
//Equal(pair.ai.GetData(), pair.bi.GetData());
}
}*/
}
}
}
}
public static T ViaJson<T>(T i)
{
var settings = new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.All
};
return JsonConvert.DeserializeObject<T>(JsonConvert.SerializeObject(i, settings), settings);
}
private static void Equal(HashSet<string> a, HashSet<string> b)
{
Equal(a.Count, b.Count);

2
Compression.BSA.Test/packages.config
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Newtonsoft.Json" version="12.0.2" targetFramework="net472" />
<package id="SharpZipLib" version="1.2.0" targetFramework="net472" />
</packages>

152
Compression.BSA/BA2Builder.cs Normal file
View File

@ -0,0 +1,152 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
namespace Compression.BSA
{
interface IFileBuilder
{
uint FileHash { get; }
uint DirHash { get; }
string FullName { get; }
int Index { get; }
void WriteData(BinaryWriter wtr);
void WriteHeader(BinaryWriter wtr);
}
public class BA2Builder : IBSABuilder
{
private BA2StateObject _state;
private List<IFileBuilder> _entries = new List<IFileBuilder>();
public BA2Builder(BA2StateObject state)
{
_state = state;
}
public void Dispose()
{
}
public void AddFile(FileStateObject state, Stream src)
{
switch (_state.Type)
{
case EntryType.GNRL:
var result = new BA2FileEntryBuilder((BA2FileEntryState)state, src);
lock(_entries) _entries.Add(result);
break;
}
}
public void Build(string filename)
{
SortEntries();
using (var fs = File.OpenWrite(filename))
using (var bw = new BinaryWriter(fs))
{
bw.Write(Encoding.ASCII.GetBytes(_state.HeaderMagic));
bw.Write(_state.Version);
bw.Write(Encoding.ASCII.GetBytes(Enum.GetName(typeof(EntryType), _state.Type)));
bw.Write((uint)_entries.Count);
var table_offset_loc = bw.BaseStream.Position;
bw.Write((ulong)0);
foreach (var entry in _entries)
{
entry.WriteHeader(bw);
}
foreach (var entry in _entries)
{
entry.WriteData(bw);
}
if (_state.HasNameTable)
{
var pos = bw.BaseStream.Position;
bw.BaseStream.Seek(table_offset_loc, SeekOrigin.Begin);
bw.Write((ulong) pos);
bw.BaseStream.Seek(pos, SeekOrigin.Begin);
foreach (var entry in _entries)
{
var bytes = Encoding.UTF7.GetBytes(entry.FullName);
bw.Write((ushort)bytes.Length);
bw.Write(bytes);
}
}
}
}
private void SortEntries()
{
_entries = _entries.OrderBy(e => e.Index).ToList();
}
}
public class BA2FileEntryBuilder : IFileBuilder
{
private byte[] _data;
private int _rawSize;
private int _size;
private BA2FileEntryState _state;
private long _offsetOffset;
public BA2FileEntryBuilder(BA2FileEntryState state, Stream src)
{
_state = state;
using (var ms = new MemoryStream())
{
src.CopyTo(ms);
_data = ms.ToArray();
}
_rawSize = _data.Length;
if (state.Compressed)
{
using (var ms = new MemoryStream())
using (var ds = new DeflaterOutputStream(ms))
{
ds.Write(_data, 0, _data.Length);
}
_size = _data.Length;
}
}
public uint FileHash => _state.NameHash;
public uint DirHash => _state.DirHash;
public string FullName => _state.FullPath;
public int Index => _state.Index;
public void WriteHeader(BinaryWriter wtr)
{
wtr.Write(_state.NameHash);
wtr.Write(Encoding.ASCII.GetBytes(_state.Extension));
wtr.Write(_state.DirHash);
wtr.Write(_state.Flags);
_offsetOffset = wtr.BaseStream.Position;
wtr.Write((ulong)0);
wtr.Write(_size);
wtr.Write(_rawSize);
wtr.Write(_state.Align);
}
public void WriteData(BinaryWriter wtr)
{
var pos = wtr.BaseStream.Position;
wtr.BaseStream.Seek(_offsetOffset, SeekOrigin.Begin);
wtr.Write((ulong)pos);
wtr.BaseStream.Position = pos;
wtr.Write(_data);
}
}
}

117
Compression.BSA/BA2Reader.cs
View File

@ -5,14 +5,16 @@ using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using ICSharpCode.SharpZipLib.Zip;
using ICSharpCode.SharpZipLib.Zip.Compression;
using File = Alphaleonis.Win32.Filesystem.File;
namespace Compression.BSA
{
enum EntryType
public enum EntryType
{
GNRL,
DX10,
@ -30,11 +32,11 @@ namespace Compression.BSA
internal string _filename;
private Stream _stream;
internal BinaryReader _rdr;
private uint _version;
private string _headerMagic;
private EntryType _type;
private uint _numFiles;
private ulong _nameTableOffset;
internal uint _version;
internal string _headerMagic;
internal EntryType _type;
internal uint _numFiles;
internal ulong _nameTableOffset;
public bool UseATIFourCC { get; set; } = false;
public bool HasNameTable => _nameTableOffset > 0;
@ -80,7 +82,7 @@ namespace Compression.BSA
switch (_type)
{
case EntryType.GNRL:
files.Add(new BA2FileEntry(this));
files.Add(new BA2FileEntry(this, idx));
break;
case EntryType.DX10:
files.Add(new BA2DX10Entry(this));
@ -108,6 +110,31 @@ namespace Compression.BSA
}
public IEnumerable<IFile> Files { get; private set; }
public ArchiveStateObject State => new BA2StateObject(this);
}
public class BA2StateObject : ArchiveStateObject
{
public BA2StateObject()
{
}
public BA2StateObject(BA2Reader ba2Reader)
{
Version = ba2Reader._version;
HeaderMagic = ba2Reader._headerMagic;
Type = ba2Reader._type;
HasNameTable = ba2Reader.HasNameTable;
}
public bool HasNameTable { get; set; }
public EntryType Type { get; set; }
public string HeaderMagic { get; set; }
public uint Version { get; set; }
public override IBSABuilder MakeBuilder()
{
return new BA2Builder(this);
}
}
public class BA2DX10Entry : IFileEntry
@ -153,26 +180,24 @@ namespace Compression.BSA
public string Path => FullPath;
public uint Size => (uint)_chunks.Sum(f => f._fullSz) + HeaderSize + sizeof(uint);
public FileStateObject State { get; }
public uint HeaderSize
{
get
{
unsafe
switch ((DXGI_FORMAT) _format)
{
switch ((DXGI_FORMAT) _format)
{
case DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM_SRGB:
case DXGI_FORMAT.DXGI_FORMAT_BC3_UNORM_SRGB:
case DXGI_FORMAT.DXGI_FORMAT_BC4_UNORM:
case DXGI_FORMAT.DXGI_FORMAT_BC5_SNORM:
case DXGI_FORMAT.DXGI_FORMAT_BC6H_UF16:
case DXGI_FORMAT.DXGI_FORMAT_BC7_UNORM:
case DXGI_FORMAT.DXGI_FORMAT_BC7_UNORM_SRGB:
return DDS_HEADER_DXT10.Size + DDS_HEADER.Size;
default:
return DDS_HEADER.Size;
}
case DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM_SRGB:
case DXGI_FORMAT.DXGI_FORMAT_BC3_UNORM_SRGB:
case DXGI_FORMAT.DXGI_FORMAT_BC4_UNORM:
case DXGI_FORMAT.DXGI_FORMAT_BC5_SNORM:
case DXGI_FORMAT.DXGI_FORMAT_BC6H_UF16:
case DXGI_FORMAT.DXGI_FORMAT_BC7_UNORM:
case DXGI_FORMAT.DXGI_FORMAT_BC7_UNORM_SRGB:
return DDS_HEADER_DXT10.Size + DDS_HEADER.Size;
default:
return DDS_HEADER.Size;
}
}
}
@ -341,20 +366,22 @@ namespace Compression.BSA
public class BA2FileEntry : IFileEntry
{
private uint _nameHash;
private string _extension;
private uint _dirHash;
private uint _flags;
private ulong _offset;
private uint _size;
private uint _realSize;
private uint _align;
private BA2Reader _bsa;
internal uint _nameHash;
internal string _extension;
internal uint _dirHash;
internal uint _flags;
internal ulong _offset;
internal uint _size;
internal uint _realSize;
internal uint _align;
internal BA2Reader _bsa;
internal int _index;
private bool Compressed => _size != 0;
public bool Compressed => _size != 0;
public BA2FileEntry(BA2Reader ba2Reader)
public BA2FileEntry(BA2Reader ba2Reader, int index)
{
_index = index;
_bsa = ba2Reader;
var _rdr = ba2Reader._rdr;
_nameHash = _rdr.ReadUInt32();
@ -372,6 +399,7 @@ namespace Compression.BSA
public string Path => FullPath;
public uint Size => _realSize;
public FileStateObject State => new BA2FileEntryState(this);
public void CopyDataTo(Stream output)
{
@ -400,4 +428,29 @@ namespace Compression.BSA
}
}
}
public class BA2FileEntryState : FileStateObject
{
public BA2FileEntryState() { }
public BA2FileEntryState(BA2FileEntry ba2FileEntry)
{
NameHash = ba2FileEntry._nameHash;
DirHash = ba2FileEntry._dirHash;
Flags = ba2FileEntry._flags;
Align = ba2FileEntry._align;
Compressed = ba2FileEntry.Compressed;
FullPath = ba2FileEntry.FullPath;
Extension = ba2FileEntry._extension;
Index = ba2FileEntry._index;
}
public string Extension { get; set; }
public string FullPath { get; set; }
public bool Compressed { get; set; }
public uint Align { get; set; }
public uint Flags { get; set; }
public uint DirHash { get; set; }
public uint NameHash { get; set; }
}
}

3
Compression.BSA/BSAReader.cs
View File

@ -84,6 +84,8 @@ namespace Compression.BSA
}
}
public ArchiveStateObject State { get; }
public VersionType HeaderType => (VersionType) _version;
public ArchiveFlags ArchiveFlags => (ArchiveFlags) _archiveFlags;
@ -260,6 +262,7 @@ namespace Compression.BSA
}
public uint Size => _dataSize;
public FileStateObject State { get; }
public ulong Hash { get; }

1
Compression.BSA/Compression.BSA.csproj
View File

@ -93,6 +93,7 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="BA2Builder.cs" />
<Compile Include="BA2Reader.cs" />
<Compile Include="BSABuilder.cs" />
<Compile Include="BSADispatch.cs" />

26
Compression.BSA/IBSAReader.cs
View File

@ -13,8 +13,29 @@ namespace Compression.BSA
/// The files defined by the archive
/// </summary>
IEnumerable<IFile> Files { get; }
ArchiveStateObject State { get; }
}
public interface IBSABuilder : IDisposable
{
void AddFile(FileStateObject state, Stream src);
void Build(string filename);
}
public class ArchiveStateObject
{
public virtual IBSABuilder MakeBuilder()
{
throw new NotImplementedException();
}
}
public class FileStateObject
{
public int Index { get; set; }
}
public interface IFile
{
/// <summary>
@ -27,6 +48,11 @@ namespace Compression.BSA
/// </summary>
uint Size { get; }
/// <summary>
/// Get the metadata for the file.
/// </summary>
FileStateObject State { get; }
/// <summary>
/// Copies this entry to the given stream. 100% thread safe, the .bsa will be opened multiple times
/// in order to maintain thread-safe access.