Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #609 from wabbajack-tools/issue-606

Browse Source 
Use virtual memory during BSA creation
This commit is contained in:
Timothy Baldridge 2020-03-04 23:01:13 -07:00 committed by GitHub
commit 9eaaf9e5d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 185 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -6,6 +6,7 @@
* Fix a bug with bad data in inferred game INI files.
* Added download support for YouTube
* Slideshow can now display mods from non-Nexus sites
* Building BSAs now leverage Virtual Memory resulting in a 32x reduction in memory usage during installation (#609)
#### Verison - 1.0.0.0 - 2/29/2020
* 1.0, first non-beta release

45
Compression.BSA.Test/BSATests.cs
View File

@ -9,6 +9,7 @@ using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.VirtualFileSystem;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
@ -61,24 +62,22 @@ namespace Compression.BSA.Test
private static async Task<string> DownloadMod((Game, int) info)
{
using (var client = await NexusApiClient.Get())
using var client = await NexusApiClient.Get();
var results = await client.GetModFiles(info.Item1, info.Item2);
var file = results.files.FirstOrDefault(f => f.is_primary) ??
results.files.OrderByDescending(f => f.uploaded_timestamp).First();
var src = Path.Combine(_stagingFolder, file.file_name);
if (File.Exists(src)) return src;
var state = new NexusDownloader.State
{
var results = await client.GetModFiles(info.Item1, info.Item2);
var file = results.files.FirstOrDefault(f => f.is_primary) ??
results.files.OrderByDescending(f => f.uploaded_timestamp).First();
var src = Path.Combine(_stagingFolder, file.file_name);
if (File.Exists(src)) return src;
var state = new NexusDownloader.State
{
ModID = info.Item2.ToString(),
GameName = info.Item1.MetaData().NexusName,
FileID = file.file_id.ToString()
};
await state.Download(src);
return src;
}
ModID = info.Item2.ToString(),
GameName = info.Item1.MetaData().NexusName,
FileID = file.file_id.ToString()
};
await state.Download(src);
return src;
}
public static IEnumerable<object[]> BSAs()
@ -123,15 +122,15 @@ namespace Compression.BSA.Test
using (var w = ViaJson(a.State).MakeBuilder())
{
await a.Files.PMap(Queue, file =>
var streams = await a.Files.PMap(Queue, file =>
{
var absPath = Path.Combine(_tempDir, file.Path);
using (var str = File.OpenRead(absPath))
{
w.AddFile(ViaJson(file.State), str);
}
var str = File.OpenRead(absPath);
w.AddFile(ViaJson(file.State), str);
return str;
});
w.Build(tempFile);
streams.Do(s => s.Dispose());
}
Console.WriteLine($"Verifying {bsa}");
@ -155,7 +154,7 @@ namespace Compression.BSA.Test
Assert.AreEqual(pair.ai.Path, pair.bi.Path);
//Equal(pair.ai.Compressed, pair.bi.Compressed);
Assert.AreEqual(pair.ai.Size, pair.bi.Size);
CollectionAssert.AreEqual(GetData(pair.ai), GetData(pair.bi));
CollectionAssert.AreEqual(GetData(pair.ai), GetData(pair.bi), $"{pair.ai.Path} {JsonConvert.SerializeObject(pair.ai.State)}");
});
}
}

87
Compression.BSA/BA2Builder.cs
View File

@ -1,9 +1,11 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.MemoryMappedFiles;
using System.Linq;
using System.Text;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using Wabbajack.Common;
namespace Compression.BSA
{
@ -23,14 +25,17 @@ namespace Compression.BSA
{
private BA2StateObject _state;
private List<IFileBuilder> _entries = new List<IFileBuilder>();
private DiskSlabAllocator _slab;
public BA2Builder(BA2StateObject state)
{
_state = state;
_slab = new DiskSlabAllocator();
}
public void Dispose()
{
_slab.Dispose();
}
public void AddFile(FileStateObject state, Stream src)
@ -38,11 +43,11 @@ namespace Compression.BSA
switch (_state.Type)
{
case EntryType.GNRL:
var result = BA2FileEntryBuilder.Create((BA2FileEntryState)state, src);
var result = BA2FileEntryBuilder.Create((BA2FileEntryState)state, src, _slab);
lock(_entries) _entries.Add(result);
break;
case EntryType.DX10:
var resultdx10 = BA2DX10FileEntryBuilder.Create((BA2DX10EntryState)state, src);
var resultdx10 = BA2DX10FileEntryBuilder.Create((BA2DX10EntryState)state, src, _slab);
lock(_entries) _entries.Add(resultdx10);
break;
}
@ -99,7 +104,7 @@ namespace Compression.BSA
private BA2DX10EntryState _state;
private List<ChunkBuilder> _chunks;
public static BA2DX10FileEntryBuilder Create(BA2DX10EntryState state, Stream src)
public static BA2DX10FileEntryBuilder Create(BA2DX10EntryState state, Stream src, DiskSlabAllocator slab)
{
var builder = new BA2DX10FileEntryBuilder {_state = state};
@ -110,7 +115,7 @@ namespace Compression.BSA
builder._chunks = new List<ChunkBuilder>();
foreach (var chunk in state.Chunks)
builder._chunks.Add(ChunkBuilder.Create(state, chunk, src));
builder._chunks.Add(ChunkBuilder.Create(state, chunk, src, slab));
return builder;
}
@ -149,33 +154,34 @@ namespace Compression.BSA
public class ChunkBuilder
{
private ChunkState _chunk;
private byte[] _data;
private uint _packSize;
private long _offsetOffset;
private Stream _dataSlab;
public static ChunkBuilder Create(BA2DX10EntryState state, ChunkState chunk, Stream src)
public static ChunkBuilder Create(BA2DX10EntryState state, ChunkState chunk, Stream src, DiskSlabAllocator slab)
{
var builder = new ChunkBuilder {_chunk = chunk};
using (var ms = new MemoryStream())
{
src.CopyToLimit(ms, (int)chunk.FullSz);
builder._data = ms.ToArray();
if (!chunk.Compressed)
{
builder._dataSlab = slab.Allocate(chunk.FullSz);
src.CopyToLimit(builder._dataSlab, (int)chunk.FullSz);
}
if (!chunk.Compressed) return builder;
using (var ms = new MemoryStream())
else
{
using var ms = new MemoryStream();
using (var ds = new DeflaterOutputStream(ms))
{
ds.Write(builder._data, 0, builder._data.Length);
ds.IsStreamOwner = false;
src.CopyToLimit(ds, (int)chunk.FullSz);
}
builder._data = ms.ToArray();
builder._dataSlab = slab.Allocate(ms.Length);
ms.Position = 0;
ms.CopyTo(builder._dataSlab);
builder._packSize = (uint)ms.Length;
}
builder._packSize = (uint) builder._data.Length;
builder._dataSlab.Position = 0;
return builder;
}
@ -198,40 +204,43 @@ namespace Compression.BSA
bw.BaseStream.Position = _offsetOffset;
bw.Write((ulong)pos);
bw.BaseStream.Position = pos;
bw.BaseStream.Write(_data, 0, _data.Length);
_dataSlab.CopyToLimit(bw.BaseStream, (int)_dataSlab.Length);
_dataSlab.Dispose();
}
}
public class BA2FileEntryBuilder : IFileBuilder
{
private byte[] _data;
private int _rawSize;
private int _size;
private BA2FileEntryState _state;
private long _offsetOffset;
private Stream _dataSrc;
public static BA2FileEntryBuilder Create(BA2FileEntryState state, Stream src)
public static BA2FileEntryBuilder Create(BA2FileEntryState state, Stream src, DiskSlabAllocator slab)
{
var builder = new BA2FileEntryBuilder {_state = state};
var builder = new BA2FileEntryBuilder
{
_state = state,
_rawSize = (int)src.Length,
_dataSrc = src
};
if (!state.Compressed)
return builder;
using (var ms = new MemoryStream())
{
src.CopyTo(ms);
builder._data = ms.ToArray();
}
builder._rawSize = builder._data.Length;
if (state.Compressed)
{
using (var ms = new MemoryStream())
using (var ds = new DeflaterOutputStream(ms))
{
using (var ds = new DeflaterOutputStream(ms))
{
ds.Write(builder._data, 0, builder._data.Length);
}
builder._data = ms.ToArray();
ds.IsStreamOwner = false;
builder._dataSrc.CopyTo(ds);
}
builder._size = builder._data.Length;
builder._dataSrc = slab.Allocate(ms.Length);
ms.Position = 0;
ms.CopyTo(builder._dataSrc);
builder._dataSrc.Position = 0;
builder._size = (int)ms.Length;
}
return builder;
}
@ -257,10 +266,12 @@ namespace Compression.BSA
public void WriteData(BinaryWriter wtr)
{
var pos = wtr.BaseStream.Position;
wtr.BaseStream.Seek(_offsetOffset, SeekOrigin.Begin);
wtr.BaseStream.Position = _offsetOffset;
wtr.Write((ulong)pos);
wtr.BaseStream.Position = pos;
wtr.BaseStream.Write(_data, 0, _data.Length);
_dataSrc.Position = 0;
_dataSrc.CopyToLimit(wtr.BaseStream, (int)_dataSrc.Length);
_dataSrc.Dispose();
}
}
}

67
Compression.BSA/BSABuilder.cs
View File

@ -6,6 +6,7 @@ using System.Text;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using K4os.Compression.LZ4;
using K4os.Compression.LZ4.Streams;
using Wabbajack.Common;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
@ -22,11 +23,13 @@ namespace Compression.BSA
internal uint _offset;
internal uint _totalFileNameLength;
internal uint _version;
internal DiskSlabAllocator _slab;
public BSABuilder()
{
_fileId = Encoding.ASCII.GetBytes("BSA\0");
_offset = 0x24;
_slab = new DiskSlabAllocator();
}
public BSABuilder(BSAStateObject bsaStateObject) : this()
@ -74,6 +77,7 @@ namespace Compression.BSA
public void Dispose()
{
_slab.Dispose();
}
public void AddFile(FileStateObject state, Stream src)
{
@ -231,7 +235,7 @@ namespace Compression.BSA
internal string _path;
private byte[] _pathBSBytes;
internal byte[] _pathBytes;
internal byte[] _rawData;
private Stream _srcData;
public static FileEntry Create(BSABuilder bsa, string path, Stream src, bool flipCompression)
{
@ -244,11 +248,9 @@ namespace Compression.BSA
entry._pathBytes = entry._path.ToTermString(bsa.HeaderType);
entry._pathBSBytes = entry._path.ToBSString();
entry._flipCompression = flipCompression;
entry._srcData = src;
var ms = new MemoryStream();
src.CopyTo(ms);
entry._rawData = ms.ToArray();
entry._originalSize = entry._rawData.Length;
entry._originalSize = (int)entry._srcData.Length;
if (entry.Compressed)
entry.CompressData();
@ -275,29 +277,38 @@ namespace Compression.BSA
private void CompressData()
{
if (_bsa.HeaderType == VersionType.SSE)
switch (_bsa.HeaderType)
{
var r = new MemoryStream();
using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L10_OPT}))
case VersionType.SSE:
{
new MemoryStream(_rawData).CopyTo(w);
var r = new MemoryStream();
using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L10_OPT}, true))
{
_srcData.CopyTo(w);
}
_srcData = _bsa._slab.Allocate(r.Length);
r.Position = 0;
r.CopyTo(_srcData);
_srcData.Position = 0;
break;
}
_rawData = r.ToArray();
}
else if (_bsa.HeaderType == VersionType.FO3 || _bsa.HeaderType == VersionType.TES4)
{
var r = new MemoryStream();
using (var w = new DeflaterOutputStream(r))
case VersionType.FO3:
case VersionType.TES4:
{
new MemoryStream(_rawData).CopyTo(w);
var r = new MemoryStream();
using (var w = new DeflaterOutputStream(r))
{
w.IsStreamOwner = false;
_srcData.CopyTo(w);
}
_srcData = _bsa._slab.Allocate(r.Length);
r.Position = 0;
r.CopyTo(_srcData);
_srcData.Position = 0;
break;
}
_rawData = r.ToArray();
}
else
{
throw new NotImplementedException($"Can't compress data for {_bsa.HeaderType} BSAs.");
default:
throw new NotImplementedException($"Can't compress data for {_bsa.HeaderType} BSAs.");
}
}
@ -305,7 +316,7 @@ namespace Compression.BSA
{
wtr.Write(_hash);
var size = _rawData.Length;
var size = _srcData.Length;
if (_bsa.HasNameBlobs) size += _pathBSBytes.Length;
if (Compressed) size += 4;
if (_flipCompression)
@ -329,11 +340,15 @@ namespace Compression.BSA
if (Compressed)
{
wtr.Write((uint) _originalSize);
wtr.BaseStream.Write(_rawData, 0, _rawData.Length);
_srcData.Position = 0;
_srcData.CopyToLimit(wtr.BaseStream, (int)_srcData.Length);
_srcData.Dispose();
}
else
{
wtr.BaseStream.Write(_rawData, 0, _rawData.Length);
_srcData.Position = 0;
_srcData.CopyToLimit(wtr.BaseStream, (int)_srcData.Length);
_srcData.Dispose();
}
}
}

3
Compression.BSA/Compression.BSA.csproj
View File

@ -15,4 +15,7 @@
<ItemGroup>
<Folder Include="Properties\" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
</ItemGroup>
</Project>

10
Compression.BSA/TES3Builder.cs
View File

@ -8,19 +8,18 @@ namespace Compression.BSA
public class TES3Builder : IBSABuilder
{
private TES3ArchiveState _state;
private (TES3FileState state, byte[] data)[] _files;
private (TES3FileState state, Stream data)[] _files;
public TES3Builder(TES3ArchiveState state)
{
_state = state;
_files = new (TES3FileState state, byte[] data)[_state.FileCount];
_files = new (TES3FileState state, Stream data)[_state.FileCount];
}
public void AddFile(FileStateObject state, Stream src)
{
using var br = new BinaryReader(src);
var cstate = (TES3FileState)state;
_files[state.Index] = (cstate, br.ReadBytes((int)cstate.Size));
_files[state.Index] = (cstate, src);
}
public void Build(string filename)
@ -66,7 +65,8 @@ namespace Compression.BSA
foreach (var (state, data) in _files)
{
bw.BaseStream.Position = _state.DataOffset + state.Offset;
bw.Write(data);
data.CopyTo(bw.BaseStream);
data.Dispose();
}
}

2
Compression.BSA/Utils.cs
View File

@ -156,6 +156,8 @@ namespace Compression.BSA
{
var to_read = Math.Min(buff.Length, limit);
var read = frm.Read(buff, 0, to_read);
if (read == 0)
throw new Exception("End of stream before end of limit");
tw.Write(buff, 0, read);
limit -= read;
}

39
Wabbajack.Common/Util/DiskSlabAllocator.cs Normal file
View File

@ -0,0 +1,39 @@
using System;
using System.IO;
using System.IO.MemoryMappedFiles;
namespace Wabbajack.Common
{
/// <summary>
/// Memory allocator that stores data via memory mapping to a on-disk file. Disposing of this object
/// deletes the memory mapped file
/// </summary>
public class DiskSlabAllocator : IDisposable
{
private TempFile _file;
private MemoryMappedFile _mmap;
private long _head = 0;
private string _name;
public DiskSlabAllocator()
{
_name = Guid.NewGuid().ToString();
_mmap = MemoryMappedFile.CreateNew(null, (long)1 << 34);
}
public Stream Allocate(long size)
{
lock (this)
{
var startAt = _head;
_head += size;
return _mmap.CreateViewStream(startAt, size, MemoryMappedFileAccess.ReadWrite);
}
}
public void Dispose()
{
_mmap?.Dispose();
}
}
}

11
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -16,15 +16,6 @@
<None Update="7Zip\7z.exe">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Extractors\innounp.exe">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Extractors\7z.exe">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Extractors\7z.dll">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<Folder Include="KnownFolders\" />
@ -37,7 +28,6 @@
<PackageReference Include="Microsoft.Win32.Registry" Version="4.7.0" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="Octodiff" Version="1.2.1" />
<PackageReference Include="OMODFramework" Version="2.0.0" />
<PackageReference Include="ReactiveUI" Version="11.1.23" />
<PackageReference Include="SharpZipLib" Version="1.2.0" />
<PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
@ -47,7 +37,6 @@
<PackageReference Include="YamlDotNet" Version="8.1.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Compression.BSA\Compression.BSA.csproj" />
<ProjectReference Include="..\Wabbajack.Common.CSP\Wabbajack.Common.CSP.csproj" />
</ItemGroup>
</Project>

10
Wabbajack.Lib/MO2Installer.cs
View File

@ -239,17 +239,17 @@ namespace Wabbajack.Lib
using (var a = bsa.State.MakeBuilder())
{
await bsa.FileStates.PMap(Queue, state =>
var streams = await bsa.FileStates.PMap(Queue, state =>
{
Status($"Adding {state.Path} to BSA");
using (var fs = File.OpenRead(Path.Combine(sourceDir, state.Path)))
{
a.AddFile(state, fs);
}
var fs = File.OpenRead(Path.Combine(sourceDir, state.Path));
a.AddFile(state, fs);
return fs;
});
Info($"Writing {bsa.To}");
a.Build(Path.Combine(OutputFolder, bsa.To));
streams.Do(s => s.Dispose());
}
}

1
Wabbajack.Test/EndToEndTests.cs
View File

@ -9,6 +9,7 @@ using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Util;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.Test
{

0
Wabbajack.Common/Extractors/7z.dll → Wabbajack.VirtualFileSystem/Extractors/7z.dll
View File

0
Wabbajack.Common/Extractors/7z.exe → Wabbajack.VirtualFileSystem/Extractors/7z.exe
View File

0
Wabbajack.Common/Extractors/innounp.exe → Wabbajack.VirtualFileSystem/Extractors/innounp.exe
View File

11
Wabbajack.Common/FileExtractor.cs → Wabbajack.VirtualFileSystem/FileExtractor.cs
View File

@ -1,20 +1,17 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Security.Cryptography;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Compression.BSA;
using ICSharpCode.SharpZipLib.GZip;
using Newtonsoft.Json;
using OMODFramework;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Common.StatusFeed.Errors;
using Wabbajack.Common;
using Utils = Wabbajack.Common.Utils;
namespace Wabbajack.Common
namespace Wabbajack.VirtualFileSystem
{
public class FileExtractor
{

13
Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj
View File

@ -6,11 +6,24 @@
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Compression.BSA\Compression.BSA.csproj" />
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
<PackageReference Include="K4os.Hash.Crc" Version="1.1.4" />
<PackageReference Include="OMODFramework" Version="2.0.0" />
<PackageReference Include="System.Collections.Immutable" Version="1.7.0" />
</ItemGroup>
<ItemGroup>
<None Update="Extractors\7z.dll">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Extractors\7z.exe">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Extractors\innounp.exe">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>