diff --git a/CHANGELOG.md b/CHANGELOG.md index b9e28411..cb470294 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * Fix a bug with bad data in inferred game INI files. * Added download support for YouTube * Slideshow can now display mods from non-Nexus sites +* Building BSAs now leverage Virtual Memory resulting in a 32x reduction in memory usage during installation (#609) #### Verison - 1.0.0.0 - 2/29/2020 * 1.0, first non-beta release diff --git a/Compression.BSA.Test/BSATests.cs b/Compression.BSA.Test/BSATests.cs index 93abf867..51a0bbb5 100644 --- a/Compression.BSA.Test/BSATests.cs +++ b/Compression.BSA.Test/BSATests.cs @@ -9,6 +9,7 @@ using Newtonsoft.Json; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.NexusApi; +using Wabbajack.VirtualFileSystem; using Directory = Alphaleonis.Win32.Filesystem.Directory; using File = Alphaleonis.Win32.Filesystem.File; using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo; @@ -61,24 +62,22 @@ namespace Compression.BSA.Test private static async Task DownloadMod((Game, int) info) { - using (var client = await NexusApiClient.Get()) + using var client = await NexusApiClient.Get(); + var results = await client.GetModFiles(info.Item1, info.Item2); + var file = results.files.FirstOrDefault(f => f.is_primary) ?? + results.files.OrderByDescending(f => f.uploaded_timestamp).First(); + var src = Path.Combine(_stagingFolder, file.file_name); + + if (File.Exists(src)) return src; + + var state = new NexusDownloader.State { - var results = await client.GetModFiles(info.Item1, info.Item2); - var file = results.files.FirstOrDefault(f => f.is_primary) ?? - results.files.OrderByDescending(f => f.uploaded_timestamp).First(); - var src = Path.Combine(_stagingFolder, file.file_name); - - if (File.Exists(src)) return src; - - var state = new NexusDownloader.State - { - ModID = info.Item2.ToString(), - GameName = info.Item1.MetaData().NexusName, - FileID = file.file_id.ToString() - }; - await state.Download(src); - return src; - } + ModID = info.Item2.ToString(), + GameName = info.Item1.MetaData().NexusName, + FileID = file.file_id.ToString() + }; + await state.Download(src); + return src; } public static IEnumerable BSAs() @@ -123,15 +122,15 @@ namespace Compression.BSA.Test using (var w = ViaJson(a.State).MakeBuilder()) { - await a.Files.PMap(Queue, file => + var streams = await a.Files.PMap(Queue, file => { var absPath = Path.Combine(_tempDir, file.Path); - using (var str = File.OpenRead(absPath)) - { - w.AddFile(ViaJson(file.State), str); - } + var str = File.OpenRead(absPath); + w.AddFile(ViaJson(file.State), str); + return str; }); w.Build(tempFile); + streams.Do(s => s.Dispose()); } Console.WriteLine($"Verifying {bsa}"); @@ -155,7 +154,7 @@ namespace Compression.BSA.Test Assert.AreEqual(pair.ai.Path, pair.bi.Path); //Equal(pair.ai.Compressed, pair.bi.Compressed); Assert.AreEqual(pair.ai.Size, pair.bi.Size); - CollectionAssert.AreEqual(GetData(pair.ai), GetData(pair.bi)); + CollectionAssert.AreEqual(GetData(pair.ai), GetData(pair.bi), $"{pair.ai.Path} {JsonConvert.SerializeObject(pair.ai.State)}"); }); } } diff --git a/Compression.BSA/BA2Builder.cs b/Compression.BSA/BA2Builder.cs index 7e9e578e..7b77e54a 100644 --- a/Compression.BSA/BA2Builder.cs +++ b/Compression.BSA/BA2Builder.cs @@ -1,9 +1,11 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.MemoryMappedFiles; using System.Linq; using System.Text; using ICSharpCode.SharpZipLib.Zip.Compression.Streams; +using Wabbajack.Common; namespace Compression.BSA { @@ -23,14 +25,17 @@ namespace Compression.BSA { private BA2StateObject _state; private List _entries = new List(); + private DiskSlabAllocator _slab; public BA2Builder(BA2StateObject state) { _state = state; + _slab = new DiskSlabAllocator(); } public void Dispose() { + _slab.Dispose(); } public void AddFile(FileStateObject state, Stream src) @@ -38,11 +43,11 @@ namespace Compression.BSA switch (_state.Type) { case EntryType.GNRL: - var result = BA2FileEntryBuilder.Create((BA2FileEntryState)state, src); + var result = BA2FileEntryBuilder.Create((BA2FileEntryState)state, src, _slab); lock(_entries) _entries.Add(result); break; case EntryType.DX10: - var resultdx10 = BA2DX10FileEntryBuilder.Create((BA2DX10EntryState)state, src); + var resultdx10 = BA2DX10FileEntryBuilder.Create((BA2DX10EntryState)state, src, _slab); lock(_entries) _entries.Add(resultdx10); break; } @@ -99,7 +104,7 @@ namespace Compression.BSA private BA2DX10EntryState _state; private List _chunks; - public static BA2DX10FileEntryBuilder Create(BA2DX10EntryState state, Stream src) + public static BA2DX10FileEntryBuilder Create(BA2DX10EntryState state, Stream src, DiskSlabAllocator slab) { var builder = new BA2DX10FileEntryBuilder {_state = state}; @@ -110,7 +115,7 @@ namespace Compression.BSA builder._chunks = new List(); foreach (var chunk in state.Chunks) - builder._chunks.Add(ChunkBuilder.Create(state, chunk, src)); + builder._chunks.Add(ChunkBuilder.Create(state, chunk, src, slab)); return builder; } @@ -149,33 +154,34 @@ namespace Compression.BSA public class ChunkBuilder { private ChunkState _chunk; - private byte[] _data; private uint _packSize; private long _offsetOffset; + private Stream _dataSlab; - public static ChunkBuilder Create(BA2DX10EntryState state, ChunkState chunk, Stream src) + public static ChunkBuilder Create(BA2DX10EntryState state, ChunkState chunk, Stream src, DiskSlabAllocator slab) { var builder = new ChunkBuilder {_chunk = chunk}; - using (var ms = new MemoryStream()) - { - src.CopyToLimit(ms, (int)chunk.FullSz); - builder._data = ms.ToArray(); + if (!chunk.Compressed) + { + builder._dataSlab = slab.Allocate(chunk.FullSz); + src.CopyToLimit(builder._dataSlab, (int)chunk.FullSz); } - - if (!chunk.Compressed) return builder; - - using (var ms = new MemoryStream()) + else { + using var ms = new MemoryStream(); using (var ds = new DeflaterOutputStream(ms)) { - ds.Write(builder._data, 0, builder._data.Length); + ds.IsStreamOwner = false; + src.CopyToLimit(ds, (int)chunk.FullSz); } - builder._data = ms.ToArray(); + builder._dataSlab = slab.Allocate(ms.Length); + ms.Position = 0; + ms.CopyTo(builder._dataSlab); + builder._packSize = (uint)ms.Length; } - - builder._packSize = (uint) builder._data.Length; + builder._dataSlab.Position = 0; return builder; } @@ -198,40 +204,43 @@ namespace Compression.BSA bw.BaseStream.Position = _offsetOffset; bw.Write((ulong)pos); bw.BaseStream.Position = pos; - bw.BaseStream.Write(_data, 0, _data.Length); + _dataSlab.CopyToLimit(bw.BaseStream, (int)_dataSlab.Length); + _dataSlab.Dispose(); } } public class BA2FileEntryBuilder : IFileBuilder { - private byte[] _data; private int _rawSize; private int _size; private BA2FileEntryState _state; private long _offsetOffset; + private Stream _dataSrc; - public static BA2FileEntryBuilder Create(BA2FileEntryState state, Stream src) + public static BA2FileEntryBuilder Create(BA2FileEntryState state, Stream src, DiskSlabAllocator slab) { - var builder = new BA2FileEntryBuilder {_state = state}; + var builder = new BA2FileEntryBuilder + { + _state = state, + _rawSize = (int)src.Length, + _dataSrc = src + }; + if (!state.Compressed) + return builder; using (var ms = new MemoryStream()) { - src.CopyTo(ms); - builder._data = ms.ToArray(); - } - builder._rawSize = builder._data.Length; - - if (state.Compressed) - { - using (var ms = new MemoryStream()) + using (var ds = new DeflaterOutputStream(ms)) { - using (var ds = new DeflaterOutputStream(ms)) - { - ds.Write(builder._data, 0, builder._data.Length); - } - builder._data = ms.ToArray(); + ds.IsStreamOwner = false; + builder._dataSrc.CopyTo(ds); } - builder._size = builder._data.Length; + + builder._dataSrc = slab.Allocate(ms.Length); + ms.Position = 0; + ms.CopyTo(builder._dataSrc); + builder._dataSrc.Position = 0; + builder._size = (int)ms.Length; } return builder; } @@ -257,10 +266,12 @@ namespace Compression.BSA public void WriteData(BinaryWriter wtr) { var pos = wtr.BaseStream.Position; - wtr.BaseStream.Seek(_offsetOffset, SeekOrigin.Begin); + wtr.BaseStream.Position = _offsetOffset; wtr.Write((ulong)pos); wtr.BaseStream.Position = pos; - wtr.BaseStream.Write(_data, 0, _data.Length); + _dataSrc.Position = 0; + _dataSrc.CopyToLimit(wtr.BaseStream, (int)_dataSrc.Length); + _dataSrc.Dispose(); } } } diff --git a/Compression.BSA/BSABuilder.cs b/Compression.BSA/BSABuilder.cs index 375f530c..628a21df 100644 --- a/Compression.BSA/BSABuilder.cs +++ b/Compression.BSA/BSABuilder.cs @@ -6,6 +6,7 @@ using System.Text; using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using K4os.Compression.LZ4; using K4os.Compression.LZ4.Streams; +using Wabbajack.Common; using File = Alphaleonis.Win32.Filesystem.File; using Path = Alphaleonis.Win32.Filesystem.Path; @@ -22,11 +23,13 @@ namespace Compression.BSA internal uint _offset; internal uint _totalFileNameLength; internal uint _version; + internal DiskSlabAllocator _slab; public BSABuilder() { _fileId = Encoding.ASCII.GetBytes("BSA\0"); _offset = 0x24; + _slab = new DiskSlabAllocator(); } public BSABuilder(BSAStateObject bsaStateObject) : this() @@ -74,6 +77,7 @@ namespace Compression.BSA public void Dispose() { + _slab.Dispose(); } public void AddFile(FileStateObject state, Stream src) { @@ -231,7 +235,7 @@ namespace Compression.BSA internal string _path; private byte[] _pathBSBytes; internal byte[] _pathBytes; - internal byte[] _rawData; + private Stream _srcData; public static FileEntry Create(BSABuilder bsa, string path, Stream src, bool flipCompression) { @@ -244,11 +248,9 @@ namespace Compression.BSA entry._pathBytes = entry._path.ToTermString(bsa.HeaderType); entry._pathBSBytes = entry._path.ToBSString(); entry._flipCompression = flipCompression; + entry._srcData = src; - var ms = new MemoryStream(); - src.CopyTo(ms); - entry._rawData = ms.ToArray(); - entry._originalSize = entry._rawData.Length; + entry._originalSize = (int)entry._srcData.Length; if (entry.Compressed) entry.CompressData(); @@ -275,29 +277,38 @@ namespace Compression.BSA private void CompressData() { - if (_bsa.HeaderType == VersionType.SSE) + switch (_bsa.HeaderType) { - var r = new MemoryStream(); - using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L10_OPT})) + case VersionType.SSE: { - new MemoryStream(_rawData).CopyTo(w); + var r = new MemoryStream(); + using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L10_OPT}, true)) + { + _srcData.CopyTo(w); + } + _srcData = _bsa._slab.Allocate(r.Length); + r.Position = 0; + r.CopyTo(_srcData); + _srcData.Position = 0; + break; } - - _rawData = r.ToArray(); - } - else if (_bsa.HeaderType == VersionType.FO3 || _bsa.HeaderType == VersionType.TES4) - { - var r = new MemoryStream(); - using (var w = new DeflaterOutputStream(r)) + case VersionType.FO3: + case VersionType.TES4: { - new MemoryStream(_rawData).CopyTo(w); + var r = new MemoryStream(); + using (var w = new DeflaterOutputStream(r)) + { + w.IsStreamOwner = false; + _srcData.CopyTo(w); + } + _srcData = _bsa._slab.Allocate(r.Length); + r.Position = 0; + r.CopyTo(_srcData); + _srcData.Position = 0; + break; } - - _rawData = r.ToArray(); - } - else - { - throw new NotImplementedException($"Can't compress data for {_bsa.HeaderType} BSAs."); + default: + throw new NotImplementedException($"Can't compress data for {_bsa.HeaderType} BSAs."); } } @@ -305,7 +316,7 @@ namespace Compression.BSA { wtr.Write(_hash); - var size = _rawData.Length; + var size = _srcData.Length; if (_bsa.HasNameBlobs) size += _pathBSBytes.Length; if (Compressed) size += 4; if (_flipCompression) @@ -329,11 +340,15 @@ namespace Compression.BSA if (Compressed) { wtr.Write((uint) _originalSize); - wtr.BaseStream.Write(_rawData, 0, _rawData.Length); + _srcData.Position = 0; + _srcData.CopyToLimit(wtr.BaseStream, (int)_srcData.Length); + _srcData.Dispose(); } else { - wtr.BaseStream.Write(_rawData, 0, _rawData.Length); + _srcData.Position = 0; + _srcData.CopyToLimit(wtr.BaseStream, (int)_srcData.Length); + _srcData.Dispose(); } } } diff --git a/Compression.BSA/Compression.BSA.csproj b/Compression.BSA/Compression.BSA.csproj index 1836fee8..50859329 100644 --- a/Compression.BSA/Compression.BSA.csproj +++ b/Compression.BSA/Compression.BSA.csproj @@ -15,4 +15,7 @@ + + + \ No newline at end of file diff --git a/Compression.BSA/TES3Builder.cs b/Compression.BSA/TES3Builder.cs index 1bf7b8c3..3f85bd49 100644 --- a/Compression.BSA/TES3Builder.cs +++ b/Compression.BSA/TES3Builder.cs @@ -8,19 +8,18 @@ namespace Compression.BSA public class TES3Builder : IBSABuilder { private TES3ArchiveState _state; - private (TES3FileState state, byte[] data)[] _files; + private (TES3FileState state, Stream data)[] _files; public TES3Builder(TES3ArchiveState state) { _state = state; - _files = new (TES3FileState state, byte[] data)[_state.FileCount]; + _files = new (TES3FileState state, Stream data)[_state.FileCount]; } public void AddFile(FileStateObject state, Stream src) { - using var br = new BinaryReader(src); var cstate = (TES3FileState)state; - _files[state.Index] = (cstate, br.ReadBytes((int)cstate.Size)); + _files[state.Index] = (cstate, src); } public void Build(string filename) @@ -66,7 +65,8 @@ namespace Compression.BSA foreach (var (state, data) in _files) { bw.BaseStream.Position = _state.DataOffset + state.Offset; - bw.Write(data); + data.CopyTo(bw.BaseStream); + data.Dispose(); } } diff --git a/Compression.BSA/Utils.cs b/Compression.BSA/Utils.cs index 64026327..9fd36c77 100644 --- a/Compression.BSA/Utils.cs +++ b/Compression.BSA/Utils.cs @@ -156,6 +156,8 @@ namespace Compression.BSA { var to_read = Math.Min(buff.Length, limit); var read = frm.Read(buff, 0, to_read); + if (read == 0) + throw new Exception("End of stream before end of limit"); tw.Write(buff, 0, read); limit -= read; } diff --git a/Wabbajack.Common/Util/DiskSlabAllocator.cs b/Wabbajack.Common/Util/DiskSlabAllocator.cs new file mode 100644 index 00000000..effdee3b --- /dev/null +++ b/Wabbajack.Common/Util/DiskSlabAllocator.cs @@ -0,0 +1,39 @@ +using System; +using System.IO; +using System.IO.MemoryMappedFiles; + +namespace Wabbajack.Common +{ + /// + /// Memory allocator that stores data via memory mapping to a on-disk file. Disposing of this object + /// deletes the memory mapped file + /// + public class DiskSlabAllocator : IDisposable + { + private TempFile _file; + private MemoryMappedFile _mmap; + private long _head = 0; + private string _name; + + public DiskSlabAllocator() + { + _name = Guid.NewGuid().ToString(); + _mmap = MemoryMappedFile.CreateNew(null, (long)1 << 34); + } + + public Stream Allocate(long size) + { + lock (this) + { + var startAt = _head; + _head += size; + return _mmap.CreateViewStream(startAt, size, MemoryMappedFileAccess.ReadWrite); + } + } + + public void Dispose() + { + _mmap?.Dispose(); + } + } +} diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index ae13b6a0..b033a928 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -16,15 +16,6 @@ Always - - Always - - - Always - - - Always - @@ -37,7 +28,6 @@ - @@ -47,7 +37,6 @@ - \ No newline at end of file diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index 55f97b8d..6a7b23f2 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -239,17 +239,17 @@ namespace Wabbajack.Lib using (var a = bsa.State.MakeBuilder()) { - await bsa.FileStates.PMap(Queue, state => + var streams = await bsa.FileStates.PMap(Queue, state => { Status($"Adding {state.Path} to BSA"); - using (var fs = File.OpenRead(Path.Combine(sourceDir, state.Path))) - { - a.AddFile(state, fs); - } + var fs = File.OpenRead(Path.Combine(sourceDir, state.Path)); + a.AddFile(state, fs); + return fs; }); Info($"Writing {bsa.To}"); a.Build(Path.Combine(OutputFolder, bsa.To)); + streams.Do(s => s.Dispose()); } } diff --git a/Wabbajack.Test/EndToEndTests.cs b/Wabbajack.Test/EndToEndTests.cs index 02801164..0257634b 100644 --- a/Wabbajack.Test/EndToEndTests.cs +++ b/Wabbajack.Test/EndToEndTests.cs @@ -9,6 +9,7 @@ using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.NexusApi; using Wabbajack.Util; +using Wabbajack.VirtualFileSystem; namespace Wabbajack.Test { diff --git a/Wabbajack.Common/Extractors/7z.dll b/Wabbajack.VirtualFileSystem/Extractors/7z.dll similarity index 100% rename from Wabbajack.Common/Extractors/7z.dll rename to Wabbajack.VirtualFileSystem/Extractors/7z.dll diff --git a/Wabbajack.Common/Extractors/7z.exe b/Wabbajack.VirtualFileSystem/Extractors/7z.exe similarity index 100% rename from Wabbajack.Common/Extractors/7z.exe rename to Wabbajack.VirtualFileSystem/Extractors/7z.exe diff --git a/Wabbajack.Common/Extractors/innounp.exe b/Wabbajack.VirtualFileSystem/Extractors/innounp.exe similarity index 100% rename from Wabbajack.Common/Extractors/innounp.exe rename to Wabbajack.VirtualFileSystem/Extractors/innounp.exe diff --git a/Wabbajack.Common/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor.cs similarity index 98% rename from Wabbajack.Common/FileExtractor.cs rename to Wabbajack.VirtualFileSystem/FileExtractor.cs index 18c574bb..769c60af 100644 --- a/Wabbajack.Common/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor.cs @@ -1,20 +1,17 @@ using System; -using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using System.Net.Http; -using System.Reflection; -using System.Security.Cryptography; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; using Compression.BSA; -using ICSharpCode.SharpZipLib.GZip; -using Newtonsoft.Json; using OMODFramework; using Wabbajack.Common.StatusFeed; using Wabbajack.Common.StatusFeed.Errors; +using Wabbajack.Common; +using Utils = Wabbajack.Common.Utils; -namespace Wabbajack.Common + +namespace Wabbajack.VirtualFileSystem { public class FileExtractor { diff --git a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj index a2ec35cc..1a2fe92d 100644 --- a/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj +++ b/Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj @@ -6,11 +6,24 @@ win10-x64 + + + + + Always + + + Always + + + Always + + \ No newline at end of file