Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move around namespaces so BSA routines can use Common namespaces. Make BA2 creation use memory mapped files instead of memory streams

Browse Source
This commit is contained in:
Timothy Baldridge 2020-03-04 17:02:16 -07:00
parent 186facb066
commit 2a14932092
11 changed files with 127 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
Compression.BSA.Test/BSATests.cs
View File

@ -9,6 +9,7 @@ using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.VirtualFileSystem;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using FileInfo = Alphaleonis.Win32.Filesystem.FileInfo;
@ -61,24 +62,22 @@ namespace Compression.BSA.Test
private static async Task<string> DownloadMod((Game, int) info)
{
using (var client = await NexusApiClient.Get())
using var client = await NexusApiClient.Get();
var results = await client.GetModFiles(info.Item1, info.Item2);
var file = results.files.FirstOrDefault(f => f.is_primary) ??
results.files.OrderByDescending(f => f.uploaded_timestamp).First();
var src = Path.Combine(_stagingFolder, file.file_name);
if (File.Exists(src)) return src;
var state = new NexusDownloader.State
{
var results = await client.GetModFiles(info.Item1, info.Item2);
var file = results.files.FirstOrDefault(f => f.is_primary) ??
results.files.OrderByDescending(f => f.uploaded_timestamp).First();
var src = Path.Combine(_stagingFolder, file.file_name);
if (File.Exists(src)) return src;
var state = new NexusDownloader.State
{
ModID = info.Item2.ToString(),
GameName = info.Item1.MetaData().NexusName,
FileID = file.file_id.ToString()
};
await state.Download(src);
return src;
}
ModID = info.Item2.ToString(),
GameName = info.Item1.MetaData().NexusName,
FileID = file.file_id.ToString()
};
await state.Download(src);
return src;
}
public static IEnumerable<object[]> BSAs()
@ -123,15 +122,15 @@ namespace Compression.BSA.Test
using (var w = ViaJson(a.State).MakeBuilder())
{
await a.Files.PMap(Queue, file =>
var streams = await a.Files.PMap(Queue, file =>
{
var absPath = Path.Combine(_tempDir, file.Path);
using (var str = File.OpenRead(absPath))
{
w.AddFile(ViaJson(file.State), str);
}
var str = File.OpenRead(absPath);
w.AddFile(ViaJson(file.State), str);
return str;
});
w.Build(tempFile);
streams.Do(s => s.Dispose());
}
Console.WriteLine($"Verifying {bsa}");
@ -155,7 +154,7 @@ namespace Compression.BSA.Test
Assert.AreEqual(pair.ai.Path, pair.bi.Path);
//Equal(pair.ai.Compressed, pair.bi.Compressed);
Assert.AreEqual(pair.ai.Size, pair.bi.Size);
CollectionAssert.AreEqual(GetData(pair.ai), GetData(pair.bi));
CollectionAssert.AreEqual(GetData(pair.ai), GetData(pair.bi), $"{pair.ai.Path} {JsonConvert.SerializeObject(pair.ai.State)}");
});
}
}

83
Compression.BSA/BA2Builder.cs
View File

@ -1,9 +1,11 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.MemoryMappedFiles;
using System.Linq;
using System.Text;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using Wabbajack.Common;
namespace Compression.BSA
{
@ -23,14 +25,17 @@ namespace Compression.BSA
{
private BA2StateObject _state;
private List<IFileBuilder> _entries = new List<IFileBuilder>();
private DiskSlabAllocator _slab;
public BA2Builder(BA2StateObject state)
{
_state = state;
_slab = new DiskSlabAllocator();
}
public void Dispose()
{
_slab.Dispose();
}
public void AddFile(FileStateObject state, Stream src)
@ -38,11 +43,11 @@ namespace Compression.BSA
switch (_state.Type)
{
case EntryType.GNRL:
var result = BA2FileEntryBuilder.Create((BA2FileEntryState)state, src);
var result = BA2FileEntryBuilder.Create((BA2FileEntryState)state, src, _slab);
lock(_entries) _entries.Add(result);
break;
case EntryType.DX10:
var resultdx10 = BA2DX10FileEntryBuilder.Create((BA2DX10EntryState)state, src);
var resultdx10 = BA2DX10FileEntryBuilder.Create((BA2DX10EntryState)state, src, _slab);
lock(_entries) _entries.Add(resultdx10);
break;
}
@ -99,7 +104,7 @@ namespace Compression.BSA
private BA2DX10EntryState _state;
private List<ChunkBuilder> _chunks;
public static BA2DX10FileEntryBuilder Create(BA2DX10EntryState state, Stream src)
public static BA2DX10FileEntryBuilder Create(BA2DX10EntryState state, Stream src, DiskSlabAllocator slab)
{
var builder = new BA2DX10FileEntryBuilder {_state = state};
@ -110,7 +115,7 @@ namespace Compression.BSA
builder._chunks = new List<ChunkBuilder>();
foreach (var chunk in state.Chunks)
builder._chunks.Add(ChunkBuilder.Create(state, chunk, src));
builder._chunks.Add(ChunkBuilder.Create(state, chunk, src, slab));
return builder;
}
@ -149,33 +154,33 @@ namespace Compression.BSA
public class ChunkBuilder
{
private ChunkState _chunk;
private byte[] _data;
private uint _packSize;
private long _offsetOffset;
private Stream _dataSlab;
public static ChunkBuilder Create(BA2DX10EntryState state, ChunkState chunk, Stream src)
public static ChunkBuilder Create(BA2DX10EntryState state, ChunkState chunk, Stream src, DiskSlabAllocator slab)
{
var builder = new ChunkBuilder {_chunk = chunk};
using (var ms = new MemoryStream())
{
src.CopyToLimit(ms, (int)chunk.FullSz);
builder._data = ms.ToArray();
if (!chunk.Compressed)
{
builder._dataSlab = slab.Allocate(chunk.FullSz);
src.CopyToLimit(builder._dataSlab, (int)chunk.FullSz);
}
if (!chunk.Compressed) return builder;
using (var ms = new MemoryStream())
else
{
using var ms = new MemoryStream();
using (var ds = new DeflaterOutputStream(ms))
{
ds.Write(builder._data, 0, builder._data.Length);
src.CopyToLimit(ds, (int)chunk.FullSz);
}
builder._data = ms.ToArray();
builder._dataSlab = slab.Allocate(ms.Length);
ms.Position = 0;
ms.CopyTo(builder._dataSlab);
builder._packSize = (uint)ms.Length;
}
builder._packSize = (uint) builder._data.Length;
builder._dataSlab.Position = 0;
return builder;
}
@ -198,40 +203,41 @@ namespace Compression.BSA
bw.BaseStream.Position = _offsetOffset;
bw.Write((ulong)pos);
bw.BaseStream.Position = pos;
bw.BaseStream.Write(_data, 0, _data.Length);
_dataSlab.CopyTo(bw.BaseStream);
}
}
public class BA2FileEntryBuilder : IFileBuilder
{
private byte[] _data;
private int _rawSize;
private int _size;
private BA2FileEntryState _state;
private long _offsetOffset;
private Stream _dataSrc;
public static BA2FileEntryBuilder Create(BA2FileEntryState state, Stream src)
public static BA2FileEntryBuilder Create(BA2FileEntryState state, Stream src, DiskSlabAllocator slab)
{
var builder = new BA2FileEntryBuilder {_state = state};
var builder = new BA2FileEntryBuilder
{
_state = state,
_rawSize = (int)src.Length,
_dataSrc = src
};
if (!state.Compressed)
return builder;
using (var ms = new MemoryStream())
{
src.CopyTo(ms);
builder._data = ms.ToArray();
}
builder._rawSize = builder._data.Length;
if (state.Compressed)
{
using (var ms = new MemoryStream())
using (var ds = new DeflaterOutputStream(ms))
{
using (var ds = new DeflaterOutputStream(ms))
{
ds.Write(builder._data, 0, builder._data.Length);
}
builder._data = ms.ToArray();
builder._dataSrc.CopyTo(ds);
}
builder._size = builder._data.Length;
builder._dataSrc = slab.Allocate(ms.Length);
ms.Position = 0;
ms.CopyTo(builder._dataSrc);
builder._dataSrc.Position = 0;
builder._size = (int)ms.Length;
}
return builder;
}
@ -257,10 +263,11 @@ namespace Compression.BSA
public void WriteData(BinaryWriter wtr)
{
var pos = wtr.BaseStream.Position;
wtr.BaseStream.Seek(_offsetOffset, SeekOrigin.Begin);
wtr.BaseStream.Position = _offsetOffset;
wtr.Write((ulong)pos);
wtr.BaseStream.Position = pos;
wtr.BaseStream.Write(_data, 0, _data.Length);
_dataSrc.Position = 0;
_dataSrc.CopyTo(wtr.BaseStream);
}
}
}

3
Compression.BSA/Compression.BSA.csproj
View File

@ -15,4 +15,7 @@
<ItemGroup>
<Folder Include="Properties\" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
</ItemGroup>
</Project>

39
Wabbajack.Common/Util/DiskSlabAllocator.cs Normal file
View File

@ -0,0 +1,39 @@
using System;
using System.IO;
using System.IO.MemoryMappedFiles;
namespace Wabbajack.Common
{
/// <summary>
/// Memory allocator that stores data via memory mapping to a on-disk file. Disposing of this object
/// deletes the memory mapped file
/// </summary>
public class DiskSlabAllocator : IDisposable
{
private TempFile _file;
private MemoryMappedFile _mmap;
private long _head = 0;
private string _name;
public DiskSlabAllocator()
{
_name = Guid.NewGuid().ToString();
_mmap = MemoryMappedFile.CreateNew(_name, (long)1 << 34);
}
public Stream Allocate(long size)
{
lock (this)
{
var startAt = _head;
_head += size;
return _mmap.CreateViewStream(startAt, size, MemoryMappedFileAccess.ReadWrite);
}
}
public void Dispose()
{
_mmap?.Dispose();
}
}
}

11
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -16,15 +16,6 @@
<None Update="7Zip\7z.exe">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Extractors\innounp.exe">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Extractors\7z.exe">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Extractors\7z.dll">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<Folder Include="KnownFolders\" />
@ -37,7 +28,6 @@
<PackageReference Include="Microsoft.Win32.Registry" Version="4.7.0" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="Octodiff" Version="1.2.1" />
<PackageReference Include="OMODFramework" Version="2.0.0" />
<PackageReference Include="ReactiveUI" Version="11.1.23" />
<PackageReference Include="SharpZipLib" Version="1.2.0" />
<PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
@ -47,7 +37,6 @@
<PackageReference Include="YamlDotNet" Version="8.1.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Compression.BSA\Compression.BSA.csproj" />
<ProjectReference Include="..\Wabbajack.Common.CSP\Wabbajack.Common.CSP.csproj" />
</ItemGroup>
</Project>

1
Wabbajack.Test/EndToEndTests.cs
View File

@ -9,6 +9,7 @@ using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Util;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.Test
{

0
Wabbajack.Common/Extractors/7z.dll → Wabbajack.VirtualFileSystem/Extractors/7z.dll
View File

0
Wabbajack.Common/Extractors/7z.exe → Wabbajack.VirtualFileSystem/Extractors/7z.exe
View File

0
Wabbajack.Common/Extractors/innounp.exe → Wabbajack.VirtualFileSystem/Extractors/innounp.exe
View File

11
Wabbajack.Common/FileExtractor.cs → Wabbajack.VirtualFileSystem/FileExtractor.cs
View File

@ -1,20 +1,17 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Security.Cryptography;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Compression.BSA;
using ICSharpCode.SharpZipLib.GZip;
using Newtonsoft.Json;
using OMODFramework;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Common.StatusFeed.Errors;
using Wabbajack.Common;
using Utils = Wabbajack.Common.Utils;
namespace Wabbajack.Common
namespace Wabbajack.VirtualFileSystem
{
public class FileExtractor
{

13
Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj
View File

@ -6,11 +6,24 @@
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Compression.BSA\Compression.BSA.csproj" />
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
<PackageReference Include="K4os.Hash.Crc" Version="1.1.4" />
<PackageReference Include="OMODFramework" Version="2.0.0" />
<PackageReference Include="System.Collections.Immutable" Version="1.7.0" />
</ItemGroup>
<ItemGroup>
<None Update="Extractors\7z.dll">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Extractors\7z.exe">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Extractors\innounp.exe">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>