Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #177 from wabbajack-tools/make-it-sync

Browse Source 
Make it sync
This commit is contained in:
Timothy Baldridge 2019-11-15 17:41:17 -07:00 committed by GitHub
commit 2d3d92068a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 308 additions and 317 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
Compression.BSA.Test/BSATests.cs
View File

@ -2,12 +2,10 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem; using Alphaleonis.Win32.Filesystem;
using Microsoft.VisualStudio.TestTools.UnitTesting; using Microsoft.VisualStudio.TestTools.UnitTesting;
using Newtonsoft.Json; using Newtonsoft.Json;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.Common.CSP;
using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.NexusApi;
using Directory = Alphaleonis.Win32.Filesystem.Directory; using Directory = Alphaleonis.Win32.Filesystem.Directory;
@ -28,7 +26,7 @@ namespace Compression.BSA.Test
public TestContext TestContext { get; set; } public TestContext TestContext { get; set; }
[ClassInitialize] [ClassInitialize]
public static async Task Setup(TestContext TestContext) public static void Setup(TestContext TestContext)
{ {
Utils.LogMessages.Subscribe(f => TestContext.WriteLine(f)); Utils.LogMessages.Subscribe(f => TestContext.WriteLine(f));
@ -52,7 +50,7 @@ namespace Compression.BSA.Test
var folder = Path.Combine(BSAFolder, info.Item1.ToString(), info.Item2.ToString()); var folder = Path.Combine(BSAFolder, info.Item1.ToString(), info.Item2.ToString());
if (!Directory.Exists(folder)) if (!Directory.Exists(folder))
Directory.CreateDirectory(folder); Directory.CreateDirectory(folder);
await FileExtractor.ExtractAll(filename, folder); FileExtractor.ExtractAll(filename, folder);
} }
} }
@ -88,7 +86,7 @@ namespace Compression.BSA.Test
[TestMethod] [TestMethod]
[DataTestMethod] [DataTestMethod]
[DynamicData(nameof(BSAs), DynamicDataSourceType.Method)] [DynamicData(nameof(BSAs), DynamicDataSourceType.Method)]
public async Task BSACompressionRecompression(string bsa) public void BSACompressionRecompression(string bsa)
{ {
TestContext.WriteLine($"From {bsa}"); TestContext.WriteLine($"From {bsa}");
TestContext.WriteLine("Cleaning Output Dir"); TestContext.WriteLine("Cleaning Output Dir");
@ -97,9 +95,10 @@ namespace Compression.BSA.Test
Directory.CreateDirectory(TempDir); Directory.CreateDirectory(TempDir);
TestContext.WriteLine($"Reading {bsa}"); TestContext.WriteLine($"Reading {bsa}");
using (var a = await BSADispatch.OpenRead(bsa)) string TempFile = Path.Combine("tmp.bsa");
using (var a = BSADispatch.OpenRead(bsa))
{ {
await a.Files.UnorderedParallelDo(async file => a.Files.PMap(file =>
{ {
var abs_name = Path.Combine(TempDir, file.Path); var abs_name = Path.Combine(TempDir, file.Path);
ViaJson(file.State); ViaJson(file.State);
@ -110,30 +109,29 @@ namespace Compression.BSA.Test
using (var fs = File.OpenWrite(abs_name)) using (var fs = File.OpenWrite(abs_name))
{ {
await file.CopyDataToAsync(fs); file.CopyDataTo(fs);
} }
Assert.AreEqual(file.Size, new FileInfo(abs_name).Length); Assert.AreEqual(file.Size, new FileInfo(abs_name).Length);
}); });
Console.WriteLine($"Building {bsa}"); Console.WriteLine($"Building {bsa}");
string TempFile = Path.Combine("tmp.bsa");
using (var w = ViaJson(a.State).MakeBuilder()) using (var w = ViaJson(a.State).MakeBuilder())
{ {
await a.Files.UnorderedParallelDo(async file => a.Files.PMap(file =>
{ {
var abs_path = Path.Combine(TempDir, file.Path); var abs_path = Path.Combine(TempDir, file.Path);
using (var str = File.OpenRead(abs_path)) using (var str = File.OpenRead(abs_path))
{ {
await w.AddFile(ViaJson(file.State), str); w.AddFile(ViaJson(file.State), str);
} }
}); });
await w.Build(TempFile); w.Build(TempFile);
} }
Console.WriteLine($"Verifying {bsa}"); Console.WriteLine($"Verifying {bsa}");
using (var b = await BSADispatch.OpenRead(TempFile)) using (var b = BSADispatch.OpenRead(TempFile))
{ {
Console.WriteLine($"Performing A/B tests on {bsa}"); Console.WriteLine($"Performing A/B tests on {bsa}");
@ -143,8 +141,8 @@ namespace Compression.BSA.Test
Assert.AreEqual(a.Files.Count(), b.Files.Count()); Assert.AreEqual(a.Files.Count(), b.Files.Count());
var idx = 0; var idx = 0;
await a.Files.Zip(b.Files, (ai, bi) => (ai, bi)) a.Files.Zip(b.Files, (ai, bi) => (ai, bi))
.UnorderedParallelDo(async pair => .PMap(pair =>
{ {
idx++; idx++;
Assert.AreEqual(JsonConvert.SerializeObject(pair.ai.State), Assert.AreEqual(JsonConvert.SerializeObject(pair.ai.State),
@ -153,17 +151,17 @@ namespace Compression.BSA.Test
Assert.AreEqual(pair.ai.Path, pair.bi.Path); Assert.AreEqual(pair.ai.Path, pair.bi.Path);
//Equal(pair.ai.Compressed, pair.bi.Compressed); //Equal(pair.ai.Compressed, pair.bi.Compressed);
Assert.AreEqual(pair.ai.Size, pair.bi.Size); Assert.AreEqual(pair.ai.Size, pair.bi.Size);
CollectionAssert.AreEqual(await GetData(pair.ai), await GetData(pair.bi)); CollectionAssert.AreEqual(GetData(pair.ai), GetData(pair.bi));
}); });
} }
} }
} }
private static async Task<byte[]> GetData(IFile pairAi) private static byte[] GetData(IFile pairAi)
{ {
using (var ms = new MemoryStream()) using (var ms = new MemoryStream())
{ {
await pairAi.CopyDataToAsync(ms); pairAi.CopyDataTo(ms);
return ms.ToArray(); return ms.ToArray();
} }
} }

50
Compression.BSA/BA2Builder.cs
View File

@ -1,13 +1,9 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Dynamic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using Wabbajack.Common.CSP;
namespace Compression.BSA namespace Compression.BSA
{ {
@ -19,7 +15,7 @@ namespace Compression.BSA
int Index { get; } int Index { get; }
Task WriteData(BinaryWriter wtr); void WriteData(BinaryWriter wtr);
void WriteHeader(BinaryWriter wtr); void WriteHeader(BinaryWriter wtr);
} }
@ -37,22 +33,22 @@ namespace Compression.BSA
{ {
} }
public async Task AddFile(FileStateObject state, Stream src) public void AddFile(FileStateObject state, Stream src)
{ {
switch (_state.Type) switch (_state.Type)
{ {
case EntryType.GNRL: case EntryType.GNRL:
var result = await BA2FileEntryBuilder.Create((BA2FileEntryState)state, src); var result = BA2FileEntryBuilder.Create((BA2FileEntryState)state, src);
lock(_entries) _entries.Add(result); lock(_entries) _entries.Add(result);
break; break;
case EntryType.DX10: case EntryType.DX10:
var resultdx10 = await BA2DX10FileEntryBuilder.Create((BA2DX10EntryState)state, src); var resultdx10 = BA2DX10FileEntryBuilder.Create((BA2DX10EntryState)state, src);
lock(_entries) _entries.Add(resultdx10); lock(_entries) _entries.Add(resultdx10);
break; break;
} }
} }
public async Task Build(string filename) public void Build(string filename)
{ {
SortEntries(); SortEntries();
using (var fs = File.OpenWrite(filename)) using (var fs = File.OpenWrite(filename))
@ -72,7 +68,7 @@ namespace Compression.BSA
foreach (var entry in _entries) foreach (var entry in _entries)
{ {
await entry.WriteData(bw); entry.WriteData(bw);
} }
if (_state.HasNameTable) if (_state.HasNameTable)
@ -86,7 +82,7 @@ namespace Compression.BSA
{ {
var bytes = Encoding.UTF7.GetBytes(entry.FullName); var bytes = Encoding.UTF7.GetBytes(entry.FullName);
bw.Write((ushort)bytes.Length); bw.Write((ushort)bytes.Length);
await bw.BaseStream.WriteAsync(bytes, 0, bytes.Length); bw.BaseStream.Write(bytes, 0, bytes.Length);
} }
} }
} }
@ -103,10 +99,9 @@ namespace Compression.BSA
private BA2DX10EntryState _state; private BA2DX10EntryState _state;
private List<ChunkBuilder> _chunks; private List<ChunkBuilder> _chunks;
public static async Task<BA2DX10FileEntryBuilder> Create(BA2DX10EntryState state, Stream src) public static BA2DX10FileEntryBuilder Create(BA2DX10EntryState state, Stream src)
{ {
var builder = new BA2DX10FileEntryBuilder(); var builder = new BA2DX10FileEntryBuilder {_state = state};
builder._state = state;
var header_size = DDS.HeaderSizeForFormat((DXGI_FORMAT) state.PixelFormat) + 4; var header_size = DDS.HeaderSizeForFormat((DXGI_FORMAT) state.PixelFormat) + 4;
new BinaryReader(src).ReadBytes((int)header_size); new BinaryReader(src).ReadBytes((int)header_size);
@ -115,7 +110,7 @@ namespace Compression.BSA
builder._chunks = new List<ChunkBuilder>(); builder._chunks = new List<ChunkBuilder>();
foreach (var chunk in state.Chunks) foreach (var chunk in state.Chunks)
builder._chunks.Add(await ChunkBuilder.Create(state, chunk, src)); builder._chunks.Add(ChunkBuilder.Create(state, chunk, src));
return builder; return builder;
} }
@ -143,10 +138,10 @@ namespace Compression.BSA
chunk.WriteHeader(bw); chunk.WriteHeader(bw);
} }
public async Task WriteData(BinaryWriter wtr) public void WriteData(BinaryWriter wtr)
{ {
foreach (var chunk in _chunks) foreach (var chunk in _chunks)
await chunk.WriteData(wtr); chunk.WriteData(wtr);
} }
} }
@ -158,13 +153,13 @@ namespace Compression.BSA
private uint _packSize; private uint _packSize;
private long _offsetOffset; private long _offsetOffset;
public static async Task<ChunkBuilder> Create(BA2DX10EntryState state, ChunkState chunk, Stream src) public static ChunkBuilder Create(BA2DX10EntryState state, ChunkState chunk, Stream src)
{ {
var builder = new ChunkBuilder {_chunk = chunk}; var builder = new ChunkBuilder {_chunk = chunk};
using (var ms = new MemoryStream()) using (var ms = new MemoryStream())
{ {
await src.CopyToLimitAsync(ms, (int)chunk.FullSz); src.CopyToLimit(ms, (int)chunk.FullSz);
builder._data = ms.ToArray(); builder._data = ms.ToArray();
} }
@ -197,13 +192,13 @@ namespace Compression.BSA
} }
public async Task WriteData(BinaryWriter bw) public void WriteData(BinaryWriter bw)
{ {
var pos = bw.BaseStream.Position; var pos = bw.BaseStream.Position;
bw.BaseStream.Position = _offsetOffset; bw.BaseStream.Position = _offsetOffset;
bw.Write((ulong)pos); bw.Write((ulong)pos);
bw.BaseStream.Position = pos; bw.BaseStream.Position = pos;
await bw.BaseStream.WriteAsync(_data, 0, _data.Length); bw.BaseStream.Write(_data, 0, _data.Length);
} }
} }
@ -215,14 +210,13 @@ namespace Compression.BSA
private BA2FileEntryState _state; private BA2FileEntryState _state;
private long _offsetOffset; private long _offsetOffset;
public static async Task<BA2FileEntryBuilder> Create(BA2FileEntryState state, Stream src) public static BA2FileEntryBuilder Create(BA2FileEntryState state, Stream src)
{ {
var builder = new BA2FileEntryBuilder(); var builder = new BA2FileEntryBuilder {_state = state};
builder._state = state;
using (var ms = new MemoryStream()) using (var ms = new MemoryStream())
{ {
await src.CopyToAsync(ms); src.CopyTo(ms);
builder._data = ms.ToArray(); builder._data = ms.ToArray();
} }
builder._rawSize = builder._data.Length; builder._rawSize = builder._data.Length;
@ -233,7 +227,7 @@ namespace Compression.BSA
{ {
using (var ds = new DeflaterOutputStream(ms)) using (var ds = new DeflaterOutputStream(ms))
{ {
await ds.WriteAsync(builder._data, 0, builder._data.Length); ds.Write(builder._data, 0, builder._data.Length);
} }
builder._data = ms.ToArray(); builder._data = ms.ToArray();
} }
@ -260,13 +254,13 @@ namespace Compression.BSA
wtr.Write(_state.Align); wtr.Write(_state.Align);
} }
public async Task WriteData(BinaryWriter wtr) public void WriteData(BinaryWriter wtr)
{ {
var pos = wtr.BaseStream.Position; var pos = wtr.BaseStream.Position;
wtr.BaseStream.Seek(_offsetOffset, SeekOrigin.Begin); wtr.BaseStream.Seek(_offsetOffset, SeekOrigin.Begin);
wtr.Write((ulong)pos); wtr.Write((ulong)pos);
wtr.BaseStream.Position = pos; wtr.BaseStream.Position = pos;
await wtr.BaseStream.WriteAsync(_data, 0, _data.Length); wtr.BaseStream.Write(_data, 0, _data.Length);
} }
} }
} }

24
Compression.BSA/BA2Reader.cs
View File

@ -1,17 +1,9 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Runtime.InteropServices;
using System.Text; using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using ICSharpCode.SharpZipLib.Zip;
using ICSharpCode.SharpZipLib.Zip.Compression; using ICSharpCode.SharpZipLib.Zip.Compression;
using Microsoft.SqlServer.Server;
using File = Alphaleonis.Win32.Filesystem.File; using File = Alphaleonis.Win32.Filesystem.File;
namespace Compression.BSA namespace Compression.BSA
@ -188,7 +180,7 @@ namespace Compression.BSA
public uint HeaderSize => DDS.HeaderSizeForFormat((DXGI_FORMAT)_format); public uint HeaderSize => DDS.HeaderSizeForFormat((DXGI_FORMAT)_format);
public async Task CopyDataToAsync(Stream output) public void CopyDataTo(Stream output)
{ {
var bw = new BinaryWriter(output); var bw = new BinaryWriter(output);
@ -206,18 +198,18 @@ namespace Compression.BSA
if (!isCompressed) if (!isCompressed)
{ {
await br.BaseStream.ReadAsync(full, 0, full.Length); br.BaseStream.Read(full, 0, full.Length);
} }
else else
{ {
byte[] compressed = new byte[chunk._packSz]; byte[] compressed = new byte[chunk._packSz];
await br.BaseStream.ReadAsync(compressed, 0, compressed.Length); br.BaseStream.Read(compressed, 0, compressed.Length);
var inflater = new Inflater(); var inflater = new Inflater();
inflater.SetInput(compressed); inflater.SetInput(compressed);
inflater.Inflate(full); inflater.Inflate(full);
} }
await bw.BaseStream.WriteAsync(full, 0, full.Length); bw.BaseStream.Write(full, 0, full.Length);
} }
} }
@ -450,7 +442,7 @@ namespace Compression.BSA
public uint Size => _realSize; public uint Size => _realSize;
public FileStateObject State => new BA2FileEntryState(this); public FileStateObject State => new BA2FileEntryState(this);
public async Task CopyDataToAsync(Stream output) public void CopyDataTo(Stream output)
{ {
using (var fs = File.OpenRead(_bsa._filename)) using (var fs = File.OpenRead(_bsa._filename))
{ {
@ -458,11 +450,11 @@ namespace Compression.BSA
uint len = Compressed ? _size : _realSize; uint len = Compressed ? _size : _realSize;
var bytes = new byte[len]; var bytes = new byte[len];
await fs.ReadAsync(bytes, 0, (int) len); fs.Read(bytes, 0, (int) len);
if (!Compressed) if (!Compressed)
{ {
await output.WriteAsync(bytes, 0, bytes.Length); output.Write(bytes, 0, bytes.Length);
} }
else else
{ {
@ -470,7 +462,7 @@ namespace Compression.BSA
var inflater = new Inflater(); var inflater = new Inflater();
inflater.SetInput(bytes); inflater.SetInput(bytes);
inflater.Inflate(uncompressed); inflater.Inflate(uncompressed);
await output.WriteAsync(uncompressed, 0, uncompressed.Length); output.Write(uncompressed, 0, uncompressed.Length);
} }
} }
} }

19
Compression.BSA/BSABuilder.cs
View File

@ -3,7 +3,6 @@ using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using K4os.Compression.LZ4; using K4os.Compression.LZ4;
using K4os.Compression.LZ4.Streams; using K4os.Compression.LZ4.Streams;
@ -77,11 +76,11 @@ namespace Compression.BSA
public void Dispose() public void Dispose()
{ {
} }
public async Task AddFile(FileStateObject state, Stream src) public void AddFile(FileStateObject state, Stream src)
{ {
var ostate = (BSAFileStateObject) state; var ostate = (BSAFileStateObject) state;
var r = await FileEntry.Create(this, ostate.Path, src, ostate.FlipCompression); var r = FileEntry.Create(this, ostate.Path, src, ostate.FlipCompression);
lock (this) lock (this)
{ {
@ -89,7 +88,7 @@ namespace Compression.BSA
} }
} }
public async Task Build(string outputName) public void Build(string outputName)
{ {
RegenFolderRecords(); RegenFolderRecords();
if (File.Exists(outputName)) File.Delete(outputName); if (File.Exists(outputName)) File.Delete(outputName);
@ -122,7 +121,7 @@ namespace Compression.BSA
foreach (var file in _files) wtr.Write(file._nameBytes); foreach (var file in _files) wtr.Write(file._nameBytes);
foreach (var file in _files) foreach (var file in _files)
await file.WriteData(wtr); file.WriteData(wtr);
} }
} }
@ -237,7 +236,7 @@ namespace Compression.BSA
internal byte[] _pathBytes; internal byte[] _pathBytes;
internal byte[] _rawData; internal byte[] _rawData;
public static async Task<FileEntry> Create(BSABuilder bsa, string path, Stream src, bool flipCompression) public static FileEntry Create(BSABuilder bsa, string path, Stream src, bool flipCompression)
{ {
var entry = new FileEntry(); var entry = new FileEntry();
entry._bsa = bsa; entry._bsa = bsa;
@ -250,7 +249,7 @@ namespace Compression.BSA
entry._flipCompression = flipCompression; entry._flipCompression = flipCompression;
var ms = new MemoryStream(); var ms = new MemoryStream();
await src.CopyToAsync(ms); src.CopyTo(ms);
entry._rawData = ms.ToArray(); entry._rawData = ms.ToArray();
entry._originalSize = entry._rawData.Length; entry._originalSize = entry._rawData.Length;
@ -321,7 +320,7 @@ namespace Compression.BSA
wtr.Write(0xDEADBEEF); wtr.Write(0xDEADBEEF);
} }
internal async Task WriteData(BinaryWriter wtr) internal void WriteData(BinaryWriter wtr)
{ {
var offset = (uint) wtr.BaseStream.Position; var offset = (uint) wtr.BaseStream.Position;
wtr.BaseStream.Position = _offsetOffset; wtr.BaseStream.Position = _offsetOffset;
@ -333,11 +332,11 @@ namespace Compression.BSA
if (Compressed) if (Compressed)
{ {
wtr.Write((uint) _originalSize); wtr.Write((uint) _originalSize);
await wtr.BaseStream.WriteAsync(_rawData, 0, _rawData.Length); wtr.BaseStream.Write(_rawData, 0, _rawData.Length);
} }
else else
{ {
await wtr.BaseStream.WriteAsync(_rawData, 0, _rawData.Length); wtr.BaseStream.Write(_rawData, 0, _rawData.Length);
} }
} }
} }

31
Compression.BSA/BSADispatch.cs
View File

@ -1,32 +1,23 @@
using System; using System.IO;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common.CSP;
namespace Compression.BSA namespace Compression.BSA
{ {
public static class BSADispatch public static class BSADispatch
{ {
public static Task<IBSAReader> OpenRead(string filename) public static IBSAReader OpenRead(string filename)
{ {
return CSPExtensions.ThreadedTask<IBSAReader>(() => var fourcc = "";
using (var file = File.OpenRead(filename))
{ {
string fourcc = ""; fourcc = Encoding.ASCII.GetString(new BinaryReader(file).ReadBytes(4));
using (var file = File.OpenRead(filename)) }
{
fourcc = Encoding.ASCII.GetString(new BinaryReader(file).ReadBytes(4));
}
if (fourcc == "BSA\0") if (fourcc == "BSA\0")
return new BSAReader(filename); return new BSAReader(filename);
if (fourcc == "BTDX") if (fourcc == "BTDX")
return new BA2Reader(filename); return new BA2Reader(filename);
throw new InvalidDataException("Filename is not a .bsa or .ba2, magic " + fourcc); throw new InvalidDataException("Filename is not a .bsa or .ba2, magic " + fourcc);
});
} }
} }
} }

16
Compression.BSA/BSAReader.cs
View File

@ -2,8 +2,6 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Text; using System.Text;
using System.Threading;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using K4os.Compression.LZ4.Streams; using K4os.Compression.LZ4.Streams;
using File = Alphaleonis.Win32.Filesystem.File; using File = Alphaleonis.Win32.Filesystem.File;
@ -303,7 +301,7 @@ namespace Compression.BSA
_name = rdr.ReadStringTerm(_bsa.HeaderType); _name = rdr.ReadStringTerm(_bsa.HeaderType);
} }
public async Task CopyDataToAsync(Stream output) public void CopyDataTo(Stream output)
{ {
using (var in_file = File.OpenRead(_bsa._fileName)) using (var in_file = File.OpenRead(_bsa._fileName))
using (var rdr = new BinaryReader(in_file)) using (var rdr = new BinaryReader(in_file))
@ -315,11 +313,11 @@ namespace Compression.BSA
if (Compressed) if (Compressed)
{ {
var r = LZ4Stream.Decode(rdr.BaseStream); var r = LZ4Stream.Decode(rdr.BaseStream);
await r.CopyToLimitAsync(output, (int) _originalSize); r.CopyToLimit(output, (int) _originalSize);
} }
else else
{ {
await rdr.BaseStream.CopyToLimitAsync(output, (int) _onDiskSize); rdr.BaseStream.CopyToLimit(output, (int) _onDiskSize);
} }
} }
else else
@ -327,18 +325,18 @@ namespace Compression.BSA
if (Compressed) if (Compressed)
using (var z = new InflaterInputStream(rdr.BaseStream)) using (var z = new InflaterInputStream(rdr.BaseStream))
{ {
await z.CopyToLimitAsync(output, (int) _originalSize); z.CopyToLimit(output, (int) _originalSize);
} }
else else
await rdr.BaseStream.CopyToLimitAsync(output, (int) _onDiskSize); rdr.BaseStream.CopyToLimit(output, (int) _onDiskSize);
} }
} }
} }
public async Task<byte[]> GetData() public byte[] GetData()
{ {
var ms = new MemoryStream(); var ms = new MemoryStream();
await CopyDataToAsync(ms); CopyDataTo(ms);
return ms.ToArray(); return ms.ToArray();
} }
} }

6
Compression.BSA/Compression.BSA.csproj
View File

@ -110,11 +110,5 @@
<Version>1.2.0</Version> <Version>1.2.0</Version>
</PackageReference> </PackageReference>
</ItemGroup> </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Common.CSP\Wabbajack.Common.CSP.csproj">
<Project>{9e69bc98-1512-4977-b683-6e7e5292c0b8}</Project>
<Name>Wabbajack.Common.CSP</Name>
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project> </Project>

8
Compression.BSA/DDS.cs
View File

@ -1,10 +1,4 @@
using System; using System.Runtime.InteropServices;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace Compression.BSA namespace Compression.BSA
{ {

9
Compression.BSA/IBSAReader.cs
View File

@ -1,9 +1,6 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Compression.BSA namespace Compression.BSA
{ {
@ -19,8 +16,8 @@ namespace Compression.BSA
public interface IBSABuilder : IDisposable public interface IBSABuilder : IDisposable
{ {
Task AddFile(FileStateObject state, Stream src); void AddFile(FileStateObject state, Stream src);
Task Build(string filename); void Build(string filename);
} }
public class ArchiveStateObject public class ArchiveStateObject
@ -59,6 +56,6 @@ namespace Compression.BSA
/// in order to maintain thread-safe access. /// in order to maintain thread-safe access.
/// </summary> /// </summary>
/// <param name="output"></param> /// <param name="output"></param>
Task CopyDataToAsync(Stream output); void CopyDataTo(Stream output);
} }
} }

15
Compression.BSA/Utils.cs
View File

@ -2,7 +2,6 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Text; using System.Text;
using System.Threading.Tasks;
using Path = Alphaleonis.Win32.Filesystem.Path; using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Compression.BSA namespace Compression.BSA
@ -142,20 +141,6 @@ namespace Compression.BSA
return ((ulong) (hash2 + hash3) << 32) + hash1; return ((ulong) (hash2 + hash3) << 32) + hash1;
} }
public static async Task CopyToLimitAsync(this Stream frm, Stream tw, int limit)
{
var buff = new byte[1024];
while (limit > 0)
{
var to_read = Math.Min(buff.Length, limit);
var read = await frm.ReadAsync(buff, 0, to_read);
await tw.WriteAsync(buff, 0, read);
limit -= read;
}
tw.Flush();
}
public static void CopyToLimit(this Stream frm, Stream tw, int limit) public static void CopyToLimit(this Stream frm, Stream tw, int limit)
{ {
var buff = new byte[1024]; var buff = new byte[1024];

73
Wabbajack.Common/FileExtractor.cs
View File

@ -2,12 +2,10 @@
using System.Diagnostics; using System.Diagnostics;
using System.Linq; using System.Linq;
using System.Reflection; using System.Reflection;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem; using Alphaleonis.Win32.Filesystem;
using Compression.BSA; using Compression.BSA;
using ICSharpCode.SharpZipLib.GZip; using ICSharpCode.SharpZipLib.GZip;
using OMODFramework; using OMODFramework;
using Wabbajack.Common.CSP;
namespace Wabbajack.Common namespace Wabbajack.Common
{ {
@ -33,16 +31,16 @@ namespace Wabbajack.Common
} }
public static async Task ExtractAll(string source, string dest) public static void ExtractAll(string source, string dest)
{ {
try try
{ {
if (Consts.SupportedBSAs.Any(b => source.ToLower().EndsWith(b))) if (Consts.SupportedBSAs.Any(b => source.ToLower().EndsWith(b)))
await ExtractAllWithBSA(source, dest); ExtractAllWithBSA(source, dest);
else if (source.EndsWith(".omod")) else if (source.EndsWith(".omod"))
await ExtractAllWithOMOD(source, dest); ExtractAllWithOMOD(source, dest);
else else
await ExtractAllWith7Zip(source, dest); ExtractAllWith7Zip(source, dest);
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -51,48 +49,41 @@ namespace Wabbajack.Common
} }
} }
private static Task ExtractAllWithOMOD(string source, string dest) private static string ExtractAllWithOMOD(string source, string dest)
{ {
return CSPExtensions.ThreadedTask(() => Utils.Log($"Extracting {Path.GetFileName(source)}");
{ var f = new Framework();
Utils.Log($"Extracting {Path.GetFileName(source)}"); f.SetTempDirectory(dest);
var f = new Framework(); var omod = new OMOD(source, ref f);
f.SetTempDirectory(dest); omod.ExtractDataFiles();
var omod = new OMOD(source, ref f); omod.ExtractPlugins();
omod.ExtractDataFiles(); return dest;
omod.ExtractPlugins();
return dest;
});
} }
private static async Task ExtractAllWithBSA(string source, string dest) private static void ExtractAllWithBSA(string source, string dest)
{ {
try try
{ {
using (var arch = await BSADispatch.OpenRead(source)) using (var arch = BSADispatch.OpenRead(source))
{ {
await arch.Files.ToChannel() arch.Files
.UnorderedPipeline( .PMap(f =>
Channel.CreateSink<IFile>(), {
async f => var path = f.Path;
{ if (f.Path.StartsWith("\\"))
var path = f.Path; path = f.Path.Substring(1);
if (f.Path.StartsWith("\\")) Utils.Status($"Extracting {path}");
path = f.Path.Substring(1); var out_path = Path.Combine(dest, path);
Utils.Status($"Extracting {path}"); var parent = Path.GetDirectoryName(out_path);
var out_path = Path.Combine(dest, path);
var parent = Path.GetDirectoryName(out_path);
if (!Directory.Exists(parent)) if (!Directory.Exists(parent))
Directory.CreateDirectory(parent); Directory.CreateDirectory(parent);
using (var fs = File.OpenWrite(out_path)) using (var fs = File.OpenWrite(out_path))
{ {
await f.CopyDataToAsync(fs); f.CopyDataTo(fs);
} }
});
return f;
});
} }
} }
catch (Exception ex) catch (Exception ex)
@ -102,7 +93,7 @@ namespace Wabbajack.Common
} }
} }
private static async Task ExtractAllWith7Zip(string source, string dest) private static void ExtractAllWith7Zip(string source, string dest)
{ {
Utils.Log($"Extracting {Path.GetFileName(source)}"); Utils.Log($"Extracting {Path.GetFileName(source)}");
@ -137,7 +128,7 @@ namespace Wabbajack.Common
{ {
while (!p.HasExited) while (!p.HasExited)
{ {
var line = await p.StandardOutput.ReadLineAsync(); var line = p.StandardOutput.ReadLine();
if (line == null) if (line == null)
break; break;

16
Wabbajack.Lib/ACompiler.cs
View File

@ -1,6 +1,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Reactive.Subjects;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using Wabbajack.Common; using Wabbajack.Common;
@ -11,7 +12,16 @@ namespace Wabbajack.Lib
{ {
public abstract class ACompiler public abstract class ACompiler
{ {
protected static string _vfsCacheName = "vfs_compile_cache.bin";
/// <summary>
/// A stream of tuples of ("Update Title", 0.25) which represent the name of the current task
/// and the current progress.
/// </summary>
public IObservable<(string, float)> ProgressUpdates => _progressUpdates;
protected readonly Subject<(string, float)> _progressUpdates = new Subject<(string, float)>();
public Context VFS { get; internal set; } = new Context(); public Context VFS { get; internal set; } = new Context();
public ModManager ModManager; public ModManager ModManager;
public string GamePath; public string GamePath;
@ -39,5 +49,11 @@ namespace Wabbajack.Lib
public abstract Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source); public abstract Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source);
public abstract IEnumerable<ICompilationStep> GetStack(); public abstract IEnumerable<ICompilationStep> GetStack();
public abstract IEnumerable<ICompilationStep> MakeStack(); public abstract IEnumerable<ICompilationStep> MakeStack();
protected ACompiler()
{
ProgressUpdates.Subscribe(itm => Utils.Log($"{itm.Item2} - {itm.Item1}"));
VFS.LogSpam.Subscribe(itm => Utils.Status(itm));
}
} }
} }

2
Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs
View File

@ -77,7 +77,7 @@ namespace Wabbajack.Lib.CompilationSteps
} }
CreateBSA directive; CreateBSA directive;
using (var bsa = BSADispatch.OpenRead(source.AbsolutePath).Result) using (var bsa = BSADispatch.OpenRead(source.AbsolutePath))
{ {
directive = new CreateBSA directive = new CreateBSA
{ {

28
Wabbajack.Lib/Compiler.cs
View File

@ -28,6 +28,7 @@ namespace Wabbajack.Lib
{ {
public class Compiler : ACompiler public class Compiler : ACompiler
{ {
private string _mo2DownloadsFolder; private string _mo2DownloadsFolder;
public Dictionary<string, IEnumerable<IndexedFileMatch>> DirectMatchIndex; public Dictionary<string, IEnumerable<IndexedFileMatch>> DirectMatchIndex;
@ -49,6 +50,8 @@ namespace Wabbajack.Lib
ModListOutputFolder = "output_folder"; ModListOutputFolder = "output_folder";
ModListOutputFile = MO2Profile + ExtensionManager.Extension; ModListOutputFile = MO2Profile + ExtensionManager.Extension;
VFS.ProgressUpdates.Debounce(new TimeSpan(0, 0, 0, 0, 100))
.Subscribe(itm => _progressUpdates.OnNext(itm));
} }
public dynamic MO2Ini { get; } public dynamic MO2Ini { get; }
@ -119,13 +122,24 @@ namespace Wabbajack.Lib
Info("Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p))); Info("Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p)));
VFS.IntegrateFromFile(_vfsCacheName);
Info($"Indexing {MO2Folder}"); Info($"Indexing {MO2Folder}");
VFS.AddRoot(MO2Folder).Wait(); VFS.AddRoot(MO2Folder);
VFS.WriteToFile(_vfsCacheName);
Info($"Indexing {GamePath}"); Info($"Indexing {GamePath}");
VFS.AddRoot(GamePath).Wait(); VFS.AddRoot(GamePath);
VFS.WriteToFile(_vfsCacheName);
Info($"Indexing {MO2DownloadsFolder}"); Info($"Indexing {MO2DownloadsFolder}");
VFS.AddRoot(MO2DownloadsFolder).Wait(); VFS.AddRoot(MO2DownloadsFolder);
VFS.WriteToFile(_vfsCacheName);
Info("Cleaning output folder"); Info("Cleaning output folder");
if (Directory.Exists(ModListOutputFolder)) if (Directory.Exists(ModListOutputFolder))
@ -150,7 +164,9 @@ namespace Wabbajack.Lib
if (Directory.Exists(loot_path)) if (Directory.Exists(loot_path))
{ {
Info($"Indexing {loot_path}"); Info($"Indexing {loot_path}");
VFS.AddRoot(loot_path).Wait(); VFS.AddRoot(loot_path);
VFS.WriteToFile(_vfsCacheName);
loot_files = Directory.EnumerateFiles(loot_path, "userlist.yaml", SearchOption.AllDirectories) loot_files = Directory.EnumerateFiles(loot_path, "userlist.yaml", SearchOption.AllDirectories)
.Where(p => p.FileExists()) .Where(p => p.FileExists())
@ -452,13 +468,13 @@ namespace Wabbajack.Lib
var bsa_id = to.Split('\\')[1]; var bsa_id = to.Split('\\')[1];
var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsa_id); var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsa_id);
using (var a = await BSADispatch.OpenRead(Path.Combine(MO2Folder, bsa.To))) using (var a = BSADispatch.OpenRead(Path.Combine(MO2Folder, bsa.To)))
{ {
var find = Path.Combine(to.Split('\\').Skip(2).ToArray()); var find = Path.Combine(to.Split('\\').Skip(2).ToArray());
var file = a.Files.First(e => e.Path.Replace('/', '\\') == find); var file = a.Files.First(e => e.Path.Replace('/', '\\') == find);
using (var ms = new MemoryStream()) using (var ms = new MemoryStream())
{ {
await file.CopyDataToAsync(ms); file.CopyDataTo(ms);
return ms.ToArray(); return ms.ToArray();
} }
} }

8
Wabbajack.Lib/Installer.cs
View File

@ -150,7 +150,7 @@ namespace Wabbajack.Lib
Error("Cannot continue, was unable to download one or more archives"); Error("Cannot continue, was unable to download one or more archives");
} }
PrimeVFS().Wait(); PrimeVFS();
BuildFolderStructure(); BuildFolderStructure();
InstallArchives(); InstallArchives();
@ -233,7 +233,7 @@ namespace Wabbajack.Lib
/// We don't want to make the installer index all the archives, that's just a waste of time, so instead /// We don't want to make the installer index all the archives, that's just a waste of time, so instead
/// we'll pass just enough information to VFS to let it know about the files we have. /// we'll pass just enough information to VFS to let it know about the files we have.
/// </summary> /// </summary>
private async Task PrimeVFS() private void PrimeVFS()
{ {
VFS.AddKnown(HashedArchives.Select(a => new KnownFile VFS.AddKnown(HashedArchives.Select(a => new KnownFile
{ {
@ -247,7 +247,7 @@ namespace Wabbajack.Lib
.OfType<FromArchive>() .OfType<FromArchive>()
.Select(f => new KnownFile { Paths = f.ArchiveHashPath})); .Select(f => new KnownFile { Paths = f.ArchiveHashPath}));
await VFS.BackfillMissing(); VFS.BackfillMissing();
} }
private void BuildBSAs() private void BuildBSAs()
@ -386,7 +386,7 @@ namespace Wabbajack.Lib
return g; return g;
}).ToList(); }).ToList();
var on_finish = VFS.Stage(vfiles.Select(f => f.FromFile).Distinct()).Result; var on_finish = VFS.Stage(vfiles.Select(f => f.FromFile).Distinct());
Status($"Copying files for {archive.Name}"); Status($"Copying files for {archive.Name}");

6
Wabbajack.Lib/VortexCompiler.cs
View File

@ -95,13 +95,13 @@ namespace Wabbajack.Lib
CreateMetaFiles(); CreateMetaFiles();
Info($"Indexing {StagingFolder}"); Info($"Indexing {StagingFolder}");
VFS.AddRoot(StagingFolder).Wait(); VFS.AddRoot(StagingFolder);
Info($"Indexing {GamePath}"); Info($"Indexing {GamePath}");
VFS.AddRoot(GamePath).Wait(); VFS.AddRoot(GamePath);
Info($"Indexing {DownloadsFolder}"); Info($"Indexing {DownloadsFolder}");
VFS.AddRoot(DownloadsFolder).Wait(); VFS.AddRoot(DownloadsFolder);
AddExternalFolder(); AddExternalFolder();

2
Wabbajack.Lib/VortexInstaller.cs
View File

@ -156,7 +156,7 @@ namespace Wabbajack.Lib
return g; return g;
}).ToList(); }).ToList();
var onFinish = VFS.Stage(vFiles.Select(f => f.FromFile).Distinct()).Result; var onFinish = VFS.Stage(vFiles.Select(f => f.FromFile).Distinct());
Status($"Copying files for {archive.Name}"); Status($"Copying files for {archive.Name}");

6
Wabbajack.Test/EndToEndTests.cs
View File

@ -90,9 +90,9 @@ namespace Wabbajack.Test
File.Copy(src, Path.Combine(utils.DownloadsFolder, filename)); File.Copy(src, Path.Combine(utils.DownloadsFolder, filename));
if (mod_name == null) if (mod_name == null)
FileExtractor.ExtractAll(src, utils.MO2Folder).Wait(); FileExtractor.ExtractAll(src, utils.MO2Folder);
else else
FileExtractor.ExtractAll(src, Path.Combine(utils.ModsFolder, mod_name)).Wait(); FileExtractor.ExtractAll(src, Path.Combine(utils.ModsFolder, mod_name));
} }
@ -127,7 +127,7 @@ namespace Wabbajack.Test
var dest = Path.Combine(utils.DownloadsFolder, file.file_name); var dest = Path.Combine(utils.DownloadsFolder, file.file_name);
File.Copy(src, dest); File.Copy(src, dest);
FileExtractor.ExtractAll(src, Path.Combine(utils.ModsFolder, mod_name)).Wait(); FileExtractor.ExtractAll(src, Path.Combine(utils.ModsFolder, mod_name));
File.WriteAllText(dest + ".meta", ini); File.WriteAllText(dest + ".meta", ini);
} }

46
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
View File

@ -29,10 +29,10 @@ namespace Wabbajack.VirtualFileSystem.Test
} }
[TestMethod] [TestMethod]
public async Task FilesAreIndexed() public void FilesAreIndexed()
{ {
AddFile("test.txt", "This is a test"); AddFile("test.txt", "This is a test");
await AddTestRoot(); AddTestRoot();
var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")]; var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
Assert.IsNotNull(file); Assert.IsNotNull(file);
@ -41,11 +41,11 @@ namespace Wabbajack.VirtualFileSystem.Test
Assert.AreEqual(file.Hash, "qX0GZvIaTKM="); Assert.AreEqual(file.Hash, "qX0GZvIaTKM=");
} }
private async Task AddTestRoot() private void AddTestRoot()
{ {
await context.AddRoot(VFS_TEST_DIR_FULL); context.AddRoot(VFS_TEST_DIR_FULL);
await context.WriteToFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin")); context.WriteToFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
await context.IntegrateFromFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin")); context.IntegrateFromFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
} }
@ -54,7 +54,7 @@ namespace Wabbajack.VirtualFileSystem.Test
{ {
AddFile("archive/test.txt", "This is a test"); AddFile("archive/test.txt", "This is a test");
ZipUpFolder("archive", "test.zip"); ZipUpFolder("archive", "test.zip");
await AddTestRoot(); AddTestRoot();
var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip"); var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip");
var file = context.Index.ByFullPath[abs_path]; var file = context.Index.ByFullPath[abs_path];
@ -77,7 +77,7 @@ namespace Wabbajack.VirtualFileSystem.Test
ZipUpFolder("archive", "test.zip"); ZipUpFolder("archive", "test.zip");
AddFile("test.txt", "This is a test"); AddFile("test.txt", "This is a test");
await AddTestRoot(); AddTestRoot();
var files = context.Index.ByHash["qX0GZvIaTKM="]; var files = context.Index.ByHash["qX0GZvIaTKM="];
@ -88,7 +88,7 @@ namespace Wabbajack.VirtualFileSystem.Test
public async Task DeletedFilesAreRemoved() public async Task DeletedFilesAreRemoved()
{ {
AddFile("test.txt", "This is a test"); AddFile("test.txt", "This is a test");
await AddTestRoot(); AddTestRoot();
var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")]; var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
Assert.IsNotNull(file); Assert.IsNotNull(file);
@ -98,21 +98,21 @@ namespace Wabbajack.VirtualFileSystem.Test
File.Delete(Path.Combine(VFS_TEST_DIR_FULL, "test.txt")); File.Delete(Path.Combine(VFS_TEST_DIR_FULL, "test.txt"));
await AddTestRoot(); AddTestRoot();
CollectionAssert.DoesNotContain(context.Index.ByFullPath, Path.Combine(VFS_TEST_DIR_FULL, "test.txt")); CollectionAssert.DoesNotContain(context.Index.ByFullPath, Path.Combine(VFS_TEST_DIR_FULL, "test.txt"));
} }
[TestMethod] [TestMethod]
public async Task UnmodifiedFilesAreNotReIndexed() public void UnmodifiedFilesAreNotReIndexed()
{ {
AddFile("test.txt", "This is a test"); AddFile("test.txt", "This is a test");
await AddTestRoot(); AddTestRoot();
var old_file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")]; var old_file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
var old_time = old_file.LastAnalyzed; var old_time = old_file.LastAnalyzed;
await AddTestRoot(); AddTestRoot();
var new_file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")]; var new_file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
@ -120,23 +120,23 @@ namespace Wabbajack.VirtualFileSystem.Test
} }
[TestMethod] [TestMethod]
public async Task CanStageSimpleArchives() public void CanStageSimpleArchives()
{ {
AddFile("archive/test.txt", "This is a test"); AddFile("archive/test.txt", "This is a test");
ZipUpFolder("archive", "test.zip"); ZipUpFolder("archive", "test.zip");
await AddTestRoot(); AddTestRoot();
var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip"); var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip");
var file = context.Index.ByFullPath[abs_path + "|test.txt"]; var file = context.Index.ByFullPath[abs_path + "|test.txt"];
var cleanup = await context.Stage(new List<VirtualFile> {file}); var cleanup = context.Stage(new List<VirtualFile> {file});
Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath)); Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));
cleanup(); cleanup();
} }
[TestMethod] [TestMethod]
public async Task CanStageNestedArchives() public void CanStageNestedArchives()
{ {
AddFile("archive/test.txt", "This is a test"); AddFile("archive/test.txt", "This is a test");
ZipUpFolder("archive", "test.zip"); ZipUpFolder("archive", "test.zip");
@ -146,11 +146,11 @@ namespace Wabbajack.VirtualFileSystem.Test
Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir\nested.zip")); Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir\nested.zip"));
ZipUpFolder("archive", "test.zip"); ZipUpFolder("archive", "test.zip");
await AddTestRoot(); AddTestRoot();
var files = context.Index.ByHash["qX0GZvIaTKM="]; var files = context.Index.ByHash["qX0GZvIaTKM="];
var cleanup = await context.Stage(files); var cleanup = context.Stage(files);
foreach (var file in files) foreach (var file in files)
Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath)); Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));
@ -159,7 +159,7 @@ namespace Wabbajack.VirtualFileSystem.Test
} }
[TestMethod] [TestMethod]
public async Task CanRequestPortableFileTrees() public void CanRequestPortableFileTrees()
{ {
AddFile("archive/test.txt", "This is a test"); AddFile("archive/test.txt", "This is a test");
ZipUpFolder("archive", "test.zip"); ZipUpFolder("archive", "test.zip");
@ -169,7 +169,7 @@ namespace Wabbajack.VirtualFileSystem.Test
Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir\nested.zip")); Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir\nested.zip"));
ZipUpFolder("archive", "test.zip"); ZipUpFolder("archive", "test.zip");
await AddTestRoot(); AddTestRoot();
var files = context.Index.ByHash["qX0GZvIaTKM="]; var files = context.Index.ByHash["qX0GZvIaTKM="];
var archive = context.Index.ByRootPath[Path.Combine(VFS_TEST_DIR_FULL, "test.zip")]; var archive = context.Index.ByRootPath[Path.Combine(VFS_TEST_DIR_FULL, "test.zip")];
@ -178,12 +178,12 @@ namespace Wabbajack.VirtualFileSystem.Test
var new_context = new Context(); var new_context = new Context();
await new_context.IntegrateFromPortable(state, new_context.IntegrateFromPortable(state,
new Dictionary<string, string> {{archive.Hash, archive.FullPath}}); new Dictionary<string, string> {{archive.Hash, archive.FullPath}});
var new_files = new_context.Index.ByHash["qX0GZvIaTKM="]; var new_files = new_context.Index.ByHash["qX0GZvIaTKM="];
var close = await new_context.Stage(new_files); var close = new_context.Stage(new_files);
foreach (var file in new_files) foreach (var file in new_files)
Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath)); Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));

201
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -4,6 +4,7 @@ using System.Collections.Immutable;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Reactive.Linq; using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem; using Alphaleonis.Win32.Filesystem;
@ -24,44 +25,54 @@ namespace Wabbajack.VirtualFileSystem
private readonly string _stagingFolder = "vfs_staging"; private readonly string _stagingFolder = "vfs_staging";
public IndexRoot Index { get; private set; } = IndexRoot.Empty; public IndexRoot Index { get; private set; } = IndexRoot.Empty;
/// <summary>
/// A stream of tuples of ("Update Title", 0.25) which represent the name of the current task
/// and the current progress.
/// </summary>
public IObservable<(string, float)> ProgressUpdates => _progressUpdates;
private readonly Subject<(string, float)> _progressUpdates = new Subject<(string, float)>();
/// <summary>
/// A high throughput firehose of updates from the VFS. These go into more detail on the status
/// of what's happening in the context, but is probably too high bandwidth to tie driectly to the
/// UI.
/// </summary>
public IObservable<string> LogSpam => _logSpam;
internal readonly Subject<string> _logSpam = new Subject<string>();
public TemporaryDirectory GetTemporaryFolder() public TemporaryDirectory GetTemporaryFolder()
{ {
return new TemporaryDirectory(Path.Combine(_stagingFolder, Guid.NewGuid().ToString())); return new TemporaryDirectory(Path.Combine(_stagingFolder, Guid.NewGuid().ToString()));
} }
public async Task<IndexRoot> AddRoot(string root) public IndexRoot AddRoot(string root)
{ {
if (!Path.IsPathRooted(root)) if (!Path.IsPathRooted(root))
throw new InvalidDataException($"Path is not absolute: {root}"); throw new InvalidDataException($"Path is not absolute: {root}");
var filtered = await Index.AllFiles var filtered = Index.AllFiles.Where(file => File.Exists(file.Name)).ToList();
.ToChannel()
.UnorderedPipelineRx(o => o.Where(file => File.Exists(file.Name)))
.TakeAll();
var byPath = filtered.ToImmutableDictionary(f => f.Name); var byPath = filtered.ToImmutableDictionary(f => f.Name);
var results = Channel.Create<VirtualFile>(1024); var filesToIndex = Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive).ToList();
var pipeline = Directory.EnumerateFiles(root, "*", DirectoryEnumerationOptions.Recursive)
.ToChannel() var results = Channel.Create(1024, ProgressUpdater<VirtualFile>($"Indexing {root}", filesToIndex.Count));
.UnorderedPipeline(results, async f =>
{
if (byPath.TryGetValue(f, out var found))
{
var fi = new FileInfo(f);
if (found.LastModified == fi.LastWriteTimeUtc.Ticks && found.Size == fi.Length)
return found;
}
return await VirtualFile.Analyze(this, null, f, f); var allFiles= filesToIndex
}); .PMap(f =>
{
if (byPath.TryGetValue(f, out var found))
{
var fi = new FileInfo(f);
if (found.LastModified == fi.LastWriteTimeUtc.Ticks && found.Size == fi.Length)
return found;
}
var allFiles = await results.TakeAll(); return VirtualFile.Analyze(this, null, f, f);
});
// Should already be done but let's make the async tracker happy var newIndex = IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
await pipeline;
var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
lock (this) lock (this)
{ {
@ -71,7 +82,26 @@ namespace Wabbajack.VirtualFileSystem
return newIndex; return newIndex;
} }
public async Task WriteToFile(string filename) class Box<T>
{
public T Value { get; set; }
}
private Func<IObservable<T>, IObservable<T>> ProgressUpdater<T>(string s, float totalCount)
{
if (totalCount == 0)
totalCount = 1;
var box = new Box<float>();
return sub => sub.Select(itm =>
{
box.Value += 1;
_progressUpdates.OnNext((s, box.Value / totalCount));
return itm;
});
}
public void WriteToFile(string filename)
{ {
using (var fs = File.OpenWrite(filename)) using (var fs = File.OpenWrite(filename))
using (var bw = new BinaryWriter(fs, Encoding.UTF8, true)) using (var bw = new BinaryWriter(fs, Encoding.UTF8, true))
@ -82,66 +112,62 @@ namespace Wabbajack.VirtualFileSystem
bw.Write(FileVersion); bw.Write(FileVersion);
bw.Write((ulong) Index.AllFiles.Count); bw.Write((ulong) Index.AllFiles.Count);
var sizes = await Index.AllFiles var sizes = Index.AllFiles
.ToChannel() .PMap(f =>
.UnorderedPipelineSync(f =>
{ {
var ms = new MemoryStream(); var ms = new MemoryStream();
f.Write(ms); f.Write(ms);
return ms; return ms;
}) })
.Select(async ms => .Select(ms =>
{ {
var size = ms.Position; var size = ms.Position;
ms.Position = 0; ms.Position = 0;
bw.Write((ulong) size); bw.Write((ulong) size);
await ms.CopyToAsync(fs); ms.CopyTo(fs);
return ms.Position; return ms.Position;
}) });
.TakeAll();
Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}"); Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}");
} }
} }
public async Task IntegrateFromFile(string filename) public void IntegrateFromFile(string filename)
{ {
using (var fs = File.OpenRead(filename)) try
using (var br = new BinaryReader(fs, Encoding.UTF8, true))
{ {
var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length)); using (var fs = File.OpenRead(filename))
var fileVersion = br.ReadUInt64(); using (var br = new BinaryReader(fs, Encoding.UTF8, true))
if (fileVersion != FileVersion || magic != magic)
throw new InvalidDataException("Bad Data Format");
var numFiles = br.ReadUInt64();
var input = Channel.Create<byte[]>(1024);
var pipeline = input.UnorderedPipelineSync(
data => VirtualFile.Read(this, data))
.TakeAll();
Utils.Log($"Loading {numFiles} files from {filename}");
for (ulong idx = 0; idx < numFiles; idx++)
{ {
var size = br.ReadUInt64(); var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length));
var bytes = new byte[size]; var fileVersion = br.ReadUInt64();
await br.BaseStream.ReadAsync(bytes, 0, (int) size); if (fileVersion != FileVersion || magic != magic)
await input.Put(bytes); throw new InvalidDataException("Bad Data Format");
}
input.Close(); var numFiles = br.ReadUInt64();
var files = await pipeline; var files = Enumerable.Range(0, (int) numFiles)
var newIndex = await Index.Integrate(files); .Select(idx =>
lock (this) {
{ var size = br.ReadUInt64();
Index = newIndex; var bytes = new byte[size];
br.BaseStream.Read(bytes, 0, (int) size);
return VirtualFile.Read(this, bytes);
}).ToList();
var newIndex = Index.Integrate(files);
lock (this)
{
Index = newIndex;
}
} }
} }
catch (IOException)
{
if (File.Exists(filename))
File.Delete(filename);
}
} }
public async Task<Action> Stage(IEnumerable<VirtualFile> files) public Action Stage(IEnumerable<VirtualFile> files)
{ {
var grouped = files.SelectMany(f => f.FilesInFullPath) var grouped = files.SelectMany(f => f.FilesInFullPath)
.Distinct() .Distinct()
@ -155,7 +181,7 @@ namespace Wabbajack.VirtualFileSystem
foreach (var group in grouped) foreach (var group in grouped)
{ {
var tmpPath = Path.Combine(_stagingFolder, Guid.NewGuid().ToString()); var tmpPath = Path.Combine(_stagingFolder, Guid.NewGuid().ToString());
await FileExtractor.ExtractAll(group.Key.StagedPath, tmpPath); FileExtractor.ExtractAll(group.Key.StagedPath, tmpPath);
paths.Add(tmpPath); paths.Add(tmpPath);
foreach (var file in group) foreach (var file in group)
file.StagedPath = Path.Combine(tmpPath, file.Name); file.StagedPath = Path.Combine(tmpPath, file.Name);
@ -184,16 +210,14 @@ namespace Wabbajack.VirtualFileSystem
}).ToList(); }).ToList();
} }
public async Task IntegrateFromPortable(List<PortableFile> state, Dictionary<string, string> links) public void IntegrateFromPortable(List<PortableFile> state, Dictionary<string, string> links)
{ {
var indexedState = state.GroupBy(f => f.ParentHash) var indexedState = state.GroupBy(f => f.ParentHash)
.ToDictionary(f => f.Key ?? "", f => (IEnumerable<PortableFile>) f); .ToDictionary(f => f.Key ?? "", f => (IEnumerable<PortableFile>) f);
var parents = await indexedState[""] var parents = indexedState[""]
.ToChannel() .PMap(f => VirtualFile.CreateFromPortable(this, indexedState, links, f));
.UnorderedPipelineSync(f => VirtualFile.CreateFromPortable(this, indexedState, links, f))
.TakeAll();
var newIndex = await Index.Integrate(parents); var newIndex = Index.Integrate(parents);
lock (this) lock (this)
{ {
Index = newIndex; Index = newIndex;
@ -202,7 +226,7 @@ namespace Wabbajack.VirtualFileSystem
public async Task<DisposableList<VirtualFile>> StageWith(IEnumerable<VirtualFile> files) public async Task<DisposableList<VirtualFile>> StageWith(IEnumerable<VirtualFile> files)
{ {
return new DisposableList<VirtualFile>(await Stage(files), files); return new DisposableList<VirtualFile>(Stage(files), files);
} }
@ -214,7 +238,7 @@ namespace Wabbajack.VirtualFileSystem
_knownFiles.AddRange(known); _knownFiles.AddRange(known);
} }
public async Task BackfillMissing() public void BackfillMissing()
{ {
var newFiles = _knownFiles.Where(f => f.Paths.Length == 1) var newFiles = _knownFiles.Where(f => f.Paths.Length == 1)
.GroupBy(f => f.Hash) .GroupBy(f => f.Hash)
@ -248,7 +272,7 @@ namespace Wabbajack.VirtualFileSystem
} }
_knownFiles.Where(f => f.Paths.Length > 1).Do(BackFillOne); _knownFiles.Where(f => f.Paths.Length > 1).Do(BackFillOne);
var newIndex = await Index.Integrate(newFiles.Values.ToList()); var newIndex = Index.Integrate(newFiles.Values.ToList());
lock (this) lock (this)
Index = newIndex; Index = newIndex;
@ -258,7 +282,6 @@ namespace Wabbajack.VirtualFileSystem
} }
#endregion #endregion
} }
public class KnownFile public class KnownFile
@ -315,30 +338,30 @@ namespace Wabbajack.VirtualFileSystem
public ImmutableDictionary<string, ImmutableStack<VirtualFile>> ByName { get; set; } public ImmutableDictionary<string, ImmutableStack<VirtualFile>> ByName { get; set; }
public ImmutableDictionary<string, VirtualFile> ByRootPath { get; } public ImmutableDictionary<string, VirtualFile> ByRootPath { get; }
public async Task<IndexRoot> Integrate(List<VirtualFile> files) public IndexRoot Integrate(List<VirtualFile> files)
{ {
Utils.Log($"Integrating");
var allFiles = AllFiles.Concat(files).GroupBy(f => f.Name).Select(g => g.Last()).ToImmutableList(); var allFiles = AllFiles.Concat(files).GroupBy(f => f.Name).Select(g => g.Last()).ToImmutableList();
var byFullPath = Task.Run(() => var byFullPath = allFiles.SelectMany(f => f.ThisAndAllChildren)
allFiles.SelectMany(f => f.ThisAndAllChildren) .ToImmutableDictionary(f => f.FullPath);
.ToImmutableDictionary(f => f.FullPath));
var byHash = Task.Run(() => var byHash = allFiles.SelectMany(f => f.ThisAndAllChildren)
allFiles.SelectMany(f => f.ThisAndAllChildren) .Where(f => f.Hash != null)
.Where(f => f.Hash != null) .ToGroupedImmutableDictionary(f => f.Hash);
.ToGroupedImmutableDictionary(f => f.Hash));
var byName = Task.Run(() => var byName = allFiles.SelectMany(f => f.ThisAndAllChildren)
allFiles.SelectMany(f => f.ThisAndAllChildren) .ToGroupedImmutableDictionary(f => f.Name);
.ToGroupedImmutableDictionary(f => f.Name));
var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.Name)); var byRootPath = allFiles.ToImmutableDictionary(f => f.Name);
return new IndexRoot(allFiles, var result = new IndexRoot(allFiles,
await byFullPath, byFullPath,
await byHash, byHash,
await byRootPath, byRootPath,
await byName); byName);
Utils.Log($"Done integrating");
return result;
} }
public VirtualFile FileForArchiveHashPath(string[] argArchiveHashPath) public VirtualFile FileForArchiveHashPath(string[] argArchiveHashPath)

25
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -117,11 +117,11 @@ namespace Wabbajack.VirtualFileSystem
} }
} }
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, string abs_path, public static VirtualFile Analyze(Context context, VirtualFile parent, string abs_path,
string rel_path) string rel_path)
{ {
var hasher = abs_path.FileHashAsync();
var fi = new FileInfo(abs_path); var fi = new FileInfo(abs_path);
context._logSpam.OnNext($"Analyzing {rel_path}");
var self = new VirtualFile var self = new VirtualFile
{ {
Context = context, Context = context,
@ -129,23 +129,26 @@ namespace Wabbajack.VirtualFileSystem
Parent = parent, Parent = parent,
Size = fi.Length, Size = fi.Length,
LastModified = fi.LastWriteTimeUtc.Ticks, LastModified = fi.LastWriteTimeUtc.Ticks,
LastAnalyzed = DateTime.Now.Ticks LastAnalyzed = DateTime.Now.Ticks,
Hash = abs_path.FileHash()
}; };
if (FileExtractor.CanExtract(Path.GetExtension(abs_path))) if (FileExtractor.CanExtract(Path.GetExtension(abs_path)))
{
context._logSpam.OnNext($"Extracting {rel_path}");
using (var tempFolder = context.GetTemporaryFolder()) using (var tempFolder = context.GetTemporaryFolder())
{ {
await FileExtractor.ExtractAll(abs_path, tempFolder.FullName); FileExtractor.ExtractAll(abs_path, tempFolder.FullName);
var results = Channel.Create<VirtualFile>(1024); context._logSpam.OnNext($"Analyzing Contents {rel_path}");
var files = Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories) self.Children = Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories)
.ToChannel() .PMap(abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName)))
.UnorderedPipeline(results, .ToImmutableList();
async abs_src => await Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName)));
self.Children = (await results.TakeAll()).ToImmutableList();
} }
self.Hash = await hasher; }
return self; return self;
} }