Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #876 from wabbajack-tools/retry-failed-io-ops

Browse Source 
Retry failed io ops
This commit is contained in:
Timothy Baldridge 2020-05-26 04:19:05 -07:00 committed by GitHub
commit b8f4fc2e49
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
68 changed files with 548 additions and 512 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Compression.BSA.Test/BSATests.cs
View File

@ -96,16 +96,16 @@ namespace Compression.BSA.Test
var tempFile = ((RelativePath)"tmp.bsa").RelativeToEntryPoint(); var tempFile = ((RelativePath)"tmp.bsa").RelativeToEntryPoint();
var size = bsa.Size; var size = bsa.Size;
await using var a = BSADispatch.OpenRead(bsa); await using var a = await BSADispatch.OpenRead(bsa);
await a.Files.PMap(Queue, file => await a.Files.PMap(Queue, async file =>
{ {
var absName = _tempDir.Combine(file.Path); var absName = _tempDir.Combine(file.Path);
ViaJson(file.State); ViaJson(file.State);
absName.Parent.CreateDirectory(); absName.Parent.CreateDirectory();
using (var fs = absName.Create()) await using (var fs = await absName.Create())
{ {
file.CopyDataTo(fs); await file.CopyDataTo(fs);
} }
Assert.Equal(file.Size, absName.Size); Assert.Equal(file.Size, absName.Size);
@ -123,7 +123,7 @@ namespace Compression.BSA.Test
var streams = await a.Files.PMap(Queue, async file => var streams = await a.Files.PMap(Queue, async file =>
{ {
var absPath = _tempDir.Combine(file.Path); var absPath = _tempDir.Combine(file.Path);
var str = absPath.OpenRead(); var str = await absPath.OpenRead();
await w.AddFile(ViaJson(file.State), str); await w.AddFile(ViaJson(file.State), str);
return str; return str;
}); });
@ -132,7 +132,7 @@ namespace Compression.BSA.Test
} }
TestContext.WriteLine($"Verifying {bsa}"); TestContext.WriteLine($"Verifying {bsa}");
await using var b = BSADispatch.OpenRead(tempFile); await using var b = await BSADispatch.OpenRead(tempFile);
TestContext.WriteLine($"Performing A/B tests on {bsa}"); TestContext.WriteLine($"Performing A/B tests on {bsa}");
Assert.Equal(a.State.ToJson(), b.State.ToJson()); Assert.Equal(a.State.ToJson(), b.State.ToJson());
@ -141,22 +141,22 @@ namespace Compression.BSA.Test
await a.Files.Zip(b.Files, (ai, bi) => (ai, bi)) await a.Files.Zip(b.Files, (ai, bi) => (ai, bi))
.PMap(Queue, pair => .PMap(Queue, async pair =>
{ {
Assert.Equal(pair.ai.State.ToJson(), pair.bi.State.ToJson()); Assert.Equal(pair.ai.State.ToJson(), pair.bi.State.ToJson());
//Console.WriteLine($" - {pair.ai.Path}"); //Console.WriteLine($" - {pair.ai.Path}");
Assert.Equal(pair.ai.Path, pair.bi.Path); Assert.Equal(pair.ai.Path, pair.bi.Path);
//Equal(pair.ai.Compressed, pair.bi.Compressed); //Equal(pair.ai.Compressed, pair.bi.Compressed);
Assert.Equal(pair.ai.Size, pair.bi.Size); Assert.Equal(pair.ai.Size, pair.bi.Size);
Assert.Equal(GetData(pair.ai), GetData(pair.bi)); Assert.Equal(await GetData(pair.ai), await GetData(pair.bi));
}); });
} }
} }
private static byte[] GetData(IFile pairAi) private static async ValueTask<byte[]> GetData(IFile pairAi)
{ {
using var ms = new MemoryStream(); await using var ms = new MemoryStream();
pairAi.CopyDataTo(ms); await pairAi.CopyDataTo(ms);
return ms.ToArray(); return ms.ToArray();
} }

2
Compression.BSA/BA2Builder.cs
View File

@ -58,7 +58,7 @@ namespace Compression.BSA
public async Task Build(AbsolutePath filename) public async Task Build(AbsolutePath filename)
{ {
SortEntries(); SortEntries();
await using var fs = filename.Create(); await using var fs = await filename.Create();
await using var bw = new BinaryWriter(fs); await using var bw = new BinaryWriter(fs);
bw.Write(Encoding.ASCII.GetBytes(_state.HeaderMagic)); bw.Write(Encoding.ASCII.GetBytes(_state.HeaderMagic));

97
Compression.BSA/BA2Reader.cs
View File

@ -38,19 +38,22 @@ namespace Compression.BSA
public bool HasNameTable => _nameTableOffset > 0; public bool HasNameTable => _nameTableOffset > 0;
public BA2Reader(AbsolutePath filename) : this(filename.OpenRead())
public static async Task<BA2Reader> Load(AbsolutePath filename)
{ {
_filename = filename; var rdr = new BA2Reader(await filename.OpenShared()) {_filename = filename};
await rdr.LoadHeaders();
return rdr;
} }
public BA2Reader(Stream stream) private BA2Reader(Stream stream)
{ {
_stream = stream; _stream = stream;
_rdr = new BinaryReader(_stream, Encoding.UTF7); _rdr = new BinaryReader(_stream, Encoding.UTF7);
LoadHeaders();
} }
public void LoadHeaders() private async Task LoadHeaders()
{ {
_headerMagic = Encoding.ASCII.GetString(_rdr.ReadBytes(4)); _headerMagic = Encoding.ASCII.GetString(_rdr.ReadBytes(4));
@ -196,39 +199,36 @@ namespace Compression.BSA
public uint HeaderSize => DDS.HeaderSizeForFormat((DXGI_FORMAT)_format); public uint HeaderSize => DDS.HeaderSizeForFormat((DXGI_FORMAT)_format);
public void CopyDataTo(Stream output) public async ValueTask CopyDataTo(Stream output)
{ {
var bw = new BinaryWriter(output); var bw = new BinaryWriter(output);
WriteHeader(bw); WriteHeader(bw);
using (var fs = _bsa._filename.OpenRead()) await using var fs = await _bsa._filename.OpenRead();
using (var br = new BinaryReader(fs)) using var br = new BinaryReader(fs);
foreach (var chunk in _chunks)
{ {
foreach (var chunk in _chunks) var full = new byte[chunk._fullSz];
var isCompressed = chunk._packSz != 0;
br.BaseStream.Seek((long)chunk._offset, SeekOrigin.Begin);
if (!isCompressed)
{ {
var full = new byte[chunk._fullSz]; await br.BaseStream.ReadAsync(full, 0, full.Length);
var isCompressed = chunk._packSz != 0; }
else
br.BaseStream.Seek((long)chunk._offset, SeekOrigin.Begin); {
byte[] compressed = new byte[chunk._packSz];
if (!isCompressed) await br.BaseStream.ReadAsync(compressed, 0, compressed.Length);
{ var inflater = new Inflater();
br.BaseStream.Read(full, 0, full.Length); inflater.SetInput(compressed);
} inflater.Inflate(full);
else
{
byte[] compressed = new byte[chunk._packSz];
br.BaseStream.Read(compressed, 0, compressed.Length);
var inflater = new Inflater();
inflater.SetInput(compressed);
inflater.Inflate(full);
}
bw.BaseStream.Write(full, 0, full.Length);
} }
}
await bw.BaseStream.WriteAsync(full, 0, full.Length);
}
} }
public void Dump(Action<string> print) public void Dump(Action<string> print)
@ -480,31 +480,28 @@ namespace Compression.BSA
public uint Size => _realSize; public uint Size => _realSize;
public FileStateObject State => new BA2FileEntryState(this); public FileStateObject State => new BA2FileEntryState(this);
public void CopyDataTo(Stream output) public async ValueTask CopyDataTo(Stream output)
{ {
using (var fs = _bsa._filename.OpenRead()) await using var fs = await _bsa._filename.OpenRead();
fs.Seek((long) _offset, SeekOrigin.Begin);
uint len = Compressed ? _size : _realSize;
var bytes = new byte[len];
fs.Read(bytes, 0, (int) len);
if (!Compressed)
{ {
fs.Seek((long) _offset, SeekOrigin.Begin); await output.WriteAsync(bytes, 0, bytes.Length);
uint len = Compressed ? _size : _realSize; }
else
var bytes = new byte[len]; {
fs.Read(bytes, 0, (int) len); var uncompressed = new byte[_realSize];
var inflater = new Inflater();
if (!Compressed) inflater.SetInput(bytes);
{ inflater.Inflate(uncompressed);
output.Write(bytes, 0, bytes.Length); await output.WriteAsync(uncompressed, 0, uncompressed.Length);
}
else
{
var uncompressed = new byte[_realSize];
var inflater = new Inflater();
inflater.SetInput(bytes);
inflater.Inflate(uncompressed);
output.Write(uncompressed, 0, uncompressed.Length);
}
} }
} }
} }
[JsonName("BA2FileEntryState")] [JsonName("BA2FileEntryState")]

2
Compression.BSA/BSABuilder.cs
View File

@ -95,7 +95,7 @@ namespace Compression.BSA
public async Task Build(AbsolutePath outputName) public async Task Build(AbsolutePath outputName)
{ {
RegenFolderRecords(); RegenFolderRecords();
await using var fs = outputName.Create(); await using var fs = await outputName.Create();
await using var wtr = new BinaryWriter(fs); await using var wtr = new BinaryWriter(fs);
wtr.Write(_fileId); wtr.Write(_fileId);

15
Compression.BSA/BSADispatch.cs
View File

@ -1,33 +1,34 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Text; using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common; using Wabbajack.Common;
namespace Compression.BSA namespace Compression.BSA
{ {
public static class BSADispatch public static class BSADispatch
{ {
public static IBSAReader OpenRead(AbsolutePath filename) public static async ValueTask<IBSAReader> OpenRead(AbsolutePath filename)
{ {
var fourcc = ""; var fourcc = "";
using (var file = filename.OpenRead()) using (var file = await filename.OpenRead())
{ {
fourcc = Encoding.ASCII.GetString(new BinaryReader(file).ReadBytes(4)); fourcc = Encoding.ASCII.GetString(new BinaryReader(file).ReadBytes(4));
} }
if (fourcc == TES3Reader.TES3_MAGIC) if (fourcc == TES3Reader.TES3_MAGIC)
return new TES3Reader(filename); return await TES3Reader.Load(filename);
if (fourcc == "BSA\0") if (fourcc == "BSA\0")
return new BSAReader(filename); return await BSAReader.Load(filename);
if (fourcc == "BTDX") if (fourcc == "BTDX")
return new BA2Reader(filename); return await BA2Reader.Load(filename);
throw new InvalidDataException("Filename is not a .bsa or .ba2, magic " + fourcc); throw new InvalidDataException("Filename is not a .bsa or .ba2, magic " + fourcc);
} }
private static HashSet<string> MagicStrings = new HashSet<string> {TES3Reader.TES3_MAGIC, "BSA\0", "BTDX"}; private static HashSet<string> MagicStrings = new HashSet<string> {TES3Reader.TES3_MAGIC, "BSA\0", "BTDX"};
public static bool MightBeBSA(AbsolutePath filename) public static async ValueTask<bool> MightBeBSA(AbsolutePath filename)
{ {
using var file = filename.OpenRead(); using var file = await filename.OpenRead();
var fourcc = Encoding.ASCII.GetString(new BinaryReader(file).ReadBytes(4)); var fourcc = Encoding.ASCII.GetString(new BinaryReader(file).ReadBytes(4));
return MagicStrings.Contains(fourcc); return MagicStrings.Contains(fourcc);
} }

70
Compression.BSA/BSAReader.cs
View File

@ -60,8 +60,8 @@ namespace Compression.BSA
internal uint _folderRecordOffset; internal uint _folderRecordOffset;
private List<FolderRecord> _folders; private List<FolderRecord> _folders;
internal string _magic; internal string _magic;
private readonly BinaryReader _rdr; private BinaryReader _rdr;
private readonly Stream _stream; private Stream _stream;
internal uint _totalFileNameLength; internal uint _totalFileNameLength;
internal uint _totalFolderNameLength; internal uint _totalFolderNameLength;
internal uint _version; internal uint _version;
@ -80,16 +80,15 @@ namespace Compression.BSA
} }
public BSAReader(AbsolutePath filename) public static async ValueTask<BSAReader> Load(AbsolutePath filename)
{ {
_fileName = filename; using var stream = await filename.OpenRead();
using var stream = filename.OpenRead();
using var br = new BinaryReader(stream); using var br = new BinaryReader(stream);
_rdr = br; var bsa = new BSAReader {_rdr = br, _stream = stream, _fileName = filename};
_stream = stream; await bsa.LoadHeaders();
LoadHeaders(); bsa._rdr = null;
_rdr = null; bsa._stream = null;
_stream = null; return bsa;
} }
public IEnumerable<IFile> Files public IEnumerable<IFile> Files
@ -132,7 +131,7 @@ namespace Compression.BSA
{ {
} }
private void LoadHeaders() private async ValueTask LoadHeaders()
{ {
var fourcc = Encoding.ASCII.GetString(_rdr.ReadBytes(4)); var fourcc = Encoding.ASCII.GetString(_rdr.ReadBytes(4));
@ -330,43 +329,34 @@ namespace Compression.BSA
_name = rdr.ReadStringTerm(_bsa.HeaderType); _name = rdr.ReadStringTerm(_bsa.HeaderType);
} }
public void CopyDataTo(Stream output) public async ValueTask CopyDataTo(Stream output)
{ {
using (var in_file = _bsa._fileName.OpenRead()) await using var in_file = await _bsa._fileName.OpenRead();
using (var rdr = new BinaryReader(in_file)) using var rdr = new BinaryReader(in_file);
{ rdr.BaseStream.Position = _dataOffset;
rdr.BaseStream.Position = _dataOffset;
if (_bsa.HeaderType == VersionType.SSE) if (_bsa.HeaderType == VersionType.SSE)
{
if (Compressed)
{ {
if (Compressed) using var r = LZ4Stream.Decode(rdr.BaseStream);
{ await r.CopyToLimitAsync(output, (int) _originalSize);
using var r = LZ4Stream.Decode(rdr.BaseStream);
r.CopyToLimit(output, (int) _originalSize);
}
else
{
rdr.BaseStream.CopyToLimit(output, (int) _onDiskSize);
}
} }
else else
{ {
if (Compressed) await rdr.BaseStream.CopyToLimitAsync(output, (int) _onDiskSize);
{
using var z = new InflaterInputStream(rdr.BaseStream);
z.CopyToLimit(output, (int) _originalSize);
}
else
rdr.BaseStream.CopyToLimit(output, (int) _onDiskSize);
} }
} }
} else
{
public byte[] GetData() if (Compressed)
{ {
var ms = new MemoryStream(); await using var z = new InflaterInputStream(rdr.BaseStream);
CopyDataTo(ms); await z.CopyToLimitAsync(output, (int) _originalSize);
return ms.ToArray(); }
else
await rdr.BaseStream.CopyToLimitAsync(output, (int) _onDiskSize);
}
} }
} }

2
Compression.BSA/IBSAReader.cs
View File

@ -60,7 +60,7 @@ namespace Compression.BSA
/// in order to maintain thread-safe access. /// in order to maintain thread-safe access.
/// </summary> /// </summary>
/// <param name="output"></param> /// <param name="output"></param>
void CopyDataTo(Stream output); ValueTask CopyDataTo(Stream output);
void Dump(Action<string> print); void Dump(Action<string> print);
} }

2
Compression.BSA/TES3Builder.cs
View File

@ -26,7 +26,7 @@ namespace Compression.BSA
public async Task Build(AbsolutePath filename) public async Task Build(AbsolutePath filename)
{ {
await using var fs = filename.Create(); await using var fs = await filename.Create();
await using var bw = new BinaryWriter(fs); await using var bw = new BinaryWriter(fs);
bw.Write(_state.VersionNumber); bw.Write(_state.VersionNumber);

50
Compression.BSA/TES3Reader.cs
View File

@ -18,48 +18,52 @@ namespace Compression.BSA
internal long _dataOffset; internal long _dataOffset;
internal AbsolutePath _filename; internal AbsolutePath _filename;
public TES3Reader(AbsolutePath filename) public static async ValueTask<TES3Reader> Load(AbsolutePath filename)
{ {
_filename = filename; await using var fs = await filename.OpenRead();
using var fs = filename.OpenRead();
using var br = new BinaryReader(fs); using var br = new BinaryReader(fs);
_versionNumber = br.ReadUInt32(); var rdr = new TES3Reader
_hashTableOffset = br.ReadUInt32(); {
_fileCount = br.ReadUInt32(); _filename = filename,
_versionNumber = br.ReadUInt32(),
_hashTableOffset = br.ReadUInt32(),
_fileCount = br.ReadUInt32()
};
_files = new TES3FileEntry[_fileCount]; rdr._files = new TES3FileEntry[rdr._fileCount];
for (int i = 0; i < _fileCount; i++) for (int i = 0; i < rdr._fileCount; i++)
{ {
var file = new TES3FileEntry { var file = new TES3FileEntry {
Index = i, Index = i,
Archive = this, Archive = rdr,
Size = br.ReadUInt32(), Size = br.ReadUInt32(),
Offset = br.ReadUInt32() Offset = br.ReadUInt32()
}; };
_files[i] = file; rdr._files[i] = file;
} }
for (int i = 0; i < _fileCount; i++) for (int i = 0; i < rdr._fileCount; i++)
{ {
_files[i].NameOffset = br.ReadUInt32(); rdr._files[i].NameOffset = br.ReadUInt32();
} }
var origPos = br.BaseStream.Position; var origPos = br.BaseStream.Position;
for (int i = 0; i < _fileCount; i++) for (int i = 0; i < rdr._fileCount; i++)
{ {
br.BaseStream.Position = origPos + _files[i].NameOffset; br.BaseStream.Position = origPos + rdr._files[i].NameOffset;
_files[i].Path = new RelativePath(br.ReadStringTerm(VersionType.TES3)); rdr._files[i].Path = new RelativePath(br.ReadStringTerm(VersionType.TES3));
} }
br.BaseStream.Position = _hashTableOffset + 12; br.BaseStream.Position = rdr._hashTableOffset + 12;
for (int i = 0; i < _fileCount; i++) for (int i = 0; i < rdr._fileCount; i++)
{ {
_files[i].Hash1 = br.ReadUInt32(); rdr._files[i].Hash1 = br.ReadUInt32();
_files[i].Hash2 = br.ReadUInt32(); rdr._files[i].Hash2 = br.ReadUInt32();
} }
_dataOffset = br.BaseStream.Position; rdr._dataOffset = br.BaseStream.Position;
return rdr;
} }
public async ValueTask DisposeAsync() public async ValueTask DisposeAsync()
@ -118,11 +122,11 @@ namespace Compression.BSA
Hash2 = Hash2 Hash2 = Hash2
}; };
public void CopyDataTo(Stream output) public async ValueTask CopyDataTo(Stream output)
{ {
using var fs = Archive._filename.OpenRead(); await using var fs = await Archive._filename.OpenRead();
fs.Position = Archive._dataOffset + Offset; fs.Position = Archive._dataOffset + Offset;
fs.CopyToLimit(output, (int)Size); await fs.CopyToLimitAsync(output, (int)Size);
} }
public void Dump(Action<string> print) public void Dump(Action<string> print)

8
Wabbajack.App.Test/SettingsTests.cs
View File

@ -8,16 +8,16 @@ namespace Wabbajack.Test
[Fact] [Fact]
public async Task CanSaveAndLoadSettings() public async Task CanSaveAndLoadSettings()
{ {
MainSettings.TryLoadTypicalSettings(out var settings); var (settings, loadedSettings) = await MainSettings.TryLoadTypicalSettings();
if (settings == null) if (settings == null || !loadedSettings)
{ {
settings = new MainSettings(); settings = new MainSettings();
} }
MainSettings.SaveSettings(settings); await MainSettings.SaveSettings(settings);
Assert.True(MainSettings.TryLoadTypicalSettings(out settings)); Assert.True((await MainSettings.TryLoadTypicalSettings()).loaded);
} }
} }

4
Wabbajack.CLI/Verbs/BSADump.cs
View File

@ -14,8 +14,8 @@ namespace Wabbajack.CLI.Verbs
protected override async Task<ExitCode> Run() protected override async Task<ExitCode> Run()
{ {
await using var bsa = BSADispatch.OpenRead(Input.RelativeTo(AbsolutePath.GetCurrentDirectory())); await using var bsa = await BSADispatch.OpenRead(Input.RelativeTo(AbsolutePath.GetCurrentDirectory()));
bsa.Dump(line => Console.WriteLine(line)); bsa.Dump(Console.WriteLine);
return ExitCode.Ok; return ExitCode.Ok;
} }
} }

2
Wabbajack.CLI/Verbs/Decrypt.cs
View File

@ -16,7 +16,7 @@ namespace Wabbajack.CLI.Verbs
protected override async Task<ExitCode> Run() protected override async Task<ExitCode> Run()
{ {
File.WriteAllBytes(Output, Utils.FromEncryptedData(Name)); await Output.RelativeTo(AbsolutePath.EntryPoint).WriteAllBytesAsync(await Utils.FromEncryptedData(Name));
return 0; return 0;
} }
} }

2
Wabbajack.CLI/Verbs/Encrypt.cs
View File

@ -18,7 +18,7 @@ namespace Wabbajack.CLI.Verbs
protected override async Task<ExitCode> Run() protected override async Task<ExitCode> Run()
{ {
File.ReadAllBytes(Input).ToEcryptedData(Name); await File.ReadAllBytes(Input).ToEcryptedData(Name);
return 0; return 0;
} }
} }

10
Wabbajack.Common.Test/EncryptedDataTests.cs
View File

@ -11,20 +11,20 @@ namespace Wabbajack.Common.Test
[Fact] [Fact]
public async Task CanDetectNewEncryptedData() public async Task CanDetectNewEncryptedData()
{ {
var test_string = Guid.NewGuid().ToString(); var testString = Guid.NewGuid().ToString();
var data = new ConcurrentBag<string>(); var data = new ConcurrentBag<string>();
var events = Utils.HaveEncryptedJsonObservable(test_string).Subscribe(e => var events = Utils.HaveEncryptedJsonObservable(testString).Subscribe(e =>
{ {
if (e) if (e)
data.Add(test_string); data.Add(testString);
else else
data.Clear(); data.Clear();
}); });
test_string.ToEcryptedJson(test_string); await testString.ToEcryptedJson(testString);
await Task.Delay(100); await Task.Delay(100);
Assert.Contains(test_string, data); Assert.Contains(testString, data);
} }

13
Wabbajack.Common.Test/PathTests.cs
View File

@ -1,28 +1,29 @@
using Xunit; using System.Threading.Tasks;
using Xunit;
namespace Wabbajack.Common.Test namespace Wabbajack.Common.Test
{ {
public class PathTests public class PathTests
{ {
[Fact] [Fact]
public void CanDeleteReadOnlyFile() public async Task CanDeleteReadOnlyFile()
{ {
var tempFile = new TempFile(); var tempFile = new TempFile();
tempFile.Path.WriteAllText("Test"); await tempFile.Path.WriteAllTextAsync("Test");
tempFile.Path.SetReadOnly(true); tempFile.Path.SetReadOnly(true);
tempFile.Path.Delete(); tempFile.Path.Delete();
} }
[Fact] [Fact]
public void CanMoveReadOnlyFiles() public async Task CanMoveReadOnlyFiles()
{ {
var tempFile = new TempFile(); var tempFile = new TempFile();
var tempFile2 = new TempFile(); var tempFile2 = new TempFile();
tempFile.Path.WriteAllText("Test"); await tempFile.Path.WriteAllTextAsync("Test");
tempFile.Path.SetReadOnly(true); tempFile.Path.SetReadOnly(true);
tempFile.Path.MoveTo(tempFile2.Path); await tempFile.Path.MoveToAsync(tempFile2.Path);
} }
[Fact] [Fact]

37
Wabbajack.Common/CircuitBreaker/WithAutoRetry.cs Normal file
View File

@ -0,0 +1,37 @@
using System;
using System.Threading.Tasks;
namespace Wabbajack.Common
{
public static class CircuitBreaker
{
public static TimeSpan DEFAULT_DELAY = TimeSpan.FromMilliseconds(100);
public static int DEFAULT_DELAY_MULTIPLIER = 2;
public static int DEFAULT_RETRIES = 5;
public static async ValueTask<TR> WithAutoRetry<TR, TE>(Func<ValueTask<TR>> f, TimeSpan? delay = null, int? multipler = null, int? maxRetries = null) where TE : Exception
{
int retries = 0;
delay ??= DEFAULT_DELAY;
multipler ??= DEFAULT_DELAY_MULTIPLIER;
maxRetries ??= DEFAULT_RETRIES;
TOP:
try
{
return await f();
}
catch (TE ex)
{
retries += 1;
if (retries > maxRetries)
throw;
Utils.Log($"(Retry {retries} of {maxRetries}), got exception {ex.Message}, waiting {delay.Value.TotalMilliseconds}ms");
await Task.Delay(delay.Value);
delay = delay * multipler;
goto TOP;
}
}
}
}

31
Wabbajack.Common/Hash.cs
View File

@ -129,21 +129,6 @@ namespace Wabbajack.Common
return sha.Hash.ToHex(); return sha.Hash.ToHex();
} }
public static Hash FileHash(this AbsolutePath file, bool nullOnIoError = false)
{
try
{
using var fs = file.OpenRead();
var config = new xxHashConfig {HashSizeInBits = 64};
using var f = new StatusFileStream(fs, $"Hashing {(string)file.FileName}");
return new Hash(BitConverter.ToUInt64(xxHashFactory.Instance.Create(config).ComputeHash(f).Hash));
}
catch (IOException)
{
if (nullOnIoError) return Hash.Empty;
throw;
}
}
public static Hash xxHash(this byte[] data) public static Hash xxHash(this byte[] data)
{ {
var hash = new xxHashConfig(); var hash = new xxHashConfig();
@ -166,16 +151,14 @@ namespace Wabbajack.Common
var value = xxHashFactory.Instance.Create(config).ComputeHash(f); var value = xxHashFactory.Instance.Create(config).ComputeHash(f);
return Hash.FromULong(BitConverter.ToUInt64(value.Hash)); return Hash.FromULong(BitConverter.ToUInt64(value.Hash));
} }
public static Hash FileHashCached(this AbsolutePath file, bool nullOnIoError = false)
public static async Task<Hash> xxHashAsync(this Stream stream)
{ {
if (TryGetHashCache(file, out var foundHash)) return foundHash; var config = new xxHashConfig {HashSizeInBits = 64};
await using var f = new StatusFileStream(stream, $"Hashing memory stream");
var hash = file.FileHash(nullOnIoError); var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(f);
if (hash != Hash.Empty) return Hash.FromULong(BitConverter.ToUInt64(value.Hash));
WriteHashCache(file, hash);
return hash;
} }
public static bool TryGetHashCache(AbsolutePath file, out Hash hash) public static bool TryGetHashCache(AbsolutePath file, out Hash hash)
{ {
var normPath = Encoding.UTF8.GetBytes(file.Normalize()); var normPath = Encoding.UTF8.GetBytes(file.Normalize());
@ -223,7 +206,7 @@ namespace Wabbajack.Common
{ {
try try
{ {
await using var fs = file.OpenRead(); await using var fs = await file.OpenRead();
var config = new xxHashConfig {HashSizeInBits = 64}; var config = new xxHashConfig {HashSizeInBits = 64};
await using var hs = new StatusFileStream(fs, $"Hashing {file}"); await using var hs = new StatusFileStream(fs, $"Hashing {file}");
var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(hs); var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(hs);

52
Wabbajack.Common/IAsyncEnumerableExtensions.cs Normal file
View File

@ -0,0 +1,52 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace Wabbajack.Common
{
public static class IAsyncEnumerableExtensions
{
/// <summary>
/// Same as .Select but expects a function that returns an async result
/// </summary>
/// <param name="coll"></param>
/// <param name="mapFn"></param>
/// <typeparam name="TIn"></typeparam>
/// <typeparam name="TOut"></typeparam>
/// <returns></returns>
public static async IAsyncEnumerable<TOut> SelectAsync<TIn, TOut>(this IEnumerable<TIn> coll,
Func<TIn, ValueTask<TOut>> mapFn)
{
foreach (var itm in coll)
{
yield return await mapFn(itm);
}
}
/// <summary>
/// Same as .Select but expects a function that returns an async result
/// </summary>
/// <param name="coll"></param>
/// <param name="mapFn"></param>
/// <typeparam name="TIn"></typeparam>
/// <typeparam name="TOut"></typeparam>
/// <returns></returns>
public static async ValueTask DoAsync<TIn, TOut>(this IEnumerable<TIn> coll,
Func<TIn, ValueTask<TOut>> mapFn)
{
foreach (var itm in coll)
{
await mapFn(itm);
}
}
public static async ValueTask<List<T>> ToList<T>(this IAsyncEnumerable<T> coll)
{
var list =new List<T>();
await foreach (var itm in coll)
list.Add(itm);
return list;
}
}
}

5
Wabbajack.Common/Json.cs
View File

@ -5,6 +5,7 @@ using System.IO;
using System.Linq; using System.Linq;
using System.Reflection; using System.Reflection;
using System.Text; using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json; using Newtonsoft.Json;
using Newtonsoft.Json.Serialization; using Newtonsoft.Json.Serialization;
using Wabbajack.Common.Serialization.Json; using Wabbajack.Common.Serialization.Json;
@ -57,9 +58,9 @@ namespace Wabbajack.Common
ser.Serialize(writer, obj); ser.Serialize(writer, obj);
} }
public static void ToJson<T>(this T obj, AbsolutePath path) public static async ValueTask ToJsonAsync<T>(this T obj, AbsolutePath path)
{ {
using var fs = path.Create(); await using var fs = await path.Create();
obj.ToJson(fs); obj.ToJson(fs);
} }

4
Wabbajack.Common/Metrics.cs
View File

@ -17,7 +17,7 @@ namespace Wabbajack.Common
{ {
if (!Utils.HaveEncryptedJson(Consts.MetricsKeyHeader)) if (!Utils.HaveEncryptedJson(Consts.MetricsKeyHeader))
{ {
Utils.ToEcryptedJson(Utils.MakeRandomKey(), Consts.MetricsKeyHeader); Utils.MakeRandomKey().ToEcryptedJson(Consts.MetricsKeyHeader).AsTask().Wait();
} }
} }
/// <summary> /// <summary>
@ -32,7 +32,7 @@ namespace Wabbajack.Common
try try
{ {
client.DefaultRequestHeaders.Add(Consts.MetricsKeyHeader, client.DefaultRequestHeaders.Add(Consts.MetricsKeyHeader,
Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader)); await Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader));
await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{value}"); await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{value}");
} }
catch (Exception) catch (Exception)

89
Wabbajack.Common/Paths.cs
View File

@ -87,19 +87,21 @@ namespace Wabbajack.Common
public Extension Extension => Extension.FromPath(_path); public Extension Extension => Extension.FromPath(_path);
public FileStream OpenRead() public ValueTask<FileStream> OpenRead()
{ {
return File.OpenRead(_path); return OpenShared();
} }
public FileStream Create() public ValueTask<FileStream> Create()
{ {
return File.Create(_path); var path = _path;
return CircuitBreaker.WithAutoRetry<FileStream, IOException>(async () => File.Create(path));
} }
public FileStream OpenWrite() public ValueTask<FileStream> OpenWrite()
{ {
return File.OpenWrite(_path); var path = _path;
return CircuitBreaker.WithAutoRetry<FileStream, IOException>(async () => File.OpenWrite(path));
} }
public async Task WriteAllTextAsync(string text) public async Task WriteAllTextAsync(string text)
@ -107,12 +109,6 @@ namespace Wabbajack.Common
await using var fs = File.Create(_path); await using var fs = File.Create(_path);
await fs.WriteAsync(Encoding.UTF8.GetBytes(text)); await fs.WriteAsync(Encoding.UTF8.GetBytes(text));
} }
public void WriteAllText(string text)
{
using var fs = File.Create(_path);
fs.Write(Encoding.UTF8.GetBytes(text));
}
public bool Exists => File.Exists(_path) || Directory.Exists(_path); public bool Exists => File.Exists(_path) || Directory.Exists(_path);
public bool IsFile => File.Exists(_path); public bool IsFile => File.Exists(_path);
@ -175,24 +171,6 @@ namespace Wabbajack.Common
public AbsolutePath Root => (AbsolutePath)Path.GetPathRoot(_path); public AbsolutePath Root => (AbsolutePath)Path.GetPathRoot(_path);
/// <summary>
/// Moves this file to the specified location, will use Copy if required
/// </summary>
/// <param name="otherPath"></param>
/// <param name="overwrite">Replace the destination file if it exists</param>
public void MoveTo(AbsolutePath otherPath, bool overwrite = false)
{
if (Root != otherPath.Root)
{
if (otherPath.Exists && overwrite)
otherPath.Delete();
CopyTo(otherPath);
return;
}
File.Move(_path, otherPath._path, overwrite ? MoveOptions.ReplaceExisting : MoveOptions.None);
}
/// <summary> /// <summary>
/// Moves this file to the specified location, will use Copy if required /// Moves this file to the specified location, will use Copy if required
/// </summary> /// </summary>
@ -286,7 +264,7 @@ namespace Wabbajack.Common
public async Task<byte[]> ReadAllBytesAsync() public async Task<byte[]> ReadAllBytesAsync()
{ {
await using var f = OpenRead(); await using var f = await OpenShared();
return await f.ReadAllAsync(); return await f.ReadAllAsync();
} }
@ -325,37 +303,19 @@ namespace Wabbajack.Common
return File.ReadAllLines(_path); return File.ReadAllLines(_path);
} }
public void WriteAllBytes(byte[] data)
{
using var fs = Create();
fs.Write(data);
}
public async Task WriteAllBytesAsync(byte[] data) public async Task WriteAllBytesAsync(byte[] data)
{ {
await using var fs = Create(); await using var fs = await Create();
await fs.WriteAsync(data); await fs.WriteAsync(data);
} }
public async Task WriteAllAsync(Stream data, bool disposeAfter = true) public async Task WriteAllAsync(Stream data, bool disposeAfter = true)
{ {
await using var fs = Create(); await using var fs = await Create();
await data.CopyToAsync(fs); await data.CopyToAsync(fs);
if (disposeAfter) await data.DisposeAsync(); if (disposeAfter) await data.DisposeAsync();
} }
public void AppendAllText(string text)
{
File.AppendAllText(_path, text);
}
public void CopyTo(AbsolutePath dest)
{
File.Copy(_path, dest._path);
}
[DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Auto)] [DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Auto)]
private static extern bool CreateHardLink(string lpFileName, string lpExistingFileName, IntPtr lpSecurityAttributes); private static extern bool CreateHardLink(string lpFileName, string lpExistingFileName, IntPtr lpSecurityAttributes);
@ -384,12 +344,6 @@ namespace Wabbajack.Common
return (await ReadAllTextAsync()).Split(new[] {'\n', '\r'}, StringSplitOptions.RemoveEmptyEntries); return (await ReadAllTextAsync()).Split(new[] {'\n', '\r'}, StringSplitOptions.RemoveEmptyEntries);
} }
public byte[] ReadAllBytes()
{
using var file = OpenShared();
return file.ReadAll();
}
public static AbsolutePath GetCurrentDirectory() public static AbsolutePath GetCurrentDirectory()
{ {
return new AbsolutePath(Directory.GetCurrentDirectory()); return new AbsolutePath(Directory.GetCurrentDirectory());
@ -397,8 +351,8 @@ namespace Wabbajack.Common
public async Task CopyToAsync(AbsolutePath destFile) public async Task CopyToAsync(AbsolutePath destFile)
{ {
await using var src = OpenRead(); await using var src = await OpenRead();
await using var dest = destFile.Create(); await using var dest = await destFile.Create();
await src.CopyToAsync(dest); await src.CopyToAsync(dest);
} }
@ -413,11 +367,6 @@ namespace Wabbajack.Common
await WriteAllTextAsync(string.Join("\r\n",strings)); await WriteAllTextAsync(string.Join("\r\n",strings));
} }
public void WriteAllLines(params string[] strings)
{
WriteAllText(string.Join("\n",strings));
}
public int CompareTo(AbsolutePath other) public int CompareTo(AbsolutePath other)
{ {
return string.Compare(_path, other._path, StringComparison.Ordinal); return string.Compare(_path, other._path, StringComparison.Ordinal);
@ -428,14 +377,18 @@ namespace Wabbajack.Common
return File.ReadAllText(_path); return File.ReadAllText(_path);
} }
public FileStream OpenShared() public ValueTask<FileStream> OpenShared()
{ {
return File.Open(_path, FileMode.Open, FileAccess.Read, FileShare.Read); var path = _path;
return CircuitBreaker.WithAutoRetry<FileStream, IOException>(async () =>
File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read));
} }
public FileStream WriteShared() public ValueTask<FileStream> WriteShared()
{ {
return File.Open(_path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite); var path = _path;
return CircuitBreaker.WithAutoRetry<FileStream, IOException>(async () =>
File.Open(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite));
} }
public async Task CopyDirectoryToAsync(AbsolutePath destination) public async Task CopyDirectoryToAsync(AbsolutePath destination)

53
Wabbajack.Common/Utils.cs
View File

@ -68,7 +68,7 @@ namespace Wabbajack.Common
if (LogFile.Exists) if (LogFile.Exists)
{ {
var newPath = Consts.LogsFolder.Combine(Consts.EntryPoint.FileNameWithoutExtension + LogFile.LastModified.ToString(" yyyy-MM-dd HH_mm_ss") + ".log"); var newPath = Consts.LogsFolder.Combine(Consts.EntryPoint.FileNameWithoutExtension + LogFile.LastModified.ToString(" yyyy-MM-dd HH_mm_ss") + ".log");
LogFile.MoveTo(newPath, true); LogFile.MoveToAsync(newPath, true).Wait();
} }
var logFiles = LogFolder.EnumerateFiles(false).ToList(); var logFiles = LogFolder.EnumerateFiles(false).ToList();
@ -165,7 +165,7 @@ namespace Wabbajack.Common
if (LogFile == default) return; if (LogFile == default) return;
lock (_lock) lock (_lock)
{ {
LogFile.AppendAllText($"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}\r\n"); File.AppendAllText(LogFile.ToString(), $"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}\r\n");
} }
} }
@ -300,6 +300,17 @@ namespace Wabbajack.Common
{ {
foreach (var i in coll) f(i); foreach (var i in coll) f(i);
} }
/// <summary>
/// Executes the action for every item in coll
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="coll"></param>
/// <param name="f"></param>
public static async Task DoAsync<T>(this IEnumerable<T> coll, Func<T, Task> f)
{
foreach (var i in coll) await f(i);
}
public static void DoIndexed<T>(this IEnumerable<T> coll, Action<int, T> f) public static void DoIndexed<T>(this IEnumerable<T> coll, Action<int, T> f)
{ {
@ -726,7 +737,7 @@ namespace Wabbajack.Common
RETRY_OPEN: RETRY_OPEN:
try try
{ {
await using var f = cacheFile.OpenRead(); await using var f = await cacheFile.OpenRead();
await f.CopyToAsync(output); await f.CopyToAsync(output);
} }
catch (IOException) catch (IOException)
@ -741,7 +752,7 @@ namespace Wabbajack.Common
{ {
var tmpName = Consts.PatchCacheFolder.Combine(Guid.NewGuid() + ".tmp"); var tmpName = Consts.PatchCacheFolder.Combine(Guid.NewGuid() + ".tmp");
await using (var f = tmpName.Create()) await using (var f = await tmpName.Create())
{ {
Status("Creating Patch"); Status("Creating Patch");
OctoDiff.Create(a, b, f); OctoDiff.Create(a, b, f);
@ -750,7 +761,7 @@ namespace Wabbajack.Common
RETRY: RETRY:
try try
{ {
tmpName.MoveTo(cacheFile, true); await tmpName.MoveToAsync(cacheFile, true);
} }
catch (UnauthorizedAccessException) catch (UnauthorizedAccessException)
{ {
@ -775,7 +786,7 @@ namespace Wabbajack.Common
patchStream.Position = 0; patchStream.Position = 0;
var tmpName = Consts.PatchCacheFolder.Combine(Guid.NewGuid() + ".tmp"); var tmpName = Consts.PatchCacheFolder.Combine(Guid.NewGuid() + ".tmp");
await using (var f = tmpName.Create()) await using (var f = await tmpName.Create())
{ {
await patchStream.CopyToAsync(f); await patchStream.CopyToAsync(f);
patchStream.Position = 0; patchStream.Position = 0;
@ -786,7 +797,7 @@ namespace Wabbajack.Common
var cacheFile = Consts.PatchCacheFolder.Combine($"{srcHash.ToHex()}_{destHash.ToHex()}.patch"); var cacheFile = Consts.PatchCacheFolder.Combine($"{srcHash.ToHex()}_{destHash.ToHex()}.patch");
Consts.PatchCacheFolder.CreateDirectory(); Consts.PatchCacheFolder.CreateDirectory();
tmpName.MoveTo(cacheFile, true); await tmpName.MoveToAsync(cacheFile, true);
} }
catch (UnauthorizedAccessException) catch (UnauthorizedAccessException)
{ {
@ -794,16 +805,16 @@ namespace Wabbajack.Common
} }
} }
public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch) public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out AbsolutePath ePatch)
{ {
var patchName = Consts.PatchCacheFolder.Combine($"{foundHash.ToHex()}_{fileHash.ToHex()}.patch"); var patchName = Consts.PatchCacheFolder.Combine($"{foundHash.ToHex()}_{fileHash.ToHex()}.patch");
if (patchName.Exists) if (patchName.Exists)
{ {
ePatch = patchName.ReadAllBytes(); ePatch = patchName;
return true; return true;
} }
ePatch = Array.Empty<byte>(); ePatch = default;
return false; return false;
} }
@ -900,7 +911,7 @@ namespace Wabbajack.Common
var startTime = DateTime.Now; var startTime = DateTime.Now;
var seconds = 2; var seconds = 2;
var results = await Enumerable.Range(0, queue.DesiredNumWorkers) var results = await Enumerable.Range(0, queue.DesiredNumWorkers)
.PMap(queue, idx => .PMap(queue, async idx =>
{ {
var random = new Random(); var random = new Random();
@ -908,7 +919,7 @@ namespace Wabbajack.Common
long size = 0; long size = 0;
byte[] buffer = new byte[1024 * 8]; byte[] buffer = new byte[1024 * 8];
random.NextBytes(buffer); random.NextBytes(buffer);
using (var fs = file.Create()) await using (var fs = await file.Create())
{ {
while (DateTime.Now < startTime + new TimeSpan(0, 0, seconds)) while (DateTime.Now < startTime + new TimeSpan(0, 0, seconds))
{ {
@ -939,7 +950,7 @@ namespace Wabbajack.Common
} }
} }
var speed = await TestDiskSpeedInner(queue, path); var speed = await TestDiskSpeedInner(queue, path);
speed.ToJson(benchmarkFile); await speed.ToJsonAsync(benchmarkFile);
return speed; return speed;
} }
@ -1033,29 +1044,29 @@ namespace Wabbajack.Common
/// <typeparam name="T"></typeparam> /// <typeparam name="T"></typeparam>
/// <param name="key"></param> /// <param name="key"></param>
/// <param name="data"></param> /// <param name="data"></param>
public static void ToEcryptedJson<T>(this T data, string key) public static async ValueTask ToEcryptedJson<T>(this T data, string key)
{ {
var bytes = Encoding.UTF8.GetBytes(data.ToJson()); var bytes = Encoding.UTF8.GetBytes(data.ToJson());
bytes.ToEcryptedData(key); await bytes.ToEcryptedData(key);
} }
public static T FromEncryptedJson<T>(string key) public static async Task<T> FromEncryptedJson<T>(string key)
{ {
var decoded = FromEncryptedData(key); var decoded = await FromEncryptedData(key);
return Encoding.UTF8.GetString(decoded).FromJsonString<T>(); return Encoding.UTF8.GetString(decoded).FromJsonString<T>();
} }
public static void ToEcryptedData(this byte[] bytes, string key) public static async ValueTask ToEcryptedData(this byte[] bytes, string key)
{ {
var encoded = ProtectedData.Protect(bytes, Encoding.UTF8.GetBytes(key), DataProtectionScope.LocalMachine); var encoded = ProtectedData.Protect(bytes, Encoding.UTF8.GetBytes(key), DataProtectionScope.LocalMachine);
Consts.LocalAppDataPath.CreateDirectory(); Consts.LocalAppDataPath.CreateDirectory();
Consts.LocalAppDataPath.Combine(key).WriteAllBytes(encoded); await Consts.LocalAppDataPath.Combine(key).WriteAllBytesAsync(encoded);
} }
public static byte[] FromEncryptedData(string key) public static async Task<byte[]> FromEncryptedData(string key)
{ {
var bytes = Consts.LocalAppDataPath.Combine(key).ReadAllBytes(); var bytes = await Consts.LocalAppDataPath.Combine(key).ReadAllBytesAsync();
return ProtectedData.Unprotect(bytes, Encoding.UTF8.GetBytes(key), DataProtectionScope.LocalMachine); return ProtectedData.Unprotect(bytes, Encoding.UTF8.GetBytes(key), DataProtectionScope.LocalMachine);
} }

28
Wabbajack.Lib/ACompiler.cs
View File

@ -81,10 +81,10 @@ namespace Wabbajack.Lib
return id; return id;
} }
internal FileStream IncludeFile(out RelativePath id) internal AbsolutePath IncludeFile(out RelativePath id)
{ {
id = IncludeId(); id = IncludeId();
return ModListOutputFolder.Combine(id).Create(); return ModListOutputFolder.Combine(id);
} }
internal async Task<RelativePath> IncludeFile(string data) internal async Task<RelativePath> IncludeFile(string data)
@ -144,34 +144,32 @@ namespace Wabbajack.Lib
ModList.Image = (RelativePath)"modlist-image.png"; ModList.Image = (RelativePath)"modlist-image.png";
} }
using (var of = ModListOutputFolder.Combine("modlist").Create()) using (var of = await ModListOutputFolder.Combine("modlist").Create())
ModList.ToJson(of); ModList.ToJson(of);
ModListOutputFile.Delete(); ModListOutputFile.Delete();
using (var fs = ModListOutputFile.Create()) using (var fs = await ModListOutputFile.Create())
{ {
using (var za = new ZipArchive(fs, ZipArchiveMode.Create)) using (var za = new ZipArchive(fs, ZipArchiveMode.Create))
{ {
ModListOutputFolder.EnumerateFiles() await ModListOutputFolder.EnumerateFiles()
.DoProgress("Compressing ModList", .DoProgress("Compressing ModList",
f => async f =>
{ {
var ze = za.CreateEntry((string)f.FileName); var ze = za.CreateEntry((string)f.FileName);
using var os = ze.Open(); await using var os = ze.Open();
using var ins = f.OpenRead(); await using var ins = await f.OpenRead();
ins.CopyTo(os); await ins.CopyToAsync(os);
}); });
// Copy in modimage // Copy in modimage
if (ModListImage.Exists) if (ModListImage.Exists)
{ {
var ze = za.CreateEntry((string)ModList.Image); var ze = za.CreateEntry((string)ModList.Image);
using (var os = ze.Open()) await using var os = ze.Open();
using (var ins = ModListImage.OpenRead()) await using var ins = await ModListImage.OpenRead();
{ await ins.CopyToAsync(os);
ins.CopyTo(os);
}
} }
} }
} }
@ -180,7 +178,7 @@ namespace Wabbajack.Lib
var metadata = new DownloadMetadata var metadata = new DownloadMetadata
{ {
Size = ModListOutputFile.Size, Size = ModListOutputFile.Size,
Hash = ModListOutputFile.FileHash(), Hash = await ModListOutputFile.FileHashAsync(),
NumberOfArchives = ModList.Archives.Count, NumberOfArchives = ModList.Archives.Count,
SizeOfArchives = ModList.Archives.Sum(a => a.Size), SizeOfArchives = ModList.Archives.Sum(a => a.Size),
NumberOfInstalledFiles = ModList.Directives.Count, NumberOfInstalledFiles = ModList.Directives.Count,

4
Wabbajack.Lib/AInstaller.cs
View File

@ -71,7 +71,7 @@ namespace Wabbajack.Lib
public async Task<byte[]> LoadBytesFromPath(RelativePath path) public async Task<byte[]> LoadBytesFromPath(RelativePath path)
{ {
await using var e = ExtractedModListFiles![path].OpenRead(); await using var e = await ExtractedModListFiles![path].OpenRead();
return await e.ReadAllAsync(); return await e.ReadAllAsync();
} }
@ -213,7 +213,7 @@ namespace Wabbajack.Lib
toFile.Delete(); toFile.Delete();
// Patch it // Patch it
await using (var outStream = toFile.Create()) await using (var outStream = await toFile.Create())
{ {
Utils.ApplyPatch(oldData, () => new MemoryStream(patchData), outStream); Utils.ApplyPatch(oldData, () => new MemoryStream(patchData), outStream);
} }

4
Wabbajack.Lib/AuthorApi/Client.cs
View File

@ -63,7 +63,7 @@ namespace Wabbajack.Lib.AuthorApi
{ {
progressFn("Hashing file parts", Percent.FactoryPutInRange(part.Index, parts.Length)); progressFn("Hashing file parts", Percent.FactoryPutInRange(part.Index, parts.Length));
var buffer = new byte[part.Size]; var buffer = new byte[part.Size];
await using (var fs = path.OpenShared()) await using (var fs = await path.OpenShared())
{ {
fs.Position = part.Offset; fs.Position = part.Offset;
await fs.ReadAsync(buffer); await fs.ReadAsync(buffer);
@ -91,7 +91,7 @@ namespace Wabbajack.Lib.AuthorApi
{ {
progressFn("Uploading Part", Percent.FactoryPutInRange(part.Index, definition.Parts.Length)); progressFn("Uploading Part", Percent.FactoryPutInRange(part.Index, definition.Parts.Length));
var buffer = new byte[part.Size]; var buffer = new byte[part.Size];
await using (var fs = path.OpenShared()) await using (var fs = await path.OpenShared())
{ {
fs.Position = part.Offset; fs.Position = part.Offset;
await fs.ReadAsync(buffer); await fs.ReadAsync(buffer);

8
Wabbajack.Lib/ClientAPI.cs
View File

@ -43,11 +43,11 @@ namespace Wabbajack.Lib
public class ClientAPI public class ClientAPI
{ {
public static Common.Http.Client GetClient() public static async Task<Common.Http.Client> GetClient()
{ {
var client = new Common.Http.Client(); var client = new Common.Http.Client();
if (Utils.HaveEncryptedJson(Consts.MetricsKeyHeader)) if (Utils.HaveEncryptedJson(Consts.MetricsKeyHeader))
client.Headers.Add((Consts.MetricsKeyHeader, Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader))); client.Headers.Add((Consts.MetricsKeyHeader, await Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader)));
return client; return client;
} }
@ -63,7 +63,7 @@ namespace Wabbajack.Lib
RETRY: RETRY:
var response = await GetClient() var response = await (await GetClient())
.PostAsync($"{Consts.WabbajackBuildServerUri}mod_upgrade", new StringContent(request.ToJson(), Encoding.UTF8, "application/json")); .PostAsync($"{Consts.WabbajackBuildServerUri}mod_upgrade", new StringContent(request.ToJson(), Encoding.UTF8, "application/json"));
if (response.IsSuccessStatusCode) if (response.IsSuccessStatusCode)
@ -114,7 +114,7 @@ namespace Wabbajack.Lib
public static async Task<NexusCacheStats> GetNexusCacheStats() public static async Task<NexusCacheStats> GetNexusCacheStats()
{ {
return await GetClient() return await (await GetClient())
.GetJsonAsync<NexusCacheStats>($"{Consts.WabbajackBuildServerUri}nexus_cache/stats"); .GetJsonAsync<NexusCacheStats>($"{Consts.WabbajackBuildServerUri}nexus_cache/stats");
} }

4
Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs
View File

@ -62,7 +62,7 @@ namespace Wabbajack.Lib.CompilationSteps
if (source.AbsolutePath.Size >= (long) 2 << 31) if (source.AbsolutePath.Size >= (long) 2 << 31)
{ {
await using var bsa = BSADispatch.OpenRead(source.AbsolutePath); await using var bsa = await BSADispatch.OpenRead(source.AbsolutePath);
if (bsa.State is BSAStateObject) if (bsa.State is BSAStateObject)
{ {
Utils.Error( Utils.Error(
@ -95,7 +95,7 @@ namespace Wabbajack.Lib.CompilationSteps
} }
CreateBSA directive; CreateBSA directive;
await using (var bsa = BSADispatch.OpenRead(source.AbsolutePath)) await using (var bsa = await BSADispatch.OpenRead(source.AbsolutePath))
{ {
directive = new CreateBSA( directive = new CreateBSA(
state: bsa.State, state: bsa.State,

2
Wabbajack.Lib/CompilationSteps/IncludeAll.cs
View File

@ -14,7 +14,7 @@ namespace Wabbajack.Lib.CompilationSteps
public override async ValueTask<Directive?> Run(RawSourceFile source) public override async ValueTask<Directive?> Run(RawSourceFile source)
{ {
var inline = source.EvolveTo<InlineFile>(); var inline = source.EvolveTo<InlineFile>();
await using var file = source.File.StagedFile.OpenRead(); await using var file = await source.File.StagedFile.OpenRead();
inline.SourceDataID = await _compiler.IncludeFile(await file.ReadAllAsync()); inline.SourceDataID = await _compiler.IncludeFile(await file.ReadAllAsync());
return inline; return inline;
} }

2
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -124,7 +124,7 @@ namespace Wabbajack.Lib.Downloaders
using var stream = await ResolveDownloadStream(a); using var stream = await ResolveDownloadStream(a);
if (stream == null) return false; if (stream == null) return false;
await using var fromStream = await stream.Content.ReadAsStreamAsync(); await using var fromStream = await stream.Content.ReadAsStreamAsync();
await using var toStream = destination.Create(); await using var toStream = await destination.Create();
await fromStream.CopyToAsync(toStream); await fromStream.CopyToAsync(toStream);
return true; return true;
} }

4
Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs
View File

@ -81,7 +81,7 @@ namespace Wabbajack.Lib.Downloaders
await Task.Delay(500, cancel); await Task.Delay(500, cancel);
} }
cookies.ToEcryptedJson(_encryptedKeyName); await cookies.ToEcryptedJson(_encryptedKeyName);
return cookies; return cookies;
} }
@ -91,7 +91,7 @@ namespace Wabbajack.Lib.Downloaders
Helpers.Cookie[] cookies; Helpers.Cookie[] cookies;
try try
{ {
cookies = Utils.FromEncryptedJson<Helpers.Cookie[]>(_encryptedKeyName); cookies = await Utils.FromEncryptedJson<Helpers.Cookie[]>(_encryptedKeyName);
if (cookies != null) if (cookies != null)
return Helpers.GetClient(cookies, SiteURL.ToString()); return Helpers.GetClient(cookies, SiteURL.ToString());
} }

8
Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs
View File

@ -110,7 +110,7 @@ namespace Wabbajack.Lib.Downloaders
try try
{ {
var result = last_line.FromJsonString<BethesdaNetData>(); var result = last_line.FromJsonString<BethesdaNetData>();
result.ToEcryptedJson(DataName); await result.ToEcryptedJson(DataName);
return result; return result;
} }
catch (Exception ex) catch (Exception ex)
@ -180,7 +180,7 @@ namespace Wabbajack.Lib.Downloaders
private const uint CKM_Magic = 0x52415442; // BTAR private const uint CKM_Magic = 0x52415442; // BTAR
private async Task ConvertCKMToZip(AbsolutePath src, AbsolutePath dest) private async Task ConvertCKMToZip(AbsolutePath src, AbsolutePath dest)
{ {
using var reader = new BinaryReader(src.OpenRead()); using var reader = new BinaryReader(await src.OpenRead());
var magic = reader.ReadUInt32(); var magic = reader.ReadUInt32();
if (magic != CKM_Magic) if (magic != CKM_Magic)
throw new InvalidDataException("Invalid magic format in CKM parsing"); throw new InvalidDataException("Invalid magic format in CKM parsing");
@ -193,7 +193,7 @@ namespace Wabbajack.Lib.Downloaders
if (minorVersion < 2 || minorVersion > 4) if (minorVersion < 2 || minorVersion > 4)
throw new InvalidDataException("Archive minor version is unknown. Should be 2, 3, or 4."); throw new InvalidDataException("Archive minor version is unknown. Should be 2, 3, or 4.");
await using var fos = dest.Create(); await using var fos = await dest.Create();
using var archive = new ZipArchive(fos, ZipArchiveMode.Create); using var archive = new ZipArchive(fos, ZipArchiveMode.Create);
while (reader.PeekChar() != -1) while (reader.PeekChar() != -1)
{ {
@ -220,7 +220,7 @@ namespace Wabbajack.Lib.Downloaders
{ {
var info = new CollectedBNetInfo(); var info = new CollectedBNetInfo();
var login_info = Utils.FromEncryptedJson<BethesdaNetData>(DataName); var login_info = await Utils.FromEncryptedJson<BethesdaNetData>(DataName);
var client = new Common.Http.Client(); var client = new Common.Http.Client();

6
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -128,10 +128,10 @@ namespace Wabbajack.Lib.Downloaders
response.Dispose(); response.Dispose();
Utils.Log($"Applying patch to {archive.Name}"); Utils.Log($"Applying patch to {archive.Name}");
await using(var src = result.NewFile.Path.OpenShared()) await using(var src = await result.NewFile.Path.OpenShared())
await using (var final = destination.Create()) await using (var final = await destination.Create())
{ {
Utils.ApplyPatch(src, () => tempFile.Path.OpenShared(), final); Utils.ApplyPatch(src, () => tempFile.Path.OpenShared().Result, final);
} }
var hash = await destination.FileHashCachedAsync(); var hash = await destination.FileHashCachedAsync();

4
Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
View File

@ -71,8 +71,8 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Download(Archive a, AbsolutePath destination) public override async Task<bool> Download(Archive a, AbsolutePath destination)
{ {
await using var src = SourcePath.OpenRead(); await using var src = await SourcePath.OpenRead();
await using var dest = destination.Create(); await using var dest = await destination.Create();
var size = SourcePath.Size; var size = SourcePath.Size;
await src.CopyToWithStatusAsync(size, dest, "Copying from Game folder"); await src.CopyToWithStatusAsync(size, dest, "Copying from Game folder");

2
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -81,7 +81,7 @@ namespace Wabbajack.Lib.Downloaders
destination.Parent.CreateDirectory(); destination.Parent.CreateDirectory();
} }
using (var fs = download ? destination.Create() : null) using (var fs = download ? await destination.Create() : null)
{ {
var client = Client ?? new Common.Http.Client(); var client = Client ?? new Common.Http.Client();
client.Headers.Add(("User-Agent", Consts.UserAgent)); client.Headers.Add(("User-Agent", Consts.UserAgent));

2
Wabbajack.Lib/Downloaders/MEGADownloader.cs
View File

@ -153,7 +153,7 @@ namespace Wabbajack.Lib.Downloaders
else else
{ {
Utils.Status("Logging into MEGA with saved credentials."); Utils.Status("Logging into MEGA with saved credentials.");
var infos = Utils.FromEncryptedJson<MEGAAuthInfos>(DataName); var infos = await Utils.FromEncryptedJson<MEGAAuthInfos>(DataName);
var authInfo = infos.ToAuthInfos(); var authInfo = infos.ToAuthInfos();
await MegaApiClient.LoginAsync(authInfo); await MegaApiClient.LoginAsync(authInfo);
} }

2
Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs
View File

@ -57,7 +57,7 @@ namespace Wabbajack.Lib.Downloaders
{ {
destination.Parent.CreateDirectory(); destination.Parent.CreateDirectory();
var definition = await GetDefinition(); var definition = await GetDefinition();
using var fs = destination.Create(); await using var fs = await destination.Create();
using var mmfile = MemoryMappedFile.CreateFromFile(fs, null, definition.Size, MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, false); using var mmfile = MemoryMappedFile.CreateFromFile(fs, null, definition.Size, MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, false);
var client = new Common.Http.Client(); var client = new Common.Http.Client();
using var queue = new WorkQueue(); using var queue = new WorkQueue();

6
Wabbajack.Lib/Downloaders/YouTubeDownloader.cs
View File

@ -114,7 +114,7 @@ namespace Wabbajack.Lib.Downloaders
var trackFolder = folder.Dir.Combine("tracks"); var trackFolder = folder.Dir.Combine("tracks");
await using (var fs = initialDownload.Create()) await using (var fs = await initialDownload.Create())
{ {
await client.Videos.Streams.CopyToAsync(stream, fs, new Progress($"Downloading {a.Name}"), await client.Videos.Streams.CopyToAsync(stream, fs, new Progress($"Downloading {a.Name}"),
CancellationToken.None); CancellationToken.None);
@ -128,7 +128,7 @@ namespace Wabbajack.Lib.Downloaders
await ExtractTrack(initialDownload, trackFolder, track); await ExtractTrack(initialDownload, trackFolder, track);
}); });
await using var dest = destination.Create(); await using var dest = await destination.Create();
using var ar = new ZipArchive(dest, ZipArchiveMode.Create); using var ar = new ZipArchive(dest, ZipArchiveMode.Create);
foreach (var track in trackFolder.EnumerateFiles().OrderBy(e => e)) foreach (var track in trackFolder.EnumerateFiles().OrderBy(e => e))
{ {
@ -136,7 +136,7 @@ namespace Wabbajack.Lib.Downloaders
var entry = ar.CreateEntry(Path.Combine("Data", "tracks", (string)track.RelativeTo(trackFolder)), CompressionLevel.NoCompression); var entry = ar.CreateEntry(Path.Combine("Data", "tracks", (string)track.RelativeTo(trackFolder)), CompressionLevel.NoCompression);
entry.LastWriteTime = meta.UploadDate; entry.LastWriteTime = meta.UploadDate;
await using var es = entry.Open(); await using var es = entry.Open();
await using var ins = track.OpenRead(); await using var ins = await track.OpenRead();
await ins.CopyToAsync(es); await ins.CopyToAsync(es);
} }

16
Wabbajack.Lib/Extensions/ReactiveUIExt.cs
View File

@ -4,6 +4,8 @@ using System.Linq.Expressions;
using System.Reactive; using System.Reactive;
using System.Reactive.Disposables; using System.Reactive.Disposables;
using System.Reactive.Linq; using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Threading.Tasks;
using DynamicData; using DynamicData;
using DynamicData.Kernel; using DynamicData.Kernel;
using ReactiveUI; using ReactiveUI;
@ -38,6 +40,20 @@ namespace Wabbajack
return source.ObserveOn(RxApp.MainThreadScheduler); return source.ObserveOn(RxApp.MainThreadScheduler);
} }
/// <summary>
/// Like IObservable.Select but supports async map functions
/// </summary>
/// <param name="source"></param>
/// <param name="f"></param>
/// <typeparam name="T"></typeparam>
/// <returns></returns>
public static IObservable<TOut> SelectAsync<TIn, TOut>(this IObservable<TIn> source, Func<TIn, Task<TOut>> f)
{
return source.Select(itm => Observable.FromAsync(async () => await f(itm))).Merge(10);
}
public static IObservable<Unit> StartingExecution(this IReactiveCommand cmd) public static IObservable<Unit> StartingExecution(this IReactiveCommand cmd)
{ {
return cmd.IsExecuting return cmd.IsExecuting

10
Wabbajack.Lib/MO2Compiler.cs
View File

@ -484,8 +484,8 @@ namespace Wabbajack.Lib
Info($"Patching {entry.To}"); Info($"Patching {entry.To}");
Status($"Patching {entry.To}"); Status($"Patching {entry.To}");
var srcFile = byPath[string.Join("|", entry.ArchiveHashPath.Paths)]; var srcFile = byPath[string.Join("|", entry.ArchiveHashPath.Paths)];
await using var srcStream = srcFile.OpenRead(); await using var srcStream = await srcFile.OpenRead();
await using var outputStream = IncludeFile(out var id); await using var outputStream = await IncludeFile(out var id).Create();
entry.PatchID = id; entry.PatchID = id;
await using var destStream = await LoadDataForTo(entry.To, absolutePaths); await using var destStream = await LoadDataForTo(entry.To, absolutePaths);
await Utils.CreatePatch(srcStream, srcFile.Hash, destStream, entry.Hash, outputStream); await Utils.CreatePatch(srcStream, srcFile.Hash, destStream, entry.Hash, outputStream);
@ -496,18 +496,18 @@ namespace Wabbajack.Lib
private async Task<FileStream> LoadDataForTo(RelativePath to, Dictionary<RelativePath, AbsolutePath> absolutePaths) private async Task<FileStream> LoadDataForTo(RelativePath to, Dictionary<RelativePath, AbsolutePath> absolutePaths)
{ {
if (absolutePaths.TryGetValue(to, out var absolute)) if (absolutePaths.TryGetValue(to, out var absolute))
return absolute.OpenRead(); return await absolute.OpenRead();
if (to.StartsWith(Consts.BSACreationDir)) if (to.StartsWith(Consts.BSACreationDir))
{ {
var bsaId = (RelativePath)((string)to).Split('\\')[1]; var bsaId = (RelativePath)((string)to).Split('\\')[1];
var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsaId); var bsa = InstallDirectives.OfType<CreateBSA>().First(b => b.TempID == bsaId);
await using var a = BSADispatch.OpenRead(MO2Folder.Combine(bsa.To)); await using var a = await BSADispatch.OpenRead(MO2Folder.Combine(bsa.To));
var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray()); var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray());
var file = a.Files.First(e => e.Path == find); var file = a.Files.First(e => e.Path == find);
var returnStream = new TempStream(); var returnStream = new TempStream();
file.CopyDataTo(returnStream); await file.CopyDataTo(returnStream);
returnStream.Position = 0; returnStream.Position = 0;
return returnStream; return returnStream;
} }

12
Wabbajack.Lib/MO2Installer.cs
View File

@ -84,7 +84,7 @@ namespace Wabbajack.Lib
if (cancel.IsCancellationRequested) return false; if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Validating Game ESMs"); UpdateTracker.NextStep("Validating Game ESMs");
ValidateGameESMs(); await ValidateGameESMs();
if (cancel.IsCancellationRequested) return false; if (cancel.IsCancellationRequested) return false;
UpdateTracker.NextStep("Validating Modlist"); UpdateTracker.NextStep("Validating Modlist");
@ -238,14 +238,14 @@ namespace Wabbajack.Lib
}); });
} }
private void ValidateGameESMs() private async ValueTask ValidateGameESMs()
{ {
foreach (var esm in ModList.Directives.OfType<CleanedESM>().ToList()) foreach (var esm in ModList.Directives.OfType<CleanedESM>().ToList())
{ {
var filename = esm.To.FileName; var filename = esm.To.FileName;
var gameFile = GameFolder!.Value.Combine((RelativePath)"Data", filename); var gameFile = GameFolder!.Value.Combine((RelativePath)"Data", filename);
Utils.Log($"Validating {filename}"); Utils.Log($"Validating {filename}");
var hash = gameFile.FileHash(); var hash = await gameFile.FileHashAsync();
if (hash != esm.SourceESMHash) if (hash != esm.SourceESMHash)
{ {
Utils.ErrorThrow(new InvalidGameESMError(esm, hash, gameFile)); Utils.ErrorThrow(new InvalidGameESMError(esm, hash, gameFile));
@ -269,7 +269,7 @@ namespace Wabbajack.Lib
var streams = await bsa.FileStates.PMap(Queue, async state => var streams = await bsa.FileStates.PMap(Queue, async state =>
{ {
Status($"Adding {state.Path} to BSA"); Status($"Adding {state.Path} to BSA");
var fs = sourceDir.Combine(state.Path).OpenRead(); var fs = await sourceDir.Combine(state.Path).OpenRead();
await a.AddFile(state, fs); await a.AddFile(state, fs);
return fs; return fs;
}); });
@ -328,8 +328,8 @@ namespace Wabbajack.Lib
var patchData = await LoadBytesFromPath(directive.SourceDataID); var patchData = await LoadBytesFromPath(directive.SourceDataID);
var toFile = OutputFolder.Combine(directive.To); var toFile = OutputFolder.Combine(directive.To);
Status($"Patching {filename}"); Status($"Patching {filename}");
using var output = toFile.Create(); await using var output = await toFile.Create();
using var input = gameFile.OpenRead(); await using var input = await gameFile.OpenRead();
Utils.ApplyPatch(input, () => new MemoryStream(patchData), output); Utils.ApplyPatch(input, () => new MemoryStream(patchData), output);
} }

4
Wabbajack.Lib/ModListRegistry/ModListMetadata.cs
View File

@ -84,14 +84,14 @@ namespace Wabbajack.Lib.ModListRegistry
return metadata.OrderBy(m => (m.ValidationSummary?.HasFailures ?? false ? 1 : 0, m.Title)).ToList(); return metadata.OrderBy(m => (m.ValidationSummary?.HasFailures ?? false ? 1 : 0, m.Title)).ToList();
} }
public bool NeedsDownload(AbsolutePath modlistPath) public async ValueTask<bool> NeedsDownload(AbsolutePath modlistPath)
{ {
if (!modlistPath.Exists) return true; if (!modlistPath.Exists) return true;
if (DownloadMetadata?.Hash == null) if (DownloadMetadata?.Hash == null)
{ {
return true; return true;
} }
return DownloadMetadata.Hash != modlistPath.FileHashCached(true); return DownloadMetadata.Hash != await modlistPath.FileHashCachedAsync(true);
} }
} }

4
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -58,7 +58,7 @@ namespace Wabbajack.Lib.NexusApi
try try
{ {
return Utils.FromEncryptedJson<string>("nexusapikey"); return await Utils.FromEncryptedJson<string>("nexusapikey");
} }
catch (Exception) catch (Exception)
{ {
@ -77,7 +77,7 @@ namespace Wabbajack.Lib.NexusApi
public static async Task<string> RequestAndCacheAPIKey() public static async Task<string> RequestAndCacheAPIKey()
{ {
var result = await Utils.Log(new RequestNexusAuthorization()).Task; var result = await Utils.Log(new RequestNexusAuthorization()).Task;
result.ToEcryptedJson("nexusapikey"); await result.ToEcryptedJson("nexusapikey");
return result; return result;
} }

7
Wabbajack.Lib/zEditIntegration.cs
View File

@ -185,7 +185,7 @@ namespace Wabbajack.Lib
}; };
})); }));
var srcData = result.Sources.Select(f => _mo2Compiler.MO2Folder.Combine(f.RelativePath).ReadAllBytes()) var srcData = (await result.Sources.SelectAsync(async f => await _mo2Compiler.MO2Folder.Combine(f.RelativePath).ReadAllBytesAsync()).ToList())
.ConcatArrays(); .ConcatArrays();
var dstData = await source.AbsolutePath.ReadAllBytesAsync(); var dstData = await source.AbsolutePath.ReadAllBytesAsync();
@ -264,12 +264,13 @@ namespace Wabbajack.Lib
{ {
Utils.LogStatus($"Generating zEdit merge: {m.To}"); Utils.LogStatus($"Generating zEdit merge: {m.To}");
var srcData = m.Sources.Select(s => installer.OutputFolder.Combine(s.RelativePath).ReadAllBytes()) var srcData = (await m.Sources.SelectAsync(async s => await installer.OutputFolder.Combine(s.RelativePath).ReadAllBytesAsync())
.ToList())
.ConcatArrays(); .ConcatArrays();
var patchData = await installer.LoadBytesFromPath(m.PatchID); var patchData = await installer.LoadBytesFromPath(m.PatchID);
await using var fs = installer.OutputFolder.Combine(m.To).Create(); await using var fs = await installer.OutputFolder.Combine(m.To).Create();
Utils.ApplyPatch(new MemoryStream(srcData), () => new MemoryStream(patchData), fs); Utils.ApplyPatch(new MemoryStream(srcData), () => new MemoryStream(patchData), fs);
}); });
} }

9
Wabbajack.Server.Test/ABuildServerSystemTest.cs
View File

@ -36,8 +36,9 @@ namespace Wabbajack.BuildServer.Test
public AbsolutePath ServerUpdatesFolder => "updates".RelativeTo(AbsolutePath.EntryPoint); public AbsolutePath ServerUpdatesFolder => "updates".RelativeTo(AbsolutePath.EntryPoint);
public BuildServerFixture() public override async Task InitializeAsync()
{ {
await base.InitializeAsync();
ServerArchivesFolder.DeleteDirectory().Wait(); ServerArchivesFolder.DeleteDirectory().Wait();
ServerArchivesFolder.CreateDirectory(); ServerArchivesFolder.CreateDirectory();
@ -61,7 +62,7 @@ namespace Wabbajack.BuildServer.Test
_task = _host.RunAsync(_token.Token); _task = _host.RunAsync(_token.Token);
Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080"); Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080");
"ServerWhitelist.yaml".RelativeTo(ServerPublicFolder).WriteAllText( await "ServerWhitelist.yaml".RelativeTo(ServerPublicFolder).WriteAllTextAsync(
"GoogleIDs:\nAllowedPrefixes:\n - http://localhost"); "GoogleIDs:\nAllowedPrefixes:\n - http://localhost");
} }
@ -208,7 +209,7 @@ namespace Wabbajack.BuildServer.Test
var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder); var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder);
await using (var fs = modListPath.Create()) await using (var fs = await modListPath.Create())
{ {
using var za = new ZipArchive(fs, ZipArchiveMode.Create); using var za = new ZipArchive(fs, ZipArchiveMode.Create);
var entry = za.CreateEntry("modlist"); var entry = za.CreateEntry("modlist");
@ -254,7 +255,7 @@ namespace Wabbajack.BuildServer.Test
var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder); var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder);
ModListMetaData.ToJson(metadataPath); await ModListMetaData.ToJsonAsync(metadataPath);
return new Uri(MakeURL("test_mod_list_metadata.json")); return new Uri(MakeURL("test_mod_list_metadata.json"));
} }

2
Wabbajack.Server.Test/ADBTest.cs
View File

@ -32,7 +32,7 @@ namespace Wabbajack.BuildServer.Test
public string APIKey { get; } public string APIKey { get; }
public string User { get; } public string User { get; }
public async Task InitializeAsync() public virtual async Task InitializeAsync()
{ {
await CreateSchema(); await CreateSchema();
} }

2
Wabbajack.Server.Test/ModListValidationTests.cs
View File

@ -141,7 +141,7 @@ namespace Wabbajack.BuildServer.Test
Assert.Equal(1, data.ValidationSummary.Updating); Assert.Equal(1, data.ValidationSummary.Updating);
var patcher = Fixture.GetService<PatchBuilder>(); var patcher = Fixture.GetService<PatchBuilder>();
Assert.Equal(1, await patcher.Execute()); Assert.True(await patcher.Execute() > 1);
await RevalidateLists(false); await RevalidateLists(false);

2
Wabbajack.Server.Test/ModlistUpdater.cs
View File

@ -64,7 +64,7 @@ namespace Wabbajack.Server.Test
await Assert.ThrowsAsync<HttpException>(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); await Assert.ThrowsAsync<HttpException>(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
Assert.Equal(1, await patcher.Execute()); Assert.True(await patcher.Execute() > 1);
Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_updates/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_updates/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
} }

2
Wabbajack.Server/Controllers/AuthoredFiles.cs
View File

@ -99,7 +99,7 @@ namespace Wabbajack.BuildServer.Controllers
private async Task<FtpClient> GetBunnyCdnFtpClient() private async Task<FtpClient> GetBunnyCdnFtpClient()
{ {
var info = Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info"); var info = await Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");
var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)}; var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
await client.ConnectAsync(); await client.ConnectAsync();
return client; return client;

2
Wabbajack.Server/Services/ModListDownloader.cs
View File

@ -99,7 +99,7 @@ namespace Wabbajack.Server.Services
_maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath); _maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath);
ModList modlist; ModList modlist;
await using (var fs = modlistPath.OpenRead()) await using (var fs = await modlistPath.OpenRead())
using (var zip = new ZipArchive(fs, ZipArchiveMode.Read)) using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
await using (var entry = zip.GetEntry("modlist")?.Open()) await using (var entry = zip.GetEntry("modlist")?.Open())
{ {

10
Wabbajack.Server/Services/PatchBuilder.cs
View File

@ -70,10 +70,10 @@ namespace Wabbajack.Server.Services
using var sigFile = new TempFile(); using var sigFile = new TempFile();
using var patchFile = new TempFile(); using var patchFile = new TempFile();
await using var srcStream = srcPath.OpenShared(); await using var srcStream = await srcPath.OpenShared();
await using var destStream = destPath.OpenShared(); await using var destStream = await destPath.OpenShared();
await using var sigStream = sigFile.Path.Create(); await using var sigStream = await sigFile.Path.Create();
await using var patchOutput = patchFile.Path.Create(); await using var patchOutput = await patchFile.Path.Create();
OctoDiff.Create(destStream, srcStream, sigStream, patchOutput); OctoDiff.Create(destStream, srcStream, sigStream, patchOutput);
await patchOutput.DisposeAsync(); await patchOutput.DisposeAsync();
var size = patchFile.Path.Size; var size = patchFile.Path.Size;
@ -132,7 +132,7 @@ namespace Wabbajack.Server.Services
private async Task<FtpClient> GetBunnyCdnFtpClient() private async Task<FtpClient> GetBunnyCdnFtpClient()
{ {
var info = Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info"); var info = await Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");
var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)}; var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
await client.ConnectAsync(); await client.ConnectAsync();
return client; return client;

2
Wabbajack.Test/DownloaderTests.cs
View File

@ -436,7 +436,7 @@ namespace Wabbajack.Test
await converted.Download(new Archive(state: null!) { Name = "mod.zip" }, filename.Path); await converted.Download(new Archive(state: null!) { Name = "mod.zip" }, filename.Path);
await using var fs = filename.Path.OpenRead(); await using var fs = await filename.Path.OpenRead();
using var archive = new ZipArchive(fs); using var archive = new ZipArchive(fs);
var entries = archive.Entries.Select(e => e.FullName).ToList(); var entries = archive.Entries.Select(e => e.FullName).ToList();
Assert.Equal(entries, new List<string> {@"Data\TestCK.esp", @"Data\TestCK.ini"}); Assert.Equal(entries, new List<string> {@"Data\TestCK.esp", @"Data\TestCK.ini"});

6
Wabbajack.Test/EndToEndTests.cs
View File

@ -47,7 +47,7 @@ namespace Wabbajack.Test
public async Task CreateModlist() public async Task CreateModlist()
{ {
var profile = utils.AddProfile("Default"); var profile = utils.AddProfile("Default");
var mod = utils.AddMod(); var mod = await utils.AddMod();
await DownloadAndInstall( await DownloadAndInstall(
"https://github.com/ModOrganizer2/modorganizer/releases/download/v2.2.1/Mod.Organizer.2.2.1.7z", "https://github.com/ModOrganizer2/modorganizer/releases/download/v2.2.1/Mod.Organizer.2.2.1.7z",
@ -79,7 +79,7 @@ namespace Wabbajack.Test
await CompileAndInstall(profile); await CompileAndInstall(profile);
utils.VerifyAllFiles(); await utils.VerifyAllFiles();
await utils.InstallFolder.Combine(Consts.LOOTFolderFilesDir).DeleteDirectory(); await utils.InstallFolder.Combine(Consts.LOOTFolderFilesDir).DeleteDirectory();
@ -111,7 +111,7 @@ namespace Wabbajack.Test
private async Task<(AbsolutePath Download, AbsolutePath ModFolder)> DownloadAndInstall(Game game, int modId, string modName) private async Task<(AbsolutePath Download, AbsolutePath ModFolder)> DownloadAndInstall(Game game, int modId, string modName)
{ {
utils.AddMod(modName); await utils.AddMod(modName);
var client = await NexusApiClient.Get(); var client = await NexusApiClient.Get();
var resp = await client.GetModFiles(game, modId); var resp = await client.GetModFiles(game, modId);
var file = resp.files.FirstOrDefault(f => f.is_primary) ?? resp.files.FirstOrDefault(f => !string.IsNullOrEmpty(f.category_name)); var file = resp.files.FirstOrDefault(f => f.is_primary) ?? resp.files.FirstOrDefault(f => !string.IsNullOrEmpty(f.category_name));

10
Wabbajack.Test/MO2Tests.cs
View File

@ -27,8 +27,8 @@ namespace Wabbajack.Test
public async Task CheckValidInstallPath_HasModlist() public async Task CheckValidInstallPath_HasModlist()
{ {
await using var tempDir = await TempFolder.Create(); await using var tempDir = await TempFolder.Create();
await using var mo2 = tempDir.Dir.Combine("ModOrganizer.exe").Create(); await using var mo2 = await tempDir.Dir.Combine("ModOrganizer.exe").Create();
await using var molist = tempDir.Dir.Combine(((RelativePath)"modlist")).WithExtension(Consts.ModListExtension).Create(); await using var molist = await tempDir.Dir.Combine(((RelativePath)"modlist")).WithExtension(Consts.ModListExtension).Create();
Assert.False(MO2Installer.CheckValidInstallPath(tempDir.Dir, downloadFolder: null).Succeeded); Assert.False(MO2Installer.CheckValidInstallPath(tempDir.Dir, downloadFolder: null).Succeeded);
} }
@ -36,7 +36,7 @@ namespace Wabbajack.Test
public async Task CheckValidInstallPath_ProperOverwrite() public async Task CheckValidInstallPath_ProperOverwrite()
{ {
await using var tempDir = await TempFolder.Create(); await using var tempDir = await TempFolder.Create();
await using var tmp = tempDir.Dir.Combine(Consts.ModOrganizer2Exe).Create(); await using var tmp = await tempDir.Dir.Combine(Consts.ModOrganizer2Exe).Create();
Assert.True(MO2Installer.CheckValidInstallPath(tempDir.Dir, downloadFolder: null).Succeeded); Assert.True(MO2Installer.CheckValidInstallPath(tempDir.Dir, downloadFolder: null).Succeeded);
} }
@ -46,7 +46,7 @@ namespace Wabbajack.Test
await using var tempDir = await TempFolder.Create(); await using var tempDir = await TempFolder.Create();
await tempDir.Dir.DeleteDirectory(); await tempDir.Dir.DeleteDirectory();
tempDir.Dir.CreateDirectory(); tempDir.Dir.CreateDirectory();
await using var tmp = tempDir.Dir.Combine($"someFile.txt").Create(); await using var tmp = await tempDir.Dir.Combine($"someFile.txt").Create();
Assert.False(MO2Installer.CheckValidInstallPath(tempDir.Dir, downloadFolder: null).Succeeded); Assert.False(MO2Installer.CheckValidInstallPath(tempDir.Dir, downloadFolder: null).Succeeded);
} }
@ -56,7 +56,7 @@ namespace Wabbajack.Test
await using var tempDir = await TempFolder.Create(); await using var tempDir = await TempFolder.Create();
var downloadsFolder = tempDir.Dir.Combine("downloads"); var downloadsFolder = tempDir.Dir.Combine("downloads");
downloadsFolder.CreateDirectory(); downloadsFolder.CreateDirectory();
await using var tmp = tempDir.Dir.Combine($"downloads/someFile.txt").Create(); await using var tmp = await tempDir.Dir.Combine($"downloads/someFile.txt").Create();
Assert.True(MO2Installer.CheckValidInstallPath(tempDir.Dir, downloadFolder: downloadsFolder).Succeeded); Assert.True(MO2Installer.CheckValidInstallPath(tempDir.Dir, downloadFolder: downloadsFolder).Succeeded);
} }
#endregion #endregion

132
Wabbajack.Test/SanityTests.cs
View File

@ -27,17 +27,17 @@ namespace Wabbajack.Test
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod(); var mod = await utils.AddMod();
var testPex = utils.AddModFile(mod, @"Data\scripts\test.pex", 10); var testPex = await utils.AddModFile(mod, @"Data\scripts\test.pex", 10);
await utils.Configure(); await utils.Configure();
utils.AddManualDownload( await utils.AddManualDownload(
new Dictionary<string, byte[]> {{"/baz/biz.pex", await testPex.ReadAllBytesAsync()}}); new Dictionary<string, byte[]> {{"/baz/biz.pex", await testPex.ReadAllBytesAsync()}});
await CompileAndInstall(profile); await CompileAndInstall(profile);
utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex"); await utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex");
} }
[Fact] [Fact]
@ -45,19 +45,19 @@ namespace Wabbajack.Test
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod(); var mod = await utils.AddMod();
var testPex = utils.AddModFile(mod, @"Data\scripts\test.pex", 10); var testPex = await utils.AddModFile(mod, @"Data\scripts\test.pex", 10);
await utils.Configure(); await utils.Configure();
utils.AddManualDownload( await utils.AddManualDownload(
new Dictionary<string, byte[]> {{"/baz/biz.pex", await testPex.ReadAllBytesAsync()}}); new Dictionary<string, byte[]> {{"/baz/biz.pex", await testPex.ReadAllBytesAsync()}});
await utils.DownloadsFolder.Combine("some_other_file.7z").WriteAllTextAsync("random data"); await utils.DownloadsFolder.Combine("some_other_file.7z").WriteAllTextAsync("random data");
await CompileAndInstall(profile); await CompileAndInstall(profile);
utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex"); await utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex");
} }
[Fact] [Fact]
@ -65,17 +65,17 @@ namespace Wabbajack.Test
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod(); var mod = await utils.AddMod();
var testPex = utils.AddGameFile(@"enbstuff\test.pex", 10); var testPex = await utils.AddGameFile(@"enbstuff\test.pex", 10);
await utils.Configure(); await utils.Configure();
utils.AddManualDownload( await utils.AddManualDownload(
new Dictionary<string, byte[]> {{"/baz/biz.pex", await testPex.ReadAllBytesAsync()}}); new Dictionary<string, byte[]> {{"/baz/biz.pex", await testPex.ReadAllBytesAsync()}});
await CompileAndInstall(profile); await CompileAndInstall(profile);
utils.VerifyInstalledGameFile(@"enbstuff\test.pex"); await utils.VerifyInstalledGameFile(@"enbstuff\test.pex");
} }
[Fact] [Fact]
@ -83,14 +83,14 @@ namespace Wabbajack.Test
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod(); var mod = await utils.AddMod();
var testPex = utils.AddGameFile(@"enbstuff\test.pex", 10); var testPex = await utils.AddGameFile(@"enbstuff\test.pex", 10);
await utils.Configure(); await utils.Configure();
utils.MO2Folder.Combine(Consts.GameFolderFilesDir).CreateDirectory(); utils.MO2Folder.Combine(Consts.GameFolderFilesDir).CreateDirectory();
utils.AddManualDownload( await utils.AddManualDownload(
new Dictionary<string, byte[]> {{"/baz/biz.pex", await testPex.ReadAllBytesAsync()}}); new Dictionary<string, byte[]> {{"/baz/biz.pex", await testPex.ReadAllBytesAsync()}});
await CompileAndInstall(profile); await CompileAndInstall(profile);
@ -103,21 +103,21 @@ namespace Wabbajack.Test
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod(); var mod = await utils.AddMod();
var testPex = utils.AddModFile(mod, @"Data\scripts\test.pex", 10); var testPex = await utils.AddModFile(mod, @"Data\scripts\test.pex", 10);
// Make a copy to make sure it gets picked up and moved around. // Make a copy to make sure it gets picked up and moved around.
testPex.CopyTo(testPex.WithExtension(new Extension(".copy"))); await testPex.CopyToAsync(testPex.WithExtension(new Extension(".copy")));
await utils.Configure(); await utils.Configure();
utils.AddManualDownload( await utils.AddManualDownload(
new Dictionary<string, byte[]> { { "/baz/biz.pex", await testPex.ReadAllBytesAsync() } }); new Dictionary<string, byte[]> { { "/baz/biz.pex", await testPex.ReadAllBytesAsync() } });
await CompileAndInstall(profile); await CompileAndInstall(profile);
utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex"); await utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex");
utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex.copy"); await utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex.copy");
} }
[Fact] [Fact]
@ -125,14 +125,14 @@ namespace Wabbajack.Test
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod(); var mod = await utils.AddMod();
var unchanged = utils.AddModFile(mod, @"Data\scripts\unchanged.pex", 10); var unchanged = await utils.AddModFile(mod, @"Data\scripts\unchanged.pex", 10);
var deleted = utils.AddModFile(mod, @"Data\scripts\deleted.pex", 10); var deleted = await utils.AddModFile(mod, @"Data\scripts\deleted.pex", 10);
var modified = utils.AddModFile(mod, @"Data\scripts\modified.pex", 10); var modified = await utils.AddModFile(mod, @"Data\scripts\modified.pex", 10);
await utils.Configure(); await utils.Configure();
utils.AddManualDownload( await utils.AddManualDownload(
new Dictionary<string, byte[]> new Dictionary<string, byte[]>
{ {
{ "/baz/unchanged.pex", await unchanged.ReadAllBytesAsync() }, { "/baz/unchanged.pex", await unchanged.ReadAllBytesAsync() },
@ -142,9 +142,9 @@ namespace Wabbajack.Test
await CompileAndInstall(profile); await CompileAndInstall(profile);
utils.VerifyInstalledFile(mod, @"Data\scripts\unchanged.pex"); await utils.VerifyInstalledFile(mod, @"Data\scripts\unchanged.pex");
utils.VerifyInstalledFile(mod, @"Data\scripts\deleted.pex"); await utils.VerifyInstalledFile(mod, @"Data\scripts\deleted.pex");
utils.VerifyInstalledFile(mod, @"Data\scripts\modified.pex"); await utils.VerifyInstalledFile(mod, @"Data\scripts\modified.pex");
var unchangedPath = utils.PathOfInstalledFile(mod, @"Data\scripts\unchanged.pex"); var unchangedPath = utils.PathOfInstalledFile(mod, @"Data\scripts\unchanged.pex");
var deletedPath = utils.PathOfInstalledFile(mod, @"Data\scripts\deleted.pex"); var deletedPath = utils.PathOfInstalledFile(mod, @"Data\scripts\deleted.pex");
@ -170,9 +170,9 @@ namespace Wabbajack.Test
await CompileAndInstall(profile); await CompileAndInstall(profile);
utils.VerifyInstalledFile(mod, @"Data\scripts\unchanged.pex"); await utils.VerifyInstalledFile(mod, @"Data\scripts\unchanged.pex");
utils.VerifyInstalledFile(mod, @"Data\scripts\deleted.pex"); await utils.VerifyInstalledFile(mod, @"Data\scripts\deleted.pex");
utils.VerifyInstalledFile(mod, @"Data\scripts\modified.pex"); await utils.VerifyInstalledFile(mod, @"Data\scripts\modified.pex");
Assert.Equal(unchangedModified, unchangedPath.LastModified); Assert.Equal(unchangedModified, unchangedPath.LastModified);
Assert.NotEqual(modifiedModified, modifiedPath.LastModified); Assert.NotEqual(modifiedModified, modifiedPath.LastModified);
@ -184,7 +184,7 @@ namespace Wabbajack.Test
public async Task SetScreenSizeTest() public async Task SetScreenSizeTest()
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod("dummy"); var mod = await utils.AddMod("dummy");
await utils.Configure(); await utils.Configure();
await utils.MO2Folder.Combine("profiles", profile, "somegameprefs.ini").WriteAllLinesAsync( await utils.MO2Folder.Combine("profiles", profile, "somegameprefs.ini").WriteAllLinesAsync(
@ -216,11 +216,11 @@ namespace Wabbajack.Test
public async Task UnmodifiedInlinedFilesArePulledFromArchives() public async Task UnmodifiedInlinedFilesArePulledFromArchives()
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod(); var mod = await utils.AddMod();
var ini = utils.AddModFile(mod, @"foo.ini", 10); var ini = await utils.AddModFile(mod, @"foo.ini", 10);
await utils.Configure(); await utils.Configure();
utils.AddManualDownload( await utils.AddManualDownload(
new Dictionary<string, byte[]> { { "/baz/biz.pex", await ini.ReadAllBytesAsync() } }); new Dictionary<string, byte[]> { { "/baz/biz.pex", await ini.ReadAllBytesAsync() } });
var modlist = await CompileAndInstall(profile); var modlist = await CompileAndInstall(profile);
@ -234,9 +234,9 @@ namespace Wabbajack.Test
public async Task ModifiedIniFilesArePatchedAgainstFileWithSameName() public async Task ModifiedIniFilesArePatchedAgainstFileWithSameName()
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod(); var mod = await utils.AddMod();
var ini = utils.AddModFile(mod, @"foo.ini", 10); var ini = await utils.AddModFile(mod, @"foo.ini", 10);
var meta = utils.AddModFile(mod, "meta.ini"); var meta = await utils.AddModFile(mod, "meta.ini");
await utils.Configure(); await utils.Configure();
@ -262,8 +262,8 @@ namespace Wabbajack.Test
public async Task CanPatchFilesSourcedFromBSAs() public async Task CanPatchFilesSourcedFromBSAs()
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod(); var mod = await utils.AddMod();
var file = utils.AddModFile(mod, @"baz.bin", 10); var file = await utils.AddModFile(mod, @"baz.bin", 10);
await utils.Configure(); await utils.Configure();
@ -287,7 +287,7 @@ namespace Wabbajack.Test
new Dictionary<string, byte[]> { { "/stuff/files.bsa", await tempFile.Path.ReadAllBytesAsync() } }); new Dictionary<string, byte[]> { { "/stuff/files.bsa", await tempFile.Path.ReadAllBytesAsync() } });
await CompileAndInstall(profile); await CompileAndInstall(profile);
utils.VerifyInstalledFile(mod, @"baz.bin"); await utils.VerifyInstalledFile(mod, @"baz.bin");
} }
@ -295,8 +295,8 @@ namespace Wabbajack.Test
public async Task CanNoMatchIncludeFilesFromBSAs() public async Task CanNoMatchIncludeFilesFromBSAs()
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod(); var mod = await utils.AddMod();
var file = utils.AddModFile(mod, @"baz.bsa", 10); var file = await utils.AddModFile(mod, @"baz.bsa", 10);
await file.Parent.Combine("meta.ini").WriteAllLinesAsync(new[] await file.Parent.Combine("meta.ini").WriteAllLinesAsync(new[]
{ {
@ -334,7 +334,7 @@ namespace Wabbajack.Test
new Dictionary<string, byte[]> { { "/stuff/matching_file_data.bin", tempFileData } }); new Dictionary<string, byte[]> { { "/stuff/matching_file_data.bin", tempFileData } });
await CompileAndInstall(profile); await CompileAndInstall(profile);
utils.VerifyInstalledFile(mod, @"baz.bsa"); await utils.VerifyInstalledFile(mod, @"baz.bsa");
} }
@ -342,8 +342,8 @@ namespace Wabbajack.Test
public async Task CanInstallFilesFromBSAAndBSA() public async Task CanInstallFilesFromBSAAndBSA()
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod(); var mod = await utils.AddMod();
var file = utils.AddModFile(mod, @"baz.bin", 128); var file = await utils.AddModFile(mod, @"baz.bin", 128);
await utils.Configure(); await utils.Configure();
@ -362,14 +362,14 @@ namespace Wabbajack.Test
}, new MemoryStream(await file.ReadAllBytesAsync())); }, new MemoryStream(await file.ReadAllBytesAsync()));
await bsa.Build(tempFile.Path); await bsa.Build(tempFile.Path);
} }
tempFile.Path.CopyTo(file.Parent.Combine("bsa_data.bsa")); await tempFile.Path.CopyToAsync(file.Parent.Combine("bsa_data.bsa"));
var archive = utils.AddManualDownload( var archive = utils.AddManualDownload(
new Dictionary<string, byte[]> { { "/stuff/files.bsa", await tempFile.Path.ReadAllBytesAsync() } }); new Dictionary<string, byte[]> { { "/stuff/files.bsa", await tempFile.Path.ReadAllBytesAsync() } });
await CompileAndInstall(profile); await CompileAndInstall(profile);
utils.VerifyInstalledFile(mod, @"baz.bin"); await utils.VerifyInstalledFile(mod, @"baz.bin");
utils.VerifyInstalledFile(mod, @"bsa_data.bsa"); await utils.VerifyInstalledFile(mod, @"bsa_data.bsa");
} }
@ -377,8 +377,8 @@ namespace Wabbajack.Test
public async Task CanRecreateBSAsFromFilesSourcedInOtherBSAs() public async Task CanRecreateBSAsFromFilesSourcedInOtherBSAs()
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod(); var mod = await utils.AddMod();
var file = utils.AddModFile(mod, @"baz.bsa", 10); var file = await utils.AddModFile(mod, @"baz.bsa", 10);
await utils.Configure(); await utils.Configure();
@ -414,7 +414,7 @@ namespace Wabbajack.Test
await CompileAndInstall(profile); await CompileAndInstall(profile);
utils.VerifyInstalledFile(mod, @"baz.bsa"); await utils.VerifyInstalledFile(mod, @"baz.bsa");
} }
@ -425,8 +425,8 @@ namespace Wabbajack.Test
Consts.TestMode = false; Consts.TestMode = false;
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod(); var mod = await utils.AddMod();
var skyrimExe = utils.AddModFile(mod, @"Data\test.exe", 10); var skyrimExe = await utils.AddModFile(mod, @"Data\test.exe", 10);
var gameFolder = Consts.GameFolderFilesDir.RelativeTo(utils.MO2Folder); var gameFolder = Consts.GameFolderFilesDir.RelativeTo(utils.MO2Folder);
gameFolder.CreateDirectory(); gameFolder.CreateDirectory();
@ -459,18 +459,18 @@ namespace Wabbajack.Test
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var mod = utils.AddMod(); var mod = await utils.AddMod();
var testPex = utils.AddModFile(mod, @"Data\scripts\test.pex", 10); var testPex = await utils.AddModFile(mod, @"Data\scripts\test.pex", 10);
await utils.Configure(); await utils.Configure();
await utils.AddModFile(mod, "meta.ini").WriteAllLinesAsync(new[] await (await utils.AddModFile(mod, "meta.ini")).WriteAllLinesAsync(new[]
{ {
"[General]", "notes= fsdaf WABBAJACK_NOMATCH_INCLUDE fadsfsad", "[General]", "notes= fsdaf WABBAJACK_NOMATCH_INCLUDE fadsfsad",
}); });
await CompileAndInstall(profile); await CompileAndInstall(profile);
utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex"); await utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex");
} }
@ -483,11 +483,11 @@ namespace Wabbajack.Test
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var enabledMod = utils.AddMod(); var enabledMod = await utils.AddMod();
var enabledTestPex = utils.AddModFile(enabledMod, @"Data\scripts\enabledTestPex.pex", 10); var enabledTestPex = await utils.AddModFile(enabledMod, @"Data\scripts\enabledTestPex.pex", 10);
var disabledMod = utils.AddMod(); var disabledMod = await utils.AddMod();
var disabledTestPex = utils.AddModFile(disabledMod, @"Data\scripts\disabledTestPex.pex", 10); var disabledTestPex = await utils.AddModFile(disabledMod, @"Data\scripts\disabledTestPex.pex", 10);
await disabledMod.RelativeTo(utils.ModsFolder).Combine("meta.ini").WriteAllLinesAsync( await disabledMod.RelativeTo(utils.ModsFolder).Combine("meta.ini").WriteAllLinesAsync(
"[General]", "[General]",
@ -499,7 +499,7 @@ namespace Wabbajack.Test
(enabledMod, true) (enabledMod, true)
}); });
utils.AddManualDownload( await utils.AddManualDownload(
new Dictionary<string, byte[]> new Dictionary<string, byte[]>
{ {
{"/file1.pex", await enabledTestPex.ReadAllBytesAsync()}, {"/file1.pex", await enabledTestPex.ReadAllBytesAsync()},
@ -508,8 +508,8 @@ namespace Wabbajack.Test
await CompileAndInstall(profile); await CompileAndInstall(profile);
utils.VerifyInstalledFile(enabledMod, @"Data\scripts\enabledTestPex.pex"); await utils.VerifyInstalledFile(enabledMod, @"Data\scripts\enabledTestPex.pex");
utils.VerifyInstalledFile(disabledMod, @"Data\scripts\disabledTestPex.pex"); await utils.VerifyInstalledFile(disabledMod, @"Data\scripts\disabledTestPex.pex");
var modlistTxt = await utils.InstallFolder.Combine("profiles", profile, "modlist.txt").ReadAllLinesAsync(); var modlistTxt = await utils.InstallFolder.Combine("profiles", profile, "modlist.txt").ReadAllLinesAsync();
Assert.Equal(new string[] Assert.Equal(new string[]

61
Wabbajack.Test/TestUtils.cs
View File

@ -57,7 +57,7 @@ namespace Wabbajack.Test
{ {
Profiles.Do(profile => Profiles.Do(profile =>
{ {
MO2Folder.Combine("profiles", profile, "modlist.txt").WriteAllLines( MO2Folder.Combine("profiles", profile, "modlist.txt").WriteAllLinesAsync(
Mods.Select(s => $"+{s}").ToArray()); Mods.Select(s => $"+{s}").ToArray());
}); });
} }
@ -65,7 +65,7 @@ namespace Wabbajack.Test
{ {
Profiles.Do(profile => Profiles.Do(profile =>
{ {
MO2Folder.Combine("profiles", profile, "modlist.txt").WriteAllLines( MO2Folder.Combine("profiles", profile, "modlist.txt").WriteAllLinesAsync(
enabledMods.Select(s => $"{(s.IsEnabled ? "+" : "-")}{s.ModName}").ToArray()); enabledMods.Select(s => $"{(s.IsEnabled ? "+" : "-")}{s.ModName}").ToArray());
}); });
} }
@ -79,17 +79,14 @@ namespace Wabbajack.Test
return profile_name; return profile_name;
} }
public string AddMod(string name = null) public async Task<string> AddMod(string name = null)
{ {
lock (this) string mod_name = name ?? RandomName();
{ var mod_folder = MO2Folder.Combine(Consts.MO2ModFolderName, (RelativePath)mod_name);
string mod_name = name ?? RandomName(); mod_folder.CreateDirectory();
var mod_folder = MO2Folder.Combine(Consts.MO2ModFolderName, (RelativePath)mod_name); await mod_folder.Combine("meta.ini").WriteAllTextAsync("[General]");
mod_folder.CreateDirectory(); Mods.Add(mod_name);
mod_folder.Combine("meta.ini").WriteAllText("[General]"); return mod_name;
Mods.Add(mod_name);
return mod_name;
}
} }
/// <summary> /// <summary>
@ -100,17 +97,15 @@ namespace Wabbajack.Test
/// <param name="path"></param> /// <param name="path"></param>
/// <param name="random_fill"></param> /// <param name="random_fill"></param>
/// <returns></returns> /// <returns></returns>
public AbsolutePath AddModFile(string mod_name, string path, int random_fill=128) public async Task<AbsolutePath> AddModFile(string mod_name, string path, int random_fill=128)
{ {
var full_path = ModsFolder.Combine(mod_name, path); var full_path = ModsFolder.Combine(mod_name, path);
full_path.Parent.CreateDirectory(); full_path.Parent.CreateDirectory();
GenerateRandomFileData(full_path, random_fill); await GenerateRandomFileData(full_path, random_fill);
return full_path; return full_path;
} }
public void GenerateRandomFileData(AbsolutePath full_path, int random_fill) public async Task GenerateRandomFileData(AbsolutePath full_path, int random_fill)
{ {
byte[] bytes = new byte[0]; byte[] bytes = new byte[0];
if (random_fill != 0) if (random_fill != 0)
@ -118,7 +113,7 @@ namespace Wabbajack.Test
bytes = new byte[random_fill]; bytes = new byte[random_fill];
RNG.NextBytes(bytes); RNG.NextBytes(bytes);
} }
full_path.WriteAllBytes(bytes); await full_path.WriteAllBytesAsync(bytes);
} }
public static byte[] RandomData(int? size = null, int maxSize = 1024) public static byte[] RandomData(int? size = null, int maxSize = 1024)
@ -162,20 +157,20 @@ namespace Wabbajack.Test
return data; return data;
} }
public string AddManualDownload(Dictionary<string, byte[]> contents) public async ValueTask<string> AddManualDownload(Dictionary<string, byte[]> contents)
{ {
var name = RandomName() + ".zip"; var name = RandomName() + ".zip";
using FileStream fs = DownloadsFolder.Combine(name).Create(); await using FileStream fs = await DownloadsFolder.Combine(name).Create();
using ZipArchive archive = new ZipArchive(fs, ZipArchiveMode.Create); using ZipArchive archive = new ZipArchive(fs, ZipArchiveMode.Create);
contents.Do(kv => contents.Do(kv =>
{ {
var entry = archive.CreateEntry(kv.Key); var entry = archive.CreateEntry(kv.Key);
using (var os = entry.Open()) using var os = entry.Open();
os.Write(kv.Value, 0, kv.Value.Length); os.Write(kv.Value, 0, kv.Value.Length);
}); });
DownloadsFolder.Combine(name + Consts.MetaFileExtension).WriteAllLines( await DownloadsFolder.Combine(name + Consts.MetaFileExtension).WriteAllLinesAsync(
"[General]", "[General]",
"manualURL=<TESTING>" "manualURL=<TESTING>"
); );
@ -183,7 +178,7 @@ namespace Wabbajack.Test
return name; return name;
} }
public void VerifyInstalledFile(string mod, string file) public async Task VerifyInstalledFile(string mod, string file)
{ {
var src = MO2Folder.Combine((string)Consts.MO2ModFolderName, mod, file); var src = MO2Folder.Combine((string)Consts.MO2ModFolderName, mod, file);
Assert.True(src.Exists); Assert.True(src.Exists);
@ -191,8 +186,8 @@ namespace Wabbajack.Test
var dest = InstallFolder.Combine((string)Consts.MO2ModFolderName, mod, file); var dest = InstallFolder.Combine((string)Consts.MO2ModFolderName, mod, file);
Assert.True(dest.Exists, $"Destination {dest} doesn't exist"); Assert.True(dest.Exists, $"Destination {dest} doesn't exist");
var srcData = src.ReadAllBytes(); var srcData = await src.ReadAllBytesAsync();
var destData = dest.ReadAllBytes(); var destData = await dest.ReadAllBytesAsync();
Assert.Equal(srcData.Length, destData.Length); Assert.Equal(srcData.Length, destData.Length);
@ -203,7 +198,7 @@ namespace Wabbajack.Test
} }
} }
public void VerifyInstalledGameFile(string file) public async Task VerifyInstalledGameFile(string file)
{ {
var src = GameFolder.Combine(file); var src = GameFolder.Combine(file);
Assert.True(src.Exists); Assert.True(src.Exists);
@ -211,8 +206,8 @@ namespace Wabbajack.Test
var dest = InstallFolder.Combine((string)Consts.GameFolderFilesDir, file); var dest = InstallFolder.Combine((string)Consts.GameFolderFilesDir, file);
Assert.True(dest.Exists); Assert.True(dest.Exists);
var srcData = src.ReadAllBytes(); var srcData = await src.ReadAllBytesAsync();
var destData = dest.ReadAllBytes(); var destData = await dest.ReadAllBytesAsync();
Assert.Equal(srcData.Length, destData.Length); Assert.Equal(srcData.Length, destData.Length);
@ -227,7 +222,7 @@ namespace Wabbajack.Test
return InstallFolder.Combine((string)Consts.MO2ModFolderName, mod, file); return InstallFolder.Combine((string)Consts.MO2ModFolderName, mod, file);
} }
public void VerifyAllFiles(bool gameFileShouldNotExistInGameFolder = true) public async ValueTask VerifyAllFiles(bool gameFileShouldNotExistInGameFolder = true)
{ {
if (gameFileShouldNotExistInGameFolder) if (gameFileShouldNotExistInGameFolder)
{ {
@ -264,16 +259,16 @@ namespace Wabbajack.Test
if (!skipExtensions.Contains(srcFile.Extension)) if (!skipExtensions.Contains(srcFile.Extension))
{ {
Assert.Equal(srcFile.Size, destFile.Size); Assert.Equal(srcFile.Size, destFile.Size);
Assert.Equal(srcFile.FileHash(), destFile.FileHash()); Assert.Equal(await srcFile.FileHashAsync(), await destFile.FileHashAsync());
} }
} }
} }
public AbsolutePath AddGameFile(string path, int i) public async ValueTask<AbsolutePath> AddGameFile(string path, int i)
{ {
var fullPath = GameFolder.Combine(path); var fullPath = GameFolder.Combine(path);
fullPath.Parent.CreateDirectory(); fullPath.Parent.CreateDirectory();
GenerateRandomFileData(fullPath, i); await GenerateRandomFileData(fullPath, i);
return fullPath; return fullPath;
} }
} }

14
Wabbajack.Test/ZEditIntegrationTests.cs
View File

@ -15,13 +15,13 @@ namespace Wabbajack.Test
public async Task CanCreatezEditPatches() public async Task CanCreatezEditPatches()
{ {
var profile = utils.AddProfile(); var profile = utils.AddProfile();
var moda = utils.AddMod(); var moda = await utils.AddMod();
var modb = utils.AddMod(); var modb = await utils.AddMod();
var moddest = utils.AddMod(); var moddest = await utils.AddMod();
var srca = utils.AddModFile(moda, @"srca.esp", 10); var srca = await utils.AddModFile(moda, @"srca.esp", 10);
var srcb = utils.AddModFile(moda, @"srcb.esp.mohidden", 10); var srcb = await utils.AddModFile(moda, @"srcb.esp.mohidden", 10);
var srcc = utils.AddModFile(modb, @"optional\srcc.esp", 10); var srcc = await utils.AddModFile(modb, @"optional\srcc.esp", 10);
var dest = utils.AddModFile(moddest, @"merged.esp", 20); var dest = await utils.AddModFile(moddest, @"merged.esp", 20);
var srcs = new List<string> {srca, srcb, srcc}; var srcs = new List<string> {srca, srcb, srcc};

8
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
View File

@ -72,7 +72,7 @@ namespace Wabbajack.VirtualFileSystem.Test
Assert.NotNull(file); Assert.NotNull(file);
Assert.Equal(128, file.Size); Assert.Equal(128, file.Size);
Assert.Equal(absPath.FileHash(), file.Hash); Assert.Equal(await absPath.FileHashAsync(), file.Hash);
Assert.True(file.IsArchive); Assert.True(file.IsArchive);
var innerFile = file.Children.First(); var innerFile = file.Children.First();
@ -143,7 +143,7 @@ namespace Wabbajack.VirtualFileSystem.Test
var cleanup = await context.Stage(new List<VirtualFile> {file}); var cleanup = await context.Stage(new List<VirtualFile> {file});
await using var stream = file.StagedFile.OpenRead(); await using var stream = await file.StagedFile.OpenRead();
Assert.Equal("This is a test", await stream.ReadAllTextAsync()); Assert.Equal("This is a test", await stream.ReadAllTextAsync());
@ -158,7 +158,7 @@ namespace Wabbajack.VirtualFileSystem.Test
var inner_dir = @"archive\other\dir".RelativeTo(VFS_TEST_DIR); var inner_dir = @"archive\other\dir".RelativeTo(VFS_TEST_DIR);
inner_dir.CreateDirectory(); inner_dir.CreateDirectory();
TEST_ZIP.MoveTo( @"archive\other\dir\nested.zip".RelativeTo(VFS_TEST_DIR)); await TEST_ZIP.MoveToAsync( @"archive\other\dir\nested.zip".RelativeTo(VFS_TEST_DIR));
await ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP); await ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
await AddTestRoot(); await AddTestRoot();
@ -169,7 +169,7 @@ namespace Wabbajack.VirtualFileSystem.Test
foreach (var file in files) foreach (var file in files)
{ {
await using var stream = file.StagedFile.OpenRead(); await using var stream = await file.StagedFile.OpenRead();
Assert.Equal("This is a test", await stream.ReadAllTextAsync()); Assert.Equal("This is a test", await stream.ReadAllTextAsync());
} }

10
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -140,7 +140,7 @@ namespace Wabbajack.VirtualFileSystem
public async Task WriteToFile(AbsolutePath filename) public async Task WriteToFile(AbsolutePath filename)
{ {
await using var fs = filename.Create(); await using var fs = await filename.Create();
await using var bw = new BinaryWriter(fs, Encoding.UTF8, true); await using var bw = new BinaryWriter(fs, Encoding.UTF8, true);
fs.SetLength(0); fs.SetLength(0);
@ -148,7 +148,7 @@ namespace Wabbajack.VirtualFileSystem
bw.Write(FileVersion); bw.Write(FileVersion);
bw.Write((ulong) Index.AllFiles.Count); bw.Write((ulong) Index.AllFiles.Count);
(await Index.AllFiles await (await Index.AllFiles
.PMap(Queue, f => .PMap(Queue, f =>
{ {
var ms = new MemoryStream(); var ms = new MemoryStream();
@ -156,12 +156,12 @@ namespace Wabbajack.VirtualFileSystem
f.Write(ibw); f.Write(ibw);
return ms; return ms;
})) }))
.Do(ms => .DoAsync(async ms =>
{ {
var size = ms.Position; var size = ms.Position;
ms.Position = 0; ms.Position = 0;
bw.Write((ulong) size); bw.Write((ulong) size);
ms.CopyTo(fs); await ms.CopyToAsync(fs);
}); });
Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}"); Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}");
} }
@ -170,7 +170,7 @@ namespace Wabbajack.VirtualFileSystem
{ {
try try
{ {
await using var fs = filename.OpenRead(); await using var fs = await filename.OpenRead();
using var br = new BinaryReader(fs, Encoding.UTF8, true); using var br = new BinaryReader(fs, Encoding.UTF8, true);
var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length)); var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length));
var fileVersion = br.ReadUInt64(); var fileVersion = br.ReadUInt64();

12
Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs
View File

@ -19,15 +19,15 @@ namespace Wabbajack.VirtualFileSystem
public async Task<Hash> HashAsync() public async Task<Hash> HashAsync()
{ {
await using var stream = OpenRead(); await using var stream = await OpenRead();
return stream.xxHash(); return await stream.xxHashAsync();
} }
public DateTime LastModifiedUtc => DateTime.UtcNow; public DateTime LastModifiedUtc => DateTime.UtcNow;
public long Size => _file.Size; public long Size => _file.Size;
public Stream OpenRead() public async ValueTask<Stream> OpenRead()
{ {
var ms = new MemoryStream(); var ms = new MemoryStream();
_file.CopyDataTo(ms); await _file.CopyDataTo(ms);
ms.Position = 0; ms.Position = 0;
return ms; return ms;
} }
@ -44,8 +44,8 @@ namespace Wabbajack.VirtualFileSystem
public async Task MoveTo(AbsolutePath path) public async Task MoveTo(AbsolutePath path)
{ {
await using var fs = path.Create(); await using var fs = await path.Create();
_file.CopyDataTo(fs); await _file.CopyDataTo(fs);
} }
} }
} }

4
Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs
View File

@ -23,9 +23,9 @@ namespace Wabbajack.VirtualFileSystem
} }
public DateTime LastModifiedUtc => _path.LastModifiedUtc; public DateTime LastModifiedUtc => _path.LastModifiedUtc;
public long Size => _path.Size; public long Size => _path.Size;
public Stream OpenRead() public async ValueTask<Stream> OpenRead()
{ {
return _path.OpenRead(); return await _path.OpenRead();
} }
public async Task<bool> CanExtract() public async Task<bool> CanExtract()

4
Wabbajack.VirtualFileSystem/FileExtractor.cs
View File

@ -22,7 +22,7 @@ namespace Wabbajack.VirtualFileSystem
{ {
try try
{ {
if (BSADispatch.MightBeBSA(source)) if (await BSADispatch.MightBeBSA(source))
return await ExtractAllWithBSA(queue, source); return await ExtractAllWithBSA(queue, source);
else if (source.Extension == Consts.OMOD) else if (source.Extension == Consts.OMOD)
return await ExtractAllWithOMOD(source); return await ExtractAllWithOMOD(source);
@ -114,7 +114,7 @@ namespace Wabbajack.VirtualFileSystem
{ {
try try
{ {
await using var arch = BSADispatch.OpenRead(source); await using var arch = await BSADispatch.OpenRead(source);
var files = arch.Files.ToDictionary(f => f.Path, f => (IExtractedFile)new ExtractedBSAFile(f)); var files = arch.Files.ToDictionary(f => f.Path, f => (IExtractedFile)new ExtractedBSAFile(f));
return new ExtractedFiles(files, arch); return new ExtractedFiles(files, arch);
} }

2
Wabbajack.VirtualFileSystem/IExtractedFile.cs
View File

@ -12,7 +12,7 @@ namespace Wabbajack.VirtualFileSystem
public DateTime LastModifiedUtc { get; } public DateTime LastModifiedUtc { get; }
public long Size { get; } public long Size { get; }
public Stream OpenRead(); public ValueTask<Stream> OpenRead();
public Task<bool> CanExtract(); public Task<bool> CanExtract();

10
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -220,7 +220,7 @@ namespace Wabbajack.VirtualFileSystem
self.FillFullPath(depth); self.FillFullPath(depth);
if (context.UseExtendedHashes) if (context.UseExtendedHashes)
self.ExtendedHashes = ExtendedHashes.FromFile(extractedFile); self.ExtendedHashes = await ExtendedHashes.FromFile(extractedFile);
if (!await extractedFile.CanExtract()) return self; if (!await extractedFile.CanExtract()) return self;
@ -386,9 +386,9 @@ namespace Wabbajack.VirtualFileSystem
return path; return path;
} }
public Stream OpenRead() public async ValueTask<Stream> OpenRead()
{ {
return StagedFile.OpenRead(); return await StagedFile.OpenRead();
} }
} }
@ -399,10 +399,10 @@ namespace Wabbajack.VirtualFileSystem
public string MD5 { get; set; } public string MD5 { get; set; }
public string CRC { get; set; } public string CRC { get; set; }
public static ExtendedHashes FromFile(IExtractedFile file) public static async ValueTask<ExtendedHashes> FromFile(IExtractedFile file)
{ {
var hashes = new ExtendedHashes(); var hashes = new ExtendedHashes();
using var stream = file.OpenRead(); await using var stream = await file.OpenRead();
hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex(); hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex();
stream.Position = 0; stream.Position = 0;
hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex(); hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex();

19
Wabbajack/Settings.cs
View File

@ -4,6 +4,7 @@ using System.Collections.Generic;
using System.IO; using System.IO;
using System.Reactive; using System.Reactive;
using System.Reactive.Subjects; using System.Reactive.Subjects;
using System.Threading.Tasks;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json; using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib; using Wabbajack.Lib;
@ -29,20 +30,19 @@ namespace Wabbajack
[JsonIgnore] [JsonIgnore]
public IObservable<Unit> SaveSignal => _saveSignal; public IObservable<Unit> SaveSignal => _saveSignal;
public static bool TryLoadTypicalSettings(out MainSettings settings) public static async ValueTask<(MainSettings settings, bool loaded)> TryLoadTypicalSettings()
{ {
if (!Consts.SettingsFile.Exists) if (!Consts.SettingsFile.Exists)
{ {
settings = default; return default;
return false;
} }
// Version check // Version check
try try
{ {
settings = Consts.SettingsFile.FromJson<MainSettings>(); var settings = Consts.SettingsFile.FromJson<MainSettings>();
if (settings.Version == Consts.SettingsVersion) if (settings.Version == Consts.SettingsVersion)
return true; return (settings, true);
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -52,14 +52,13 @@ namespace Wabbajack
var backup = Consts.SettingsFile.AppendToName("-backup"); var backup = Consts.SettingsFile.AppendToName("-backup");
backup.Delete(); backup.Delete();
Consts.SettingsFile.CopyTo(backup); await Consts.SettingsFile.CopyToAsync(backup);
Consts.SettingsFile.Delete(); Consts.SettingsFile.Delete();
settings = default; return default;
return false;
} }
public static void SaveSettings(MainSettings settings) public static async ValueTask SaveSettings(MainSettings settings)
{ {
settings._saveSignal.OnNext(Unit.Default); settings._saveSignal.OnNext(Unit.Default);
@ -68,7 +67,7 @@ namespace Wabbajack
//settings._saveSignal.OnCompleted(); //settings._saveSignal.OnCompleted();
//await settings._saveSignal; //await settings._saveSignal;
settings.ToJson(Consts.SettingsFile); await settings.ToJsonAsync(Consts.SettingsFile);
} }
} }

6
Wabbajack/View Models/Gallery/ModListMetadataVM.cs
View File

@ -144,11 +144,11 @@ namespace Wabbajack
.Unit() .Unit()
.StartWith(Unit.Default) .StartWith(Unit.Default)
.FlowSwitch(_parent.WhenAny(x => x.IsActive)) .FlowSwitch(_parent.WhenAny(x => x.IsActive))
.Select(_ => .SelectAsync(async _ =>
{ {
try try
{ {
return !metadata.NeedsDownload(Location); return !(await metadata.NeedsDownload(Location));
} }
catch (Exception) catch (Exception)
{ {
@ -185,7 +185,7 @@ namespace Wabbajack
var downloader = DownloadDispatcher.ResolveArchive(Metadata.Links.Download); var downloader = DownloadDispatcher.ResolveArchive(Metadata.Links.Download);
var result = await downloader.Download(new Archive(state: null!) { Name = Metadata.Title, Size = Metadata.DownloadMetadata?.Size ?? 0 }, Location); var result = await downloader.Download(new Archive(state: null!) { Name = Metadata.Title, Size = Metadata.DownloadMetadata?.Size ?? 0 }, Location);
// Want to rehash to current file, even if failed? // Want to rehash to current file, even if failed?
Location.FileHashCached(); await Location.FileHashCachedAsync();
tcs.SetResult(result); tcs.SetResult(result);
} }
catch (Exception ex) catch (Exception ex)

2
Wabbajack/View Models/MainWindowVM.cs
View File

@ -199,7 +199,7 @@ namespace Wabbajack
Settings.PosY = MainWindow.Top; Settings.PosY = MainWindow.Top;
Settings.Width = MainWindow.Width; Settings.Width = MainWindow.Width;
Settings.Height = MainWindow.Height; Settings.Height = MainWindow.Height;
MainSettings.SaveSettings(Settings); MainSettings.SaveSettings(Settings).AsTask().Wait();
Application.Current.Shutdown(); Application.Current.Shutdown();
} }
} }

15
Wabbajack/View Models/ModListVM.cs
View File

@ -45,19 +45,17 @@ namespace Wabbajack
ImageObservable = Observable.Return(Unit.Default) ImageObservable = Observable.Return(Unit.Default)
// Download and retrieve bytes on background thread // Download and retrieve bytes on background thread
.ObserveOn(RxApp.TaskpoolScheduler) .ObserveOn(RxApp.TaskpoolScheduler)
.Select(filePath => .SelectAsync(async filePath =>
{ {
try try
{ {
using var fs = ModListPath.OpenShared(); await using var fs = await ModListPath.OpenShared();
using var ar = new ZipArchive(fs, ZipArchiveMode.Read); using var ar = new ZipArchive(fs, ZipArchiveMode.Read);
var ms = new MemoryStream(); var ms = new MemoryStream();
var entry = ar.GetEntry("modlist-image.png"); var entry = ar.GetEntry("modlist-image.png");
if (entry == null) return default(MemoryStream); if (entry == null) return default(MemoryStream);
using (var e = entry.Open()) await using var e = entry.Open();
{ e.CopyTo(ms);
e.CopyTo(ms);
}
return ms; return ms;
} }
catch (Exception ex) catch (Exception ex)
@ -82,10 +80,7 @@ namespace Wabbajack
} }
}) })
// If ever would return null, show WJ logo instead // If ever would return null, show WJ logo instead
.Select(x => .Select(x => x ?? ResourceLinks.WabbajackLogoNoText.Value)
{
return x ?? ResourceLinks.WabbajackLogoNoText.Value;
})
.Replay(1) .Replay(1)
.RefCount(); .RefCount();
} }

3
Wabbajack/Views/MainWindow.xaml.cs
View File

@ -45,8 +45,9 @@ namespace Wabbajack
Warmup(); Warmup();
var (settings, loadedSettings) = MainSettings.TryLoadTypicalSettings().AsTask().Result;
// Load settings // Load settings
if (CLIArguments.NoSettings || !MainSettings.TryLoadTypicalSettings(out var settings)) if (CLIArguments.NoSettings || !loadedSettings)
{ {
_settings = new MainSettings _settings = new MainSettings
{ {