From f9dc9148e76914b9d133d2f0575d59069114e059 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 25 May 2020 08:31:56 -0600 Subject: [PATCH 1/8] Purge all non async IO routines from `Paths.cs` --- Wabbajack.CLI/Verbs/Decrypt.cs | 2 +- .../CircuitBreaker/WithAutoRetry.cs | 37 ++++++++ .../IAsyncEnumerableExtensions.cs | 35 ++++++++ Wabbajack.Common/Metrics.cs | 2 +- Wabbajack.Common/Paths.cs | 89 +++++-------------- Wabbajack.Common/Utils.cs | 16 ++-- Wabbajack.Lib/AuthorApi/Client.cs | 4 +- Wabbajack.Lib/ClientAPI.cs | 8 +- .../AbstractNeedsLoginDownloader.cs | 2 +- .../Downloaders/BethesdaNetDownloader.cs | 2 +- .../Downloaders/DownloadDispatcher.cs | 4 +- Wabbajack.Lib/Downloaders/MEGADownloader.cs | 2 +- Wabbajack.Lib/NexusApi/NexusApi.cs | 2 +- Wabbajack.Lib/zEditIntegration.cs | 5 +- 14 files changed, 118 insertions(+), 92 deletions(-) create mode 100644 Wabbajack.Common/CircuitBreaker/WithAutoRetry.cs create mode 100644 Wabbajack.Common/IAsyncEnumerableExtensions.cs diff --git a/Wabbajack.CLI/Verbs/Decrypt.cs b/Wabbajack.CLI/Verbs/Decrypt.cs index 6488820a..8a5340c1 100644 --- a/Wabbajack.CLI/Verbs/Decrypt.cs +++ b/Wabbajack.CLI/Verbs/Decrypt.cs @@ -16,7 +16,7 @@ namespace Wabbajack.CLI.Verbs protected override async Task Run() { - File.WriteAllBytes(Output, Utils.FromEncryptedData(Name)); + File.WriteAllBytes(Output, await Utils.FromEncryptedData(Name)); return 0; } } diff --git a/Wabbajack.Common/CircuitBreaker/WithAutoRetry.cs b/Wabbajack.Common/CircuitBreaker/WithAutoRetry.cs new file mode 100644 index 00000000..c70a1b43 --- /dev/null +++ b/Wabbajack.Common/CircuitBreaker/WithAutoRetry.cs @@ -0,0 +1,37 @@ +using System; +using System.Threading.Tasks; + +namespace Wabbajack.Common +{ + public static class CircuitBreaker + { + public static TimeSpan DEFAULT_DELAY = TimeSpan.FromMilliseconds(100); + public static int DEFAULT_DELAY_MULTIPLIER = 2; + public static int DEFAULT_RETRIES = 5; + + public static async ValueTask WithAutoRetry(Func> f, TimeSpan? delay = null, int? multipler = null, int? maxRetries = null) where TE : Exception + { + int retries = 0; + delay ??= DEFAULT_DELAY; + multipler ??= DEFAULT_DELAY_MULTIPLIER; + maxRetries ??= DEFAULT_RETRIES; + + TOP: + try + { + return await f(); + } + catch (TE ex) + { + retries += 1; + if (retries > maxRetries) + throw; + Utils.Log($"(Retry {retries} of {maxRetries}), got exception {ex.Message}, waiting {delay.Value.TotalMilliseconds}ms"); + await Task.Delay(delay.Value); + delay = delay * multipler; + goto TOP; + } + } + + } +} diff --git a/Wabbajack.Common/IAsyncEnumerableExtensions.cs b/Wabbajack.Common/IAsyncEnumerableExtensions.cs new file mode 100644 index 00000000..9fd167dd --- /dev/null +++ b/Wabbajack.Common/IAsyncEnumerableExtensions.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Wabbajack.Common +{ + public static class IAsyncEnumerableExtensions + { + /// + /// Same as .Select but expects a function that returns an async result + /// + /// + /// + /// + /// + /// + public static async IAsyncEnumerable SelectAsync(this IEnumerable coll, + Func> mapFn) + { + foreach (var itm in coll) + { + yield return await mapFn(itm); + } + } + + public static async ValueTask> ToList(this IAsyncEnumerable coll) + { + var list =new List(); + await foreach (var itm in coll) + list.Add(itm); + return list; + } + + } +} diff --git a/Wabbajack.Common/Metrics.cs b/Wabbajack.Common/Metrics.cs index 58eaa5f8..f04ad216 100644 --- a/Wabbajack.Common/Metrics.cs +++ b/Wabbajack.Common/Metrics.cs @@ -32,7 +32,7 @@ namespace Wabbajack.Common try { client.DefaultRequestHeaders.Add(Consts.MetricsKeyHeader, - Utils.FromEncryptedJson(Consts.MetricsKeyHeader)); + await Utils.FromEncryptedJson(Consts.MetricsKeyHeader)); await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{value}"); } catch (Exception) diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index eae25994..88daac5f 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -87,19 +87,21 @@ namespace Wabbajack.Common public Extension Extension => Extension.FromPath(_path); - public FileStream OpenRead() + public ValueTask OpenRead() { - return File.OpenRead(_path); + return OpenShared(); } - public FileStream Create() + public ValueTask Create() { - return File.Create(_path); + var path = _path; + return CircuitBreaker.WithAutoRetry(async () => File.Create(path)); } - public FileStream OpenWrite() + public ValueTask OpenWrite() { - return File.OpenWrite(_path); + var path = _path; + return CircuitBreaker.WithAutoRetry(async () => File.OpenWrite(path)); } public async Task WriteAllTextAsync(string text) @@ -107,12 +109,6 @@ namespace Wabbajack.Common await using var fs = File.Create(_path); await fs.WriteAsync(Encoding.UTF8.GetBytes(text)); } - - public void WriteAllText(string text) - { - using var fs = File.Create(_path); - fs.Write(Encoding.UTF8.GetBytes(text)); - } public bool Exists => File.Exists(_path) || Directory.Exists(_path); public bool IsFile => File.Exists(_path); @@ -175,24 +171,6 @@ namespace Wabbajack.Common public AbsolutePath Root => (AbsolutePath)Path.GetPathRoot(_path); - /// - /// Moves this file to the specified location, will use Copy if required - /// - /// - /// Replace the destination file if it exists - public void MoveTo(AbsolutePath otherPath, bool overwrite = false) - { - if (Root != otherPath.Root) - { - if (otherPath.Exists && overwrite) - otherPath.Delete(); - - CopyTo(otherPath); - return; - } - File.Move(_path, otherPath._path, overwrite ? MoveOptions.ReplaceExisting : MoveOptions.None); - } - /// /// Moves this file to the specified location, will use Copy if required /// @@ -286,7 +264,7 @@ namespace Wabbajack.Common public async Task ReadAllBytesAsync() { - await using var f = OpenRead(); + await using var f = await OpenShared(); return await f.ReadAllAsync(); } @@ -325,37 +303,19 @@ namespace Wabbajack.Common return File.ReadAllLines(_path); } - public void WriteAllBytes(byte[] data) - { - using var fs = Create(); - fs.Write(data); - } - public async Task WriteAllBytesAsync(byte[] data) { - await using var fs = Create(); + await using var fs = await Create(); await fs.WriteAsync(data); } public async Task WriteAllAsync(Stream data, bool disposeAfter = true) { - await using var fs = Create(); + await using var fs = await Create(); await data.CopyToAsync(fs); if (disposeAfter) await data.DisposeAsync(); } - public void AppendAllText(string text) - { - File.AppendAllText(_path, text); - } - - public void CopyTo(AbsolutePath dest) - { - File.Copy(_path, dest._path); - } - - - [DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Auto)] private static extern bool CreateHardLink(string lpFileName, string lpExistingFileName, IntPtr lpSecurityAttributes); @@ -384,12 +344,6 @@ namespace Wabbajack.Common return (await ReadAllTextAsync()).Split(new[] {'\n', '\r'}, StringSplitOptions.RemoveEmptyEntries); } - public byte[] ReadAllBytes() - { - using var file = OpenShared(); - return file.ReadAll(); - } - public static AbsolutePath GetCurrentDirectory() { return new AbsolutePath(Directory.GetCurrentDirectory()); @@ -397,8 +351,8 @@ namespace Wabbajack.Common public async Task CopyToAsync(AbsolutePath destFile) { - await using var src = OpenRead(); - await using var dest = destFile.Create(); + await using var src = await OpenRead(); + await using var dest = await destFile.Create(); await src.CopyToAsync(dest); } @@ -413,11 +367,6 @@ namespace Wabbajack.Common await WriteAllTextAsync(string.Join("\r\n",strings)); } - public void WriteAllLines(params string[] strings) - { - WriteAllText(string.Join("\n",strings)); - } - public int CompareTo(AbsolutePath other) { return string.Compare(_path, other._path, StringComparison.Ordinal); @@ -428,14 +377,18 @@ namespace Wabbajack.Common return File.ReadAllText(_path); } - public FileStream OpenShared() + public ValueTask OpenShared() { - return File.Open(_path, FileMode.Open, FileAccess.Read, FileShare.Read); + var path = _path; + return CircuitBreaker.WithAutoRetry(async () => + File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read)); } - public FileStream WriteShared() + public ValueTask WriteShared() { - return File.Open(_path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite); + var path = _path; + return CircuitBreaker.WithAutoRetry(async () => + File.Open(path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)); } public async Task CopyDirectoryToAsync(AbsolutePath destination) diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 3ebe2a98..ecefd9ed 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -165,7 +165,7 @@ namespace Wabbajack.Common if (LogFile == default) return; lock (_lock) { - LogFile.AppendAllText($"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}\r\n"); + File.AppendAllText(LogFile.ToString(), $"{(DateTime.Now - _startTime).TotalSeconds:0.##} - {msg}\r\n"); } } @@ -794,16 +794,16 @@ namespace Wabbajack.Common } } - public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch) + public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out AbsolutePath ePatch) { var patchName = Consts.PatchCacheFolder.Combine($"{foundHash.ToHex()}_{fileHash.ToHex()}.patch"); if (patchName.Exists) { - ePatch = patchName.ReadAllBytes(); + ePatch = patchName; return true; } - ePatch = Array.Empty(); + ePatch = default; return false; } @@ -1039,9 +1039,9 @@ namespace Wabbajack.Common bytes.ToEcryptedData(key); } - public static T FromEncryptedJson(string key) + public static async Task FromEncryptedJson(string key) { - var decoded = FromEncryptedData(key); + var decoded = await FromEncryptedData(key); return Encoding.UTF8.GetString(decoded).FromJsonString(); } @@ -1053,9 +1053,9 @@ namespace Wabbajack.Common Consts.LocalAppDataPath.Combine(key).WriteAllBytes(encoded); } - public static byte[] FromEncryptedData(string key) + public static async Task FromEncryptedData(string key) { - var bytes = Consts.LocalAppDataPath.Combine(key).ReadAllBytes(); + var bytes = await Consts.LocalAppDataPath.Combine(key).ReadAllBytesAsync(); return ProtectedData.Unprotect(bytes, Encoding.UTF8.GetBytes(key), DataProtectionScope.LocalMachine); } diff --git a/Wabbajack.Lib/AuthorApi/Client.cs b/Wabbajack.Lib/AuthorApi/Client.cs index e467a3b6..8c64c826 100644 --- a/Wabbajack.Lib/AuthorApi/Client.cs +++ b/Wabbajack.Lib/AuthorApi/Client.cs @@ -63,7 +63,7 @@ namespace Wabbajack.Lib.AuthorApi { progressFn("Hashing file parts", Percent.FactoryPutInRange(part.Index, parts.Length)); var buffer = new byte[part.Size]; - await using (var fs = path.OpenShared()) + await using (var fs = await path.OpenShared()) { fs.Position = part.Offset; await fs.ReadAsync(buffer); @@ -91,7 +91,7 @@ namespace Wabbajack.Lib.AuthorApi { progressFn("Uploading Part", Percent.FactoryPutInRange(part.Index, definition.Parts.Length)); var buffer = new byte[part.Size]; - await using (var fs = path.OpenShared()) + await using (var fs = await path.OpenShared()) { fs.Position = part.Offset; await fs.ReadAsync(buffer); diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index fef0d841..b56c7d0f 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -43,11 +43,11 @@ namespace Wabbajack.Lib public class ClientAPI { - public static Common.Http.Client GetClient() + public static async Task GetClient() { var client = new Common.Http.Client(); if (Utils.HaveEncryptedJson(Consts.MetricsKeyHeader)) - client.Headers.Add((Consts.MetricsKeyHeader, Utils.FromEncryptedJson(Consts.MetricsKeyHeader))); + client.Headers.Add((Consts.MetricsKeyHeader, await Utils.FromEncryptedJson(Consts.MetricsKeyHeader))); return client; } @@ -63,7 +63,7 @@ namespace Wabbajack.Lib RETRY: - var response = await GetClient() + var response = await (await GetClient()) .PostAsync($"{Consts.WabbajackBuildServerUri}mod_upgrade", new StringContent(request.ToJson(), Encoding.UTF8, "application/json")); if (response.IsSuccessStatusCode) @@ -114,7 +114,7 @@ namespace Wabbajack.Lib public static async Task GetNexusCacheStats() { - return await GetClient() + return await (await GetClient()) .GetJsonAsync($"{Consts.WabbajackBuildServerUri}nexus_cache/stats"); } diff --git a/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs b/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs index acc5e3fd..6768635d 100644 --- a/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs @@ -91,7 +91,7 @@ namespace Wabbajack.Lib.Downloaders Helpers.Cookie[] cookies; try { - cookies = Utils.FromEncryptedJson(_encryptedKeyName); + cookies = await Utils.FromEncryptedJson(_encryptedKeyName); if (cookies != null) return Helpers.GetClient(cookies, SiteURL.ToString()); } diff --git a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs index 47445128..37255561 100644 --- a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs +++ b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs @@ -220,7 +220,7 @@ namespace Wabbajack.Lib.Downloaders { var info = new CollectedBNetInfo(); - var login_info = Utils.FromEncryptedJson(DataName); + var login_info = await Utils.FromEncryptedJson(DataName); var client = new Common.Http.Client(); diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index 7a19665f..3a12ea78 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -128,10 +128,10 @@ namespace Wabbajack.Lib.Downloaders response.Dispose(); Utils.Log($"Applying patch to {archive.Name}"); - await using(var src = result.NewFile.Path.OpenShared()) + await using(var src = await result.NewFile.Path.OpenShared()) await using (var final = destination.Create()) { - Utils.ApplyPatch(src, () => tempFile.Path.OpenShared(), final); + Utils.ApplyPatch(src, () => tempFile.Path.OpenShared().Result, final); } var hash = await destination.FileHashCachedAsync(); diff --git a/Wabbajack.Lib/Downloaders/MEGADownloader.cs b/Wabbajack.Lib/Downloaders/MEGADownloader.cs index 83f172b9..657ddcdb 100644 --- a/Wabbajack.Lib/Downloaders/MEGADownloader.cs +++ b/Wabbajack.Lib/Downloaders/MEGADownloader.cs @@ -153,7 +153,7 @@ namespace Wabbajack.Lib.Downloaders else { Utils.Status("Logging into MEGA with saved credentials."); - var infos = Utils.FromEncryptedJson(DataName); + var infos = await Utils.FromEncryptedJson(DataName); var authInfo = infos.ToAuthInfos(); await MegaApiClient.LoginAsync(authInfo); } diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index e6b64131..0e251e82 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -58,7 +58,7 @@ namespace Wabbajack.Lib.NexusApi try { - return Utils.FromEncryptedJson("nexusapikey"); + return await Utils.FromEncryptedJson("nexusapikey"); } catch (Exception) { diff --git a/Wabbajack.Lib/zEditIntegration.cs b/Wabbajack.Lib/zEditIntegration.cs index 36eea8fb..237da171 100644 --- a/Wabbajack.Lib/zEditIntegration.cs +++ b/Wabbajack.Lib/zEditIntegration.cs @@ -185,7 +185,7 @@ namespace Wabbajack.Lib }; })); - var srcData = result.Sources.Select(f => _mo2Compiler.MO2Folder.Combine(f.RelativePath).ReadAllBytes()) + var srcData = (await result.Sources.SelectAsync(async f => await _mo2Compiler.MO2Folder.Combine(f.RelativePath).ReadAllBytesAsync()).ToList()) .ConcatArrays(); var dstData = await source.AbsolutePath.ReadAllBytesAsync(); @@ -264,7 +264,8 @@ namespace Wabbajack.Lib { Utils.LogStatus($"Generating zEdit merge: {m.To}"); - var srcData = m.Sources.Select(s => installer.OutputFolder.Combine(s.RelativePath).ReadAllBytes()) + var srcData = (await m.Sources.SelectAsync(async s => await installer.OutputFolder.Combine(s.RelativePath).ReadAllBytesAsync()) + .ToList()) .ConcatArrays(); var patchData = await installer.LoadBytesFromPath(m.PatchID); From ea08c9865dbbb481609242460e133b40b632a2cf Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 25 May 2020 10:24:16 -0600 Subject: [PATCH 2/8] Common and BSA routines now use async IO exclusively --- Compression.BSA/BA2Builder.cs | 2 +- Compression.BSA/BA2Reader.cs | 97 ++++++++++----------- Compression.BSA/BSABuilder.cs | 2 +- Compression.BSA/BSADispatch.cs | 15 ++-- Compression.BSA/BSAReader.cs | 70 +++++++-------- Compression.BSA/IBSAReader.cs | 2 +- Compression.BSA/TES3Builder.cs | 2 +- Compression.BSA/TES3Reader.cs | 50 ++++++----- Wabbajack.CLI/Verbs/Decrypt.cs | 2 +- Wabbajack.Common.Test/EncryptedDataTests.cs | 10 +-- Wabbajack.Common.Test/PathTests.cs | 13 +-- Wabbajack.Common/Hash.cs | 27 +----- Wabbajack.Common/Json.cs | 5 +- Wabbajack.Common/Metrics.cs | 2 +- Wabbajack.Common/Utils.cs | 26 +++--- Wabbajack.Test/TestUtils.cs | 12 +-- 16 files changed, 153 insertions(+), 184 deletions(-) diff --git a/Compression.BSA/BA2Builder.cs b/Compression.BSA/BA2Builder.cs index c74cec6a..196df729 100644 --- a/Compression.BSA/BA2Builder.cs +++ b/Compression.BSA/BA2Builder.cs @@ -58,7 +58,7 @@ namespace Compression.BSA public async Task Build(AbsolutePath filename) { SortEntries(); - await using var fs = filename.Create(); + await using var fs = await filename.Create(); await using var bw = new BinaryWriter(fs); bw.Write(Encoding.ASCII.GetBytes(_state.HeaderMagic)); diff --git a/Compression.BSA/BA2Reader.cs b/Compression.BSA/BA2Reader.cs index 0c5cfbc5..b5d20ed9 100644 --- a/Compression.BSA/BA2Reader.cs +++ b/Compression.BSA/BA2Reader.cs @@ -38,19 +38,22 @@ namespace Compression.BSA public bool HasNameTable => _nameTableOffset > 0; - public BA2Reader(AbsolutePath filename) : this(filename.OpenRead()) + + + public static async Task Load(AbsolutePath filename) { - _filename = filename; + var rdr = new BA2Reader(await filename.OpenShared()) {_filename = filename}; + await rdr.LoadHeaders(); + return rdr; } - public BA2Reader(Stream stream) + private BA2Reader(Stream stream) { _stream = stream; _rdr = new BinaryReader(_stream, Encoding.UTF7); - LoadHeaders(); } - public void LoadHeaders() + private async Task LoadHeaders() { _headerMagic = Encoding.ASCII.GetString(_rdr.ReadBytes(4)); @@ -196,39 +199,36 @@ namespace Compression.BSA public uint HeaderSize => DDS.HeaderSizeForFormat((DXGI_FORMAT)_format); - public void CopyDataTo(Stream output) + public async ValueTask CopyDataTo(Stream output) { var bw = new BinaryWriter(output); WriteHeader(bw); - using (var fs = _bsa._filename.OpenRead()) - using (var br = new BinaryReader(fs)) + await using var fs = await _bsa._filename.OpenRead(); + using var br = new BinaryReader(fs); + foreach (var chunk in _chunks) { - foreach (var chunk in _chunks) + var full = new byte[chunk._fullSz]; + var isCompressed = chunk._packSz != 0; + + br.BaseStream.Seek((long)chunk._offset, SeekOrigin.Begin); + + if (!isCompressed) { - var full = new byte[chunk._fullSz]; - var isCompressed = chunk._packSz != 0; - - br.BaseStream.Seek((long)chunk._offset, SeekOrigin.Begin); - - if (!isCompressed) - { - br.BaseStream.Read(full, 0, full.Length); - } - else - { - byte[] compressed = new byte[chunk._packSz]; - br.BaseStream.Read(compressed, 0, compressed.Length); - var inflater = new Inflater(); - inflater.SetInput(compressed); - inflater.Inflate(full); - } - - bw.BaseStream.Write(full, 0, full.Length); + await br.BaseStream.ReadAsync(full, 0, full.Length); + } + else + { + byte[] compressed = new byte[chunk._packSz]; + await br.BaseStream.ReadAsync(compressed, 0, compressed.Length); + var inflater = new Inflater(); + inflater.SetInput(compressed); + inflater.Inflate(full); } - } + await bw.BaseStream.WriteAsync(full, 0, full.Length); + } } public void Dump(Action print) @@ -480,31 +480,28 @@ namespace Compression.BSA public uint Size => _realSize; public FileStateObject State => new BA2FileEntryState(this); - public void CopyDataTo(Stream output) + public async ValueTask CopyDataTo(Stream output) { - using (var fs = _bsa._filename.OpenRead()) + await using var fs = await _bsa._filename.OpenRead(); + fs.Seek((long) _offset, SeekOrigin.Begin); + uint len = Compressed ? _size : _realSize; + + var bytes = new byte[len]; + fs.Read(bytes, 0, (int) len); + + if (!Compressed) { - fs.Seek((long) _offset, SeekOrigin.Begin); - uint len = Compressed ? _size : _realSize; - - var bytes = new byte[len]; - fs.Read(bytes, 0, (int) len); - - if (!Compressed) - { - output.Write(bytes, 0, bytes.Length); - } - else - { - var uncompressed = new byte[_realSize]; - var inflater = new Inflater(); - inflater.SetInput(bytes); - inflater.Inflate(uncompressed); - output.Write(uncompressed, 0, uncompressed.Length); - } + await output.WriteAsync(bytes, 0, bytes.Length); + } + else + { + var uncompressed = new byte[_realSize]; + var inflater = new Inflater(); + inflater.SetInput(bytes); + inflater.Inflate(uncompressed); + await output.WriteAsync(uncompressed, 0, uncompressed.Length); } } - } [JsonName("BA2FileEntryState")] diff --git a/Compression.BSA/BSABuilder.cs b/Compression.BSA/BSABuilder.cs index dcaa2b49..5ea06ba1 100644 --- a/Compression.BSA/BSABuilder.cs +++ b/Compression.BSA/BSABuilder.cs @@ -95,7 +95,7 @@ namespace Compression.BSA public async Task Build(AbsolutePath outputName) { RegenFolderRecords(); - await using var fs = outputName.Create(); + await using var fs = await outputName.Create(); await using var wtr = new BinaryWriter(fs); wtr.Write(_fileId); diff --git a/Compression.BSA/BSADispatch.cs b/Compression.BSA/BSADispatch.cs index b6fc497b..a208b03c 100644 --- a/Compression.BSA/BSADispatch.cs +++ b/Compression.BSA/BSADispatch.cs @@ -1,33 +1,34 @@ using System.Collections.Generic; using System.IO; using System.Text; +using System.Threading.Tasks; using Wabbajack.Common; namespace Compression.BSA { public static class BSADispatch { - public static IBSAReader OpenRead(AbsolutePath filename) + public static async ValueTask OpenRead(AbsolutePath filename) { var fourcc = ""; - using (var file = filename.OpenRead()) + using (var file = await filename.OpenRead()) { fourcc = Encoding.ASCII.GetString(new BinaryReader(file).ReadBytes(4)); } if (fourcc == TES3Reader.TES3_MAGIC) - return new TES3Reader(filename); + return await TES3Reader.Load(filename); if (fourcc == "BSA\0") - return new BSAReader(filename); + return await BSAReader.Load(filename); if (fourcc == "BTDX") - return new BA2Reader(filename); + return await BA2Reader.Load(filename); throw new InvalidDataException("Filename is not a .bsa or .ba2, magic " + fourcc); } private static HashSet MagicStrings = new HashSet {TES3Reader.TES3_MAGIC, "BSA\0", "BTDX"}; - public static bool MightBeBSA(AbsolutePath filename) + public static async ValueTask MightBeBSA(AbsolutePath filename) { - using var file = filename.OpenRead(); + using var file = await filename.OpenRead(); var fourcc = Encoding.ASCII.GetString(new BinaryReader(file).ReadBytes(4)); return MagicStrings.Contains(fourcc); } diff --git a/Compression.BSA/BSAReader.cs b/Compression.BSA/BSAReader.cs index f3b92722..ff1e7cea 100644 --- a/Compression.BSA/BSAReader.cs +++ b/Compression.BSA/BSAReader.cs @@ -60,8 +60,8 @@ namespace Compression.BSA internal uint _folderRecordOffset; private List _folders; internal string _magic; - private readonly BinaryReader _rdr; - private readonly Stream _stream; + private BinaryReader _rdr; + private Stream _stream; internal uint _totalFileNameLength; internal uint _totalFolderNameLength; internal uint _version; @@ -80,16 +80,15 @@ namespace Compression.BSA } - public BSAReader(AbsolutePath filename) + public static async ValueTask Load(AbsolutePath filename) { - _fileName = filename; - using var stream = filename.OpenRead(); + using var stream = await filename.OpenRead(); using var br = new BinaryReader(stream); - _rdr = br; - _stream = stream; - LoadHeaders(); - _rdr = null; - _stream = null; + var bsa = new BSAReader {_rdr = br, _stream = stream, _fileName = filename}; + await bsa.LoadHeaders(); + bsa._rdr = null; + bsa._stream = null; + return bsa; } public IEnumerable Files @@ -132,7 +131,7 @@ namespace Compression.BSA { } - private void LoadHeaders() + private async ValueTask LoadHeaders() { var fourcc = Encoding.ASCII.GetString(_rdr.ReadBytes(4)); @@ -330,43 +329,34 @@ namespace Compression.BSA _name = rdr.ReadStringTerm(_bsa.HeaderType); } - public void CopyDataTo(Stream output) + public async ValueTask CopyDataTo(Stream output) { - using (var in_file = _bsa._fileName.OpenRead()) - using (var rdr = new BinaryReader(in_file)) - { - rdr.BaseStream.Position = _dataOffset; + await using var in_file = await _bsa._fileName.OpenRead(); + using var rdr = new BinaryReader(in_file); + rdr.BaseStream.Position = _dataOffset; - if (_bsa.HeaderType == VersionType.SSE) + if (_bsa.HeaderType == VersionType.SSE) + { + if (Compressed) { - if (Compressed) - { - using var r = LZ4Stream.Decode(rdr.BaseStream); - r.CopyToLimit(output, (int) _originalSize); - } - else - { - rdr.BaseStream.CopyToLimit(output, (int) _onDiskSize); - } + using var r = LZ4Stream.Decode(rdr.BaseStream); + await r.CopyToLimitAsync(output, (int) _originalSize); } else { - if (Compressed) - { - using var z = new InflaterInputStream(rdr.BaseStream); - z.CopyToLimit(output, (int) _originalSize); - } - else - rdr.BaseStream.CopyToLimit(output, (int) _onDiskSize); + await rdr.BaseStream.CopyToLimitAsync(output, (int) _onDiskSize); } } - } - - public byte[] GetData() - { - var ms = new MemoryStream(); - CopyDataTo(ms); - return ms.ToArray(); + else + { + if (Compressed) + { + await using var z = new InflaterInputStream(rdr.BaseStream); + await z.CopyToLimitAsync(output, (int) _originalSize); + } + else + await rdr.BaseStream.CopyToLimitAsync(output, (int) _onDiskSize); + } } } diff --git a/Compression.BSA/IBSAReader.cs b/Compression.BSA/IBSAReader.cs index 0c6cb3eb..77071005 100644 --- a/Compression.BSA/IBSAReader.cs +++ b/Compression.BSA/IBSAReader.cs @@ -60,7 +60,7 @@ namespace Compression.BSA /// in order to maintain thread-safe access. /// /// - void CopyDataTo(Stream output); + ValueTask CopyDataTo(Stream output); void Dump(Action print); } diff --git a/Compression.BSA/TES3Builder.cs b/Compression.BSA/TES3Builder.cs index 36f5229d..78a42178 100644 --- a/Compression.BSA/TES3Builder.cs +++ b/Compression.BSA/TES3Builder.cs @@ -26,7 +26,7 @@ namespace Compression.BSA public async Task Build(AbsolutePath filename) { - await using var fs = filename.Create(); + await using var fs = await filename.Create(); await using var bw = new BinaryWriter(fs); bw.Write(_state.VersionNumber); diff --git a/Compression.BSA/TES3Reader.cs b/Compression.BSA/TES3Reader.cs index 63876dd0..45ffecdc 100644 --- a/Compression.BSA/TES3Reader.cs +++ b/Compression.BSA/TES3Reader.cs @@ -18,48 +18,52 @@ namespace Compression.BSA internal long _dataOffset; internal AbsolutePath _filename; - public TES3Reader(AbsolutePath filename) + public static async ValueTask Load(AbsolutePath filename) { - _filename = filename; - using var fs = filename.OpenRead(); + await using var fs = await filename.OpenRead(); using var br = new BinaryReader(fs); - _versionNumber = br.ReadUInt32(); - _hashTableOffset = br.ReadUInt32(); - _fileCount = br.ReadUInt32(); + var rdr = new TES3Reader + { + _filename = filename, + _versionNumber = br.ReadUInt32(), + _hashTableOffset = br.ReadUInt32(), + _fileCount = br.ReadUInt32() + }; - _files = new TES3FileEntry[_fileCount]; - for (int i = 0; i < _fileCount; i++) + rdr._files = new TES3FileEntry[rdr._fileCount]; + for (int i = 0; i < rdr._fileCount; i++) { var file = new TES3FileEntry { Index = i, - Archive = this, + Archive = rdr, Size = br.ReadUInt32(), Offset = br.ReadUInt32() }; - _files[i] = file; + rdr._files[i] = file; } - for (int i = 0; i < _fileCount; i++) + for (int i = 0; i < rdr._fileCount; i++) { - _files[i].NameOffset = br.ReadUInt32(); + rdr._files[i].NameOffset = br.ReadUInt32(); } var origPos = br.BaseStream.Position; - for (int i = 0; i < _fileCount; i++) + for (int i = 0; i < rdr._fileCount; i++) { - br.BaseStream.Position = origPos + _files[i].NameOffset; - _files[i].Path = new RelativePath(br.ReadStringTerm(VersionType.TES3)); + br.BaseStream.Position = origPos + rdr._files[i].NameOffset; + rdr._files[i].Path = new RelativePath(br.ReadStringTerm(VersionType.TES3)); } - br.BaseStream.Position = _hashTableOffset + 12; - for (int i = 0; i < _fileCount; i++) + br.BaseStream.Position = rdr._hashTableOffset + 12; + for (int i = 0; i < rdr._fileCount; i++) { - _files[i].Hash1 = br.ReadUInt32(); - _files[i].Hash2 = br.ReadUInt32(); + rdr._files[i].Hash1 = br.ReadUInt32(); + rdr._files[i].Hash2 = br.ReadUInt32(); } - _dataOffset = br.BaseStream.Position; + rdr._dataOffset = br.BaseStream.Position; + return rdr; } public async ValueTask DisposeAsync() @@ -118,11 +122,11 @@ namespace Compression.BSA Hash2 = Hash2 }; - public void CopyDataTo(Stream output) + public async ValueTask CopyDataTo(Stream output) { - using var fs = Archive._filename.OpenRead(); + await using var fs = await Archive._filename.OpenRead(); fs.Position = Archive._dataOffset + Offset; - fs.CopyToLimit(output, (int)Size); + await fs.CopyToLimitAsync(output, (int)Size); } public void Dump(Action print) diff --git a/Wabbajack.CLI/Verbs/Decrypt.cs b/Wabbajack.CLI/Verbs/Decrypt.cs index 8a5340c1..53aa0fb7 100644 --- a/Wabbajack.CLI/Verbs/Decrypt.cs +++ b/Wabbajack.CLI/Verbs/Decrypt.cs @@ -16,7 +16,7 @@ namespace Wabbajack.CLI.Verbs protected override async Task Run() { - File.WriteAllBytes(Output, await Utils.FromEncryptedData(Name)); + await Output.RelativeTo(AbsolutePath.EntryPoint).WriteAllBytesAsync(await Utils.FromEncryptedData(Name)); return 0; } } diff --git a/Wabbajack.Common.Test/EncryptedDataTests.cs b/Wabbajack.Common.Test/EncryptedDataTests.cs index 05ccc609..713a63c3 100644 --- a/Wabbajack.Common.Test/EncryptedDataTests.cs +++ b/Wabbajack.Common.Test/EncryptedDataTests.cs @@ -11,20 +11,20 @@ namespace Wabbajack.Common.Test [Fact] public async Task CanDetectNewEncryptedData() { - var test_string = Guid.NewGuid().ToString(); + var testString = Guid.NewGuid().ToString(); var data = new ConcurrentBag(); - var events = Utils.HaveEncryptedJsonObservable(test_string).Subscribe(e => + var events = Utils.HaveEncryptedJsonObservable(testString).Subscribe(e => { if (e) - data.Add(test_string); + data.Add(testString); else data.Clear(); }); - test_string.ToEcryptedJson(test_string); + await testString.ToEcryptedJson(testString); await Task.Delay(100); - Assert.Contains(test_string, data); + Assert.Contains(testString, data); } diff --git a/Wabbajack.Common.Test/PathTests.cs b/Wabbajack.Common.Test/PathTests.cs index 1408d3dd..c5914f1f 100644 --- a/Wabbajack.Common.Test/PathTests.cs +++ b/Wabbajack.Common.Test/PathTests.cs @@ -1,28 +1,29 @@ -using Xunit; +using System.Threading.Tasks; +using Xunit; namespace Wabbajack.Common.Test { public class PathTests { [Fact] - public void CanDeleteReadOnlyFile() + public async Task CanDeleteReadOnlyFile() { var tempFile = new TempFile(); - tempFile.Path.WriteAllText("Test"); + await tempFile.Path.WriteAllTextAsync("Test"); tempFile.Path.SetReadOnly(true); tempFile.Path.Delete(); } [Fact] - public void CanMoveReadOnlyFiles() + public async Task CanMoveReadOnlyFiles() { var tempFile = new TempFile(); var tempFile2 = new TempFile(); - tempFile.Path.WriteAllText("Test"); + await tempFile.Path.WriteAllTextAsync("Test"); tempFile.Path.SetReadOnly(true); - tempFile.Path.MoveTo(tempFile2.Path); + await tempFile.Path.MoveToAsync(tempFile2.Path); } [Fact] diff --git a/Wabbajack.Common/Hash.cs b/Wabbajack.Common/Hash.cs index eab70e4f..7a993ae9 100644 --- a/Wabbajack.Common/Hash.cs +++ b/Wabbajack.Common/Hash.cs @@ -129,21 +129,6 @@ namespace Wabbajack.Common return sha.Hash.ToHex(); } - public static Hash FileHash(this AbsolutePath file, bool nullOnIoError = false) - { - try - { - using var fs = file.OpenRead(); - var config = new xxHashConfig {HashSizeInBits = 64}; - using var f = new StatusFileStream(fs, $"Hashing {(string)file.FileName}"); - return new Hash(BitConverter.ToUInt64(xxHashFactory.Instance.Create(config).ComputeHash(f).Hash)); - } - catch (IOException) - { - if (nullOnIoError) return Hash.Empty; - throw; - } - } public static Hash xxHash(this byte[] data) { var hash = new xxHashConfig(); @@ -166,16 +151,6 @@ namespace Wabbajack.Common var value = xxHashFactory.Instance.Create(config).ComputeHash(f); return Hash.FromULong(BitConverter.ToUInt64(value.Hash)); } - public static Hash FileHashCached(this AbsolutePath file, bool nullOnIoError = false) - { - if (TryGetHashCache(file, out var foundHash)) return foundHash; - - var hash = file.FileHash(nullOnIoError); - if (hash != Hash.Empty) - WriteHashCache(file, hash); - return hash; - } - public static bool TryGetHashCache(AbsolutePath file, out Hash hash) { var normPath = Encoding.UTF8.GetBytes(file.Normalize()); @@ -223,7 +198,7 @@ namespace Wabbajack.Common { try { - await using var fs = file.OpenRead(); + await using var fs = await file.OpenRead(); var config = new xxHashConfig {HashSizeInBits = 64}; await using var hs = new StatusFileStream(fs, $"Hashing {file}"); var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(hs); diff --git a/Wabbajack.Common/Json.cs b/Wabbajack.Common/Json.cs index 090e110e..d91ff8ae 100644 --- a/Wabbajack.Common/Json.cs +++ b/Wabbajack.Common/Json.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Reflection; using System.Text; +using System.Threading.Tasks; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; using Wabbajack.Common.Serialization.Json; @@ -57,9 +58,9 @@ namespace Wabbajack.Common ser.Serialize(writer, obj); } - public static void ToJson(this T obj, AbsolutePath path) + public static async ValueTask ToJsonAsync(this T obj, AbsolutePath path) { - using var fs = path.Create(); + await using var fs = await path.Create(); obj.ToJson(fs); } diff --git a/Wabbajack.Common/Metrics.cs b/Wabbajack.Common/Metrics.cs index f04ad216..300baa9f 100644 --- a/Wabbajack.Common/Metrics.cs +++ b/Wabbajack.Common/Metrics.cs @@ -17,7 +17,7 @@ namespace Wabbajack.Common { if (!Utils.HaveEncryptedJson(Consts.MetricsKeyHeader)) { - Utils.ToEcryptedJson(Utils.MakeRandomKey(), Consts.MetricsKeyHeader); + Utils.MakeRandomKey().ToEcryptedJson(Consts.MetricsKeyHeader).AsTask().Wait(); } } /// diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index ecefd9ed..a24ed096 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -68,7 +68,7 @@ namespace Wabbajack.Common if (LogFile.Exists) { var newPath = Consts.LogsFolder.Combine(Consts.EntryPoint.FileNameWithoutExtension + LogFile.LastModified.ToString(" yyyy-MM-dd HH_mm_ss") + ".log"); - LogFile.MoveTo(newPath, true); + LogFile.MoveToAsync(newPath, true).Wait(); } var logFiles = LogFolder.EnumerateFiles(false).ToList(); @@ -726,7 +726,7 @@ namespace Wabbajack.Common RETRY_OPEN: try { - await using var f = cacheFile.OpenRead(); + await using var f = await cacheFile.OpenRead(); await f.CopyToAsync(output); } catch (IOException) @@ -741,7 +741,7 @@ namespace Wabbajack.Common { var tmpName = Consts.PatchCacheFolder.Combine(Guid.NewGuid() + ".tmp"); - await using (var f = tmpName.Create()) + await using (var f = await tmpName.Create()) { Status("Creating Patch"); OctoDiff.Create(a, b, f); @@ -750,7 +750,7 @@ namespace Wabbajack.Common RETRY: try { - tmpName.MoveTo(cacheFile, true); + await tmpName.MoveToAsync(cacheFile, true); } catch (UnauthorizedAccessException) { @@ -775,7 +775,7 @@ namespace Wabbajack.Common patchStream.Position = 0; var tmpName = Consts.PatchCacheFolder.Combine(Guid.NewGuid() + ".tmp"); - await using (var f = tmpName.Create()) + await using (var f = await tmpName.Create()) { await patchStream.CopyToAsync(f); patchStream.Position = 0; @@ -786,7 +786,7 @@ namespace Wabbajack.Common var cacheFile = Consts.PatchCacheFolder.Combine($"{srcHash.ToHex()}_{destHash.ToHex()}.patch"); Consts.PatchCacheFolder.CreateDirectory(); - tmpName.MoveTo(cacheFile, true); + await tmpName.MoveToAsync(cacheFile, true); } catch (UnauthorizedAccessException) { @@ -900,7 +900,7 @@ namespace Wabbajack.Common var startTime = DateTime.Now; var seconds = 2; var results = await Enumerable.Range(0, queue.DesiredNumWorkers) - .PMap(queue, idx => + .PMap(queue, async idx => { var random = new Random(); @@ -908,7 +908,7 @@ namespace Wabbajack.Common long size = 0; byte[] buffer = new byte[1024 * 8]; random.NextBytes(buffer); - using (var fs = file.Create()) + await using (var fs = await file.Create()) { while (DateTime.Now < startTime + new TimeSpan(0, 0, seconds)) { @@ -939,7 +939,7 @@ namespace Wabbajack.Common } } var speed = await TestDiskSpeedInner(queue, path); - speed.ToJson(benchmarkFile); + await speed.ToJsonAsync(benchmarkFile); return speed; } @@ -1033,10 +1033,10 @@ namespace Wabbajack.Common /// /// /// - public static void ToEcryptedJson(this T data, string key) + public static async ValueTask ToEcryptedJson(this T data, string key) { var bytes = Encoding.UTF8.GetBytes(data.ToJson()); - bytes.ToEcryptedData(key); + await bytes.ToEcryptedData(key); } public static async Task FromEncryptedJson(string key) @@ -1046,12 +1046,12 @@ namespace Wabbajack.Common } - public static void ToEcryptedData(this byte[] bytes, string key) + public static async ValueTask ToEcryptedData(this byte[] bytes, string key) { var encoded = ProtectedData.Protect(bytes, Encoding.UTF8.GetBytes(key), DataProtectionScope.LocalMachine); Consts.LocalAppDataPath.CreateDirectory(); - Consts.LocalAppDataPath.Combine(key).WriteAllBytes(encoded); + await Consts.LocalAppDataPath.Combine(key).WriteAllBytesAsync(encoded); } public static async Task FromEncryptedData(string key) { diff --git a/Wabbajack.Test/TestUtils.cs b/Wabbajack.Test/TestUtils.cs index f6491f5e..3fff1d7f 100644 --- a/Wabbajack.Test/TestUtils.cs +++ b/Wabbajack.Test/TestUtils.cs @@ -183,7 +183,7 @@ namespace Wabbajack.Test return name; } - public void VerifyInstalledFile(string mod, string file) + public async Task VerifyInstalledFile(string mod, string file) { var src = MO2Folder.Combine((string)Consts.MO2ModFolderName, mod, file); Assert.True(src.Exists); @@ -191,8 +191,8 @@ namespace Wabbajack.Test var dest = InstallFolder.Combine((string)Consts.MO2ModFolderName, mod, file); Assert.True(dest.Exists, $"Destination {dest} doesn't exist"); - var srcData = src.ReadAllBytes(); - var destData = dest.ReadAllBytes(); + var srcData = await src.ReadAllBytesAsync(); + var destData = await dest.ReadAllBytesAsync(); Assert.Equal(srcData.Length, destData.Length); @@ -203,7 +203,7 @@ namespace Wabbajack.Test } } - public void VerifyInstalledGameFile(string file) + public async Task VerifyInstalledGameFile(string file) { var src = GameFolder.Combine(file); Assert.True(src.Exists); @@ -211,8 +211,8 @@ namespace Wabbajack.Test var dest = InstallFolder.Combine((string)Consts.GameFolderFilesDir, file); Assert.True(dest.Exists); - var srcData = src.ReadAllBytes(); - var destData = dest.ReadAllBytes(); + var srcData = await src.ReadAllBytesAsync(); + var destData = await dest.ReadAllBytesAsync(); Assert.Equal(srcData.Length, destData.Length); From 191f321dc21b77be215c94120740a2b09dcd5158 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 25 May 2020 10:30:47 -0600 Subject: [PATCH 3/8] `VirtualFile` now useses Async IO exclusively --- Wabbajack.Common/Hash.cs | 8 ++++++++ Wabbajack.Common/IAsyncEnumerableExtensions.cs | 17 +++++++++++++++++ .../VirtualFileSystemTests.cs | 8 ++++---- Wabbajack.VirtualFileSystem/Context.cs | 8 ++++---- Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs | 12 ++++++------ .../ExtractedDiskFile.cs | 4 ++-- Wabbajack.VirtualFileSystem/FileExtractor.cs | 4 ++-- Wabbajack.VirtualFileSystem/IExtractedFile.cs | 2 +- Wabbajack.VirtualFileSystem/VirtualFile.cs | 10 +++++----- 9 files changed, 49 insertions(+), 24 deletions(-) diff --git a/Wabbajack.Common/Hash.cs b/Wabbajack.Common/Hash.cs index 7a993ae9..88b32ae0 100644 --- a/Wabbajack.Common/Hash.cs +++ b/Wabbajack.Common/Hash.cs @@ -151,6 +151,14 @@ namespace Wabbajack.Common var value = xxHashFactory.Instance.Create(config).ComputeHash(f); return Hash.FromULong(BitConverter.ToUInt64(value.Hash)); } + + public static async Task xxHashAsync(this Stream stream) + { + var config = new xxHashConfig {HashSizeInBits = 64}; + await using var f = new StatusFileStream(stream, $"Hashing memory stream"); + var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(f); + return Hash.FromULong(BitConverter.ToUInt64(value.Hash)); + } public static bool TryGetHashCache(AbsolutePath file, out Hash hash) { var normPath = Encoding.UTF8.GetBytes(file.Normalize()); diff --git a/Wabbajack.Common/IAsyncEnumerableExtensions.cs b/Wabbajack.Common/IAsyncEnumerableExtensions.cs index 9fd167dd..5270f42d 100644 --- a/Wabbajack.Common/IAsyncEnumerableExtensions.cs +++ b/Wabbajack.Common/IAsyncEnumerableExtensions.cs @@ -22,6 +22,23 @@ namespace Wabbajack.Common yield return await mapFn(itm); } } + + /// + /// Same as .Select but expects a function that returns an async result + /// + /// + /// + /// + /// + /// + public static async ValueTask DoAsync(this IEnumerable coll, + Func> mapFn) + { + foreach (var itm in coll) + { + await mapFn(itm); + } + } public static async ValueTask> ToList(this IAsyncEnumerable coll) { diff --git a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs index 0868ea56..d9bfc14d 100644 --- a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs @@ -72,7 +72,7 @@ namespace Wabbajack.VirtualFileSystem.Test Assert.NotNull(file); Assert.Equal(128, file.Size); - Assert.Equal(absPath.FileHash(), file.Hash); + Assert.Equal(await absPath.FileHashAsync(), file.Hash); Assert.True(file.IsArchive); var innerFile = file.Children.First(); @@ -143,7 +143,7 @@ namespace Wabbajack.VirtualFileSystem.Test var cleanup = await context.Stage(new List {file}); - await using var stream = file.StagedFile.OpenRead(); + await using var stream = await file.StagedFile.OpenRead(); Assert.Equal("This is a test", await stream.ReadAllTextAsync()); @@ -158,7 +158,7 @@ namespace Wabbajack.VirtualFileSystem.Test var inner_dir = @"archive\other\dir".RelativeTo(VFS_TEST_DIR); inner_dir.CreateDirectory(); - TEST_ZIP.MoveTo( @"archive\other\dir\nested.zip".RelativeTo(VFS_TEST_DIR)); + await TEST_ZIP.MoveToAsync( @"archive\other\dir\nested.zip".RelativeTo(VFS_TEST_DIR)); await ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP); await AddTestRoot(); @@ -169,7 +169,7 @@ namespace Wabbajack.VirtualFileSystem.Test foreach (var file in files) { - await using var stream = file.StagedFile.OpenRead(); + await using var stream = await file.StagedFile.OpenRead(); Assert.Equal("This is a test", await stream.ReadAllTextAsync()); } diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index 95f0ae7a..09f3f315 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -140,7 +140,7 @@ namespace Wabbajack.VirtualFileSystem public async Task WriteToFile(AbsolutePath filename) { - await using var fs = filename.Create(); + await using var fs = await filename.Create(); await using var bw = new BinaryWriter(fs, Encoding.UTF8, true); fs.SetLength(0); @@ -156,12 +156,12 @@ namespace Wabbajack.VirtualFileSystem f.Write(ibw); return ms; })) - .Do(ms => + .Do(async ms => { var size = ms.Position; ms.Position = 0; bw.Write((ulong) size); - ms.CopyTo(fs); + await ms.CopyToAsync(fs); }); Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}"); } @@ -170,7 +170,7 @@ namespace Wabbajack.VirtualFileSystem { try { - await using var fs = filename.OpenRead(); + await using var fs = await filename.OpenRead(); using var br = new BinaryReader(fs, Encoding.UTF8, true); var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length)); var fileVersion = br.ReadUInt64(); diff --git a/Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs b/Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs index f278ccb0..694a5a2a 100644 --- a/Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs +++ b/Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs @@ -19,15 +19,15 @@ namespace Wabbajack.VirtualFileSystem public async Task HashAsync() { - await using var stream = OpenRead(); - return stream.xxHash(); + await using var stream = await OpenRead(); + return await stream.xxHashAsync(); } public DateTime LastModifiedUtc => DateTime.UtcNow; public long Size => _file.Size; - public Stream OpenRead() + public async ValueTask OpenRead() { var ms = new MemoryStream(); - _file.CopyDataTo(ms); + await _file.CopyDataTo(ms); ms.Position = 0; return ms; } @@ -44,8 +44,8 @@ namespace Wabbajack.VirtualFileSystem public async Task MoveTo(AbsolutePath path) { - await using var fs = path.Create(); - _file.CopyDataTo(fs); + await using var fs = await path.Create(); + await _file.CopyDataTo(fs); } } } diff --git a/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs b/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs index 78dceaf2..04a4e242 100644 --- a/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs +++ b/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs @@ -23,9 +23,9 @@ namespace Wabbajack.VirtualFileSystem } public DateTime LastModifiedUtc => _path.LastModifiedUtc; public long Size => _path.Size; - public Stream OpenRead() + public async ValueTask OpenRead() { - return _path.OpenRead(); + return await _path.OpenRead(); } public async Task CanExtract() diff --git a/Wabbajack.VirtualFileSystem/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor.cs index 9e6a4380..7d36226e 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor.cs @@ -22,7 +22,7 @@ namespace Wabbajack.VirtualFileSystem { try { - if (BSADispatch.MightBeBSA(source)) + if (await BSADispatch.MightBeBSA(source)) return await ExtractAllWithBSA(queue, source); else if (source.Extension == Consts.OMOD) return await ExtractAllWithOMOD(source); @@ -114,7 +114,7 @@ namespace Wabbajack.VirtualFileSystem { try { - await using var arch = BSADispatch.OpenRead(source); + await using var arch = await BSADispatch.OpenRead(source); var files = arch.Files.ToDictionary(f => f.Path, f => (IExtractedFile)new ExtractedBSAFile(f)); return new ExtractedFiles(files, arch); } diff --git a/Wabbajack.VirtualFileSystem/IExtractedFile.cs b/Wabbajack.VirtualFileSystem/IExtractedFile.cs index e37e86b6..f9660274 100644 --- a/Wabbajack.VirtualFileSystem/IExtractedFile.cs +++ b/Wabbajack.VirtualFileSystem/IExtractedFile.cs @@ -12,7 +12,7 @@ namespace Wabbajack.VirtualFileSystem public DateTime LastModifiedUtc { get; } public long Size { get; } - public Stream OpenRead(); + public ValueTask OpenRead(); public Task CanExtract(); diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 3b0ce3a1..96106c6c 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -220,7 +220,7 @@ namespace Wabbajack.VirtualFileSystem self.FillFullPath(depth); if (context.UseExtendedHashes) - self.ExtendedHashes = ExtendedHashes.FromFile(extractedFile); + self.ExtendedHashes = await ExtendedHashes.FromFile(extractedFile); if (!await extractedFile.CanExtract()) return self; @@ -386,9 +386,9 @@ namespace Wabbajack.VirtualFileSystem return path; } - public Stream OpenRead() + public async ValueTask OpenRead() { - return StagedFile.OpenRead(); + return await StagedFile.OpenRead(); } } @@ -399,10 +399,10 @@ namespace Wabbajack.VirtualFileSystem public string MD5 { get; set; } public string CRC { get; set; } - public static ExtendedHashes FromFile(IExtractedFile file) + public static async ValueTask FromFile(IExtractedFile file) { var hashes = new ExtendedHashes(); - using var stream = file.OpenRead(); + await using var stream = await file.OpenRead(); hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex(); stream.Position = 0; hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex(); From b2a112bd37a257e9b4351955c6703b803890fb38 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 25 May 2020 11:34:25 -0600 Subject: [PATCH 4/8] Whole app now useses Async IO exclusively --- Compression.BSA.Test/BSATests.cs | 12 +- Wabbajack.App.Test/SettingsTests.cs | 8 +- Wabbajack.CLI/Verbs/BSADump.cs | 4 +- Wabbajack.CLI/Verbs/Encrypt.cs | 2 +- Wabbajack.Lib/ACompiler.cs | 28 ++-- Wabbajack.Lib/AInstaller.cs | 4 +- .../CompilationSteps/DeconstructBSAs.cs | 4 +- Wabbajack.Lib/CompilationSteps/IncludeAll.cs | 2 +- .../Downloaders/AbstractIPS4Downloader.cs | 2 +- .../AbstractNeedsLoginDownloader.cs | 2 +- .../Downloaders/BethesdaNetDownloader.cs | 6 +- .../Downloaders/DownloadDispatcher.cs | 2 +- .../Downloaders/GameFileSourceDownloader.cs | 4 +- Wabbajack.Lib/Downloaders/HTTPDownloader.cs | 2 +- .../Downloaders/WabbajackCDNDownloader.cs | 2 +- .../Downloaders/YouTubeDownloader.cs | 6 +- Wabbajack.Lib/Extensions/ReactiveUIExt.cs | 16 +++ Wabbajack.Lib/MO2Compiler.cs | 10 +- Wabbajack.Lib/MO2Installer.cs | 12 +- .../ModListRegistry/ModListMetadata.cs | 4 +- Wabbajack.Lib/NexusApi/NexusApi.cs | 2 +- Wabbajack.Lib/zEditIntegration.cs | 2 +- .../ABuildServerSystemTest.cs | 23 +-- Wabbajack.Server/Controllers/AuthoredFiles.cs | 2 +- .../Services/ModListDownloader.cs | 2 +- Wabbajack.Server/Services/PatchBuilder.cs | 10 +- Wabbajack.Test/DownloaderTests.cs | 2 +- Wabbajack.Test/EndToEndTests.cs | 6 +- Wabbajack.Test/MO2Tests.cs | 10 +- Wabbajack.Test/SanityTests.cs | 132 +++++++++--------- Wabbajack.Test/TestUtils.cs | 49 +++---- Wabbajack.Test/ZEditIntegrationTests.cs | 14 +- Wabbajack/Settings.cs | 19 ++- .../View Models/Gallery/ModListMetadataVM.cs | 6 +- Wabbajack/View Models/MainWindowVM.cs | 2 +- Wabbajack/View Models/ModListVM.cs | 15 +- Wabbajack/Views/MainWindow.xaml.cs | 3 +- 37 files changed, 218 insertions(+), 213 deletions(-) diff --git a/Compression.BSA.Test/BSATests.cs b/Compression.BSA.Test/BSATests.cs index 87c8eb8f..f3d6c4b2 100644 --- a/Compression.BSA.Test/BSATests.cs +++ b/Compression.BSA.Test/BSATests.cs @@ -96,16 +96,16 @@ namespace Compression.BSA.Test var tempFile = ((RelativePath)"tmp.bsa").RelativeToEntryPoint(); var size = bsa.Size; - await using var a = BSADispatch.OpenRead(bsa); - await a.Files.PMap(Queue, file => + await using var a = await BSADispatch.OpenRead(bsa); + await a.Files.PMap(Queue, async file => { var absName = _tempDir.Combine(file.Path); ViaJson(file.State); absName.Parent.CreateDirectory(); - using (var fs = absName.Create()) + await using (var fs = await absName.Create()) { - file.CopyDataTo(fs); + await file.CopyDataTo(fs); } Assert.Equal(file.Size, absName.Size); @@ -123,7 +123,7 @@ namespace Compression.BSA.Test var streams = await a.Files.PMap(Queue, async file => { var absPath = _tempDir.Combine(file.Path); - var str = absPath.OpenRead(); + var str = await absPath.OpenRead(); await w.AddFile(ViaJson(file.State), str); return str; }); @@ -132,7 +132,7 @@ namespace Compression.BSA.Test } TestContext.WriteLine($"Verifying {bsa}"); - await using var b = BSADispatch.OpenRead(tempFile); + await using var b = await BSADispatch.OpenRead(tempFile); TestContext.WriteLine($"Performing A/B tests on {bsa}"); Assert.Equal(a.State.ToJson(), b.State.ToJson()); diff --git a/Wabbajack.App.Test/SettingsTests.cs b/Wabbajack.App.Test/SettingsTests.cs index 3409919b..aa657070 100644 --- a/Wabbajack.App.Test/SettingsTests.cs +++ b/Wabbajack.App.Test/SettingsTests.cs @@ -8,16 +8,16 @@ namespace Wabbajack.Test [Fact] public async Task CanSaveAndLoadSettings() { - MainSettings.TryLoadTypicalSettings(out var settings); + var (settings, loadedSettings) = await MainSettings.TryLoadTypicalSettings(); - if (settings == null) + if (settings == null || !loadedSettings) { settings = new MainSettings(); } - MainSettings.SaveSettings(settings); + await MainSettings.SaveSettings(settings); - Assert.True(MainSettings.TryLoadTypicalSettings(out settings)); + Assert.True((await MainSettings.TryLoadTypicalSettings()).loaded); } } diff --git a/Wabbajack.CLI/Verbs/BSADump.cs b/Wabbajack.CLI/Verbs/BSADump.cs index 6b452631..49519f6f 100644 --- a/Wabbajack.CLI/Verbs/BSADump.cs +++ b/Wabbajack.CLI/Verbs/BSADump.cs @@ -14,8 +14,8 @@ namespace Wabbajack.CLI.Verbs protected override async Task Run() { - await using var bsa = BSADispatch.OpenRead(Input.RelativeTo(AbsolutePath.GetCurrentDirectory())); - bsa.Dump(line => Console.WriteLine(line)); + await using var bsa = await BSADispatch.OpenRead(Input.RelativeTo(AbsolutePath.GetCurrentDirectory())); + bsa.Dump(Console.WriteLine); return ExitCode.Ok; } } diff --git a/Wabbajack.CLI/Verbs/Encrypt.cs b/Wabbajack.CLI/Verbs/Encrypt.cs index f178e808..5a243627 100644 --- a/Wabbajack.CLI/Verbs/Encrypt.cs +++ b/Wabbajack.CLI/Verbs/Encrypt.cs @@ -18,7 +18,7 @@ namespace Wabbajack.CLI.Verbs protected override async Task Run() { - File.ReadAllBytes(Input).ToEcryptedData(Name); + await File.ReadAllBytes(Input).ToEcryptedData(Name); return 0; } } diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index e11b6c4a..49eb6cd9 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -81,10 +81,10 @@ namespace Wabbajack.Lib return id; } - internal FileStream IncludeFile(out RelativePath id) + internal AbsolutePath IncludeFile(out RelativePath id) { id = IncludeId(); - return ModListOutputFolder.Combine(id).Create(); + return ModListOutputFolder.Combine(id); } internal async Task IncludeFile(string data) @@ -144,34 +144,32 @@ namespace Wabbajack.Lib ModList.Image = (RelativePath)"modlist-image.png"; } - using (var of = ModListOutputFolder.Combine("modlist").Create()) + using (var of = await ModListOutputFolder.Combine("modlist").Create()) ModList.ToJson(of); ModListOutputFile.Delete(); - using (var fs = ModListOutputFile.Create()) + using (var fs = await ModListOutputFile.Create()) { using (var za = new ZipArchive(fs, ZipArchiveMode.Create)) { - ModListOutputFolder.EnumerateFiles() + await ModListOutputFolder.EnumerateFiles() .DoProgress("Compressing ModList", - f => + async f => { var ze = za.CreateEntry((string)f.FileName); - using var os = ze.Open(); - using var ins = f.OpenRead(); - ins.CopyTo(os); + await using var os = ze.Open(); + await using var ins = await f.OpenRead(); + await ins.CopyToAsync(os); }); // Copy in modimage if (ModListImage.Exists) { var ze = za.CreateEntry((string)ModList.Image); - using (var os = ze.Open()) - using (var ins = ModListImage.OpenRead()) - { - ins.CopyTo(os); - } + await using var os = ze.Open(); + await using var ins = await ModListImage.OpenRead(); + await ins.CopyToAsync(os); } } } @@ -180,7 +178,7 @@ namespace Wabbajack.Lib var metadata = new DownloadMetadata { Size = ModListOutputFile.Size, - Hash = ModListOutputFile.FileHash(), + Hash = await ModListOutputFile.FileHashAsync(), NumberOfArchives = ModList.Archives.Count, SizeOfArchives = ModList.Archives.Sum(a => a.Size), NumberOfInstalledFiles = ModList.Directives.Count, diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 6784c8a1..1e074ac0 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -71,7 +71,7 @@ namespace Wabbajack.Lib public async Task LoadBytesFromPath(RelativePath path) { - await using var e = ExtractedModListFiles![path].OpenRead(); + await using var e = await ExtractedModListFiles![path].OpenRead(); return await e.ReadAllAsync(); } @@ -213,7 +213,7 @@ namespace Wabbajack.Lib toFile.Delete(); // Patch it - await using (var outStream = toFile.Create()) + await using (var outStream = await toFile.Create()) { Utils.ApplyPatch(oldData, () => new MemoryStream(patchData), outStream); } diff --git a/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs b/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs index 586ecb00..b6e1f36c 100644 --- a/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs +++ b/Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs @@ -62,7 +62,7 @@ namespace Wabbajack.Lib.CompilationSteps if (source.AbsolutePath.Size >= (long) 2 << 31) { - await using var bsa = BSADispatch.OpenRead(source.AbsolutePath); + await using var bsa = await BSADispatch.OpenRead(source.AbsolutePath); if (bsa.State is BSAStateObject) { Utils.Error( @@ -95,7 +95,7 @@ namespace Wabbajack.Lib.CompilationSteps } CreateBSA directive; - await using (var bsa = BSADispatch.OpenRead(source.AbsolutePath)) + await using (var bsa = await BSADispatch.OpenRead(source.AbsolutePath)) { directive = new CreateBSA( state: bsa.State, diff --git a/Wabbajack.Lib/CompilationSteps/IncludeAll.cs b/Wabbajack.Lib/CompilationSteps/IncludeAll.cs index e837f75a..1152e99c 100644 --- a/Wabbajack.Lib/CompilationSteps/IncludeAll.cs +++ b/Wabbajack.Lib/CompilationSteps/IncludeAll.cs @@ -14,7 +14,7 @@ namespace Wabbajack.Lib.CompilationSteps public override async ValueTask Run(RawSourceFile source) { var inline = source.EvolveTo(); - await using var file = source.File.StagedFile.OpenRead(); + await using var file = await source.File.StagedFile.OpenRead(); inline.SourceDataID = await _compiler.IncludeFile(await file.ReadAllAsync()); return inline; } diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs index 5211bcf0..4567e4bc 100644 --- a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs @@ -124,7 +124,7 @@ namespace Wabbajack.Lib.Downloaders using var stream = await ResolveDownloadStream(a); if (stream == null) return false; await using var fromStream = await stream.Content.ReadAsStreamAsync(); - await using var toStream = destination.Create(); + await using var toStream = await destination.Create(); await fromStream.CopyToAsync(toStream); return true; } diff --git a/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs b/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs index 6768635d..4f197f3c 100644 --- a/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs @@ -81,7 +81,7 @@ namespace Wabbajack.Lib.Downloaders await Task.Delay(500, cancel); } - cookies.ToEcryptedJson(_encryptedKeyName); + await cookies.ToEcryptedJson(_encryptedKeyName); return cookies; } diff --git a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs index 37255561..277948ff 100644 --- a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs +++ b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs @@ -110,7 +110,7 @@ namespace Wabbajack.Lib.Downloaders try { var result = last_line.FromJsonString(); - result.ToEcryptedJson(DataName); + await result.ToEcryptedJson(DataName); return result; } catch (Exception ex) @@ -180,7 +180,7 @@ namespace Wabbajack.Lib.Downloaders private const uint CKM_Magic = 0x52415442; // BTAR private async Task ConvertCKMToZip(AbsolutePath src, AbsolutePath dest) { - using var reader = new BinaryReader(src.OpenRead()); + using var reader = new BinaryReader(await src.OpenRead()); var magic = reader.ReadUInt32(); if (magic != CKM_Magic) throw new InvalidDataException("Invalid magic format in CKM parsing"); @@ -193,7 +193,7 @@ namespace Wabbajack.Lib.Downloaders if (minorVersion < 2 || minorVersion > 4) throw new InvalidDataException("Archive minor version is unknown. Should be 2, 3, or 4."); - await using var fos = dest.Create(); + await using var fos = await dest.Create(); using var archive = new ZipArchive(fos, ZipArchiveMode.Create); while (reader.PeekChar() != -1) { diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index 3a12ea78..f350fb93 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -129,7 +129,7 @@ namespace Wabbajack.Lib.Downloaders Utils.Log($"Applying patch to {archive.Name}"); await using(var src = await result.NewFile.Path.OpenShared()) - await using (var final = destination.Create()) + await using (var final = await destination.Create()) { Utils.ApplyPatch(src, () => tempFile.Path.OpenShared().Result, final); } diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs index 25f5b5fa..2cdf66d4 100644 --- a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs @@ -71,8 +71,8 @@ namespace Wabbajack.Lib.Downloaders public override async Task Download(Archive a, AbsolutePath destination) { - await using var src = SourcePath.OpenRead(); - await using var dest = destination.Create(); + await using var src = await SourcePath.OpenRead(); + await using var dest = await destination.Create(); var size = SourcePath.Size; await src.CopyToWithStatusAsync(size, dest, "Copying from Game folder"); diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index 0beec26e..a8280490 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -81,7 +81,7 @@ namespace Wabbajack.Lib.Downloaders destination.Parent.CreateDirectory(); } - using (var fs = download ? destination.Create() : null) + using (var fs = download ? await destination.Create() : null) { var client = Client ?? new Common.Http.Client(); client.Headers.Add(("User-Agent", Consts.UserAgent)); diff --git a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs index f6097963..1f14b7c6 100644 --- a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs +++ b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs @@ -57,7 +57,7 @@ namespace Wabbajack.Lib.Downloaders { destination.Parent.CreateDirectory(); var definition = await GetDefinition(); - using var fs = destination.Create(); + await using var fs = await destination.Create(); using var mmfile = MemoryMappedFile.CreateFromFile(fs, null, definition.Size, MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, false); var client = new Common.Http.Client(); using var queue = new WorkQueue(); diff --git a/Wabbajack.Lib/Downloaders/YouTubeDownloader.cs b/Wabbajack.Lib/Downloaders/YouTubeDownloader.cs index 783bfbb7..75ee4072 100644 --- a/Wabbajack.Lib/Downloaders/YouTubeDownloader.cs +++ b/Wabbajack.Lib/Downloaders/YouTubeDownloader.cs @@ -114,7 +114,7 @@ namespace Wabbajack.Lib.Downloaders var trackFolder = folder.Dir.Combine("tracks"); - await using (var fs = initialDownload.Create()) + await using (var fs = await initialDownload.Create()) { await client.Videos.Streams.CopyToAsync(stream, fs, new Progress($"Downloading {a.Name}"), CancellationToken.None); @@ -128,7 +128,7 @@ namespace Wabbajack.Lib.Downloaders await ExtractTrack(initialDownload, trackFolder, track); }); - await using var dest = destination.Create(); + await using var dest = await destination.Create(); using var ar = new ZipArchive(dest, ZipArchiveMode.Create); foreach (var track in trackFolder.EnumerateFiles().OrderBy(e => e)) { @@ -136,7 +136,7 @@ namespace Wabbajack.Lib.Downloaders var entry = ar.CreateEntry(Path.Combine("Data", "tracks", (string)track.RelativeTo(trackFolder)), CompressionLevel.NoCompression); entry.LastWriteTime = meta.UploadDate; await using var es = entry.Open(); - await using var ins = track.OpenRead(); + await using var ins = await track.OpenRead(); await ins.CopyToAsync(es); } diff --git a/Wabbajack.Lib/Extensions/ReactiveUIExt.cs b/Wabbajack.Lib/Extensions/ReactiveUIExt.cs index eb2a79bc..e608cf3b 100644 --- a/Wabbajack.Lib/Extensions/ReactiveUIExt.cs +++ b/Wabbajack.Lib/Extensions/ReactiveUIExt.cs @@ -4,6 +4,8 @@ using System.Linq.Expressions; using System.Reactive; using System.Reactive.Disposables; using System.Reactive.Linq; +using System.Reactive.Subjects; +using System.Threading.Tasks; using DynamicData; using DynamicData.Kernel; using ReactiveUI; @@ -38,6 +40,20 @@ namespace Wabbajack return source.ObserveOn(RxApp.MainThreadScheduler); } + + /// + /// Like IObservable.Select but supports async map functions + /// + /// + /// + /// + /// + public static IObservable SelectAsync(this IObservable source, Func> f) + { + return source.Select(itm => Observable.FromAsync(async () => await f(itm))).Merge(10); + + } + public static IObservable StartingExecution(this IReactiveCommand cmd) { return cmd.IsExecuting diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 515a7c22..37628262 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -484,8 +484,8 @@ namespace Wabbajack.Lib Info($"Patching {entry.To}"); Status($"Patching {entry.To}"); var srcFile = byPath[string.Join("|", entry.ArchiveHashPath.Paths)]; - await using var srcStream = srcFile.OpenRead(); - await using var outputStream = IncludeFile(out var id); + await using var srcStream = await srcFile.OpenRead(); + await using var outputStream = await IncludeFile(out var id).Create(); entry.PatchID = id; await using var destStream = await LoadDataForTo(entry.To, absolutePaths); await Utils.CreatePatch(srcStream, srcFile.Hash, destStream, entry.Hash, outputStream); @@ -496,18 +496,18 @@ namespace Wabbajack.Lib private async Task LoadDataForTo(RelativePath to, Dictionary absolutePaths) { if (absolutePaths.TryGetValue(to, out var absolute)) - return absolute.OpenRead(); + return await absolute.OpenRead(); if (to.StartsWith(Consts.BSACreationDir)) { var bsaId = (RelativePath)((string)to).Split('\\')[1]; var bsa = InstallDirectives.OfType().First(b => b.TempID == bsaId); - await using var a = BSADispatch.OpenRead(MO2Folder.Combine(bsa.To)); + await using var a = await BSADispatch.OpenRead(MO2Folder.Combine(bsa.To)); var find = (RelativePath)Path.Combine(((string)to).Split('\\').Skip(2).ToArray()); var file = a.Files.First(e => e.Path == find); var returnStream = new TempStream(); - file.CopyDataTo(returnStream); + await file.CopyDataTo(returnStream); returnStream.Position = 0; return returnStream; } diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index dd46f679..6c0cd7ef 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -84,7 +84,7 @@ namespace Wabbajack.Lib if (cancel.IsCancellationRequested) return false; UpdateTracker.NextStep("Validating Game ESMs"); - ValidateGameESMs(); + await ValidateGameESMs(); if (cancel.IsCancellationRequested) return false; UpdateTracker.NextStep("Validating Modlist"); @@ -238,14 +238,14 @@ namespace Wabbajack.Lib }); } - private void ValidateGameESMs() + private async ValueTask ValidateGameESMs() { foreach (var esm in ModList.Directives.OfType().ToList()) { var filename = esm.To.FileName; var gameFile = GameFolder!.Value.Combine((RelativePath)"Data", filename); Utils.Log($"Validating {filename}"); - var hash = gameFile.FileHash(); + var hash = await gameFile.FileHashAsync(); if (hash != esm.SourceESMHash) { Utils.ErrorThrow(new InvalidGameESMError(esm, hash, gameFile)); @@ -269,7 +269,7 @@ namespace Wabbajack.Lib var streams = await bsa.FileStates.PMap(Queue, async state => { Status($"Adding {state.Path} to BSA"); - var fs = sourceDir.Combine(state.Path).OpenRead(); + var fs = await sourceDir.Combine(state.Path).OpenRead(); await a.AddFile(state, fs); return fs; }); @@ -328,8 +328,8 @@ namespace Wabbajack.Lib var patchData = await LoadBytesFromPath(directive.SourceDataID); var toFile = OutputFolder.Combine(directive.To); Status($"Patching {filename}"); - using var output = toFile.Create(); - using var input = gameFile.OpenRead(); + await using var output = await toFile.Create(); + await using var input = await gameFile.OpenRead(); Utils.ApplyPatch(input, () => new MemoryStream(patchData), output); } diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index 9249a6ae..d1513bc4 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -84,14 +84,14 @@ namespace Wabbajack.Lib.ModListRegistry return metadata.OrderBy(m => (m.ValidationSummary?.HasFailures ?? false ? 1 : 0, m.Title)).ToList(); } - public bool NeedsDownload(AbsolutePath modlistPath) + public async ValueTask NeedsDownload(AbsolutePath modlistPath) { if (!modlistPath.Exists) return true; if (DownloadMetadata?.Hash == null) { return true; } - return DownloadMetadata.Hash != modlistPath.FileHashCached(true); + return DownloadMetadata.Hash != await modlistPath.FileHashCachedAsync(true); } } diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index 0e251e82..0af0bcc3 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -77,7 +77,7 @@ namespace Wabbajack.Lib.NexusApi public static async Task RequestAndCacheAPIKey() { var result = await Utils.Log(new RequestNexusAuthorization()).Task; - result.ToEcryptedJson("nexusapikey"); + await result.ToEcryptedJson("nexusapikey"); return result; } diff --git a/Wabbajack.Lib/zEditIntegration.cs b/Wabbajack.Lib/zEditIntegration.cs index 237da171..c1684a97 100644 --- a/Wabbajack.Lib/zEditIntegration.cs +++ b/Wabbajack.Lib/zEditIntegration.cs @@ -270,7 +270,7 @@ namespace Wabbajack.Lib var patchData = await installer.LoadBytesFromPath(m.PatchID); - await using var fs = installer.OutputFolder.Combine(m.To).Create(); + await using var fs = await installer.OutputFolder.Combine(m.To).Create(); Utils.ApplyPatch(new MemoryStream(srcData), () => new MemoryStream(patchData), fs); }); } diff --git a/Wabbajack.Server.Test/ABuildServerSystemTest.cs b/Wabbajack.Server.Test/ABuildServerSystemTest.cs index eaaf9838..e4dbba2c 100644 --- a/Wabbajack.Server.Test/ABuildServerSystemTest.cs +++ b/Wabbajack.Server.Test/ABuildServerSystemTest.cs @@ -36,18 +36,19 @@ namespace Wabbajack.BuildServer.Test public AbsolutePath ServerUpdatesFolder => "updates".RelativeTo(AbsolutePath.EntryPoint); - public BuildServerFixture() + public static async Task Start() { - ServerArchivesFolder.DeleteDirectory().Wait(); - ServerArchivesFolder.CreateDirectory(); + var fixture = new BuildServerFixture(); + fixture.ServerArchivesFolder.DeleteDirectory().Wait(); + fixture.ServerArchivesFolder.CreateDirectory(); var builder = Program.CreateHostBuilder( new[] { $"WabbajackSettings:DownloadDir={"tmp".RelativeTo(AbsolutePath.EntryPoint)}", $"WabbajackSettings:ArchiveDir={"archives".RelativeTo(AbsolutePath.EntryPoint)}", - $"WabbajackSettings:TempFolder={ServerTempFolder}", - $"WabbajackSettings:SQLConnection={PublicConnStr}", + $"WabbajackSettings:TempFolder={fixture.ServerTempFolder}", + $"WabbajackSettings:SQLConnection={fixture.PublicConnStr}", $"WabbajackSettings:BunnyCDN_User=TEST", $"WabbajackSettings:BunnyCDN_Password=TEST", "WabbajackSettings:JobScheduler=false", @@ -56,12 +57,12 @@ namespace Wabbajack.BuildServer.Test "WabbajackSettings:RunFrontEndJobs=false", "WabbajackSettinss:DisableNexusForwarding=true" }, true); - _host = builder.Build(); - _token = new CancellationTokenSource(); - _task = _host.RunAsync(_token.Token); + fixture._host = builder.Build(); + fixture._token = new CancellationTokenSource(); + fixture._task = fixture._host.RunAsync(fixture._token.Token); Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080"); - "ServerWhitelist.yaml".RelativeTo(ServerPublicFolder).WriteAllText( + await "ServerWhitelist.yaml".RelativeTo(fixture.ServerPublicFolder).WriteAllTextAsync( "GoogleIDs:\nAllowedPrefixes:\n - http://localhost"); } @@ -208,7 +209,7 @@ namespace Wabbajack.BuildServer.Test var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder); - await using (var fs = modListPath.Create()) + await using (var fs = await modListPath.Create()) { using var za = new ZipArchive(fs, ZipArchiveMode.Create); var entry = za.CreateEntry("modlist"); @@ -254,7 +255,7 @@ namespace Wabbajack.BuildServer.Test var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder); - ModListMetaData.ToJson(metadataPath); + await ModListMetaData.ToJsonAsync(metadataPath); return new Uri(MakeURL("test_mod_list_metadata.json")); } diff --git a/Wabbajack.Server/Controllers/AuthoredFiles.cs b/Wabbajack.Server/Controllers/AuthoredFiles.cs index b4197485..8925167b 100644 --- a/Wabbajack.Server/Controllers/AuthoredFiles.cs +++ b/Wabbajack.Server/Controllers/AuthoredFiles.cs @@ -99,7 +99,7 @@ namespace Wabbajack.BuildServer.Controllers private async Task GetBunnyCdnFtpClient() { - var info = Utils.FromEncryptedJson("bunny-cdn-ftp-info"); + var info = await Utils.FromEncryptedJson("bunny-cdn-ftp-info"); var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)}; await client.ConnectAsync(); return client; diff --git a/Wabbajack.Server/Services/ModListDownloader.cs b/Wabbajack.Server/Services/ModListDownloader.cs index 2c611e3e..d73da914 100644 --- a/Wabbajack.Server/Services/ModListDownloader.cs +++ b/Wabbajack.Server/Services/ModListDownloader.cs @@ -99,7 +99,7 @@ namespace Wabbajack.Server.Services _maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath); ModList modlist; - await using (var fs = modlistPath.OpenRead()) + await using (var fs = await modlistPath.OpenRead()) using (var zip = new ZipArchive(fs, ZipArchiveMode.Read)) await using (var entry = zip.GetEntry("modlist")?.Open()) { diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs index 682296e6..6eb29e4f 100644 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -70,10 +70,10 @@ namespace Wabbajack.Server.Services using var sigFile = new TempFile(); using var patchFile = new TempFile(); - await using var srcStream = srcPath.OpenShared(); - await using var destStream = destPath.OpenShared(); - await using var sigStream = sigFile.Path.Create(); - await using var patchOutput = patchFile.Path.Create(); + await using var srcStream = await srcPath.OpenShared(); + await using var destStream = await destPath.OpenShared(); + await using var sigStream = await sigFile.Path.Create(); + await using var patchOutput = await patchFile.Path.Create(); OctoDiff.Create(destStream, srcStream, sigStream, patchOutput); await patchOutput.DisposeAsync(); var size = patchFile.Path.Size; @@ -132,7 +132,7 @@ namespace Wabbajack.Server.Services private async Task GetBunnyCdnFtpClient() { - var info = Utils.FromEncryptedJson("bunny-cdn-ftp-info"); + var info = await Utils.FromEncryptedJson("bunny-cdn-ftp-info"); var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)}; await client.ConnectAsync(); return client; diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index a9a5113a..99f34864 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -436,7 +436,7 @@ namespace Wabbajack.Test await converted.Download(new Archive(state: null!) { Name = "mod.zip" }, filename.Path); - await using var fs = filename.Path.OpenRead(); + await using var fs = await filename.Path.OpenRead(); using var archive = new ZipArchive(fs); var entries = archive.Entries.Select(e => e.FullName).ToList(); Assert.Equal(entries, new List {@"Data\TestCK.esp", @"Data\TestCK.ini"}); diff --git a/Wabbajack.Test/EndToEndTests.cs b/Wabbajack.Test/EndToEndTests.cs index d2f74495..f479f657 100644 --- a/Wabbajack.Test/EndToEndTests.cs +++ b/Wabbajack.Test/EndToEndTests.cs @@ -47,7 +47,7 @@ namespace Wabbajack.Test public async Task CreateModlist() { var profile = utils.AddProfile("Default"); - var mod = utils.AddMod(); + var mod = await utils.AddMod(); await DownloadAndInstall( "https://github.com/ModOrganizer2/modorganizer/releases/download/v2.2.1/Mod.Organizer.2.2.1.7z", @@ -79,7 +79,7 @@ namespace Wabbajack.Test await CompileAndInstall(profile); - utils.VerifyAllFiles(); + await utils.VerifyAllFiles(); await utils.InstallFolder.Combine(Consts.LOOTFolderFilesDir).DeleteDirectory(); @@ -111,7 +111,7 @@ namespace Wabbajack.Test private async Task<(AbsolutePath Download, AbsolutePath ModFolder)> DownloadAndInstall(Game game, int modId, string modName) { - utils.AddMod(modName); + await utils.AddMod(modName); var client = await NexusApiClient.Get(); var resp = await client.GetModFiles(game, modId); var file = resp.files.FirstOrDefault(f => f.is_primary) ?? resp.files.FirstOrDefault(f => !string.IsNullOrEmpty(f.category_name)); diff --git a/Wabbajack.Test/MO2Tests.cs b/Wabbajack.Test/MO2Tests.cs index 55af9685..fa075fc8 100644 --- a/Wabbajack.Test/MO2Tests.cs +++ b/Wabbajack.Test/MO2Tests.cs @@ -27,8 +27,8 @@ namespace Wabbajack.Test public async Task CheckValidInstallPath_HasModlist() { await using var tempDir = await TempFolder.Create(); - await using var mo2 = tempDir.Dir.Combine("ModOrganizer.exe").Create(); - await using var molist = tempDir.Dir.Combine(((RelativePath)"modlist")).WithExtension(Consts.ModListExtension).Create(); + await using var mo2 = await tempDir.Dir.Combine("ModOrganizer.exe").Create(); + await using var molist = await tempDir.Dir.Combine(((RelativePath)"modlist")).WithExtension(Consts.ModListExtension).Create(); Assert.False(MO2Installer.CheckValidInstallPath(tempDir.Dir, downloadFolder: null).Succeeded); } @@ -36,7 +36,7 @@ namespace Wabbajack.Test public async Task CheckValidInstallPath_ProperOverwrite() { await using var tempDir = await TempFolder.Create(); - await using var tmp = tempDir.Dir.Combine(Consts.ModOrganizer2Exe).Create(); + await using var tmp = await tempDir.Dir.Combine(Consts.ModOrganizer2Exe).Create(); Assert.True(MO2Installer.CheckValidInstallPath(tempDir.Dir, downloadFolder: null).Succeeded); } @@ -46,7 +46,7 @@ namespace Wabbajack.Test await using var tempDir = await TempFolder.Create(); await tempDir.Dir.DeleteDirectory(); tempDir.Dir.CreateDirectory(); - await using var tmp = tempDir.Dir.Combine($"someFile.txt").Create(); + await using var tmp = await tempDir.Dir.Combine($"someFile.txt").Create(); Assert.False(MO2Installer.CheckValidInstallPath(tempDir.Dir, downloadFolder: null).Succeeded); } @@ -56,7 +56,7 @@ namespace Wabbajack.Test await using var tempDir = await TempFolder.Create(); var downloadsFolder = tempDir.Dir.Combine("downloads"); downloadsFolder.CreateDirectory(); - await using var tmp = tempDir.Dir.Combine($"downloads/someFile.txt").Create(); + await using var tmp = await tempDir.Dir.Combine($"downloads/someFile.txt").Create(); Assert.True(MO2Installer.CheckValidInstallPath(tempDir.Dir, downloadFolder: downloadsFolder).Succeeded); } #endregion diff --git a/Wabbajack.Test/SanityTests.cs b/Wabbajack.Test/SanityTests.cs index 725967dc..d66ca403 100644 --- a/Wabbajack.Test/SanityTests.cs +++ b/Wabbajack.Test/SanityTests.cs @@ -27,17 +27,17 @@ namespace Wabbajack.Test { var profile = utils.AddProfile(); - var mod = utils.AddMod(); - var testPex = utils.AddModFile(mod, @"Data\scripts\test.pex", 10); + var mod = await utils.AddMod(); + var testPex = await utils.AddModFile(mod, @"Data\scripts\test.pex", 10); await utils.Configure(); - utils.AddManualDownload( + await utils.AddManualDownload( new Dictionary {{"/baz/biz.pex", await testPex.ReadAllBytesAsync()}}); await CompileAndInstall(profile); - utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex"); + await utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex"); } [Fact] @@ -45,19 +45,19 @@ namespace Wabbajack.Test { var profile = utils.AddProfile(); - var mod = utils.AddMod(); - var testPex = utils.AddModFile(mod, @"Data\scripts\test.pex", 10); + var mod = await utils.AddMod(); + var testPex = await utils.AddModFile(mod, @"Data\scripts\test.pex", 10); await utils.Configure(); - utils.AddManualDownload( + await utils.AddManualDownload( new Dictionary {{"/baz/biz.pex", await testPex.ReadAllBytesAsync()}}); await utils.DownloadsFolder.Combine("some_other_file.7z").WriteAllTextAsync("random data"); await CompileAndInstall(profile); - utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex"); + await utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex"); } [Fact] @@ -65,17 +65,17 @@ namespace Wabbajack.Test { var profile = utils.AddProfile(); - var mod = utils.AddMod(); - var testPex = utils.AddGameFile(@"enbstuff\test.pex", 10); + var mod = await utils.AddMod(); + var testPex = await utils.AddGameFile(@"enbstuff\test.pex", 10); await utils.Configure(); - utils.AddManualDownload( + await utils.AddManualDownload( new Dictionary {{"/baz/biz.pex", await testPex.ReadAllBytesAsync()}}); await CompileAndInstall(profile); - utils.VerifyInstalledGameFile(@"enbstuff\test.pex"); + await utils.VerifyInstalledGameFile(@"enbstuff\test.pex"); } [Fact] @@ -83,14 +83,14 @@ namespace Wabbajack.Test { var profile = utils.AddProfile(); - var mod = utils.AddMod(); - var testPex = utils.AddGameFile(@"enbstuff\test.pex", 10); + var mod = await utils.AddMod(); + var testPex = await utils.AddGameFile(@"enbstuff\test.pex", 10); await utils.Configure(); utils.MO2Folder.Combine(Consts.GameFolderFilesDir).CreateDirectory(); - utils.AddManualDownload( + await utils.AddManualDownload( new Dictionary {{"/baz/biz.pex", await testPex.ReadAllBytesAsync()}}); await CompileAndInstall(profile); @@ -103,21 +103,21 @@ namespace Wabbajack.Test { var profile = utils.AddProfile(); - var mod = utils.AddMod(); - var testPex = utils.AddModFile(mod, @"Data\scripts\test.pex", 10); + var mod = await utils.AddMod(); + var testPex = await utils.AddModFile(mod, @"Data\scripts\test.pex", 10); // Make a copy to make sure it gets picked up and moved around. - testPex.CopyTo(testPex.WithExtension(new Extension(".copy"))); + await testPex.CopyToAsync(testPex.WithExtension(new Extension(".copy"))); await utils.Configure(); - utils.AddManualDownload( + await utils.AddManualDownload( new Dictionary { { "/baz/biz.pex", await testPex.ReadAllBytesAsync() } }); await CompileAndInstall(profile); - utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex"); - utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex.copy"); + await utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex"); + await utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex.copy"); } [Fact] @@ -125,14 +125,14 @@ namespace Wabbajack.Test { var profile = utils.AddProfile(); - var mod = utils.AddMod(); - var unchanged = utils.AddModFile(mod, @"Data\scripts\unchanged.pex", 10); - var deleted = utils.AddModFile(mod, @"Data\scripts\deleted.pex", 10); - var modified = utils.AddModFile(mod, @"Data\scripts\modified.pex", 10); + var mod = await utils.AddMod(); + var unchanged = await utils.AddModFile(mod, @"Data\scripts\unchanged.pex", 10); + var deleted = await utils.AddModFile(mod, @"Data\scripts\deleted.pex", 10); + var modified = await utils.AddModFile(mod, @"Data\scripts\modified.pex", 10); await utils.Configure(); - utils.AddManualDownload( + await utils.AddManualDownload( new Dictionary { { "/baz/unchanged.pex", await unchanged.ReadAllBytesAsync() }, @@ -142,9 +142,9 @@ namespace Wabbajack.Test await CompileAndInstall(profile); - utils.VerifyInstalledFile(mod, @"Data\scripts\unchanged.pex"); - utils.VerifyInstalledFile(mod, @"Data\scripts\deleted.pex"); - utils.VerifyInstalledFile(mod, @"Data\scripts\modified.pex"); + await utils.VerifyInstalledFile(mod, @"Data\scripts\unchanged.pex"); + await utils.VerifyInstalledFile(mod, @"Data\scripts\deleted.pex"); + await utils.VerifyInstalledFile(mod, @"Data\scripts\modified.pex"); var unchangedPath = utils.PathOfInstalledFile(mod, @"Data\scripts\unchanged.pex"); var deletedPath = utils.PathOfInstalledFile(mod, @"Data\scripts\deleted.pex"); @@ -170,9 +170,9 @@ namespace Wabbajack.Test await CompileAndInstall(profile); - utils.VerifyInstalledFile(mod, @"Data\scripts\unchanged.pex"); - utils.VerifyInstalledFile(mod, @"Data\scripts\deleted.pex"); - utils.VerifyInstalledFile(mod, @"Data\scripts\modified.pex"); + await utils.VerifyInstalledFile(mod, @"Data\scripts\unchanged.pex"); + await utils.VerifyInstalledFile(mod, @"Data\scripts\deleted.pex"); + await utils.VerifyInstalledFile(mod, @"Data\scripts\modified.pex"); Assert.Equal(unchangedModified, unchangedPath.LastModified); Assert.NotEqual(modifiedModified, modifiedPath.LastModified); @@ -184,7 +184,7 @@ namespace Wabbajack.Test public async Task SetScreenSizeTest() { var profile = utils.AddProfile(); - var mod = utils.AddMod("dummy"); + var mod = await utils.AddMod("dummy"); await utils.Configure(); await utils.MO2Folder.Combine("profiles", profile, "somegameprefs.ini").WriteAllLinesAsync( @@ -216,11 +216,11 @@ namespace Wabbajack.Test public async Task UnmodifiedInlinedFilesArePulledFromArchives() { var profile = utils.AddProfile(); - var mod = utils.AddMod(); - var ini = utils.AddModFile(mod, @"foo.ini", 10); + var mod = await utils.AddMod(); + var ini = await utils.AddModFile(mod, @"foo.ini", 10); await utils.Configure(); - utils.AddManualDownload( + await utils.AddManualDownload( new Dictionary { { "/baz/biz.pex", await ini.ReadAllBytesAsync() } }); var modlist = await CompileAndInstall(profile); @@ -234,9 +234,9 @@ namespace Wabbajack.Test public async Task ModifiedIniFilesArePatchedAgainstFileWithSameName() { var profile = utils.AddProfile(); - var mod = utils.AddMod(); - var ini = utils.AddModFile(mod, @"foo.ini", 10); - var meta = utils.AddModFile(mod, "meta.ini"); + var mod = await utils.AddMod(); + var ini = await utils.AddModFile(mod, @"foo.ini", 10); + var meta = await utils.AddModFile(mod, "meta.ini"); await utils.Configure(); @@ -262,8 +262,8 @@ namespace Wabbajack.Test public async Task CanPatchFilesSourcedFromBSAs() { var profile = utils.AddProfile(); - var mod = utils.AddMod(); - var file = utils.AddModFile(mod, @"baz.bin", 10); + var mod = await utils.AddMod(); + var file = await utils.AddModFile(mod, @"baz.bin", 10); await utils.Configure(); @@ -287,7 +287,7 @@ namespace Wabbajack.Test new Dictionary { { "/stuff/files.bsa", await tempFile.Path.ReadAllBytesAsync() } }); await CompileAndInstall(profile); - utils.VerifyInstalledFile(mod, @"baz.bin"); + await utils.VerifyInstalledFile(mod, @"baz.bin"); } @@ -295,8 +295,8 @@ namespace Wabbajack.Test public async Task CanNoMatchIncludeFilesFromBSAs() { var profile = utils.AddProfile(); - var mod = utils.AddMod(); - var file = utils.AddModFile(mod, @"baz.bsa", 10); + var mod = await utils.AddMod(); + var file = await utils.AddModFile(mod, @"baz.bsa", 10); await file.Parent.Combine("meta.ini").WriteAllLinesAsync(new[] { @@ -334,7 +334,7 @@ namespace Wabbajack.Test new Dictionary { { "/stuff/matching_file_data.bin", tempFileData } }); await CompileAndInstall(profile); - utils.VerifyInstalledFile(mod, @"baz.bsa"); + await utils.VerifyInstalledFile(mod, @"baz.bsa"); } @@ -342,8 +342,8 @@ namespace Wabbajack.Test public async Task CanInstallFilesFromBSAAndBSA() { var profile = utils.AddProfile(); - var mod = utils.AddMod(); - var file = utils.AddModFile(mod, @"baz.bin", 128); + var mod = await utils.AddMod(); + var file = await utils.AddModFile(mod, @"baz.bin", 128); await utils.Configure(); @@ -362,14 +362,14 @@ namespace Wabbajack.Test }, new MemoryStream(await file.ReadAllBytesAsync())); await bsa.Build(tempFile.Path); } - tempFile.Path.CopyTo(file.Parent.Combine("bsa_data.bsa")); + await tempFile.Path.CopyToAsync(file.Parent.Combine("bsa_data.bsa")); var archive = utils.AddManualDownload( new Dictionary { { "/stuff/files.bsa", await tempFile.Path.ReadAllBytesAsync() } }); await CompileAndInstall(profile); - utils.VerifyInstalledFile(mod, @"baz.bin"); - utils.VerifyInstalledFile(mod, @"bsa_data.bsa"); + await utils.VerifyInstalledFile(mod, @"baz.bin"); + await utils.VerifyInstalledFile(mod, @"bsa_data.bsa"); } @@ -377,8 +377,8 @@ namespace Wabbajack.Test public async Task CanRecreateBSAsFromFilesSourcedInOtherBSAs() { var profile = utils.AddProfile(); - var mod = utils.AddMod(); - var file = utils.AddModFile(mod, @"baz.bsa", 10); + var mod = await utils.AddMod(); + var file = await utils.AddModFile(mod, @"baz.bsa", 10); await utils.Configure(); @@ -414,7 +414,7 @@ namespace Wabbajack.Test await CompileAndInstall(profile); - utils.VerifyInstalledFile(mod, @"baz.bsa"); + await utils.VerifyInstalledFile(mod, @"baz.bsa"); } @@ -425,8 +425,8 @@ namespace Wabbajack.Test Consts.TestMode = false; var profile = utils.AddProfile(); - var mod = utils.AddMod(); - var skyrimExe = utils.AddModFile(mod, @"Data\test.exe", 10); + var mod = await utils.AddMod(); + var skyrimExe = await utils.AddModFile(mod, @"Data\test.exe", 10); var gameFolder = Consts.GameFolderFilesDir.RelativeTo(utils.MO2Folder); gameFolder.CreateDirectory(); @@ -459,18 +459,18 @@ namespace Wabbajack.Test { var profile = utils.AddProfile(); - var mod = utils.AddMod(); - var testPex = utils.AddModFile(mod, @"Data\scripts\test.pex", 10); + var mod = await utils.AddMod(); + var testPex = await utils.AddModFile(mod, @"Data\scripts\test.pex", 10); await utils.Configure(); - await utils.AddModFile(mod, "meta.ini").WriteAllLinesAsync(new[] + await (await utils.AddModFile(mod, "meta.ini")).WriteAllLinesAsync(new[] { "[General]", "notes= fsdaf WABBAJACK_NOMATCH_INCLUDE fadsfsad", }); await CompileAndInstall(profile); - utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex"); + await utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex"); } @@ -483,11 +483,11 @@ namespace Wabbajack.Test { var profile = utils.AddProfile(); - var enabledMod = utils.AddMod(); - var enabledTestPex = utils.AddModFile(enabledMod, @"Data\scripts\enabledTestPex.pex", 10); + var enabledMod = await utils.AddMod(); + var enabledTestPex = await utils.AddModFile(enabledMod, @"Data\scripts\enabledTestPex.pex", 10); - var disabledMod = utils.AddMod(); - var disabledTestPex = utils.AddModFile(disabledMod, @"Data\scripts\disabledTestPex.pex", 10); + var disabledMod = await utils.AddMod(); + var disabledTestPex = await utils.AddModFile(disabledMod, @"Data\scripts\disabledTestPex.pex", 10); await disabledMod.RelativeTo(utils.ModsFolder).Combine("meta.ini").WriteAllLinesAsync( "[General]", @@ -499,7 +499,7 @@ namespace Wabbajack.Test (enabledMod, true) }); - utils.AddManualDownload( + await utils.AddManualDownload( new Dictionary { {"/file1.pex", await enabledTestPex.ReadAllBytesAsync()}, @@ -508,8 +508,8 @@ namespace Wabbajack.Test await CompileAndInstall(profile); - utils.VerifyInstalledFile(enabledMod, @"Data\scripts\enabledTestPex.pex"); - utils.VerifyInstalledFile(disabledMod, @"Data\scripts\disabledTestPex.pex"); + await utils.VerifyInstalledFile(enabledMod, @"Data\scripts\enabledTestPex.pex"); + await utils.VerifyInstalledFile(disabledMod, @"Data\scripts\disabledTestPex.pex"); var modlistTxt = await utils.InstallFolder.Combine("profiles", profile, "modlist.txt").ReadAllLinesAsync(); Assert.Equal(new string[] diff --git a/Wabbajack.Test/TestUtils.cs b/Wabbajack.Test/TestUtils.cs index 3fff1d7f..58fed6bd 100644 --- a/Wabbajack.Test/TestUtils.cs +++ b/Wabbajack.Test/TestUtils.cs @@ -57,7 +57,7 @@ namespace Wabbajack.Test { Profiles.Do(profile => { - MO2Folder.Combine("profiles", profile, "modlist.txt").WriteAllLines( + MO2Folder.Combine("profiles", profile, "modlist.txt").WriteAllLinesAsync( Mods.Select(s => $"+{s}").ToArray()); }); } @@ -65,7 +65,7 @@ namespace Wabbajack.Test { Profiles.Do(profile => { - MO2Folder.Combine("profiles", profile, "modlist.txt").WriteAllLines( + MO2Folder.Combine("profiles", profile, "modlist.txt").WriteAllLinesAsync( enabledMods.Select(s => $"{(s.IsEnabled ? "+" : "-")}{s.ModName}").ToArray()); }); } @@ -79,17 +79,14 @@ namespace Wabbajack.Test return profile_name; } - public string AddMod(string name = null) + public async Task AddMod(string name = null) { - lock (this) - { - string mod_name = name ?? RandomName(); - var mod_folder = MO2Folder.Combine(Consts.MO2ModFolderName, (RelativePath)mod_name); - mod_folder.CreateDirectory(); - mod_folder.Combine("meta.ini").WriteAllText("[General]"); - Mods.Add(mod_name); - return mod_name; - } + string mod_name = name ?? RandomName(); + var mod_folder = MO2Folder.Combine(Consts.MO2ModFolderName, (RelativePath)mod_name); + mod_folder.CreateDirectory(); + await mod_folder.Combine("meta.ini").WriteAllTextAsync("[General]"); + Mods.Add(mod_name); + return mod_name; } /// @@ -100,17 +97,15 @@ namespace Wabbajack.Test /// /// /// - public AbsolutePath AddModFile(string mod_name, string path, int random_fill=128) + public async Task AddModFile(string mod_name, string path, int random_fill=128) { - - var full_path = ModsFolder.Combine(mod_name, path); full_path.Parent.CreateDirectory(); - GenerateRandomFileData(full_path, random_fill); + await GenerateRandomFileData(full_path, random_fill); return full_path; } - public void GenerateRandomFileData(AbsolutePath full_path, int random_fill) + public async Task GenerateRandomFileData(AbsolutePath full_path, int random_fill) { byte[] bytes = new byte[0]; if (random_fill != 0) @@ -118,7 +113,7 @@ namespace Wabbajack.Test bytes = new byte[random_fill]; RNG.NextBytes(bytes); } - full_path.WriteAllBytes(bytes); + await full_path.WriteAllBytesAsync(bytes); } public static byte[] RandomData(int? size = null, int maxSize = 1024) @@ -162,20 +157,20 @@ namespace Wabbajack.Test return data; } - public string AddManualDownload(Dictionary contents) + public async ValueTask AddManualDownload(Dictionary contents) { var name = RandomName() + ".zip"; - using FileStream fs = DownloadsFolder.Combine(name).Create(); + await using FileStream fs = await DownloadsFolder.Combine(name).Create(); using ZipArchive archive = new ZipArchive(fs, ZipArchiveMode.Create); contents.Do(kv => { var entry = archive.CreateEntry(kv.Key); - using (var os = entry.Open()) - os.Write(kv.Value, 0, kv.Value.Length); + using var os = entry.Open(); + os.Write(kv.Value, 0, kv.Value.Length); }); - DownloadsFolder.Combine(name + Consts.MetaFileExtension).WriteAllLines( + await DownloadsFolder.Combine(name + Consts.MetaFileExtension).WriteAllLinesAsync( "[General]", "manualURL=" ); @@ -227,7 +222,7 @@ namespace Wabbajack.Test return InstallFolder.Combine((string)Consts.MO2ModFolderName, mod, file); } - public void VerifyAllFiles(bool gameFileShouldNotExistInGameFolder = true) + public async ValueTask VerifyAllFiles(bool gameFileShouldNotExistInGameFolder = true) { if (gameFileShouldNotExistInGameFolder) { @@ -264,16 +259,16 @@ namespace Wabbajack.Test if (!skipExtensions.Contains(srcFile.Extension)) { Assert.Equal(srcFile.Size, destFile.Size); - Assert.Equal(srcFile.FileHash(), destFile.FileHash()); + Assert.Equal(await srcFile.FileHashAsync(), await destFile.FileHashAsync()); } } } - public AbsolutePath AddGameFile(string path, int i) + public async ValueTask AddGameFile(string path, int i) { var fullPath = GameFolder.Combine(path); fullPath.Parent.CreateDirectory(); - GenerateRandomFileData(fullPath, i); + await GenerateRandomFileData(fullPath, i); return fullPath; } } diff --git a/Wabbajack.Test/ZEditIntegrationTests.cs b/Wabbajack.Test/ZEditIntegrationTests.cs index 15190c81..5cda2f0c 100644 --- a/Wabbajack.Test/ZEditIntegrationTests.cs +++ b/Wabbajack.Test/ZEditIntegrationTests.cs @@ -15,13 +15,13 @@ namespace Wabbajack.Test public async Task CanCreatezEditPatches() { var profile = utils.AddProfile(); - var moda = utils.AddMod(); - var modb = utils.AddMod(); - var moddest = utils.AddMod(); - var srca = utils.AddModFile(moda, @"srca.esp", 10); - var srcb = utils.AddModFile(moda, @"srcb.esp.mohidden", 10); - var srcc = utils.AddModFile(modb, @"optional\srcc.esp", 10); - var dest = utils.AddModFile(moddest, @"merged.esp", 20); + var moda = await utils.AddMod(); + var modb = await utils.AddMod(); + var moddest = await utils.AddMod(); + var srca = await utils.AddModFile(moda, @"srca.esp", 10); + var srcb = await utils.AddModFile(moda, @"srcb.esp.mohidden", 10); + var srcc = await utils.AddModFile(modb, @"optional\srcc.esp", 10); + var dest = await utils.AddModFile(moddest, @"merged.esp", 20); var srcs = new List {srca, srcb, srcc}; diff --git a/Wabbajack/Settings.cs b/Wabbajack/Settings.cs index 4987666f..d1204cf5 100644 --- a/Wabbajack/Settings.cs +++ b/Wabbajack/Settings.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Reactive; using System.Reactive.Subjects; +using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Common.Serialization.Json; using Wabbajack.Lib; @@ -29,20 +30,19 @@ namespace Wabbajack [JsonIgnore] public IObservable SaveSignal => _saveSignal; - public static bool TryLoadTypicalSettings(out MainSettings settings) + public static async ValueTask<(MainSettings settings, bool loaded)> TryLoadTypicalSettings() { if (!Consts.SettingsFile.Exists) { - settings = default; - return false; + return default; } // Version check try { - settings = Consts.SettingsFile.FromJson(); + var settings = Consts.SettingsFile.FromJson(); if (settings.Version == Consts.SettingsVersion) - return true; + return (settings, true); } catch (Exception ex) { @@ -52,14 +52,13 @@ namespace Wabbajack var backup = Consts.SettingsFile.AppendToName("-backup"); backup.Delete(); - Consts.SettingsFile.CopyTo(backup); + await Consts.SettingsFile.CopyToAsync(backup); Consts.SettingsFile.Delete(); - settings = default; - return false; + return default; } - public static void SaveSettings(MainSettings settings) + public static async ValueTask SaveSettings(MainSettings settings) { settings._saveSignal.OnNext(Unit.Default); @@ -68,7 +67,7 @@ namespace Wabbajack //settings._saveSignal.OnCompleted(); //await settings._saveSignal; - settings.ToJson(Consts.SettingsFile); + await settings.ToJsonAsync(Consts.SettingsFile); } } diff --git a/Wabbajack/View Models/Gallery/ModListMetadataVM.cs b/Wabbajack/View Models/Gallery/ModListMetadataVM.cs index 981a4f2f..93a3935c 100644 --- a/Wabbajack/View Models/Gallery/ModListMetadataVM.cs +++ b/Wabbajack/View Models/Gallery/ModListMetadataVM.cs @@ -144,11 +144,11 @@ namespace Wabbajack .Unit() .StartWith(Unit.Default) .FlowSwitch(_parent.WhenAny(x => x.IsActive)) - .Select(_ => + .SelectAsync(async _ => { try { - return !metadata.NeedsDownload(Location); + return !(await metadata.NeedsDownload(Location)); } catch (Exception) { @@ -185,7 +185,7 @@ namespace Wabbajack var downloader = DownloadDispatcher.ResolveArchive(Metadata.Links.Download); var result = await downloader.Download(new Archive(state: null!) { Name = Metadata.Title, Size = Metadata.DownloadMetadata?.Size ?? 0 }, Location); // Want to rehash to current file, even if failed? - Location.FileHashCached(); + await Location.FileHashCachedAsync(); tcs.SetResult(result); } catch (Exception ex) diff --git a/Wabbajack/View Models/MainWindowVM.cs b/Wabbajack/View Models/MainWindowVM.cs index ddfa5292..80a7d70a 100644 --- a/Wabbajack/View Models/MainWindowVM.cs +++ b/Wabbajack/View Models/MainWindowVM.cs @@ -199,7 +199,7 @@ namespace Wabbajack Settings.PosY = MainWindow.Top; Settings.Width = MainWindow.Width; Settings.Height = MainWindow.Height; - MainSettings.SaveSettings(Settings); + MainSettings.SaveSettings(Settings).AsTask().Wait(); Application.Current.Shutdown(); } } diff --git a/Wabbajack/View Models/ModListVM.cs b/Wabbajack/View Models/ModListVM.cs index 44368ab0..67c6f345 100644 --- a/Wabbajack/View Models/ModListVM.cs +++ b/Wabbajack/View Models/ModListVM.cs @@ -45,19 +45,17 @@ namespace Wabbajack ImageObservable = Observable.Return(Unit.Default) // Download and retrieve bytes on background thread .ObserveOn(RxApp.TaskpoolScheduler) - .Select(filePath => + .SelectAsync(async filePath => { try { - using var fs = ModListPath.OpenShared(); + await using var fs = await ModListPath.OpenShared(); using var ar = new ZipArchive(fs, ZipArchiveMode.Read); var ms = new MemoryStream(); var entry = ar.GetEntry("modlist-image.png"); if (entry == null) return default(MemoryStream); - using (var e = entry.Open()) - { - e.CopyTo(ms); - } + await using var e = entry.Open(); + e.CopyTo(ms); return ms; } catch (Exception ex) @@ -82,10 +80,7 @@ namespace Wabbajack } }) // If ever would return null, show WJ logo instead - .Select(x => - { - return x ?? ResourceLinks.WabbajackLogoNoText.Value; - }) + .Select(x => x ?? ResourceLinks.WabbajackLogoNoText.Value) .Replay(1) .RefCount(); } diff --git a/Wabbajack/Views/MainWindow.xaml.cs b/Wabbajack/Views/MainWindow.xaml.cs index 8c48ffec..dcdb46bc 100644 --- a/Wabbajack/Views/MainWindow.xaml.cs +++ b/Wabbajack/Views/MainWindow.xaml.cs @@ -45,8 +45,9 @@ namespace Wabbajack Warmup(); + var (settings, loadedSettings) = MainSettings.TryLoadTypicalSettings().AsTask().Result; // Load settings - if (CLIArguments.NoSettings || !MainSettings.TryLoadTypicalSettings(out var settings)) + if (CLIArguments.NoSettings || !loadedSettings) { _settings = new MainSettings { From 1099d68ef7c4a77c8bdadd67b8a69b8e67c7cd80 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 25 May 2020 13:41:16 -0600 Subject: [PATCH 5/8] Fix race condition with Do and async --- Wabbajack.Common/Utils.cs | 11 +++++++++++ Wabbajack.VirtualFileSystem/Context.cs | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index a24ed096..38c7f9dd 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -300,6 +300,17 @@ namespace Wabbajack.Common { foreach (var i in coll) f(i); } + + /// + /// Executes the action for every item in coll + /// + /// + /// + /// + public static async Task DoAsync(this IEnumerable coll, Func f) + { + foreach (var i in coll) await f(i); + } public static void DoIndexed(this IEnumerable coll, Action f) { diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index 09f3f315..79e9cdb0 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -148,7 +148,7 @@ namespace Wabbajack.VirtualFileSystem bw.Write(FileVersion); bw.Write((ulong) Index.AllFiles.Count); - (await Index.AllFiles + await (await Index.AllFiles .PMap(Queue, f => { var ms = new MemoryStream(); @@ -156,7 +156,7 @@ namespace Wabbajack.VirtualFileSystem f.Write(ibw); return ms; })) - .Do(async ms => + .DoAsync(async ms => { var size = ms.Position; ms.Position = 0; From e47f80eff6b299a9e4f241f19a209247350948c1 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 25 May 2020 14:27:07 -0600 Subject: [PATCH 6/8] Fix how we initialize the server test fixture --- .../ABuildServerSystemTest.cs | 20 +++++++++---------- Wabbajack.Server.Test/ADBTest.cs | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Wabbajack.Server.Test/ABuildServerSystemTest.cs b/Wabbajack.Server.Test/ABuildServerSystemTest.cs index e4dbba2c..3e028a1d 100644 --- a/Wabbajack.Server.Test/ABuildServerSystemTest.cs +++ b/Wabbajack.Server.Test/ABuildServerSystemTest.cs @@ -36,19 +36,19 @@ namespace Wabbajack.BuildServer.Test public AbsolutePath ServerUpdatesFolder => "updates".RelativeTo(AbsolutePath.EntryPoint); - public static async Task Start() + public override async Task InitializeAsync() { - var fixture = new BuildServerFixture(); - fixture.ServerArchivesFolder.DeleteDirectory().Wait(); - fixture.ServerArchivesFolder.CreateDirectory(); + await base.InitializeAsync(); + ServerArchivesFolder.DeleteDirectory().Wait(); + ServerArchivesFolder.CreateDirectory(); var builder = Program.CreateHostBuilder( new[] { $"WabbajackSettings:DownloadDir={"tmp".RelativeTo(AbsolutePath.EntryPoint)}", $"WabbajackSettings:ArchiveDir={"archives".RelativeTo(AbsolutePath.EntryPoint)}", - $"WabbajackSettings:TempFolder={fixture.ServerTempFolder}", - $"WabbajackSettings:SQLConnection={fixture.PublicConnStr}", + $"WabbajackSettings:TempFolder={ServerTempFolder}", + $"WabbajackSettings:SQLConnection={PublicConnStr}", $"WabbajackSettings:BunnyCDN_User=TEST", $"WabbajackSettings:BunnyCDN_Password=TEST", "WabbajackSettings:JobScheduler=false", @@ -57,12 +57,12 @@ namespace Wabbajack.BuildServer.Test "WabbajackSettings:RunFrontEndJobs=false", "WabbajackSettinss:DisableNexusForwarding=true" }, true); - fixture._host = builder.Build(); - fixture._token = new CancellationTokenSource(); - fixture._task = fixture._host.RunAsync(fixture._token.Token); + _host = builder.Build(); + _token = new CancellationTokenSource(); + _task = _host.RunAsync(_token.Token); Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080"); - await "ServerWhitelist.yaml".RelativeTo(fixture.ServerPublicFolder).WriteAllTextAsync( + await "ServerWhitelist.yaml".RelativeTo(ServerPublicFolder).WriteAllTextAsync( "GoogleIDs:\nAllowedPrefixes:\n - http://localhost"); } diff --git a/Wabbajack.Server.Test/ADBTest.cs b/Wabbajack.Server.Test/ADBTest.cs index 67ec54bc..bdb2cf75 100644 --- a/Wabbajack.Server.Test/ADBTest.cs +++ b/Wabbajack.Server.Test/ADBTest.cs @@ -32,7 +32,7 @@ namespace Wabbajack.BuildServer.Test public string APIKey { get; } public string User { get; } - public async Task InitializeAsync() + public virtual async Task InitializeAsync() { await CreateSchema(); } From 619da2fa86be98ccf4f8029793f0e570f6207be5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 25 May 2020 22:17:23 -0600 Subject: [PATCH 7/8] Fix bugs in BSA tests after async IO conversion --- Compression.BSA.Test/BSATests.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Compression.BSA.Test/BSATests.cs b/Compression.BSA.Test/BSATests.cs index f3d6c4b2..5f8a7036 100644 --- a/Compression.BSA.Test/BSATests.cs +++ b/Compression.BSA.Test/BSATests.cs @@ -141,22 +141,22 @@ namespace Compression.BSA.Test await a.Files.Zip(b.Files, (ai, bi) => (ai, bi)) - .PMap(Queue, pair => + .PMap(Queue, async pair => { Assert.Equal(pair.ai.State.ToJson(), pair.bi.State.ToJson()); //Console.WriteLine($" - {pair.ai.Path}"); Assert.Equal(pair.ai.Path, pair.bi.Path); //Equal(pair.ai.Compressed, pair.bi.Compressed); Assert.Equal(pair.ai.Size, pair.bi.Size); - Assert.Equal(GetData(pair.ai), GetData(pair.bi)); + Assert.Equal(await GetData(pair.ai), await GetData(pair.bi)); }); } } - private static byte[] GetData(IFile pairAi) + private static async ValueTask GetData(IFile pairAi) { - using var ms = new MemoryStream(); - pairAi.CopyDataTo(ms); + await using var ms = new MemoryStream(); + await pairAi.CopyDataTo(ms); return ms.ToArray(); } From 9c5b746f0790bcd22910aabb29249998fbc77566 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 25 May 2020 22:48:37 -0600 Subject: [PATCH 8/8] Make two flakey tests a bit less flakey --- Wabbajack.Server.Test/ModListValidationTests.cs | 2 +- Wabbajack.Server.Test/ModlistUpdater.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Wabbajack.Server.Test/ModListValidationTests.cs b/Wabbajack.Server.Test/ModListValidationTests.cs index d36b5bcd..d61cb82f 100644 --- a/Wabbajack.Server.Test/ModListValidationTests.cs +++ b/Wabbajack.Server.Test/ModListValidationTests.cs @@ -141,7 +141,7 @@ namespace Wabbajack.BuildServer.Test Assert.Equal(1, data.ValidationSummary.Updating); var patcher = Fixture.GetService(); - Assert.Equal(1, await patcher.Execute()); + Assert.True(await patcher.Execute() > 1); await RevalidateLists(false); diff --git a/Wabbajack.Server.Test/ModlistUpdater.cs b/Wabbajack.Server.Test/ModlistUpdater.cs index 870d7489..05b932c5 100644 --- a/Wabbajack.Server.Test/ModlistUpdater.cs +++ b/Wabbajack.Server.Test/ModlistUpdater.cs @@ -64,7 +64,7 @@ namespace Wabbajack.Server.Test await Assert.ThrowsAsync(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); - Assert.Equal(1, await patcher.Execute()); + Assert.True(await patcher.Execute() > 1); Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_updates/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); }