From 191f321dc21b77be215c94120740a2b09dcd5158 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 25 May 2020 10:30:47 -0600 Subject: [PATCH] `VirtualFile` now useses Async IO exclusively --- Wabbajack.Common/Hash.cs | 8 ++++++++ Wabbajack.Common/IAsyncEnumerableExtensions.cs | 17 +++++++++++++++++ .../VirtualFileSystemTests.cs | 8 ++++---- Wabbajack.VirtualFileSystem/Context.cs | 8 ++++---- Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs | 12 ++++++------ .../ExtractedDiskFile.cs | 4 ++-- Wabbajack.VirtualFileSystem/FileExtractor.cs | 4 ++-- Wabbajack.VirtualFileSystem/IExtractedFile.cs | 2 +- Wabbajack.VirtualFileSystem/VirtualFile.cs | 10 +++++----- 9 files changed, 49 insertions(+), 24 deletions(-) diff --git a/Wabbajack.Common/Hash.cs b/Wabbajack.Common/Hash.cs index 7a993ae9..88b32ae0 100644 --- a/Wabbajack.Common/Hash.cs +++ b/Wabbajack.Common/Hash.cs @@ -151,6 +151,14 @@ namespace Wabbajack.Common var value = xxHashFactory.Instance.Create(config).ComputeHash(f); return Hash.FromULong(BitConverter.ToUInt64(value.Hash)); } + + public static async Task xxHashAsync(this Stream stream) + { + var config = new xxHashConfig {HashSizeInBits = 64}; + await using var f = new StatusFileStream(stream, $"Hashing memory stream"); + var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(f); + return Hash.FromULong(BitConverter.ToUInt64(value.Hash)); + } public static bool TryGetHashCache(AbsolutePath file, out Hash hash) { var normPath = Encoding.UTF8.GetBytes(file.Normalize()); diff --git a/Wabbajack.Common/IAsyncEnumerableExtensions.cs b/Wabbajack.Common/IAsyncEnumerableExtensions.cs index 9fd167dd..5270f42d 100644 --- a/Wabbajack.Common/IAsyncEnumerableExtensions.cs +++ b/Wabbajack.Common/IAsyncEnumerableExtensions.cs @@ -22,6 +22,23 @@ namespace Wabbajack.Common yield return await mapFn(itm); } } + + /// + /// Same as .Select but expects a function that returns an async result + /// + /// + /// + /// + /// + /// + public static async ValueTask DoAsync(this IEnumerable coll, + Func> mapFn) + { + foreach (var itm in coll) + { + await mapFn(itm); + } + } public static async ValueTask> ToList(this IAsyncEnumerable coll) { diff --git a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs index 0868ea56..d9bfc14d 100644 --- a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs @@ -72,7 +72,7 @@ namespace Wabbajack.VirtualFileSystem.Test Assert.NotNull(file); Assert.Equal(128, file.Size); - Assert.Equal(absPath.FileHash(), file.Hash); + Assert.Equal(await absPath.FileHashAsync(), file.Hash); Assert.True(file.IsArchive); var innerFile = file.Children.First(); @@ -143,7 +143,7 @@ namespace Wabbajack.VirtualFileSystem.Test var cleanup = await context.Stage(new List {file}); - await using var stream = file.StagedFile.OpenRead(); + await using var stream = await file.StagedFile.OpenRead(); Assert.Equal("This is a test", await stream.ReadAllTextAsync()); @@ -158,7 +158,7 @@ namespace Wabbajack.VirtualFileSystem.Test var inner_dir = @"archive\other\dir".RelativeTo(VFS_TEST_DIR); inner_dir.CreateDirectory(); - TEST_ZIP.MoveTo( @"archive\other\dir\nested.zip".RelativeTo(VFS_TEST_DIR)); + await TEST_ZIP.MoveToAsync( @"archive\other\dir\nested.zip".RelativeTo(VFS_TEST_DIR)); await ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP); await AddTestRoot(); @@ -169,7 +169,7 @@ namespace Wabbajack.VirtualFileSystem.Test foreach (var file in files) { - await using var stream = file.StagedFile.OpenRead(); + await using var stream = await file.StagedFile.OpenRead(); Assert.Equal("This is a test", await stream.ReadAllTextAsync()); } diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index 95f0ae7a..09f3f315 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -140,7 +140,7 @@ namespace Wabbajack.VirtualFileSystem public async Task WriteToFile(AbsolutePath filename) { - await using var fs = filename.Create(); + await using var fs = await filename.Create(); await using var bw = new BinaryWriter(fs, Encoding.UTF8, true); fs.SetLength(0); @@ -156,12 +156,12 @@ namespace Wabbajack.VirtualFileSystem f.Write(ibw); return ms; })) - .Do(ms => + .Do(async ms => { var size = ms.Position; ms.Position = 0; bw.Write((ulong) size); - ms.CopyTo(fs); + await ms.CopyToAsync(fs); }); Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}"); } @@ -170,7 +170,7 @@ namespace Wabbajack.VirtualFileSystem { try { - await using var fs = filename.OpenRead(); + await using var fs = await filename.OpenRead(); using var br = new BinaryReader(fs, Encoding.UTF8, true); var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length)); var fileVersion = br.ReadUInt64(); diff --git a/Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs b/Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs index f278ccb0..694a5a2a 100644 --- a/Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs +++ b/Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs @@ -19,15 +19,15 @@ namespace Wabbajack.VirtualFileSystem public async Task HashAsync() { - await using var stream = OpenRead(); - return stream.xxHash(); + await using var stream = await OpenRead(); + return await stream.xxHashAsync(); } public DateTime LastModifiedUtc => DateTime.UtcNow; public long Size => _file.Size; - public Stream OpenRead() + public async ValueTask OpenRead() { var ms = new MemoryStream(); - _file.CopyDataTo(ms); + await _file.CopyDataTo(ms); ms.Position = 0; return ms; } @@ -44,8 +44,8 @@ namespace Wabbajack.VirtualFileSystem public async Task MoveTo(AbsolutePath path) { - await using var fs = path.Create(); - _file.CopyDataTo(fs); + await using var fs = await path.Create(); + await _file.CopyDataTo(fs); } } } diff --git a/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs b/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs index 78dceaf2..04a4e242 100644 --- a/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs +++ b/Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs @@ -23,9 +23,9 @@ namespace Wabbajack.VirtualFileSystem } public DateTime LastModifiedUtc => _path.LastModifiedUtc; public long Size => _path.Size; - public Stream OpenRead() + public async ValueTask OpenRead() { - return _path.OpenRead(); + return await _path.OpenRead(); } public async Task CanExtract() diff --git a/Wabbajack.VirtualFileSystem/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor.cs index 9e6a4380..7d36226e 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor.cs @@ -22,7 +22,7 @@ namespace Wabbajack.VirtualFileSystem { try { - if (BSADispatch.MightBeBSA(source)) + if (await BSADispatch.MightBeBSA(source)) return await ExtractAllWithBSA(queue, source); else if (source.Extension == Consts.OMOD) return await ExtractAllWithOMOD(source); @@ -114,7 +114,7 @@ namespace Wabbajack.VirtualFileSystem { try { - await using var arch = BSADispatch.OpenRead(source); + await using var arch = await BSADispatch.OpenRead(source); var files = arch.Files.ToDictionary(f => f.Path, f => (IExtractedFile)new ExtractedBSAFile(f)); return new ExtractedFiles(files, arch); } diff --git a/Wabbajack.VirtualFileSystem/IExtractedFile.cs b/Wabbajack.VirtualFileSystem/IExtractedFile.cs index e37e86b6..f9660274 100644 --- a/Wabbajack.VirtualFileSystem/IExtractedFile.cs +++ b/Wabbajack.VirtualFileSystem/IExtractedFile.cs @@ -12,7 +12,7 @@ namespace Wabbajack.VirtualFileSystem public DateTime LastModifiedUtc { get; } public long Size { get; } - public Stream OpenRead(); + public ValueTask OpenRead(); public Task CanExtract(); diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 3b0ce3a1..96106c6c 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -220,7 +220,7 @@ namespace Wabbajack.VirtualFileSystem self.FillFullPath(depth); if (context.UseExtendedHashes) - self.ExtendedHashes = ExtendedHashes.FromFile(extractedFile); + self.ExtendedHashes = await ExtendedHashes.FromFile(extractedFile); if (!await extractedFile.CanExtract()) return self; @@ -386,9 +386,9 @@ namespace Wabbajack.VirtualFileSystem return path; } - public Stream OpenRead() + public async ValueTask OpenRead() { - return StagedFile.OpenRead(); + return await StagedFile.OpenRead(); } } @@ -399,10 +399,10 @@ namespace Wabbajack.VirtualFileSystem public string MD5 { get; set; } public string CRC { get; set; } - public static ExtendedHashes FromFile(IExtractedFile file) + public static async ValueTask FromFile(IExtractedFile file) { var hashes = new ExtendedHashes(); - using var stream = file.OpenRead(); + await using var stream = await file.OpenRead(); hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex(); stream.Position = 0; hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex();