Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

VirtualFile now useses Async IO exclusively

Browse Source
This commit is contained in:
Timothy Baldridge 2020-05-25 10:30:47 -06:00
parent ea08c9865d
commit 191f321dc2
9 changed files with 49 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Wabbajack.Common/Hash.cs
View File

@ -151,6 +151,14 @@ namespace Wabbajack.Common
var value = xxHashFactory.Instance.Create(config).ComputeHash(f); var value = xxHashFactory.Instance.Create(config).ComputeHash(f);
return Hash.FromULong(BitConverter.ToUInt64(value.Hash)); return Hash.FromULong(BitConverter.ToUInt64(value.Hash));
} }
public static async Task<Hash> xxHashAsync(this Stream stream)
{
var config = new xxHashConfig {HashSizeInBits = 64};
await using var f = new StatusFileStream(stream, $"Hashing memory stream");
var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(f);
return Hash.FromULong(BitConverter.ToUInt64(value.Hash));
}
public static bool TryGetHashCache(AbsolutePath file, out Hash hash) public static bool TryGetHashCache(AbsolutePath file, out Hash hash)
{ {
var normPath = Encoding.UTF8.GetBytes(file.Normalize()); var normPath = Encoding.UTF8.GetBytes(file.Normalize());

17
Wabbajack.Common/IAsyncEnumerableExtensions.cs
View File

@ -22,6 +22,23 @@ namespace Wabbajack.Common
yield return await mapFn(itm); yield return await mapFn(itm);
} }
} }
/// <summary>
/// Same as .Select but expects a function that returns an async result
/// </summary>
/// <param name="coll"></param>
/// <param name="mapFn"></param>
/// <typeparam name="TIn"></typeparam>
/// <typeparam name="TOut"></typeparam>
/// <returns></returns>
public static async ValueTask DoAsync<TIn, TOut>(this IEnumerable<TIn> coll,
Func<TIn, ValueTask<TOut>> mapFn)
{
foreach (var itm in coll)
{
await mapFn(itm);
}
}
public static async ValueTask<List<T>> ToList<T>(this IAsyncEnumerable<T> coll) public static async ValueTask<List<T>> ToList<T>(this IAsyncEnumerable<T> coll)
{ {

8
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
View File

@ -72,7 +72,7 @@ namespace Wabbajack.VirtualFileSystem.Test
Assert.NotNull(file); Assert.NotNull(file);
Assert.Equal(128, file.Size); Assert.Equal(128, file.Size);
Assert.Equal(absPath.FileHash(), file.Hash); Assert.Equal(await absPath.FileHashAsync(), file.Hash);
Assert.True(file.IsArchive); Assert.True(file.IsArchive);
var innerFile = file.Children.First(); var innerFile = file.Children.First();
@ -143,7 +143,7 @@ namespace Wabbajack.VirtualFileSystem.Test
var cleanup = await context.Stage(new List<VirtualFile> {file}); var cleanup = await context.Stage(new List<VirtualFile> {file});
await using var stream = file.StagedFile.OpenRead(); await using var stream = await file.StagedFile.OpenRead();
Assert.Equal("This is a test", await stream.ReadAllTextAsync()); Assert.Equal("This is a test", await stream.ReadAllTextAsync());
@ -158,7 +158,7 @@ namespace Wabbajack.VirtualFileSystem.Test
var inner_dir = @"archive\other\dir".RelativeTo(VFS_TEST_DIR); var inner_dir = @"archive\other\dir".RelativeTo(VFS_TEST_DIR);
inner_dir.CreateDirectory(); inner_dir.CreateDirectory();
TEST_ZIP.MoveTo( @"archive\other\dir\nested.zip".RelativeTo(VFS_TEST_DIR)); await TEST_ZIP.MoveToAsync( @"archive\other\dir\nested.zip".RelativeTo(VFS_TEST_DIR));
await ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP); await ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
await AddTestRoot(); await AddTestRoot();
@ -169,7 +169,7 @@ namespace Wabbajack.VirtualFileSystem.Test
foreach (var file in files) foreach (var file in files)
{ {
await using var stream = file.StagedFile.OpenRead(); await using var stream = await file.StagedFile.OpenRead();
Assert.Equal("This is a test", await stream.ReadAllTextAsync()); Assert.Equal("This is a test", await stream.ReadAllTextAsync());
} }

8
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -140,7 +140,7 @@ namespace Wabbajack.VirtualFileSystem
public async Task WriteToFile(AbsolutePath filename) public async Task WriteToFile(AbsolutePath filename)
{ {
await using var fs = filename.Create(); await using var fs = await filename.Create();
await using var bw = new BinaryWriter(fs, Encoding.UTF8, true); await using var bw = new BinaryWriter(fs, Encoding.UTF8, true);
fs.SetLength(0); fs.SetLength(0);
@ -156,12 +156,12 @@ namespace Wabbajack.VirtualFileSystem
f.Write(ibw); f.Write(ibw);
return ms; return ms;
})) }))
.Do(ms => .Do(async ms =>
{ {
var size = ms.Position; var size = ms.Position;
ms.Position = 0; ms.Position = 0;
bw.Write((ulong) size); bw.Write((ulong) size);
ms.CopyTo(fs); await ms.CopyToAsync(fs);
}); });
Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}"); Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}");
} }
@ -170,7 +170,7 @@ namespace Wabbajack.VirtualFileSystem
{ {
try try
{ {
await using var fs = filename.OpenRead(); await using var fs = await filename.OpenRead();
using var br = new BinaryReader(fs, Encoding.UTF8, true); using var br = new BinaryReader(fs, Encoding.UTF8, true);
var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length)); var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length));
var fileVersion = br.ReadUInt64(); var fileVersion = br.ReadUInt64();

12
Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs
View File

@ -19,15 +19,15 @@ namespace Wabbajack.VirtualFileSystem
public async Task<Hash> HashAsync() public async Task<Hash> HashAsync()
{ {
await using var stream = OpenRead(); await using var stream = await OpenRead();
return stream.xxHash(); return await stream.xxHashAsync();
} }
public DateTime LastModifiedUtc => DateTime.UtcNow; public DateTime LastModifiedUtc => DateTime.UtcNow;
public long Size => _file.Size; public long Size => _file.Size;
public Stream OpenRead() public async ValueTask<Stream> OpenRead()
{ {
var ms = new MemoryStream(); var ms = new MemoryStream();
_file.CopyDataTo(ms); await _file.CopyDataTo(ms);
ms.Position = 0; ms.Position = 0;
return ms; return ms;
} }
@ -44,8 +44,8 @@ namespace Wabbajack.VirtualFileSystem
public async Task MoveTo(AbsolutePath path) public async Task MoveTo(AbsolutePath path)
{ {
await using var fs = path.Create(); await using var fs = await path.Create();
_file.CopyDataTo(fs); await _file.CopyDataTo(fs);
} }
} }
} }

4
Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs
View File

@ -23,9 +23,9 @@ namespace Wabbajack.VirtualFileSystem
} }
public DateTime LastModifiedUtc => _path.LastModifiedUtc; public DateTime LastModifiedUtc => _path.LastModifiedUtc;
public long Size => _path.Size; public long Size => _path.Size;
public Stream OpenRead() public async ValueTask<Stream> OpenRead()
{ {
return _path.OpenRead(); return await _path.OpenRead();
} }
public async Task<bool> CanExtract() public async Task<bool> CanExtract()

4
Wabbajack.VirtualFileSystem/FileExtractor.cs
View File

@ -22,7 +22,7 @@ namespace Wabbajack.VirtualFileSystem
{ {
try try
{ {
if (BSADispatch.MightBeBSA(source)) if (await BSADispatch.MightBeBSA(source))
return await ExtractAllWithBSA(queue, source); return await ExtractAllWithBSA(queue, source);
else if (source.Extension == Consts.OMOD) else if (source.Extension == Consts.OMOD)
return await ExtractAllWithOMOD(source); return await ExtractAllWithOMOD(source);
@ -114,7 +114,7 @@ namespace Wabbajack.VirtualFileSystem
{ {
try try
{ {
await using var arch = BSADispatch.OpenRead(source); await using var arch = await BSADispatch.OpenRead(source);
var files = arch.Files.ToDictionary(f => f.Path, f => (IExtractedFile)new ExtractedBSAFile(f)); var files = arch.Files.ToDictionary(f => f.Path, f => (IExtractedFile)new ExtractedBSAFile(f));
return new ExtractedFiles(files, arch); return new ExtractedFiles(files, arch);
} }

2
Wabbajack.VirtualFileSystem/IExtractedFile.cs
View File

@ -12,7 +12,7 @@ namespace Wabbajack.VirtualFileSystem
public DateTime LastModifiedUtc { get; } public DateTime LastModifiedUtc { get; }
public long Size { get; } public long Size { get; }
public Stream OpenRead(); public ValueTask<Stream> OpenRead();
public Task<bool> CanExtract(); public Task<bool> CanExtract();

10
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -220,7 +220,7 @@ namespace Wabbajack.VirtualFileSystem
self.FillFullPath(depth); self.FillFullPath(depth);
if (context.UseExtendedHashes) if (context.UseExtendedHashes)
self.ExtendedHashes = ExtendedHashes.FromFile(extractedFile); self.ExtendedHashes = await ExtendedHashes.FromFile(extractedFile);
if (!await extractedFile.CanExtract()) return self; if (!await extractedFile.CanExtract()) return self;
@ -386,9 +386,9 @@ namespace Wabbajack.VirtualFileSystem
return path; return path;
} }
public Stream OpenRead() public async ValueTask<Stream> OpenRead()
{ {
return StagedFile.OpenRead(); return await StagedFile.OpenRead();
} }
} }
@ -399,10 +399,10 @@ namespace Wabbajack.VirtualFileSystem
public string MD5 { get; set; } public string MD5 { get; set; }
public string CRC { get; set; } public string CRC { get; set; }
public static ExtendedHashes FromFile(IExtractedFile file) public static async ValueTask<ExtendedHashes> FromFile(IExtractedFile file)
{ {
var hashes = new ExtendedHashes(); var hashes = new ExtendedHashes();
using var stream = file.OpenRead(); await using var stream = await file.OpenRead();
hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex(); hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex();
stream.Position = 0; stream.Position = 0;
hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex(); hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex();