Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

VirtualFile now useses Async IO exclusively

Browse Source
This commit is contained in:
Timothy Baldridge 2020-05-25 10:30:47 -06:00
parent ea08c9865d
commit 191f321dc2
9 changed files with 49 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Wabbajack.Common/Hash.cs
View File

@ -151,6 +151,14 @@ namespace Wabbajack.Common
var value = xxHashFactory.Instance.Create(config).ComputeHash(f);
return Hash.FromULong(BitConverter.ToUInt64(value.Hash));
}
public static async Task<Hash> xxHashAsync(this Stream stream)
{
var config = new xxHashConfig {HashSizeInBits = 64};
await using var f = new StatusFileStream(stream, $"Hashing memory stream");
var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(f);
return Hash.FromULong(BitConverter.ToUInt64(value.Hash));
}
public static bool TryGetHashCache(AbsolutePath file, out Hash hash)
{
var normPath = Encoding.UTF8.GetBytes(file.Normalize());

17
Wabbajack.Common/IAsyncEnumerableExtensions.cs
View File

@ -22,6 +22,23 @@ namespace Wabbajack.Common
yield return await mapFn(itm);
}
}
/// <summary>
/// Same as .Select but expects a function that returns an async result
/// </summary>
/// <param name="coll"></param>
/// <param name="mapFn"></param>
/// <typeparam name="TIn"></typeparam>
/// <typeparam name="TOut"></typeparam>
/// <returns></returns>
public static async ValueTask DoAsync<TIn, TOut>(this IEnumerable<TIn> coll,
Func<TIn, ValueTask<TOut>> mapFn)
{
foreach (var itm in coll)
{
await mapFn(itm);
}
}
public static async ValueTask<List<T>> ToList<T>(this IAsyncEnumerable<T> coll)
{

8
Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs
View File

@ -72,7 +72,7 @@ namespace Wabbajack.VirtualFileSystem.Test
Assert.NotNull(file);
Assert.Equal(128, file.Size);
Assert.Equal(absPath.FileHash(), file.Hash);
Assert.Equal(await absPath.FileHashAsync(), file.Hash);
Assert.True(file.IsArchive);
var innerFile = file.Children.First();
@ -143,7 +143,7 @@ namespace Wabbajack.VirtualFileSystem.Test
var cleanup = await context.Stage(new List<VirtualFile> {file});
await using var stream = file.StagedFile.OpenRead();
await using var stream = await file.StagedFile.OpenRead();
Assert.Equal("This is a test", await stream.ReadAllTextAsync());
@ -158,7 +158,7 @@ namespace Wabbajack.VirtualFileSystem.Test
var inner_dir = @"archive\other\dir".RelativeTo(VFS_TEST_DIR);
inner_dir.CreateDirectory();
TEST_ZIP.MoveTo( @"archive\other\dir\nested.zip".RelativeTo(VFS_TEST_DIR));
await TEST_ZIP.MoveToAsync( @"archive\other\dir\nested.zip".RelativeTo(VFS_TEST_DIR));
await ZipUpFolder(ARCHIVE_TEST_TXT.Parent, TEST_ZIP);
await AddTestRoot();
@ -169,7 +169,7 @@ namespace Wabbajack.VirtualFileSystem.Test
foreach (var file in files)
{
await using var stream = file.StagedFile.OpenRead();
await using var stream = await file.StagedFile.OpenRead();
Assert.Equal("This is a test", await stream.ReadAllTextAsync());
}

8
Wabbajack.VirtualFileSystem/Context.cs
View File

@ -140,7 +140,7 @@ namespace Wabbajack.VirtualFileSystem
public async Task WriteToFile(AbsolutePath filename)
{
await using var fs = filename.Create();
await using var fs = await filename.Create();
await using var bw = new BinaryWriter(fs, Encoding.UTF8, true);
fs.SetLength(0);
@ -156,12 +156,12 @@ namespace Wabbajack.VirtualFileSystem
f.Write(ibw);
return ms;
}))
.Do(ms =>
.Do(async ms =>
{
var size = ms.Position;
ms.Position = 0;
bw.Write((ulong) size);
ms.CopyTo(fs);
await ms.CopyToAsync(fs);
});
Utils.Log($"Wrote {fs.Position.ToFileSizeString()} file as vfs cache file {filename}");
}
@ -170,7 +170,7 @@ namespace Wabbajack.VirtualFileSystem
{
try
{
await using var fs = filename.OpenRead();
await using var fs = await filename.OpenRead();
using var br = new BinaryReader(fs, Encoding.UTF8, true);
var magic = Encoding.ASCII.GetString(br.ReadBytes(Encoding.ASCII.GetBytes(Magic).Length));
var fileVersion = br.ReadUInt64();

12
Wabbajack.VirtualFileSystem/ExtractedBSAFile.cs
View File

@ -19,15 +19,15 @@ namespace Wabbajack.VirtualFileSystem
public async Task<Hash> HashAsync()
{
await using var stream = OpenRead();
return stream.xxHash();
await using var stream = await OpenRead();
return await stream.xxHashAsync();
}
public DateTime LastModifiedUtc => DateTime.UtcNow;
public long Size => _file.Size;
public Stream OpenRead()
public async ValueTask<Stream> OpenRead()
{
var ms = new MemoryStream();
_file.CopyDataTo(ms);
await _file.CopyDataTo(ms);
ms.Position = 0;
return ms;
}
@ -44,8 +44,8 @@ namespace Wabbajack.VirtualFileSystem
public async Task MoveTo(AbsolutePath path)
{
await using var fs = path.Create();
_file.CopyDataTo(fs);
await using var fs = await path.Create();
await _file.CopyDataTo(fs);
}
}
}

4
Wabbajack.VirtualFileSystem/ExtractedDiskFile.cs
View File

@ -23,9 +23,9 @@ namespace Wabbajack.VirtualFileSystem
}
public DateTime LastModifiedUtc => _path.LastModifiedUtc;
public long Size => _path.Size;
public Stream OpenRead()
public async ValueTask<Stream> OpenRead()
{
return _path.OpenRead();
return await _path.OpenRead();
}
public async Task<bool> CanExtract()

4
Wabbajack.VirtualFileSystem/FileExtractor.cs
View File

@ -22,7 +22,7 @@ namespace Wabbajack.VirtualFileSystem
{
try
{
if (BSADispatch.MightBeBSA(source))
if (await BSADispatch.MightBeBSA(source))
return await ExtractAllWithBSA(queue, source);
else if (source.Extension == Consts.OMOD)
return await ExtractAllWithOMOD(source);
@ -114,7 +114,7 @@ namespace Wabbajack.VirtualFileSystem
{
try
{
await using var arch = BSADispatch.OpenRead(source);
await using var arch = await BSADispatch.OpenRead(source);
var files = arch.Files.ToDictionary(f => f.Path, f => (IExtractedFile)new ExtractedBSAFile(f));
return new ExtractedFiles(files, arch);
}

2
Wabbajack.VirtualFileSystem/IExtractedFile.cs
View File

@ -12,7 +12,7 @@ namespace Wabbajack.VirtualFileSystem
public DateTime LastModifiedUtc { get; }
public long Size { get; }
public Stream OpenRead();
public ValueTask<Stream> OpenRead();
public Task<bool> CanExtract();

10
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -220,7 +220,7 @@ namespace Wabbajack.VirtualFileSystem
self.FillFullPath(depth);
if (context.UseExtendedHashes)
self.ExtendedHashes = ExtendedHashes.FromFile(extractedFile);
self.ExtendedHashes = await ExtendedHashes.FromFile(extractedFile);
if (!await extractedFile.CanExtract()) return self;
@ -386,9 +386,9 @@ namespace Wabbajack.VirtualFileSystem
return path;
}
public Stream OpenRead()
public async ValueTask<Stream> OpenRead()
{
return StagedFile.OpenRead();
return await StagedFile.OpenRead();
}
}
@ -399,10 +399,10 @@ namespace Wabbajack.VirtualFileSystem
public string MD5 { get; set; }
public string CRC { get; set; }
public static ExtendedHashes FromFile(IExtractedFile file)
public static async ValueTask<ExtendedHashes> FromFile(IExtractedFile file)
{
var hashes = new ExtendedHashes();
using var stream = file.OpenRead();
await using var stream = await file.OpenRead();
hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex();
stream.Position = 0;
hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex();