Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add verification to inlined files and BSAs

Browse Source
This commit is contained in:
Halgari 2022-10-23 16:07:55 -06:00
parent dbdf94a925
commit 51d8767674
3 changed files with 38 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Wabbajack.Common/AbsolutePathExtensions.cs
View File

@ -36,4 +36,10 @@ public static class AbsolutePathExtensions
srcStream.Close();
}
}
public static async Task<Hash> WriteAllHashedAsync(this AbsolutePath file, byte[] data, CancellationToken token)
{
await using var dest = file.Open(FileMode.Create, FileAccess.Write, FileShare.None);
return await new MemoryStream(data).HashingCopy(dest, token);
}
}

8
Wabbajack.Installer/AInstaller.cs
View File

@ -12,6 +12,7 @@ using Wabbajack.Common;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs;
using Wabbajack.DTOs.BSA.FileStates;
using Wabbajack.DTOs.Directives;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.DTOs.JsonConverters;
@ -295,6 +296,13 @@ public abstract class AInstaller<T>
throw new Exception($"Hashes for {file.To} did not match, expected {file.Hash} got {gotHash}");
}
protected void ThrowOnNonMatchingHash(CreateBSA bsa, Directive directive, AFile state, Hash hash)
{
_logger.LogError("Hashes for BSA don't match after extraction, {BSA}, {Directive}, {ExpectedHash}, {Hash}", bsa.To, directive.To, directive.Hash, hash);
throw new Exception($"Hashes for {bsa.To} file {directive.To} did not match, expected {directive.Hash} got {hash}");
}
public async Task DownloadArchives(CancellationToken token)
{
var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();

26
Wabbajack.Installer/StandardInstaller.cs
View File

@ -19,9 +19,11 @@ using Wabbajack.Compression.Zip;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.DTOs;
using Wabbajack.DTOs.BSA.FileStates;
using Wabbajack.DTOs.Directives;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Installer.Utilities;
using Wabbajack.Networking.WabbajackClientApi;
using Wabbajack.Paths;
@ -271,6 +273,8 @@ public class StandardInstaller : AInstaller<StandardInstaller>
private async Task BuildBSAs(CancellationToken token)
{
var bsas = ModList.Directives.OfType<CreateBSA>().ToList();
_logger.LogInformation("Generating debug caches");
var indexedByDestination = ModList.Directives.ToDictionary(d => d.To);
_logger.LogInformation("Building {bsasCount} bsa files", bsas.Count);
NextStep("Installing", "Building BSAs", bsas.Count);
@ -281,7 +285,7 @@ public class StandardInstaller : AInstaller<StandardInstaller>
var sourceDir = _configuration.Install.Combine(BSACreationDir, bsa.TempID);
await using var a = BSADispatch.CreateBuilder(bsa.State, _manager);
var streams = await bsa.FileStates.PMapAll(async state =>
var streams = await bsa.FileStates.PMapAllBatchedAsync(_limiter, async state =>
{
using var job = await _limiter.Begin($"Adding {state.Path.FileName}", 0, token);
var fs = sourceDir.Combine(state.Path).Open(FileMode.Open, FileAccess.Read, FileShare.Read);
@ -300,6 +304,22 @@ public class StandardInstaller : AInstaller<StandardInstaller>
await FileHashCache.FileHashWriteCache(outPath, bsa.Hash);
sourceDir.DeleteDirectory();
_logger.LogInformation("Verifying {bsaTo}", bsa.To);
var reader = await BSADispatch.Open(outPath);
var results = await reader.Files.PMapAllBatchedAsync(_limiter, async state =>
{
var sf = await state.GetStreamFactory(token);
await using var stream = await sf.GetStream();
var hash = await stream.Hash(token);
var astate = bsa.FileStates.First(f => f.Path == state.Path);
var srcDirective = indexedByDestination[BSACreationDir.Combine(bsa.TempID, astate.Path)];
//DX10Files are lossy
if (astate is not BA2DX10File)
ThrowOnNonMatchingHash(bsa, srcDirective, astate, hash);
return (srcDirective, hash);
}).ToHashSet();
}
var bsaDir = _configuration.Install.Combine(BSACreationDir);
@ -310,6 +330,7 @@ public class StandardInstaller : AInstaller<StandardInstaller>
}
}
private async Task InstallIncludedFiles(CancellationToken token)
{
_logger.LogInformation("Writing inline files");
@ -329,7 +350,8 @@ public class StandardInstaller : AInstaller<StandardInstaller>
await FileHashCache.FileHashCachedAsync(outPath, token);
break;
default:
await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID), token);
var hash = await outPath.WriteAllHashedAsync(await LoadBytesFromPath(directive.SourceDataID), token);
ThrowOnNonMatchingHash(directive, hash);
await FileHashCache.FileHashWriteCache(outPath, directive.Hash);
break;
}