diff --git a/Wabbajack.Common/AbsolutePathExtensions.cs b/Wabbajack.Common/AbsolutePathExtensions.cs index 91ed4e2a..d2221daa 100644 --- a/Wabbajack.Common/AbsolutePathExtensions.cs +++ b/Wabbajack.Common/AbsolutePathExtensions.cs @@ -36,4 +36,10 @@ public static class AbsolutePathExtensions srcStream.Close(); } } + + public static async Task WriteAllHashedAsync(this AbsolutePath file, byte[] data, CancellationToken token) + { + await using var dest = file.Open(FileMode.Create, FileAccess.Write, FileShare.None); + return await new MemoryStream(data).HashingCopy(dest, token); + } } \ No newline at end of file diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index 73468a93..d4438747 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -12,6 +12,7 @@ using Wabbajack.Common; using Wabbajack.Downloaders; using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; +using Wabbajack.DTOs.BSA.FileStates; using Wabbajack.DTOs.Directives; using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.JsonConverters; @@ -294,6 +295,13 @@ public abstract class AInstaller _logger.LogError("Hashes for {Path} did not match, expected {Expected} got {Got}", file.To, file.Hash, gotHash); throw new Exception($"Hashes for {file.To} did not match, expected {file.Hash} got {gotHash}"); } + + + protected void ThrowOnNonMatchingHash(CreateBSA bsa, Directive directive, AFile state, Hash hash) + { + _logger.LogError("Hashes for BSA don't match after extraction, {BSA}, {Directive}, {ExpectedHash}, {Hash}", bsa.To, directive.To, directive.Hash, hash); + throw new Exception($"Hashes for {bsa.To} file {directive.To} did not match, expected {directive.Hash} got {hash}"); + } public async Task DownloadArchives(CancellationToken token) { diff --git a/Wabbajack.Installer/StandardInstaller.cs b/Wabbajack.Installer/StandardInstaller.cs index 638708c1..ea0d87b0 100644 --- a/Wabbajack.Installer/StandardInstaller.cs +++ b/Wabbajack.Installer/StandardInstaller.cs @@ -19,9 +19,11 @@ using Wabbajack.Compression.Zip; using Wabbajack.Downloaders; using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; +using Wabbajack.DTOs.BSA.FileStates; using Wabbajack.DTOs.Directives; using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Hashing.xxHash64; using Wabbajack.Installer.Utilities; using Wabbajack.Networking.WabbajackClientApi; using Wabbajack.Paths; @@ -271,6 +273,8 @@ public class StandardInstaller : AInstaller private async Task BuildBSAs(CancellationToken token) { var bsas = ModList.Directives.OfType().ToList(); + _logger.LogInformation("Generating debug caches"); + var indexedByDestination = ModList.Directives.ToDictionary(d => d.To); _logger.LogInformation("Building {bsasCount} bsa files", bsas.Count); NextStep("Installing", "Building BSAs", bsas.Count); @@ -281,7 +285,7 @@ public class StandardInstaller : AInstaller var sourceDir = _configuration.Install.Combine(BSACreationDir, bsa.TempID); await using var a = BSADispatch.CreateBuilder(bsa.State, _manager); - var streams = await bsa.FileStates.PMapAll(async state => + var streams = await bsa.FileStates.PMapAllBatchedAsync(_limiter, async state => { using var job = await _limiter.Begin($"Adding {state.Path.FileName}", 0, token); var fs = sourceDir.Combine(state.Path).Open(FileMode.Open, FileAccess.Read, FileShare.Read); @@ -300,6 +304,22 @@ public class StandardInstaller : AInstaller await FileHashCache.FileHashWriteCache(outPath, bsa.Hash); sourceDir.DeleteDirectory(); + + _logger.LogInformation("Verifying {bsaTo}", bsa.To); + var reader = await BSADispatch.Open(outPath); + var results = await reader.Files.PMapAllBatchedAsync(_limiter, async state => + { + var sf = await state.GetStreamFactory(token); + await using var stream = await sf.GetStream(); + var hash = await stream.Hash(token); + + var astate = bsa.FileStates.First(f => f.Path == state.Path); + var srcDirective = indexedByDestination[BSACreationDir.Combine(bsa.TempID, astate.Path)]; + //DX10Files are lossy + if (astate is not BA2DX10File) + ThrowOnNonMatchingHash(bsa, srcDirective, astate, hash); + return (srcDirective, hash); + }).ToHashSet(); } var bsaDir = _configuration.Install.Combine(BSACreationDir); @@ -310,6 +330,7 @@ public class StandardInstaller : AInstaller } } + private async Task InstallIncludedFiles(CancellationToken token) { _logger.LogInformation("Writing inline files"); @@ -329,7 +350,8 @@ public class StandardInstaller : AInstaller await FileHashCache.FileHashCachedAsync(outPath, token); break; default: - await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID), token); + var hash = await outPath.WriteAllHashedAsync(await LoadBytesFromPath(directive.SourceDataID), token); + ThrowOnNonMatchingHash(directive, hash); await FileHashCache.FileHashWriteCache(outPath, directive.Hash); break; }