Add verification to inlined files and BSAs

Wabbajack.Common/AbsolutePathExtensions.cs

@@ -36,4 +36,10 @@ public static class AbsolutePathExtensions
                 srcStream.Close();
         }
     }
+    
+    public static async Task<Hash> WriteAllHashedAsync(this AbsolutePath file, byte[] data, CancellationToken token)
+    {
+        await using var dest = file.Open(FileMode.Create, FileAccess.Write, FileShare.None);
+        return await new MemoryStream(data).HashingCopy(dest, token);
+    }
 }

Wabbajack.Installer/AInstaller.cs

@@ -12,6 +12,7 @@ using Wabbajack.Common;
 using Wabbajack.Downloaders;
 using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs;
+using Wabbajack.DTOs.BSA.FileStates;
 using Wabbajack.DTOs.Directives;
 using Wabbajack.DTOs.DownloadStates;
 using Wabbajack.DTOs.JsonConverters;
@@ -295,6 +296,13 @@ public abstract class AInstaller<T>
         throw new Exception($"Hashes for {file.To} did not match, expected {file.Hash} got {gotHash}");
     }
     
+    
+    protected void ThrowOnNonMatchingHash(CreateBSA bsa, Directive directive, AFile state, Hash hash)
+    {
+        _logger.LogError("Hashes for BSA don't match after extraction, {BSA}, {Directive}, {ExpectedHash}, {Hash}", bsa.To, directive.To, directive.Hash, hash);
+        throw new Exception($"Hashes for {bsa.To} file {directive.To} did not match, expected {directive.Hash} got {hash}");
+    }
+
     public async Task DownloadArchives(CancellationToken token)
     {
         var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();

Wabbajack.Installer/StandardInstaller.cs

@@ -19,9 +19,11 @@ using Wabbajack.Compression.Zip;
 using Wabbajack.Downloaders;
 using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs;
+using Wabbajack.DTOs.BSA.FileStates;
 using Wabbajack.DTOs.Directives;
 using Wabbajack.DTOs.DownloadStates;
 using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Installer.Utilities;
 using Wabbajack.Networking.WabbajackClientApi;
 using Wabbajack.Paths;
@@ -271,6 +273,8 @@ public class StandardInstaller : AInstaller<StandardInstaller>
     private async Task BuildBSAs(CancellationToken token)
     {
         var bsas = ModList.Directives.OfType<CreateBSA>().ToList();
+        _logger.LogInformation("Generating debug caches");
+        var indexedByDestination = ModList.Directives.ToDictionary(d => d.To);
         _logger.LogInformation("Building {bsasCount} bsa files", bsas.Count);
         NextStep("Installing", "Building BSAs", bsas.Count);
 
@@ -281,7 +285,7 @@ public class StandardInstaller : AInstaller<StandardInstaller>
             var sourceDir = _configuration.Install.Combine(BSACreationDir, bsa.TempID);
 
             await using var a = BSADispatch.CreateBuilder(bsa.State, _manager);
-            var streams = await bsa.FileStates.PMapAll(async state =>
+            var streams = await bsa.FileStates.PMapAllBatchedAsync(_limiter, async state =>
             {
                 using var job = await _limiter.Begin($"Adding {state.Path.FileName}", 0, token);
                 var fs = sourceDir.Combine(state.Path).Open(FileMode.Open, FileAccess.Read, FileShare.Read);
@@ -300,6 +304,22 @@ public class StandardInstaller : AInstaller<StandardInstaller>
             
             await FileHashCache.FileHashWriteCache(outPath, bsa.Hash);
             sourceDir.DeleteDirectory();
+            
+            _logger.LogInformation("Verifying {bsaTo}", bsa.To);
+            var reader = await BSADispatch.Open(outPath);
+            var results = await reader.Files.PMapAllBatchedAsync(_limiter, async state =>
+            {
+                var sf = await state.GetStreamFactory(token);
+                await using var stream = await sf.GetStream();
+                var hash = await stream.Hash(token);
+
+                var astate = bsa.FileStates.First(f => f.Path == state.Path);
+                var srcDirective = indexedByDestination[BSACreationDir.Combine(bsa.TempID, astate.Path)];
+                //DX10Files are lossy
+                if (astate is not BA2DX10File) 
+                    ThrowOnNonMatchingHash(bsa, srcDirective, astate, hash);
+                return (srcDirective, hash);
+            }).ToHashSet();
         }
 
         var bsaDir = _configuration.Install.Combine(BSACreationDir);
@@ -310,6 +330,7 @@ public class StandardInstaller : AInstaller<StandardInstaller>
         }
     }
 
+
     private async Task InstallIncludedFiles(CancellationToken token)
     {
         _logger.LogInformation("Writing inline files");
@@ -329,7 +350,8 @@ public class StandardInstaller : AInstaller<StandardInstaller>
                         await FileHashCache.FileHashCachedAsync(outPath, token);
                         break;
                     default:
-                        await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID), token);
+                        var hash = await outPath.WriteAllHashedAsync(await LoadBytesFromPath(directive.SourceDataID), token);
+                        ThrowOnNonMatchingHash(directive, hash);
                         await FileHashCache.FileHashWriteCache(outPath, directive.Hash);
                         break;
                 }