Finished verification code

Wabbajack.CLI/Verbs/VerifyModlistInstall.cs

@@ -1,13 +1,23 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Wabbajack.CLI.Builder;
+using Wabbajack.Common;
+using Wabbajack.Compression.BSA;
+using Wabbajack.DTOs;
+using Wabbajack.DTOs.BSA.FileStates;
 using Wabbajack.DTOs.Directives;
 using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Installer;
 using Wabbajack.Paths;
 using Wabbajack.Paths.IO;
+using Wabbajack.RateLimiter;
+using Wabbajack.VFS;
+using AbsolutePathExtensions = Wabbajack.Common.AbsolutePathExtensions;
 
 namespace Wabbajack.CLI.Verbs;
 
@@ -16,8 +26,9 @@ public class VerifyModlistInstall
     private readonly DTOSerializer _dtos;
     private readonly ILogger<VerifyModlistInstall> _logger;
     
-    public VerifyModlistInstall(ILogger<VerifyModlistInstall> logger, DTOSerializer dtos)
+    public VerifyModlistInstall(ILogger<VerifyModlistInstall> logger, DTOSerializer dtos, IResource<FileHashCache> limiter)
     {
+        _limiter = limiter;
         _logger = logger;
         _dtos = dtos;
     }
@@ -30,47 +41,76 @@ public class VerifyModlistInstall
             new OptionDefinition(typeof(AbsolutePath), "i", "installFolder", "The installation folder of the modlist")
         });
 
+    private readonly IResource<FileHashCache> _limiter;
 
 
-    public async Task<int> Run(AbsolutePath modlistLocation, AbsolutePath installFolder)
+    public async Task<int> Run(AbsolutePath modlistLocation, AbsolutePath installFolder, CancellationToken token)
     {
         _logger.LogInformation("Loading modlist {ModList}", modlistLocation);
         var list = await StandardInstaller.LoadFromFile(_dtos, modlistLocation);
+        
+        _logger.LogInformation("Indexing files");
+        var byTo = list.Directives.ToDictionary(d => d.To);
 
-        var errors = new List<Result>();
 
         _logger.LogInformation("Scanning files");
-        foreach (var directive in list.Directives)
+        var errors = await list.Directives.PMapAllBatchedAsync(_limiter, async directive =>
         {
             if (directive is ArchiveMeta)
-                continue;
-            
+                return null;
+
             if (directive is RemappedInlineFile)
-                continue;
-            
+                return null;
+
             if (directive.To.InFolder(Consts.BSACreationDir))
-                continue;
-            
+                return null;
+
             var dest = directive.To.RelativeTo(installFolder);
             if (!dest.FileExists())
             {
-                errors.Add(new Result
+                return new Result
                 {
                     Path = directive.To,
                     Message = $"File does not exist directive {directive.GetType()}"
-                });
-                continue;
+                };
             }
+
+            if (Consts.KnownModifiedFiles.Contains(directive.To.FileName))
+                return null;
+
+            if (directive is CreateBSA bsa)
+            {
+                return await VerifyBSA(dest, bsa, byTo, token);
+            }
+
             if (dest.Size() != directive.Size)
             {
-                errors.Add(new Result
+                return new Result
                 {
                     Path = directive.To,
                     Message = $"Sizes do not match got {dest.Size()} expected {directive.Size}"
-                });
+                };
             }
-            
-        }
+
+            if (directive.Size > (1024 * 1024 * 128))
+            {
+                _logger.LogInformation("Hashing {Size} file at {Path}", directive.Size.ToFileSizeString(),
+                    directive.To);
+            }
+
+            var hash = await AbsolutePathExtensions.Hash(dest, token);
+            if (hash != directive.Hash)
+            {
+                return new Result
+                {
+                    Path = directive.To,
+                    Message = $"Hashes do not match, got {hash} expected {directive.Hash}"
+                };
+            }
+
+            return null;
+        }).Where(r => r != null)
+            .ToList();
         
         _logger.LogInformation("Found {Count} errors", errors.Count);
 
@@ -84,6 +124,47 @@ public class VerifyModlistInstall
         return 0;
     }
 
+    private async Task<Result?> VerifyBSA(AbsolutePath dest, CreateBSA bsa, Dictionary<RelativePath, Directive> byTo, CancellationToken token)
+    {
+        _logger.LogInformation("Verifying Created BSA {To}", bsa.To);
+        var archive = await BSADispatch.Open(dest);
+        var filesIndexed = archive.Files.ToDictionary(d => d.Path);
+        
+        if (dest.Extension == Ext.Bsa && dest.Size() >= 1024L * 1024 * 1024 * 2)
+        {
+            return new Result()
+            {
+                Path = bsa.To,
+                Message = $"BSA is over 2GB in size, this will cause crashes : {bsa.To}"
+            };
+        }
+
+        foreach (var file in bsa.FileStates)
+        {
+            if (file is BA2DX10File) continue;
+            var state = filesIndexed[file.Path];
+            var sf = await state.GetStreamFactory(token);
+            await using var stream = await sf.GetStream();
+            var hash = await stream.Hash(token);
+
+            var astate = bsa.FileStates.First(f => f.Path == state.Path);
+            var srcDirective = byTo[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)];
+
+            if (srcDirective.Hash != hash)
+            {
+                return new Result
+                {
+                    Path = bsa.To,
+                    Message =
+                        $"BSA {bsa.To} contents do not match at {file.Path} got {hash} expected {srcDirective.Hash}"
+                };
+            }
+        }
+
+        
+        return null;
+    }
+
     public class Result
     {
         public RelativePath Path { get; set; }

Wabbajack.Installer/Consts.cs

@@ -1,3 +1,5 @@
+using System.Collections.Generic;
+using System.Linq;
 using Wabbajack.Paths;
 
 namespace Wabbajack.Installer;
@@ -27,4 +29,10 @@ public static class Consts
     public const string StepHashing = "Hashing";
     public const string StepFinished = "Finished";
     public static RelativePath BSACreationDir = "TEMP_BSA_FILES".ToRelativePath();
+
+    public static HashSet<RelativePath> KnownModifiedFiles = new[]
+    {
+        "modlist.txt",
+        "SkyrimPrefs.ini"
+    }.Select(r => r.ToRelativePath()).ToHashSet();
 }

Wabbajack.Installer/StandardInstaller.cs

@@ -330,10 +330,6 @@ public class StandardInstaller : AInstaller<StandardInstaller>
         }
     }
 
-    private static HashSet<RelativePath> KnownModifiedFiles = new[]
-    {
-        "modlist.txt"
-    }.Select(r => r.ToRelativePath()).ToHashSet();
     private async Task InstallIncludedFiles(CancellationToken token)
     {
         _logger.LogInformation("Writing inline files");
@@ -354,7 +350,7 @@ public class StandardInstaller : AInstaller<StandardInstaller>
                         break;
                     default:
                         var hash = await outPath.WriteAllHashedAsync(await LoadBytesFromPath(directive.SourceDataID), token);
-                        if (!KnownModifiedFiles.Contains(directive.To.FileName))
+                        if (!Consts.KnownModifiedFiles.Contains(directive.To.FileName))
                             ThrowOnNonMatchingHash(directive, hash);
 
                         await FileHashCache.FileHashWriteCache(outPath, directive.Hash);