Merge pull request #2140 from wabbajack-tools/verify-modlist-install

Verify modlist install

Wabbajack.CLI/VerbRegistration.cs

@@ -51,6 +51,8 @@ CommandLineBuilder.RegisterCommand<UploadToNexus>(UploadToNexus.Definition, c =>
 services.AddSingleton<UploadToNexus>();
 CommandLineBuilder.RegisterCommand<ValidateLists>(ValidateLists.Definition, c => ((ValidateLists)c).Run);
 services.AddSingleton<ValidateLists>();
+CommandLineBuilder.RegisterCommand<VerifyModlistInstall>(VerifyModlistInstall.Definition, c => ((VerifyModlistInstall)c).Run);
+services.AddSingleton<VerifyModlistInstall>();
 CommandLineBuilder.RegisterCommand<VFSIndex>(VFSIndex.Definition, c => ((VFSIndex)c).Run);
 services.AddSingleton<VFSIndex>();
 }

Wabbajack.CLI/Verbs/ModlistReport.cs

@@ -70,7 +70,7 @@ public class ModlistReport
 
         string FixupTo(RelativePath path)
         {
-            if (path.GetPart(0) != StandardInstaller.BSACreationDir.ToString()) return path.ToString();
+            if (path.GetPart(0) != Consts.BSACreationDir.ToString()) return path.ToString();
             var bsaId = path.GetPart(1);
             
             if (!bsas.TryGetValue(bsaId, out var bsa))

Wabbajack.CLI/Verbs/VerifyModlistInstall.cs

@@ -0,0 +1,173 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.CLI.Builder;
+using Wabbajack.Common;
+using Wabbajack.Compression.BSA;
+using Wabbajack.DTOs;
+using Wabbajack.DTOs.BSA.FileStates;
+using Wabbajack.DTOs.Directives;
+using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Hashing.xxHash64;
+using Wabbajack.Installer;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+using Wabbajack.RateLimiter;
+using Wabbajack.VFS;
+using AbsolutePathExtensions = Wabbajack.Common.AbsolutePathExtensions;
+
+namespace Wabbajack.CLI.Verbs;
+
+public class VerifyModlistInstall
+{
+    private readonly DTOSerializer _dtos;
+    private readonly ILogger<VerifyModlistInstall> _logger;
+    
+    public VerifyModlistInstall(ILogger<VerifyModlistInstall> logger, DTOSerializer dtos, IResource<FileHashCache> limiter)
+    {
+        _limiter = limiter;
+        _logger = logger;
+        _dtos = dtos;
+    }
+    
+    public static VerbDefinition Definition = new("verify-modlist-install", "Verify a modlist installed correctly",
+        new[]
+        {
+            new OptionDefinition(typeof(AbsolutePath), "m", "modlistLocation",
+                "The .wabbajack file used to install the modlist"),
+            new OptionDefinition(typeof(AbsolutePath), "i", "installFolder", "The installation folder of the modlist")
+        });
+
+    private readonly IResource<FileHashCache> _limiter;
+
+
+    public async Task<int> Run(AbsolutePath modlistLocation, AbsolutePath installFolder, CancellationToken token)
+    {
+        _logger.LogInformation("Loading modlist {ModList}", modlistLocation);
+        var list = await StandardInstaller.LoadFromFile(_dtos, modlistLocation);
+        
+        _logger.LogInformation("Indexing files");
+        var byTo = list.Directives.ToDictionary(d => d.To);
+
+
+        _logger.LogInformation("Scanning files");
+        var errors = await list.Directives.PMapAllBatchedAsync(_limiter, async directive =>
+        {
+            if (directive is ArchiveMeta)
+                return null;
+
+            if (directive is RemappedInlineFile)
+                return null;
+
+            if (directive.To.InFolder(Consts.BSACreationDir))
+                return null;
+
+            var dest = directive.To.RelativeTo(installFolder);
+            if (!dest.FileExists())
+            {
+                return new Result
+                {
+                    Path = directive.To,
+                    Message = $"File does not exist directive {directive.GetType()}"
+                };
+            }
+
+            if (Consts.KnownModifiedFiles.Contains(directive.To.FileName))
+                return null;
+
+            if (directive is CreateBSA bsa)
+            {
+                return await VerifyBSA(dest, bsa, byTo, token);
+            }
+
+            if (dest.Size() != directive.Size)
+            {
+                return new Result
+                {
+                    Path = directive.To,
+                    Message = $"Sizes do not match got {dest.Size()} expected {directive.Size}"
+                };
+            }
+
+            if (directive.Size > (1024 * 1024 * 128))
+            {
+                _logger.LogInformation("Hashing {Size} file at {Path}", directive.Size.ToFileSizeString(),
+                    directive.To);
+            }
+
+            var hash = await AbsolutePathExtensions.Hash(dest, token);
+            if (hash != directive.Hash)
+            {
+                return new Result
+                {
+                    Path = directive.To,
+                    Message = $"Hashes do not match, got {hash} expected {directive.Hash}"
+                };
+            }
+
+            return null;
+        }).Where(r => r != null)
+            .ToList();
+        
+        _logger.LogInformation("Found {Count} errors", errors.Count);
+
+
+        foreach (var error in errors)
+        {
+            _logger.LogError("{File} | {Message}", error.Path, error.Message);
+        }
+
+
+        return 0;
+    }
+
+    private async Task<Result?> VerifyBSA(AbsolutePath dest, CreateBSA bsa, Dictionary<RelativePath, Directive> byTo, CancellationToken token)
+    {
+        _logger.LogInformation("Verifying Created BSA {To}", bsa.To);
+        var archive = await BSADispatch.Open(dest);
+        var filesIndexed = archive.Files.ToDictionary(d => d.Path);
+        
+        if (dest.Extension == Ext.Bsa && dest.Size() >= 1024L * 1024 * 1024 * 2)
+        {
+            return new Result()
+            {
+                Path = bsa.To,
+                Message = $"BSA is over 2GB in size, this will cause crashes : {bsa.To}"
+            };
+        }
+
+        foreach (var file in bsa.FileStates)
+        {
+            if (file is BA2DX10File) continue;
+            var state = filesIndexed[file.Path];
+            var sf = await state.GetStreamFactory(token);
+            await using var stream = await sf.GetStream();
+            var hash = await stream.Hash(token);
+
+            var astate = bsa.FileStates.First(f => f.Path == state.Path);
+            var srcDirective = byTo[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)];
+
+            if (srcDirective.Hash != hash)
+            {
+                return new Result
+                {
+                    Path = bsa.To,
+                    Message =
+                        $"BSA {bsa.To} contents do not match at {file.Path} got {hash} expected {srcDirective.Hash}"
+                };
+            }
+        }
+
+        
+        return null;
+    }
+
+    public class Result
+    {
+        public RelativePath Path { get; set; }
+        public string Message { get; set; }
+    }
+}

Wabbajack.Common/AbsolutePathExtensions.cs

@@ -21,4 +21,25 @@ public static class AbsolutePathExtensions
         await using var fs = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
         return await fs.FromJson<T>(dtos);
     }
+    
+    public static async Task<Hash> WriteAllHashedAsync(this AbsolutePath file, Stream srcStream, CancellationToken token,
+        bool closeWhenDone = true)
+    {
+        try
+        {
+            await using var dest = file.Open(FileMode.Create, FileAccess.Write, FileShare.None);
+            return await srcStream.HashingCopy(dest, token);
+        }
+        finally
+        {
+            if (closeWhenDone)
+                srcStream.Close();
+        }
+    }
+    
+    public static async Task<Hash> WriteAllHashedAsync(this AbsolutePath file, byte[] data, CancellationToken token)
+    {
+        await using var dest = file.Open(FileMode.Create, FileAccess.Write, FileShare.None);
+        return await new MemoryStream(data).HashingCopy(dest, token);
+    }
 }

Wabbajack.Downloaders.Nexus/NexusDownloader.cs

@@ -129,8 +129,23 @@ public class NexusDownloader : ADownloader<Nexus>, IUrlDownloader
                 var urls = await _api.DownloadLink(state.Game.MetaData().NexusName!, state.ModID, state.FileID, token);
                 _logger.LogInformation("Downloading Nexus File: {game}|{modid}|{fileid}", state.Game, state.ModID,
                     state.FileID);
-                var message = new HttpRequestMessage(HttpMethod.Get, urls.info.First().URI);
+                foreach (var link in urls.info)
-                return await _downloader.Download(message, destination, job, token);
+                {
+                    try
+                    {
+                        var message = new HttpRequestMessage(HttpMethod.Get, link.URI);
+                        return await _downloader.Download(message, destination, job, token);
+                    }
+                    catch (Exception ex)
+                    {
+                        if (link.URI == urls.info.Last().URI)
+                            throw;
+                        _logger.LogInformation(ex, "While downloading {URI}, trying another link", link.URI);
+                    }
+                }
+
+                // Should never be hit
+                throw new NotImplementedException();
             }
             catch (HttpRequestException ex)
             {

Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs

@@ -1,6 +1,8 @@
 using System.Threading;
 using System.Threading.Tasks;
+using Wabbajack.Common;
 using Wabbajack.DTOs.Streams;
+using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Paths;
 
 namespace Wabbajack.FileExtractor.ExtractedFiles;
@@ -16,4 +18,23 @@ public interface IExtractedFile : IStreamFactory
     /// <param name="newPath">destination to move the entry to</param>
     /// <returns></returns>
     public ValueTask Move(AbsolutePath newPath, CancellationToken token);
+
+
+}
+
+public static class IExtractedFileExtensions
+{
+    public static async Task<Hash> MoveHashedAsync(this IExtractedFile file, AbsolutePath destPath, CancellationToken token)
+    {
+        if (file.CanMove)
+        {
+            await file.Move(destPath, token);
+            return await destPath.Hash(token);
+        }
+        else
+        {
+            await using var s = await file.GetStream();
+            return await destPath.WriteAllHashedAsync(s, token, false);        
+        }
+    }
 }

Wabbajack.Installer/AInstaller.cs

@@ -12,9 +12,11 @@ using Wabbajack.Common;
 using Wabbajack.Downloaders;
 using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs;
+using Wabbajack.DTOs.BSA.FileStates;
 using Wabbajack.DTOs.Directives;
 using Wabbajack.DTOs.DownloadStates;
 using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.FileExtractor.ExtractedFiles;
 using Wabbajack.Hashing.PHash;
 using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Installer.Utilities;
@@ -39,7 +41,7 @@ public abstract class AInstaller<T>
     where T : AInstaller<T>
 {
     private const int _limitMS = 100;
-    public static RelativePath BSACreationDir = "TEMP_BSA_FILES".ToRelativePath();
+
     private static readonly Regex NoDeleteRegex = new(@"(?i)[\\\/]\[NoDelete\]", RegexOptions.Compiled);
 
     protected readonly InstallerConfiguration _configuration;
@@ -243,7 +245,8 @@ public abstract class AInstaller<T>
                         await using var patchDataStream = await InlinedFileStream(pfa.PatchID);
                         {
                             await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None);
-                            await BinaryPatching.ApplyPatch(s, patchDataStream, os);
+                            var hash = await BinaryPatching.ApplyPatch(s, patchDataStream, os);
+                            ThrowOnNonMatchingHash(file, hash);
                         }
                     }
                         break;
@@ -263,12 +266,14 @@ public abstract class AInstaller<T>
                     case FromArchive _:
                         if (grouped[vf].Count() == 1)
                         {
-                            await sf.Move(destPath, token);
+                            var hash = await sf.MoveHashedAsync(destPath, token);
+                            ThrowOnNonMatchingHash(file, hash);
                         }
                         else
                         {
                             await using var s = await sf.GetStream();
-                            await destPath.WriteAllAsync(s, token, false);
+                            var hash = await destPath.WriteAllHashedAsync(s, token, false);
+                            ThrowOnNonMatchingHash(file, hash);
                         }
 
                         break;
@@ -282,6 +287,25 @@ public abstract class AInstaller<T>
         }, token);
     }
 
+    protected void ThrowOnNonMatchingHash(Directive file, Hash gotHash)
+    {
+        if (file.Hash != gotHash)
+            ThrowNonMatchingError(file, gotHash);
+    }
+    private void ThrowNonMatchingError(Directive file, Hash gotHash)
+    {
+        _logger.LogError("Hashes for {Path} did not match, expected {Expected} got {Got}", file.To, file.Hash, gotHash);
+        throw new Exception($"Hashes for {file.To} did not match, expected {file.Hash} got {gotHash}");
+    }
+    
+    
+    protected void ThrowOnNonMatchingHash(CreateBSA bsa, Directive directive, AFile state, Hash hash)
+    {
+        if (hash == directive.Hash) return;
+        _logger.LogError("Hashes for BSA don't match after extraction, {BSA}, {Directive}, {ExpectedHash}, {Hash}", bsa.To, directive.To, directive.Hash, hash);
+        throw new Exception($"Hashes for {bsa.To} file {directive.To} did not match, expected {directive.Hash} got {hash}");
+    }
+
     public async Task DownloadArchives(CancellationToken token)
     {
         var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
@@ -474,8 +498,8 @@ public abstract class AInstaller<T>
                 return d switch
                 {
                     CreateBSA bsa => !bsasToNotBuild.Contains(bsa.TempID),
-                    FromArchive a when a.To.StartsWith($"{BSACreationDir}") => !bsasToNotBuild.Any(b =>
+                    FromArchive a when a.To.StartsWith($"{Consts.BSACreationDir}") => !bsasToNotBuild.Any(b =>
-                        a.To.RelativeTo(_configuration.Install).InFolder(_configuration.Install.Combine(BSACreationDir, b))),
+                        a.To.RelativeTo(_configuration.Install).InFolder(_configuration.Install.Combine(Consts.BSACreationDir, b))),
                     _ => true
                 };
             }).ToDictionary(d => d.To);

Wabbajack.Installer/Consts.cs

@@ -1,3 +1,5 @@
+using System.Collections.Generic;
+using System.Linq;
 using Wabbajack.Paths;
 
 namespace Wabbajack.Installer;
@@ -26,4 +28,11 @@ public static class Consts
     public const string StepDownloading = "Downloading";
     public const string StepHashing = "Hashing";
     public const string StepFinished = "Finished";
+    public static RelativePath BSACreationDir = "TEMP_BSA_FILES".ToRelativePath();
+
+    public static HashSet<RelativePath> KnownModifiedFiles = new[]
+    {
+        "modlist.txt",
+        "SkyrimPrefs.ini"
+    }.Select(r => r.ToRelativePath()).ToHashSet();
 }

Wabbajack.Installer/StandardInstaller.cs

@@ -19,9 +19,11 @@ using Wabbajack.Compression.Zip;
 using Wabbajack.Downloaders;
 using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs;
+using Wabbajack.DTOs.BSA.FileStates;
 using Wabbajack.DTOs.Directives;
 using Wabbajack.DTOs.DownloadStates;
 using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Installer.Utilities;
 using Wabbajack.Networking.WabbajackClientApi;
 using Wabbajack.Paths;
@@ -271,6 +273,8 @@ public class StandardInstaller : AInstaller<StandardInstaller>
     private async Task BuildBSAs(CancellationToken token)
     {
         var bsas = ModList.Directives.OfType<CreateBSA>().ToList();
+        _logger.LogInformation("Generating debug caches");
+        var indexedByDestination = UnoptimizedDirectives.ToDictionary(d => d.To);
         _logger.LogInformation("Building {bsasCount} bsa files", bsas.Count);
         NextStep("Installing", "Building BSAs", bsas.Count);
 
@@ -278,34 +282,50 @@ public class StandardInstaller : AInstaller<StandardInstaller>
         {
             UpdateProgress(1);
             _logger.LogInformation("Building {bsaTo}", bsa.To.FileName);
-            var sourceDir = _configuration.Install.Combine(BSACreationDir, bsa.TempID);
+            var sourceDir = _configuration.Install.Combine(Consts.BSACreationDir, bsa.TempID);
 
             await using var a = BSADispatch.CreateBuilder(bsa.State, _manager);
-            var streams = await bsa.FileStates.PMapAll(async state =>
+            var streams = await bsa.FileStates.PMapAllBatchedAsync(_limiter, async state =>
             {
-                using var job = await _limiter.Begin($"Adding {state.Path.FileName}", 0, token);
                 var fs = sourceDir.Combine(state.Path).Open(FileMode.Open, FileAccess.Read, FileShare.Read);
-                var size = fs.Length;
-                job.Size = size;
                 await a.AddFile(state, fs, token);
-                await job.Report((int)size, token);
                 return fs;
             }).ToList();
 
             _logger.LogInformation("Writing {bsaTo}", bsa.To);
             var outPath = _configuration.Install.Combine(bsa.To);
-            await using var outStream = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None);
+            
-            await a.Build(outStream, token);
+            await using (var outStream = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None))
+            {
+                await a.Build(outStream, token);
+            }
+            
             streams.Do(s => s.Dispose());
             
             await FileHashCache.FileHashWriteCache(outPath, bsa.Hash);
             sourceDir.DeleteDirectory();
+            
+            _logger.LogInformation("Verifying {bsaTo}", bsa.To);
+            var reader = await BSADispatch.Open(outPath);
+            var results = await reader.Files.PMapAllBatchedAsync(_limiter, async state =>
+            {
+                var sf = await state.GetStreamFactory(token);
+                await using var stream = await sf.GetStream();
+                var hash = await stream.Hash(token);
+
+                var astate = bsa.FileStates.First(f => f.Path == state.Path);
+                var srcDirective = indexedByDestination[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)];
+                //DX10Files are lossy
+                if (astate is not BA2DX10File) 
+                    ThrowOnNonMatchingHash(bsa, srcDirective, astate, hash);
+                return (srcDirective, hash);
+            }).ToHashSet();
         }
 
-        var bsaDir = _configuration.Install.Combine(BSACreationDir);
+        var bsaDir = _configuration.Install.Combine(Consts.BSACreationDir);
         if (bsaDir.DirectoryExists())
         {
-            _logger.LogInformation("Removing temp folder {bsaCreationDir}", BSACreationDir);
+            _logger.LogInformation("Removing temp folder {bsaCreationDir}", Consts.BSACreationDir);
             bsaDir.DeleteDirectory();
         }
     }
@@ -329,7 +349,10 @@ public class StandardInstaller : AInstaller<StandardInstaller>
                         await FileHashCache.FileHashCachedAsync(outPath, token);
                         break;
                     default:
-                        await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID), token);
+                        var hash = await outPath.WriteAllHashedAsync(await LoadBytesFromPath(directive.SourceDataID), token);
+                        if (!Consts.KnownModifiedFiles.Contains(directive.To.FileName))
+                            ThrowOnNonMatchingHash(directive, hash);
+
                         await FileHashCache.FileHashWriteCache(outPath, directive.Hash);
                         break;
                 }
@@ -481,7 +504,8 @@ public class StandardInstaller : AInstaller<StandardInstaller>
                 .Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None);
             try
             {
-                await BinaryPatching.ApplyPatch(new MemoryStream(srcData), new MemoryStream(patchData), fs);
+                var hash = await BinaryPatching.ApplyPatch(new MemoryStream(srcData), new MemoryStream(patchData), fs);
+                ThrowOnNonMatchingHash(m, hash);
             }
             catch (Exception ex)
             {

Wabbajack.Installer/Utilities/BinaryPatching.cs

@@ -1,16 +1,19 @@
 using System.IO;
+using System.Threading;
 using System.Threading.Tasks;
 using Octodiff.Core;
 using Octodiff.Diagnostics;
+using Wabbajack.Hashing.xxHash64;
 
 namespace Wabbajack.Installer.Utilities;
 
 public class BinaryPatching
 {
-    public static ValueTask ApplyPatch(Stream input, Stream deltaStream, Stream output)
+    public static async ValueTask<Hash> ApplyPatch(Stream input, Stream deltaStream, Stream output, CancellationToken? token = null)
     {
         var deltaApplier = new DeltaApplier();
         deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, new NullProgressReporter()), output);
-        return ValueTask.CompletedTask;
+        output.Position = 0;
+        return await output.Hash(token ?? CancellationToken.None);
     }
 }

Wabbajack.Installer/Wabbajack.Installer.csproj

@@ -24,7 +24,7 @@
 
     <ItemGroup>
         <PackageReference Include="ini-parser-netstandard" Version="2.5.2" />
-        <PackageReference Include="Octodiff" Version="1.3.23" />
+        <PackageReference Include="Octopus.Octodiff" Version="1.3.93" />
     </ItemGroup>
 
 </Project>