Merge branch 'main' into gamefinder-fix

.github/workflows/tests.yaml

@@ -39,12 +39,7 @@ jobs:
           include-prerelease: true
 
       - name: Test
-        run: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:EnableWindowsTargeting=true --filter "Category!=FlakeyNetwork"
-        
-      - uses: codecov/codecov-action@v3
-        with:
-          flags: unittests, ${{runner.os}}
-          verbose: true 
+        run: dotnet test /p:EnableWindowsTargeting=true --filter "Category!=FlakeyNetwork"
         
       #- name: Upload Test Folder on Failure
       #  if: ${{ failure() }}

CHANGELOG.md

@@ -1,8 +1,18 @@
 ### Changelog
 
-#### Version - 3.0.2.4 - TBD
+#### Version - 3.0.4.0 - TBD
 * upgrade GameFinder to 2.2.1
 
+#### Version - 3.0.3.1 - 10/30/2022
+* Fix file verification issues for CreatedBSAs
+* Fix files during verification where CreatedDate > LastModified
+
+#### Version - 3.0.3.0 - 10/26/2022
+* Verify hashes of all installed files
+* Verify contents of BSAs during installation
+* Provide a new CLI command for verifying a installed modlist
+* When downloading from one Nexus CDN server fails, WJ will now try alternate Nexus servers
+
 #### Version - 3.0.2.3 - 10/19/2022
 * HOTFIX: revert GameFinder library to 1.8 until it's a bit more forgiving of corrupt files
 

Wabbajack.App.Wpf/View Models/MainWindowVM.cs

@@ -66,10 +66,10 @@ namespace Wabbajack
         public ICommand OpenSettingsCommand { get; }
 
         public string VersionDisplay { get; }
-        
+
         [Reactive]
         public string ResourceStatus { get; set; }
-        
+
         [Reactive]
         public string AppName { get; set; }
 
@@ -98,7 +98,7 @@ namespace Wabbajack
             MessageBus.Current.Listen<NavigateToGlobal>()
                 .Subscribe(m => HandleNavigateTo(m.Screen))
                 .DisposeWith(CompositeDisposable);
-            
+
             MessageBus.Current.Listen<NavigateTo>()
                 .Subscribe(m => HandleNavigateTo(m.ViewModel))
                 .DisposeWith(CompositeDisposable);
@@ -106,7 +106,7 @@ namespace Wabbajack
             MessageBus.Current.Listen<NavigateBack>()
                 .Subscribe(HandleNavigateBack)
                 .DisposeWith(CompositeDisposable);
-            
+
             MessageBus.Current.Listen<SpawnBrowserWindow>()
                 .ObserveOnGuiThread()
                 .Subscribe(HandleSpawnBrowserWindow)
@@ -116,7 +116,7 @@ namespace Wabbajack
                 .Select(r => string.Join(", ", r.Where(r => r.Throughput > 0)
                     .Select(s => $"{s.Name} - {s.Throughput.ToFileSizeString()}/sec")))
                 .BindToStrict(this, view => view.ResourceStatus);
-            
+
 
             if (IsStartingFromModlist(out var path))
             {
@@ -134,25 +134,24 @@ namespace Wabbajack
                 var assembly = Assembly.GetExecutingAssembly();
                 var location = assembly.Location;
                 if (string.IsNullOrWhiteSpace(location))
-                    location = Process.GetCurrentProcess().MainModule?.FileName ?? "";
-                
+                    location = Process.GetCurrentProcess().MainModule?.FileName ?? throw new Exception("Assembly location is unavailable!");
+
                 _logger.LogInformation("App Location: {Location}", assembly.Location);
                 var fvi = FileVersionInfo.GetVersionInfo(location);
                 Consts.CurrentMinimumWabbajackVersion = Version.Parse(fvi.FileVersion);
                 VersionDisplay = $"v{fvi.FileVersion}";
                 AppName = "WABBAJACK " + VersionDisplay;
                 _logger.LogInformation("Wabbajack Version: {FileVersion}", fvi.FileVersion);
-                
+
                 Task.Run(() => _wjClient.SendMetric("started_wabbajack", fvi.FileVersion)).FireAndForget();
                 Task.Run(() => _wjClient.SendMetric("started_sha", ThisAssembly.Git.Sha));
-                
+
                 // setup file association
                 try
                 {
-                    var applicationRegistrationService =
-                        _serviceProvider.GetRequiredService<IApplicationRegistrationService>();
+                    var applicationRegistrationService = _serviceProvider.GetRequiredService<IApplicationRegistrationService>();
 
-                    var applicationInfo = new ApplicationInfo(assembly);
+                    var applicationInfo = new ApplicationInfo("Wabbajack", "Wabbajack", "Wabbajack", location);
                     applicationInfo.SupportedExtensions.Add("wabbajack");
                     applicationRegistrationService.RegisterApplication(applicationInfo);
                 }
@@ -181,27 +180,27 @@ namespace Wabbajack
 
             ActivePane = objViewModel;
         }
-        
+
         private void HandleNavigateBack(NavigateBack navigateBack)
         {
             ActivePane = PreviousPanes.Last();
             PreviousPanes.RemoveAt(PreviousPanes.Count - 1);
         }
-        
+
         private void HandleManualDownload(ManualDownload manualDownload)
         {
             var handler = _serviceProvider.GetRequiredService<ManualDownloadHandler>();
             handler.Intervention = manualDownload;
             //MessageBus.Current.SendMessage(new OpenBrowserTab(handler));
         }
-        
+
         private void HandleManualBlobDownload(ManualBlobDownload manualDownload)
         {
             var handler = _serviceProvider.GetRequiredService<ManualBlobDownloadHandler>();
             handler.Intervention = manualDownload;
             //MessageBus.Current.SendMessage(new OpenBrowserTab(handler));
         }
-        
+
         private void HandleSpawnBrowserWindow(SpawnBrowserWindow msg)
         {
             var window = _serviceProvider.GetRequiredService<BrowserWindow>();
@@ -213,7 +212,7 @@ namespace Wabbajack
         {
             if (s is NavigateToGlobal.ScreenType.Settings)
                 PreviousPanes.Add(ActivePane);
-            
+
             ActivePane = s switch
             {
                 NavigateToGlobal.ScreenType.ModeSelectionView => ModeSelectionVM,

Wabbajack.CLI/VerbRegistration.cs

@@ -51,6 +51,8 @@ CommandLineBuilder.RegisterCommand<UploadToNexus>(UploadToNexus.Definition, c =>
 services.AddSingleton<UploadToNexus>();
 CommandLineBuilder.RegisterCommand<ValidateLists>(ValidateLists.Definition, c => ((ValidateLists)c).Run);
 services.AddSingleton<ValidateLists>();
+CommandLineBuilder.RegisterCommand<VerifyModlistInstall>(VerifyModlistInstall.Definition, c => ((VerifyModlistInstall)c).Run);
+services.AddSingleton<VerifyModlistInstall>();
 CommandLineBuilder.RegisterCommand<VFSIndex>(VFSIndex.Definition, c => ((VFSIndex)c).Run);
 services.AddSingleton<VFSIndex>();
 }

Wabbajack.CLI/Verbs/ModlistReport.cs

@@ -70,7 +70,7 @@ public class ModlistReport
 
         string FixupTo(RelativePath path)
         {
-            if (path.GetPart(0) != StandardInstaller.BSACreationDir.ToString()) return path.ToString();
+            if (path.GetPart(0) != Consts.BSACreationDir.ToString()) return path.ToString();
             var bsaId = path.GetPart(1);
             
             if (!bsas.TryGetValue(bsaId, out var bsa))

Wabbajack.CLI/Verbs/VerifyModlistInstall.cs

@@ -0,0 +1,173 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.CLI.Builder;
+using Wabbajack.Common;
+using Wabbajack.Compression.BSA;
+using Wabbajack.DTOs;
+using Wabbajack.DTOs.BSA.FileStates;
+using Wabbajack.DTOs.Directives;
+using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Hashing.xxHash64;
+using Wabbajack.Installer;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+using Wabbajack.RateLimiter;
+using Wabbajack.VFS;
+using AbsolutePathExtensions = Wabbajack.Common.AbsolutePathExtensions;
+
+namespace Wabbajack.CLI.Verbs;
+
+public class VerifyModlistInstall
+{
+    private readonly DTOSerializer _dtos;
+    private readonly ILogger<VerifyModlistInstall> _logger;
+    
+    public VerifyModlistInstall(ILogger<VerifyModlistInstall> logger, DTOSerializer dtos, IResource<FileHashCache> limiter)
+    {
+        _limiter = limiter;
+        _logger = logger;
+        _dtos = dtos;
+    }
+    
+    public static VerbDefinition Definition = new("verify-modlist-install", "Verify a modlist installed correctly",
+        new[]
+        {
+            new OptionDefinition(typeof(AbsolutePath), "m", "modlistLocation",
+                "The .wabbajack file used to install the modlist"),
+            new OptionDefinition(typeof(AbsolutePath), "i", "installFolder", "The installation folder of the modlist")
+        });
+
+    private readonly IResource<FileHashCache> _limiter;
+
+
+    public async Task<int> Run(AbsolutePath modlistLocation, AbsolutePath installFolder, CancellationToken token)
+    {
+        _logger.LogInformation("Loading modlist {ModList}", modlistLocation);
+        var list = await StandardInstaller.LoadFromFile(_dtos, modlistLocation);
+        
+        _logger.LogInformation("Indexing files");
+        var byTo = list.Directives.ToDictionary(d => d.To);
+
+
+        _logger.LogInformation("Scanning files");
+        var errors = await list.Directives.PMapAllBatchedAsync(_limiter, async directive => 
+            {
+                if (!(directive is CreateBSA || directive.IsDeterministic))
+                    return null;
+                
+                if (directive.To.InFolder(Consts.BSACreationDir))
+                    return null;
+
+                var dest = directive.To.RelativeTo(installFolder);
+                if (!dest.FileExists())
+                {
+                    return new Result
+                    {
+                        Path = directive.To,
+                        Message = $"File does not exist directive {directive.GetType()}"
+                    };
+                }
+
+                if (Consts.KnownModifiedFiles.Contains(directive.To.FileName))
+                    return null;
+
+                if (directive is CreateBSA bsa)
+                {
+                    return await VerifyBSA(dest, bsa, byTo, token);
+                }
+
+                if (dest.Size() != directive.Size)
+                {
+                    return new Result
+                    {
+                        Path = directive.To,
+                        Message = $"Sizes do not match got {dest.Size()} expected {directive.Size}"
+                    };
+                }
+
+                if (directive.Size > (1024 * 1024 * 128))
+                {
+                    _logger.LogInformation("Hashing {Size} file at {Path}", directive.Size.ToFileSizeString(),
+                        directive.To);
+                }
+
+                var hash = await AbsolutePathExtensions.Hash(dest, token);
+                if (hash != directive.Hash)
+                {
+                    return new Result
+                    {
+                        Path = directive.To,
+                        Message = $"Hashes do not match, got {hash} expected {directive.Hash}"
+                    };
+                }
+
+                return null;
+        }).Where(r => r != null)
+            .ToList();
+        
+        _logger.LogInformation("Found {Count} errors", errors.Count);
+
+
+        foreach (var error in errors)
+        {
+            _logger.LogError("{File} | {Message}", error.Path, error.Message);
+        }
+
+
+        return 0;
+    }
+
+    private async Task<Result?> VerifyBSA(AbsolutePath dest, CreateBSA bsa, Dictionary<RelativePath, Directive> byTo, CancellationToken token)
+    {
+        _logger.LogInformation("Verifying Created BSA {To}", bsa.To);
+        var archive = await BSADispatch.Open(dest);
+        var filesIndexed = archive.Files.ToDictionary(d => d.Path);
+        
+        if (dest.Extension == Ext.Bsa && dest.Size() >= 1024L * 1024 * 1024 * 2)
+        {
+            return new Result()
+            {
+                Path = bsa.To,
+                Message = $"BSA is over 2GB in size, this will cause crashes : {bsa.To}"
+            };
+        }
+
+        foreach (var file in bsa.FileStates)
+        {
+            if (file is BA2DX10File) continue;
+            var state = filesIndexed[file.Path];
+            var sf = await state.GetStreamFactory(token);
+            await using var stream = await sf.GetStream();
+            var hash = await stream.Hash(token);
+
+            var astate = bsa.FileStates.First(f => f.Path == state.Path);
+            var srcDirective = byTo[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)];
+
+            if (!srcDirective.IsDeterministic)
+                continue;
+
+            if (srcDirective.Hash != hash)
+            {
+                return new Result
+                {
+                    Path = bsa.To,
+                    Message =
+                        $"BSA {bsa.To} contents do not match at {file.Path} got {hash} expected {srcDirective.Hash}"
+                };
+            }
+        }
+
+        
+        return null;
+    }
+
+    public class Result
+    {
+        public RelativePath Path { get; set; }
+        public string Message { get; set; }
+    }
+}

Wabbajack.Common/AbsolutePathExtensions.cs

@@ -21,4 +21,25 @@ public static class AbsolutePathExtensions
         await using var fs = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
         return await fs.FromJson<T>(dtos);
     }
+    
+    public static async Task<Hash> WriteAllHashedAsync(this AbsolutePath file, Stream srcStream, CancellationToken token,
+        bool closeWhenDone = true)
+    {
+        try
+        {
+            await using var dest = file.Open(FileMode.Create, FileAccess.Write, FileShare.None);
+            return await srcStream.HashingCopy(dest, token);
+        }
+        finally
+        {
+            if (closeWhenDone)
+                srcStream.Close();
+        }
+    }
+    
+    public static async Task<Hash> WriteAllHashedAsync(this AbsolutePath file, byte[] data, CancellationToken token)
+    {
+        await using var dest = file.Open(FileMode.Create, FileAccess.Write, FileShare.None);
+        return await new MemoryStream(data).HashingCopy(dest, token);
+    }
 }

Wabbajack.DTOs/ModList/Directive.cs

@@ -1,3 +1,4 @@
+using System.Text.Json.Serialization;
 using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Paths;
 
@@ -12,4 +13,6 @@ public abstract class Directive
     ///     location the file will be copied to, relative to the install path.
     /// </summary>
     public RelativePath To { get; set; }
+
+    [JsonIgnore] public virtual bool IsDeterministic => true;
 }

Wabbajack.DTOs/ModList/Directives/ArchiveMeta.cs

@@ -8,4 +8,6 @@ namespace Wabbajack.DTOs.Directives;
 public class ArchiveMeta : Directive
 {
     public RelativePath SourceDataID { get; set; }
+
+    public override bool IsDeterministic => false;
 }

Wabbajack.DTOs/ModList/Directives/CreateBSA.cs

@@ -13,4 +13,6 @@ public class CreateBSA : Directive
     public RelativePath TempID { get; set; }
     public IArchive State { get; set; }
     public AFile[] FileStates { get; set; } = Array.Empty<AFile>();
+
+    public override bool IsDeterministic => false;
 }

Wabbajack.DTOs/ModList/Directives/RemappedInlineFile.cs

@@ -9,4 +9,5 @@ namespace Wabbajack.DTOs.Directives;
 [JsonAlias("RemappedInlineFile, Wabbajack.Lib")]
 public class RemappedInlineFile : InlineFile
 {
+    public override bool IsDeterministic => false;
 }

Wabbajack.DTOs/ModList/Directives/TransformedTexture.cs

@@ -11,4 +11,5 @@ public class TransformedTexture : FromArchive
     ///     The file to apply to the source file to patch it
     /// </summary>
     public ImageState ImageState { get; set; } = new();
+    public override bool IsDeterministic => false;
 }

Wabbajack.Downloaders.Dispatcher.Test/VerificationCacheTests.cs

@@ -9,7 +9,6 @@ namespace Wabbajack.Downloaders.Dispatcher.Test;
 
 public class VerificationCacheTests
 {
-    private readonly TemporaryFileManager _temp;
     private readonly ILogger<VerificationCache.VerificationCache> _logger;
 
     public VerificationCacheTests(ILogger<VerificationCache.VerificationCache> logger)

Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs

@@ -40,7 +40,8 @@ public class MediaFireDownloader : ADownloader<DTOs.DownloadStates.MediaFire>, I
 
     public override bool IsAllowed(ServerAllowList allowList, IDownloadState state)
     {
-        return true;
+        var mediaFireState = (DTOs.DownloadStates.MediaFire) state;
+        return allowList.AllowedPrefixes.Any(p => mediaFireState.Url.ToString().StartsWith(p, StringComparison.OrdinalIgnoreCase));
     }
 
     public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
@@ -105,26 +106,26 @@ public class MediaFireDownloader : ADownloader<DTOs.DownloadStates.MediaFire>, I
         if (!result.IsSuccessStatusCode)
             return null;
 
-        if (job != null) 
+        if (job != null)
             job.Size = result.Content.Headers.ContentLength ?? 0;
 
         if (result.Content.Headers.ContentType!.MediaType!.StartsWith("text/html",
             StringComparison.OrdinalIgnoreCase))
         {
             var bodyData = await result.Content.ReadAsStringAsync((CancellationToken) token);
-            if (job != null) 
+            if (job != null)
                 await job.Report((int) (job.Size ?? 0), (CancellationToken) token);
             var body = new HtmlDocument();
             body.LoadHtml(bodyData);
             var node = body.DocumentNode.DescendantsAndSelf().FirstOrDefault(d => d.HasClass("input") && d.HasClass("popsok") &&
                                                                          d.GetAttributeValue("aria-label", "") ==
                                                                          "Download file");
-            if (node != null) 
+            if (node != null)
                 return new Uri(node.GetAttributeValue("href", "not-found"));
-            
+
             var startText = "window.location.href = '";
             var start = body.DocumentNode.InnerHtml.IndexOf(startText, StringComparison.CurrentCultureIgnoreCase);
-            
+
             if (start != -1)
             {
                 var end = body.DocumentNode.InnerHtml.IndexOf("\'", start + startText.Length,
@@ -141,4 +142,4 @@ public class MediaFireDownloader : ADownloader<DTOs.DownloadStates.MediaFire>, I
     {
         return new[] {$"directURL={state.Url}"};
     }
-}
+}

Wabbajack.Downloaders.Nexus/NexusDownloader.cs

@@ -129,8 +129,23 @@ public class NexusDownloader : ADownloader<Nexus>, IUrlDownloader
                 var urls = await _api.DownloadLink(state.Game.MetaData().NexusName!, state.ModID, state.FileID, token);
                 _logger.LogInformation("Downloading Nexus File: {game}|{modid}|{fileid}", state.Game, state.ModID,
                     state.FileID);
-                var message = new HttpRequestMessage(HttpMethod.Get, urls.info.First().URI);
-                return await _downloader.Download(message, destination, job, token);
+                foreach (var link in urls.info)
+                {
+                    try
+                    {
+                        var message = new HttpRequestMessage(HttpMethod.Get, link.URI);
+                        return await _downloader.Download(message, destination, job, token);
+                    }
+                    catch (Exception ex)
+                    {
+                        if (link.URI == urls.info.Last().URI)
+                            throw;
+                        _logger.LogInformation(ex, "While downloading {URI}, trying another link", link.URI);
+                    }
+                }
+
+                // Should never be hit
+                throw new NotImplementedException();
             }
             catch (HttpRequestException ex)
             {

Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs

@@ -1,6 +1,8 @@
 using System.Threading;
 using System.Threading.Tasks;
+using Wabbajack.Common;
 using Wabbajack.DTOs.Streams;
+using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Paths;
 
 namespace Wabbajack.FileExtractor.ExtractedFiles;
@@ -16,4 +18,23 @@ public interface IExtractedFile : IStreamFactory
     /// <param name="newPath">destination to move the entry to</param>
     /// <returns></returns>
     public ValueTask Move(AbsolutePath newPath, CancellationToken token);
+
+
+}
+
+public static class IExtractedFileExtensions
+{
+    public static async Task<Hash> MoveHashedAsync(this IExtractedFile file, AbsolutePath destPath, CancellationToken token)
+    {
+        if (file.CanMove)
+        {
+            await file.Move(destPath, token);
+            return await destPath.Hash(token);
+        }
+        else
+        {
+            await using var s = await file.GetStream();
+            return await destPath.WriteAllHashedAsync(s, token, false);        
+        }
+    }
 }

Wabbajack.Installer/AInstaller.cs

@@ -12,9 +12,11 @@ using Wabbajack.Common;
 using Wabbajack.Downloaders;
 using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs;
+using Wabbajack.DTOs.BSA.FileStates;
 using Wabbajack.DTOs.Directives;
 using Wabbajack.DTOs.DownloadStates;
 using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.FileExtractor.ExtractedFiles;
 using Wabbajack.Hashing.PHash;
 using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Installer.Utilities;
@@ -39,7 +41,7 @@ public abstract class AInstaller<T>
     where T : AInstaller<T>
 {
     private const int _limitMS = 100;
-    public static RelativePath BSACreationDir = "TEMP_BSA_FILES".ToRelativePath();
+
     private static readonly Regex NoDeleteRegex = new(@"(?i)[\\\/]\[NoDelete\]", RegexOptions.Compiled);
 
     protected readonly InstallerConfiguration _configuration;
@@ -243,7 +245,8 @@ public abstract class AInstaller<T>
                         await using var patchDataStream = await InlinedFileStream(pfa.PatchID);
                         {
                             await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None);
-                            await BinaryPatching.ApplyPatch(s, patchDataStream, os);
+                            var hash = await BinaryPatching.ApplyPatch(s, patchDataStream, os);
+                            ThrowOnNonMatchingHash(file, hash);
                         }
                     }
                         break;
@@ -263,12 +266,14 @@ public abstract class AInstaller<T>
                     case FromArchive _:
                         if (grouped[vf].Count() == 1)
                         {
-                            await sf.Move(destPath, token);
+                            var hash = await sf.MoveHashedAsync(destPath, token);
+                            ThrowOnNonMatchingHash(file, hash);
                         }
                         else
                         {
                             await using var s = await sf.GetStream();
-                            await destPath.WriteAllAsync(s, token, false);
+                            var hash = await destPath.WriteAllHashedAsync(s, token, false);
+                            ThrowOnNonMatchingHash(file, hash);
                         }
 
                         break;
@@ -282,6 +287,25 @@ public abstract class AInstaller<T>
         }, token);
     }
 
+    protected void ThrowOnNonMatchingHash(Directive file, Hash gotHash)
+    {
+        if (file.Hash != gotHash)
+            ThrowNonMatchingError(file, gotHash);
+    }
+    private void ThrowNonMatchingError(Directive file, Hash gotHash)
+    {
+        _logger.LogError("Hashes for {Path} did not match, expected {Expected} got {Got}", file.To, file.Hash, gotHash);
+        throw new Exception($"Hashes for {file.To} did not match, expected {file.Hash} got {gotHash}");
+    }
+    
+    
+    protected void ThrowOnNonMatchingHash(CreateBSA bsa, Directive directive, AFile state, Hash hash)
+    {
+        if (hash == directive.Hash) return;
+        _logger.LogError("Hashes for BSA don't match after extraction, {BSA}, {Directive}, {ExpectedHash}, {Hash}", bsa.To, directive.To, directive.Hash, hash);
+        throw new Exception($"Hashes for {bsa.To} file {directive.To} did not match, expected {directive.Hash} got {hash}");
+    }
+
     public async Task DownloadArchives(CancellationToken token)
     {
         var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
@@ -474,8 +498,8 @@ public abstract class AInstaller<T>
                 return d switch
                 {
                     CreateBSA bsa => !bsasToNotBuild.Contains(bsa.TempID),
-                    FromArchive a when a.To.StartsWith($"{BSACreationDir}") => !bsasToNotBuild.Any(b =>
-                        a.To.RelativeTo(_configuration.Install).InFolder(_configuration.Install.Combine(BSACreationDir, b))),
+                    FromArchive a when a.To.StartsWith($"{Consts.BSACreationDir}") => !bsasToNotBuild.Any(b =>
+                        a.To.RelativeTo(_configuration.Install).InFolder(_configuration.Install.Combine(Consts.BSACreationDir, b))),
                     _ => true
                 };
             }).ToDictionary(d => d.To);

Wabbajack.Installer/Consts.cs

@@ -1,3 +1,5 @@
+using System.Collections.Generic;
+using System.Linq;
 using Wabbajack.Paths;
 
 namespace Wabbajack.Installer;
@@ -26,4 +28,11 @@ public static class Consts
     public const string StepDownloading = "Downloading";
     public const string StepHashing = "Hashing";
     public const string StepFinished = "Finished";
+    public static RelativePath BSACreationDir = "TEMP_BSA_FILES".ToRelativePath();
+
+    public static HashSet<RelativePath> KnownModifiedFiles = new[]
+    {
+        "modlist.txt",
+        "SkyrimPrefs.ini"
+    }.Select(r => r.ToRelativePath()).ToHashSet();
 }

Wabbajack.Installer/StandardInstaller.cs

@@ -19,9 +19,11 @@ using Wabbajack.Compression.Zip;
 using Wabbajack.Downloaders;
 using Wabbajack.Downloaders.GameFile;
 using Wabbajack.DTOs;
+using Wabbajack.DTOs.BSA.FileStates;
 using Wabbajack.DTOs.Directives;
 using Wabbajack.DTOs.DownloadStates;
 using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Installer.Utilities;
 using Wabbajack.Networking.WabbajackClientApi;
 using Wabbajack.Paths;
@@ -271,6 +273,8 @@ public class StandardInstaller : AInstaller<StandardInstaller>
     private async Task BuildBSAs(CancellationToken token)
     {
         var bsas = ModList.Directives.OfType<CreateBSA>().ToList();
+        _logger.LogInformation("Generating debug caches");
+        var indexedByDestination = UnoptimizedDirectives.ToDictionary(d => d.To);
         _logger.LogInformation("Building {bsasCount} bsa files", bsas.Count);
         NextStep("Installing", "Building BSAs", bsas.Count);
 
@@ -278,34 +282,50 @@ public class StandardInstaller : AInstaller<StandardInstaller>
         {
             UpdateProgress(1);
             _logger.LogInformation("Building {bsaTo}", bsa.To.FileName);
-            var sourceDir = _configuration.Install.Combine(BSACreationDir, bsa.TempID);
+            var sourceDir = _configuration.Install.Combine(Consts.BSACreationDir, bsa.TempID);
 
             await using var a = BSADispatch.CreateBuilder(bsa.State, _manager);
-            var streams = await bsa.FileStates.PMapAll(async state =>
+            var streams = await bsa.FileStates.PMapAllBatchedAsync(_limiter, async state =>
             {
-                using var job = await _limiter.Begin($"Adding {state.Path.FileName}", 0, token);
                 var fs = sourceDir.Combine(state.Path).Open(FileMode.Open, FileAccess.Read, FileShare.Read);
-                var size = fs.Length;
-                job.Size = size;
                 await a.AddFile(state, fs, token);
-                await job.Report((int)size, token);
                 return fs;
             }).ToList();
 
             _logger.LogInformation("Writing {bsaTo}", bsa.To);
             var outPath = _configuration.Install.Combine(bsa.To);
-            await using var outStream = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None);
-            await a.Build(outStream, token);
+            
+            await using (var outStream = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None))
+            {
+                await a.Build(outStream, token);
+            }
+            
             streams.Do(s => s.Dispose());
             
             await FileHashCache.FileHashWriteCache(outPath, bsa.Hash);
             sourceDir.DeleteDirectory();
+            
+            _logger.LogInformation("Verifying {bsaTo}", bsa.To);
+            var reader = await BSADispatch.Open(outPath);
+            var results = await reader.Files.PMapAllBatchedAsync(_limiter, async state =>
+            {
+                var sf = await state.GetStreamFactory(token);
+                await using var stream = await sf.GetStream();
+                var hash = await stream.Hash(token);
+
+                var astate = bsa.FileStates.First(f => f.Path == state.Path);
+                var srcDirective = indexedByDestination[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)];
+                //DX10Files are lossy
+                if (astate is not BA2DX10File && srcDirective.IsDeterministic) 
+                    ThrowOnNonMatchingHash(bsa, srcDirective, astate, hash);
+                return (srcDirective, hash);
+            }).ToHashSet();
         }
 
-        var bsaDir = _configuration.Install.Combine(BSACreationDir);
+        var bsaDir = _configuration.Install.Combine(Consts.BSACreationDir);
         if (bsaDir.DirectoryExists())
         {
-            _logger.LogInformation("Removing temp folder {bsaCreationDir}", BSACreationDir);
+            _logger.LogInformation("Removing temp folder {bsaCreationDir}", Consts.BSACreationDir);
             bsaDir.DeleteDirectory();
         }
     }
@@ -329,7 +349,10 @@ public class StandardInstaller : AInstaller<StandardInstaller>
                         await FileHashCache.FileHashCachedAsync(outPath, token);
                         break;
                     default:
-                        await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID), token);
+                        var hash = await outPath.WriteAllHashedAsync(await LoadBytesFromPath(directive.SourceDataID), token);
+                        if (!Consts.KnownModifiedFiles.Contains(directive.To.FileName))
+                            ThrowOnNonMatchingHash(directive, hash);
+
                         await FileHashCache.FileHashWriteCache(outPath, directive.Hash);
                         break;
                 }
@@ -481,7 +504,8 @@ public class StandardInstaller : AInstaller<StandardInstaller>
                 .Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None);
             try
             {
-                await BinaryPatching.ApplyPatch(new MemoryStream(srcData), new MemoryStream(patchData), fs);
+                var hash = await BinaryPatching.ApplyPatch(new MemoryStream(srcData), new MemoryStream(patchData), fs);
+                ThrowOnNonMatchingHash(m, hash);
             }
             catch (Exception ex)
             {

Wabbajack.Installer/Utilities/BinaryPatching.cs

@@ -1,16 +1,19 @@
 using System.IO;
+using System.Threading;
 using System.Threading.Tasks;
 using Octodiff.Core;
 using Octodiff.Diagnostics;
+using Wabbajack.Hashing.xxHash64;
 
 namespace Wabbajack.Installer.Utilities;
 
 public class BinaryPatching
 {
-    public static ValueTask ApplyPatch(Stream input, Stream deltaStream, Stream output)
+    public static async ValueTask<Hash> ApplyPatch(Stream input, Stream deltaStream, Stream output, CancellationToken? token = null)
     {
         var deltaApplier = new DeltaApplier();
         deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, new NullProgressReporter()), output);
-        return ValueTask.CompletedTask;
+        output.Position = 0;
+        return await output.Hash(token ?? CancellationToken.None);
     }
 }

Wabbajack.Installer/Wabbajack.Installer.csproj

@@ -24,7 +24,7 @@
 
     <ItemGroup>
         <PackageReference Include="ini-parser-netstandard" Version="2.5.2" />
-        <PackageReference Include="Octodiff" Version="1.3.23" />
+        <PackageReference Include="Octopus.Octodiff" Version="1.3.93" />
     </ItemGroup>
 
 </Project>

Wabbajack.Networking.NexusApi/NexusApi.cs

@@ -33,8 +33,7 @@ public class NexusApi
     private (ValidateInfo info, ResponseMetadata header) _lastValidatedInfo; 
 
     public NexusApi(ITokenProvider<NexusApiState> apiKey, ILogger<NexusApi> logger, HttpClient client,
-        IResource<HttpClient> limiter,
-        ApplicationInfo appInfo, JsonSerializerOptions jsonOptions)
+        IResource<HttpClient> limiter, ApplicationInfo appInfo, JsonSerializerOptions jsonOptions)
     {
         ApiKey = apiKey;
         _logger = logger;

Wabbajack.Paths.IO/AbsolutePathExtensions.cs

@@ -55,6 +55,11 @@ public static class AbsolutePathExtensions
     {
         return new FileInfo(file.ToNativePath()).LastWriteTimeUtc;
     }
+    
+    public static DateTime CreatedUtc(this AbsolutePath file)
+    {
+        return new FileInfo(file.ToNativePath()).CreationTimeUtc;
+    }
 
     public static DateTime LastModified(this AbsolutePath file)
     {

Wabbajack.VFS/FileHashCache.cs

@@ -91,6 +91,10 @@ public class FileHashCache
         var result = await Get(file);
         if (result == default || result.Hash == default)
             return default;
+        
+        // Fix for strange issue where dates are messed up on some systems
+        if (file.LastModifiedUtc() < file.CreatedUtc())
+            file.Touch();
 
         if (result.LastModified == file.LastModifiedUtc().ToFileTimeUtc())
         {