diff --git a/Wabbajack.CLI/VerbRegistration.cs b/Wabbajack.CLI/VerbRegistration.cs index 0e7154a7..b000d201 100644 --- a/Wabbajack.CLI/VerbRegistration.cs +++ b/Wabbajack.CLI/VerbRegistration.cs @@ -51,6 +51,8 @@ CommandLineBuilder.RegisterCommand(UploadToNexus.Definition, c => services.AddSingleton(); CommandLineBuilder.RegisterCommand(ValidateLists.Definition, c => ((ValidateLists)c).Run); services.AddSingleton(); +CommandLineBuilder.RegisterCommand(VerifyModlistInstall.Definition, c => ((VerifyModlistInstall)c).Run); +services.AddSingleton(); CommandLineBuilder.RegisterCommand(VFSIndex.Definition, c => ((VFSIndex)c).Run); services.AddSingleton(); } diff --git a/Wabbajack.CLI/Verbs/ModlistReport.cs b/Wabbajack.CLI/Verbs/ModlistReport.cs index 64f31f96..dd9fdd96 100644 --- a/Wabbajack.CLI/Verbs/ModlistReport.cs +++ b/Wabbajack.CLI/Verbs/ModlistReport.cs @@ -70,7 +70,7 @@ public class ModlistReport string FixupTo(RelativePath path) { - if (path.GetPart(0) != StandardInstaller.BSACreationDir.ToString()) return path.ToString(); + if (path.GetPart(0) != Consts.BSACreationDir.ToString()) return path.ToString(); var bsaId = path.GetPart(1); if (!bsas.TryGetValue(bsaId, out var bsa)) diff --git a/Wabbajack.CLI/Verbs/VerifyModlistInstall.cs b/Wabbajack.CLI/Verbs/VerifyModlistInstall.cs new file mode 100644 index 00000000..173609ba --- /dev/null +++ b/Wabbajack.CLI/Verbs/VerifyModlistInstall.cs @@ -0,0 +1,173 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.CLI.Builder; +using Wabbajack.Common; +using Wabbajack.Compression.BSA; +using Wabbajack.DTOs; +using Wabbajack.DTOs.BSA.FileStates; +using Wabbajack.DTOs.Directives; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Installer; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; +using Wabbajack.RateLimiter; +using Wabbajack.VFS; +using AbsolutePathExtensions = Wabbajack.Common.AbsolutePathExtensions; + +namespace Wabbajack.CLI.Verbs; + +public class VerifyModlistInstall +{ + private readonly DTOSerializer _dtos; + private readonly ILogger _logger; + + public VerifyModlistInstall(ILogger logger, DTOSerializer dtos, IResource limiter) + { + _limiter = limiter; + _logger = logger; + _dtos = dtos; + } + + public static VerbDefinition Definition = new("verify-modlist-install", "Verify a modlist installed correctly", + new[] + { + new OptionDefinition(typeof(AbsolutePath), "m", "modlistLocation", + "The .wabbajack file used to install the modlist"), + new OptionDefinition(typeof(AbsolutePath), "i", "installFolder", "The installation folder of the modlist") + }); + + private readonly IResource _limiter; + + + public async Task Run(AbsolutePath modlistLocation, AbsolutePath installFolder, CancellationToken token) + { + _logger.LogInformation("Loading modlist {ModList}", modlistLocation); + var list = await StandardInstaller.LoadFromFile(_dtos, modlistLocation); + + _logger.LogInformation("Indexing files"); + var byTo = list.Directives.ToDictionary(d => d.To); + + + _logger.LogInformation("Scanning files"); + var errors = await list.Directives.PMapAllBatchedAsync(_limiter, async directive => + { + if (directive is ArchiveMeta) + return null; + + if (directive is RemappedInlineFile) + return null; + + if (directive.To.InFolder(Consts.BSACreationDir)) + return null; + + var dest = directive.To.RelativeTo(installFolder); + if (!dest.FileExists()) + { + return new Result + { + Path = directive.To, + Message = $"File does not exist directive {directive.GetType()}" + }; + } + + if (Consts.KnownModifiedFiles.Contains(directive.To.FileName)) + return null; + + if (directive is CreateBSA bsa) + { + return await VerifyBSA(dest, bsa, byTo, token); + } + + if (dest.Size() != directive.Size) + { + return new Result + { + Path = directive.To, + Message = $"Sizes do not match got {dest.Size()} expected {directive.Size}" + }; + } + + if (directive.Size > (1024 * 1024 * 128)) + { + _logger.LogInformation("Hashing {Size} file at {Path}", directive.Size.ToFileSizeString(), + directive.To); + } + + var hash = await AbsolutePathExtensions.Hash(dest, token); + if (hash != directive.Hash) + { + return new Result + { + Path = directive.To, + Message = $"Hashes do not match, got {hash} expected {directive.Hash}" + }; + } + + return null; + }).Where(r => r != null) + .ToList(); + + _logger.LogInformation("Found {Count} errors", errors.Count); + + + foreach (var error in errors) + { + _logger.LogError("{File} | {Message}", error.Path, error.Message); + } + + + return 0; + } + + private async Task VerifyBSA(AbsolutePath dest, CreateBSA bsa, Dictionary byTo, CancellationToken token) + { + _logger.LogInformation("Verifying Created BSA {To}", bsa.To); + var archive = await BSADispatch.Open(dest); + var filesIndexed = archive.Files.ToDictionary(d => d.Path); + + if (dest.Extension == Ext.Bsa && dest.Size() >= 1024L * 1024 * 1024 * 2) + { + return new Result() + { + Path = bsa.To, + Message = $"BSA is over 2GB in size, this will cause crashes : {bsa.To}" + }; + } + + foreach (var file in bsa.FileStates) + { + if (file is BA2DX10File) continue; + var state = filesIndexed[file.Path]; + var sf = await state.GetStreamFactory(token); + await using var stream = await sf.GetStream(); + var hash = await stream.Hash(token); + + var astate = bsa.FileStates.First(f => f.Path == state.Path); + var srcDirective = byTo[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)]; + + if (srcDirective.Hash != hash) + { + return new Result + { + Path = bsa.To, + Message = + $"BSA {bsa.To} contents do not match at {file.Path} got {hash} expected {srcDirective.Hash}" + }; + } + } + + + return null; + } + + public class Result + { + public RelativePath Path { get; set; } + public string Message { get; set; } + } +} \ No newline at end of file diff --git a/Wabbajack.Common/AbsolutePathExtensions.cs b/Wabbajack.Common/AbsolutePathExtensions.cs index c2b4ff36..d2221daa 100644 --- a/Wabbajack.Common/AbsolutePathExtensions.cs +++ b/Wabbajack.Common/AbsolutePathExtensions.cs @@ -21,4 +21,25 @@ public static class AbsolutePathExtensions await using var fs = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read); return await fs.FromJson(dtos); } + + public static async Task WriteAllHashedAsync(this AbsolutePath file, Stream srcStream, CancellationToken token, + bool closeWhenDone = true) + { + try + { + await using var dest = file.Open(FileMode.Create, FileAccess.Write, FileShare.None); + return await srcStream.HashingCopy(dest, token); + } + finally + { + if (closeWhenDone) + srcStream.Close(); + } + } + + public static async Task WriteAllHashedAsync(this AbsolutePath file, byte[] data, CancellationToken token) + { + await using var dest = file.Open(FileMode.Create, FileAccess.Write, FileShare.None); + return await new MemoryStream(data).HashingCopy(dest, token); + } } \ No newline at end of file diff --git a/Wabbajack.Downloaders.Nexus/NexusDownloader.cs b/Wabbajack.Downloaders.Nexus/NexusDownloader.cs index 883dd5ef..89aa9917 100644 --- a/Wabbajack.Downloaders.Nexus/NexusDownloader.cs +++ b/Wabbajack.Downloaders.Nexus/NexusDownloader.cs @@ -129,8 +129,23 @@ public class NexusDownloader : ADownloader, IUrlDownloader var urls = await _api.DownloadLink(state.Game.MetaData().NexusName!, state.ModID, state.FileID, token); _logger.LogInformation("Downloading Nexus File: {game}|{modid}|{fileid}", state.Game, state.ModID, state.FileID); - var message = new HttpRequestMessage(HttpMethod.Get, urls.info.First().URI); - return await _downloader.Download(message, destination, job, token); + foreach (var link in urls.info) + { + try + { + var message = new HttpRequestMessage(HttpMethod.Get, link.URI); + return await _downloader.Download(message, destination, job, token); + } + catch (Exception ex) + { + if (link.URI == urls.info.Last().URI) + throw; + _logger.LogInformation(ex, "While downloading {URI}, trying another link", link.URI); + } + } + + // Should never be hit + throw new NotImplementedException(); } catch (HttpRequestException ex) { diff --git a/Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs b/Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs index c96543a9..70b146ae 100644 --- a/Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs +++ b/Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs @@ -1,6 +1,8 @@ using System.Threading; using System.Threading.Tasks; +using Wabbajack.Common; using Wabbajack.DTOs.Streams; +using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; namespace Wabbajack.FileExtractor.ExtractedFiles; @@ -16,4 +18,23 @@ public interface IExtractedFile : IStreamFactory /// destination to move the entry to /// public ValueTask Move(AbsolutePath newPath, CancellationToken token); + + +} + +public static class IExtractedFileExtensions +{ + public static async Task MoveHashedAsync(this IExtractedFile file, AbsolutePath destPath, CancellationToken token) + { + if (file.CanMove) + { + await file.Move(destPath, token); + return await destPath.Hash(token); + } + else + { + await using var s = await file.GetStream(); + return await destPath.WriteAllHashedAsync(s, token, false); + } + } } \ No newline at end of file diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index 1d20a863..1fa6b156 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -12,9 +12,11 @@ using Wabbajack.Common; using Wabbajack.Downloaders; using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; +using Wabbajack.DTOs.BSA.FileStates; using Wabbajack.DTOs.Directives; using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.JsonConverters; +using Wabbajack.FileExtractor.ExtractedFiles; using Wabbajack.Hashing.PHash; using Wabbajack.Hashing.xxHash64; using Wabbajack.Installer.Utilities; @@ -39,7 +41,7 @@ public abstract class AInstaller where T : AInstaller { private const int _limitMS = 100; - public static RelativePath BSACreationDir = "TEMP_BSA_FILES".ToRelativePath(); + private static readonly Regex NoDeleteRegex = new(@"(?i)[\\\/]\[NoDelete\]", RegexOptions.Compiled); protected readonly InstallerConfiguration _configuration; @@ -243,7 +245,8 @@ public abstract class AInstaller await using var patchDataStream = await InlinedFileStream(pfa.PatchID); { await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); - await BinaryPatching.ApplyPatch(s, patchDataStream, os); + var hash = await BinaryPatching.ApplyPatch(s, patchDataStream, os); + ThrowOnNonMatchingHash(file, hash); } } break; @@ -263,12 +266,14 @@ public abstract class AInstaller case FromArchive _: if (grouped[vf].Count() == 1) { - await sf.Move(destPath, token); + var hash = await sf.MoveHashedAsync(destPath, token); + ThrowOnNonMatchingHash(file, hash); } else { await using var s = await sf.GetStream(); - await destPath.WriteAllAsync(s, token, false); + var hash = await destPath.WriteAllHashedAsync(s, token, false); + ThrowOnNonMatchingHash(file, hash); } break; @@ -282,6 +287,25 @@ public abstract class AInstaller }, token); } + protected void ThrowOnNonMatchingHash(Directive file, Hash gotHash) + { + if (file.Hash != gotHash) + ThrowNonMatchingError(file, gotHash); + } + private void ThrowNonMatchingError(Directive file, Hash gotHash) + { + _logger.LogError("Hashes for {Path} did not match, expected {Expected} got {Got}", file.To, file.Hash, gotHash); + throw new Exception($"Hashes for {file.To} did not match, expected {file.Hash} got {gotHash}"); + } + + + protected void ThrowOnNonMatchingHash(CreateBSA bsa, Directive directive, AFile state, Hash hash) + { + if (hash == directive.Hash) return; + _logger.LogError("Hashes for BSA don't match after extraction, {BSA}, {Directive}, {ExpectedHash}, {Hash}", bsa.To, directive.To, directive.Hash, hash); + throw new Exception($"Hashes for {bsa.To} file {directive.To} did not match, expected {directive.Hash} got {hash}"); + } + public async Task DownloadArchives(CancellationToken token) { var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList(); @@ -474,8 +498,8 @@ public abstract class AInstaller return d switch { CreateBSA bsa => !bsasToNotBuild.Contains(bsa.TempID), - FromArchive a when a.To.StartsWith($"{BSACreationDir}") => !bsasToNotBuild.Any(b => - a.To.RelativeTo(_configuration.Install).InFolder(_configuration.Install.Combine(BSACreationDir, b))), + FromArchive a when a.To.StartsWith($"{Consts.BSACreationDir}") => !bsasToNotBuild.Any(b => + a.To.RelativeTo(_configuration.Install).InFolder(_configuration.Install.Combine(Consts.BSACreationDir, b))), _ => true }; }).ToDictionary(d => d.To); diff --git a/Wabbajack.Installer/Consts.cs b/Wabbajack.Installer/Consts.cs index a4c81009..db5f690a 100644 --- a/Wabbajack.Installer/Consts.cs +++ b/Wabbajack.Installer/Consts.cs @@ -1,3 +1,5 @@ +using System.Collections.Generic; +using System.Linq; using Wabbajack.Paths; namespace Wabbajack.Installer; @@ -26,4 +28,11 @@ public static class Consts public const string StepDownloading = "Downloading"; public const string StepHashing = "Hashing"; public const string StepFinished = "Finished"; + public static RelativePath BSACreationDir = "TEMP_BSA_FILES".ToRelativePath(); + + public static HashSet KnownModifiedFiles = new[] + { + "modlist.txt", + "SkyrimPrefs.ini" + }.Select(r => r.ToRelativePath()).ToHashSet(); } \ No newline at end of file diff --git a/Wabbajack.Installer/StandardInstaller.cs b/Wabbajack.Installer/StandardInstaller.cs index e6a9d7cc..cd68f7fa 100644 --- a/Wabbajack.Installer/StandardInstaller.cs +++ b/Wabbajack.Installer/StandardInstaller.cs @@ -19,9 +19,11 @@ using Wabbajack.Compression.Zip; using Wabbajack.Downloaders; using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; +using Wabbajack.DTOs.BSA.FileStates; using Wabbajack.DTOs.Directives; using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Hashing.xxHash64; using Wabbajack.Installer.Utilities; using Wabbajack.Networking.WabbajackClientApi; using Wabbajack.Paths; @@ -271,6 +273,8 @@ public class StandardInstaller : AInstaller private async Task BuildBSAs(CancellationToken token) { var bsas = ModList.Directives.OfType().ToList(); + _logger.LogInformation("Generating debug caches"); + var indexedByDestination = UnoptimizedDirectives.ToDictionary(d => d.To); _logger.LogInformation("Building {bsasCount} bsa files", bsas.Count); NextStep("Installing", "Building BSAs", bsas.Count); @@ -278,34 +282,50 @@ public class StandardInstaller : AInstaller { UpdateProgress(1); _logger.LogInformation("Building {bsaTo}", bsa.To.FileName); - var sourceDir = _configuration.Install.Combine(BSACreationDir, bsa.TempID); + var sourceDir = _configuration.Install.Combine(Consts.BSACreationDir, bsa.TempID); await using var a = BSADispatch.CreateBuilder(bsa.State, _manager); - var streams = await bsa.FileStates.PMapAll(async state => + var streams = await bsa.FileStates.PMapAllBatchedAsync(_limiter, async state => { - using var job = await _limiter.Begin($"Adding {state.Path.FileName}", 0, token); var fs = sourceDir.Combine(state.Path).Open(FileMode.Open, FileAccess.Read, FileShare.Read); - var size = fs.Length; - job.Size = size; await a.AddFile(state, fs, token); - await job.Report((int)size, token); return fs; }).ToList(); _logger.LogInformation("Writing {bsaTo}", bsa.To); var outPath = _configuration.Install.Combine(bsa.To); - await using var outStream = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); - await a.Build(outStream, token); + + await using (var outStream = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None)) + { + await a.Build(outStream, token); + } + streams.Do(s => s.Dispose()); await FileHashCache.FileHashWriteCache(outPath, bsa.Hash); sourceDir.DeleteDirectory(); + + _logger.LogInformation("Verifying {bsaTo}", bsa.To); + var reader = await BSADispatch.Open(outPath); + var results = await reader.Files.PMapAllBatchedAsync(_limiter, async state => + { + var sf = await state.GetStreamFactory(token); + await using var stream = await sf.GetStream(); + var hash = await stream.Hash(token); + + var astate = bsa.FileStates.First(f => f.Path == state.Path); + var srcDirective = indexedByDestination[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)]; + //DX10Files are lossy + if (astate is not BA2DX10File) + ThrowOnNonMatchingHash(bsa, srcDirective, astate, hash); + return (srcDirective, hash); + }).ToHashSet(); } - var bsaDir = _configuration.Install.Combine(BSACreationDir); + var bsaDir = _configuration.Install.Combine(Consts.BSACreationDir); if (bsaDir.DirectoryExists()) { - _logger.LogInformation("Removing temp folder {bsaCreationDir}", BSACreationDir); + _logger.LogInformation("Removing temp folder {bsaCreationDir}", Consts.BSACreationDir); bsaDir.DeleteDirectory(); } } @@ -329,7 +349,10 @@ public class StandardInstaller : AInstaller await FileHashCache.FileHashCachedAsync(outPath, token); break; default: - await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID), token); + var hash = await outPath.WriteAllHashedAsync(await LoadBytesFromPath(directive.SourceDataID), token); + if (!Consts.KnownModifiedFiles.Contains(directive.To.FileName)) + ThrowOnNonMatchingHash(directive, hash); + await FileHashCache.FileHashWriteCache(outPath, directive.Hash); break; } @@ -481,7 +504,8 @@ public class StandardInstaller : AInstaller .Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); try { - await BinaryPatching.ApplyPatch(new MemoryStream(srcData), new MemoryStream(patchData), fs); + var hash = await BinaryPatching.ApplyPatch(new MemoryStream(srcData), new MemoryStream(patchData), fs); + ThrowOnNonMatchingHash(m, hash); } catch (Exception ex) { diff --git a/Wabbajack.Installer/Utilities/BinaryPatching.cs b/Wabbajack.Installer/Utilities/BinaryPatching.cs index 59e65cb9..03cd46a1 100644 --- a/Wabbajack.Installer/Utilities/BinaryPatching.cs +++ b/Wabbajack.Installer/Utilities/BinaryPatching.cs @@ -1,16 +1,19 @@ using System.IO; +using System.Threading; using System.Threading.Tasks; using Octodiff.Core; using Octodiff.Diagnostics; +using Wabbajack.Hashing.xxHash64; namespace Wabbajack.Installer.Utilities; public class BinaryPatching { - public static ValueTask ApplyPatch(Stream input, Stream deltaStream, Stream output) + public static async ValueTask ApplyPatch(Stream input, Stream deltaStream, Stream output, CancellationToken? token = null) { var deltaApplier = new DeltaApplier(); deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, new NullProgressReporter()), output); - return ValueTask.CompletedTask; + output.Position = 0; + return await output.Hash(token ?? CancellationToken.None); } } \ No newline at end of file diff --git a/Wabbajack.Installer/Wabbajack.Installer.csproj b/Wabbajack.Installer/Wabbajack.Installer.csproj index 23356d4d..7b3417ae 100644 --- a/Wabbajack.Installer/Wabbajack.Installer.csproj +++ b/Wabbajack.Installer/Wabbajack.Installer.csproj @@ -24,7 +24,7 @@ - +