From 2b943c0da31da667e37c9099093ccfc44cc18a1d Mon Sep 17 00:00:00 2001 From: Halgari Date: Sun, 23 Oct 2022 22:56:57 -0600 Subject: [PATCH] Fixes for installer verification --- .../NexusDownloader.cs | 19 +++++++++++++-- .../ExtractedFiles/IExtractedFile.cs | 21 +++++++++++++++++ Wabbajack.Installer/AInstaller.cs | 5 +++- Wabbajack.Installer/StandardInstaller.cs | 23 +++++++++++-------- 4 files changed, 56 insertions(+), 12 deletions(-) diff --git a/Wabbajack.Downloaders.Nexus/NexusDownloader.cs b/Wabbajack.Downloaders.Nexus/NexusDownloader.cs index 883dd5ef..89aa9917 100644 --- a/Wabbajack.Downloaders.Nexus/NexusDownloader.cs +++ b/Wabbajack.Downloaders.Nexus/NexusDownloader.cs @@ -129,8 +129,23 @@ public class NexusDownloader : ADownloader, IUrlDownloader var urls = await _api.DownloadLink(state.Game.MetaData().NexusName!, state.ModID, state.FileID, token); _logger.LogInformation("Downloading Nexus File: {game}|{modid}|{fileid}", state.Game, state.ModID, state.FileID); - var message = new HttpRequestMessage(HttpMethod.Get, urls.info.First().URI); - return await _downloader.Download(message, destination, job, token); + foreach (var link in urls.info) + { + try + { + var message = new HttpRequestMessage(HttpMethod.Get, link.URI); + return await _downloader.Download(message, destination, job, token); + } + catch (Exception ex) + { + if (link.URI == urls.info.Last().URI) + throw; + _logger.LogInformation(ex, "While downloading {URI}, trying another link", link.URI); + } + } + + // Should never be hit + throw new NotImplementedException(); } catch (HttpRequestException ex) { diff --git a/Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs b/Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs index c96543a9..70b146ae 100644 --- a/Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs +++ b/Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs @@ -1,6 +1,8 @@ using System.Threading; using System.Threading.Tasks; +using Wabbajack.Common; using Wabbajack.DTOs.Streams; +using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; namespace Wabbajack.FileExtractor.ExtractedFiles; @@ -16,4 +18,23 @@ public interface IExtractedFile : IStreamFactory /// destination to move the entry to /// public ValueTask Move(AbsolutePath newPath, CancellationToken token); + + +} + +public static class IExtractedFileExtensions +{ + public static async Task MoveHashedAsync(this IExtractedFile file, AbsolutePath destPath, CancellationToken token) + { + if (file.CanMove) + { + await file.Move(destPath, token); + return await destPath.Hash(token); + } + else + { + await using var s = await file.GetStream(); + return await destPath.WriteAllHashedAsync(s, token, false); + } + } } \ No newline at end of file diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index d4438747..28784921 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -16,6 +16,7 @@ using Wabbajack.DTOs.BSA.FileStates; using Wabbajack.DTOs.Directives; using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.JsonConverters; +using Wabbajack.FileExtractor.ExtractedFiles; using Wabbajack.Hashing.PHash; using Wabbajack.Hashing.xxHash64; using Wabbajack.Installer.Utilities; @@ -265,7 +266,8 @@ public abstract class AInstaller case FromArchive _: if (grouped[vf].Count() == 1) { - await sf.Move(destPath, token); + var hash = await sf.MoveHashedAsync(destPath, token); + ThrowOnNonMatchingHash(file, hash); } else { @@ -299,6 +301,7 @@ public abstract class AInstaller protected void ThrowOnNonMatchingHash(CreateBSA bsa, Directive directive, AFile state, Hash hash) { + if (hash == directive.Hash) return; _logger.LogError("Hashes for BSA don't match after extraction, {BSA}, {Directive}, {ExpectedHash}, {Hash}", bsa.To, directive.To, directive.Hash, hash); throw new Exception($"Hashes for {bsa.To} file {directive.To} did not match, expected {directive.Hash} got {hash}"); } diff --git a/Wabbajack.Installer/StandardInstaller.cs b/Wabbajack.Installer/StandardInstaller.cs index ea0d87b0..47f11965 100644 --- a/Wabbajack.Installer/StandardInstaller.cs +++ b/Wabbajack.Installer/StandardInstaller.cs @@ -274,7 +274,7 @@ public class StandardInstaller : AInstaller { var bsas = ModList.Directives.OfType().ToList(); _logger.LogInformation("Generating debug caches"); - var indexedByDestination = ModList.Directives.ToDictionary(d => d.To); + var indexedByDestination = UnoptimizedDirectives.ToDictionary(d => d.To); _logger.LogInformation("Building {bsasCount} bsa files", bsas.Count); NextStep("Installing", "Building BSAs", bsas.Count); @@ -287,19 +287,19 @@ public class StandardInstaller : AInstaller await using var a = BSADispatch.CreateBuilder(bsa.State, _manager); var streams = await bsa.FileStates.PMapAllBatchedAsync(_limiter, async state => { - using var job = await _limiter.Begin($"Adding {state.Path.FileName}", 0, token); var fs = sourceDir.Combine(state.Path).Open(FileMode.Open, FileAccess.Read, FileShare.Read); - var size = fs.Length; - job.Size = size; await a.AddFile(state, fs, token); - await job.Report((int)size, token); return fs; }).ToList(); _logger.LogInformation("Writing {bsaTo}", bsa.To); var outPath = _configuration.Install.Combine(bsa.To); - await using var outStream = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); - await a.Build(outStream, token); + + await using (var outStream = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None)) + { + await a.Build(outStream, token); + } + streams.Do(s => s.Dispose()); await FileHashCache.FileHashWriteCache(outPath, bsa.Hash); @@ -330,7 +330,10 @@ public class StandardInstaller : AInstaller } } - + private static HashSet KnownModifiedFiles = new[] + { + "modlist.txt" + }.Select(r => r.ToRelativePath()).ToHashSet(); private async Task InstallIncludedFiles(CancellationToken token) { _logger.LogInformation("Writing inline files"); @@ -351,7 +354,9 @@ public class StandardInstaller : AInstaller break; default: var hash = await outPath.WriteAllHashedAsync(await LoadBytesFromPath(directive.SourceDataID), token); - ThrowOnNonMatchingHash(directive, hash); + if (!KnownModifiedFiles.Contains(directive.To.FileName)) + ThrowOnNonMatchingHash(directive, hash); + await FileHashCache.FileHashWriteCache(outPath, directive.Hash); break; }