diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ca94632b..d8bee614 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -39,12 +39,7 @@ jobs: include-prerelease: true - name: Test - run: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:EnableWindowsTargeting=true --filter "Category!=FlakeyNetwork" - - - uses: codecov/codecov-action@v3 - with: - flags: unittests, ${{runner.os}} - verbose: true + run: dotnet test /p:EnableWindowsTargeting=true --filter "Category!=FlakeyNetwork" #- name: Upload Test Folder on Failure # if: ${{ failure() }} diff --git a/CHANGELOG.md b/CHANGELOG.md index bcf3197a..e69ed61e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,18 @@ ### Changelog -#### Version - 3.0.2.4 - TBD +#### Version - 3.0.4.0 - TBD * upgrade GameFinder to 2.2.1 +#### Version - 3.0.3.1 - 10/30/2022 +* Fix file verification issues for CreatedBSAs +* Fix files during verification where CreatedDate > LastModified + +#### Version - 3.0.3.0 - 10/26/2022 +* Verify hashes of all installed files +* Verify contents of BSAs during installation +* Provide a new CLI command for verifying a installed modlist +* When downloading from one Nexus CDN server fails, WJ will now try alternate Nexus servers + #### Version - 3.0.2.3 - 10/19/2022 * HOTFIX: revert GameFinder library to 1.8 until it's a bit more forgiving of corrupt files diff --git a/Wabbajack.App.Wpf/View Models/MainWindowVM.cs b/Wabbajack.App.Wpf/View Models/MainWindowVM.cs index cef0719a..2274dfab 100644 --- a/Wabbajack.App.Wpf/View Models/MainWindowVM.cs +++ b/Wabbajack.App.Wpf/View Models/MainWindowVM.cs @@ -66,10 +66,10 @@ namespace Wabbajack public ICommand OpenSettingsCommand { get; } public string VersionDisplay { get; } - + [Reactive] public string ResourceStatus { get; set; } - + [Reactive] public string AppName { get; set; } @@ -98,7 +98,7 @@ namespace Wabbajack MessageBus.Current.Listen() .Subscribe(m => HandleNavigateTo(m.Screen)) .DisposeWith(CompositeDisposable); - + MessageBus.Current.Listen() .Subscribe(m => HandleNavigateTo(m.ViewModel)) .DisposeWith(CompositeDisposable); @@ -106,7 +106,7 @@ namespace Wabbajack MessageBus.Current.Listen() .Subscribe(HandleNavigateBack) .DisposeWith(CompositeDisposable); - + MessageBus.Current.Listen() .ObserveOnGuiThread() .Subscribe(HandleSpawnBrowserWindow) @@ -116,7 +116,7 @@ namespace Wabbajack .Select(r => string.Join(", ", r.Where(r => r.Throughput > 0) .Select(s => $"{s.Name} - {s.Throughput.ToFileSizeString()}/sec"))) .BindToStrict(this, view => view.ResourceStatus); - + if (IsStartingFromModlist(out var path)) { @@ -134,25 +134,24 @@ namespace Wabbajack var assembly = Assembly.GetExecutingAssembly(); var location = assembly.Location; if (string.IsNullOrWhiteSpace(location)) - location = Process.GetCurrentProcess().MainModule?.FileName ?? ""; - + location = Process.GetCurrentProcess().MainModule?.FileName ?? throw new Exception("Assembly location is unavailable!"); + _logger.LogInformation("App Location: {Location}", assembly.Location); var fvi = FileVersionInfo.GetVersionInfo(location); Consts.CurrentMinimumWabbajackVersion = Version.Parse(fvi.FileVersion); VersionDisplay = $"v{fvi.FileVersion}"; AppName = "WABBAJACK " + VersionDisplay; _logger.LogInformation("Wabbajack Version: {FileVersion}", fvi.FileVersion); - + Task.Run(() => _wjClient.SendMetric("started_wabbajack", fvi.FileVersion)).FireAndForget(); Task.Run(() => _wjClient.SendMetric("started_sha", ThisAssembly.Git.Sha)); - + // setup file association try { - var applicationRegistrationService = - _serviceProvider.GetRequiredService(); + var applicationRegistrationService = _serviceProvider.GetRequiredService(); - var applicationInfo = new ApplicationInfo(assembly); + var applicationInfo = new ApplicationInfo("Wabbajack", "Wabbajack", "Wabbajack", location); applicationInfo.SupportedExtensions.Add("wabbajack"); applicationRegistrationService.RegisterApplication(applicationInfo); } @@ -181,27 +180,27 @@ namespace Wabbajack ActivePane = objViewModel; } - + private void HandleNavigateBack(NavigateBack navigateBack) { ActivePane = PreviousPanes.Last(); PreviousPanes.RemoveAt(PreviousPanes.Count - 1); } - + private void HandleManualDownload(ManualDownload manualDownload) { var handler = _serviceProvider.GetRequiredService(); handler.Intervention = manualDownload; //MessageBus.Current.SendMessage(new OpenBrowserTab(handler)); } - + private void HandleManualBlobDownload(ManualBlobDownload manualDownload) { var handler = _serviceProvider.GetRequiredService(); handler.Intervention = manualDownload; //MessageBus.Current.SendMessage(new OpenBrowserTab(handler)); } - + private void HandleSpawnBrowserWindow(SpawnBrowserWindow msg) { var window = _serviceProvider.GetRequiredService(); @@ -213,7 +212,7 @@ namespace Wabbajack { if (s is NavigateToGlobal.ScreenType.Settings) PreviousPanes.Add(ActivePane); - + ActivePane = s switch { NavigateToGlobal.ScreenType.ModeSelectionView => ModeSelectionVM, diff --git a/Wabbajack.CLI/VerbRegistration.cs b/Wabbajack.CLI/VerbRegistration.cs index 0e7154a7..b000d201 100644 --- a/Wabbajack.CLI/VerbRegistration.cs +++ b/Wabbajack.CLI/VerbRegistration.cs @@ -51,6 +51,8 @@ CommandLineBuilder.RegisterCommand(UploadToNexus.Definition, c => services.AddSingleton(); CommandLineBuilder.RegisterCommand(ValidateLists.Definition, c => ((ValidateLists)c).Run); services.AddSingleton(); +CommandLineBuilder.RegisterCommand(VerifyModlistInstall.Definition, c => ((VerifyModlistInstall)c).Run); +services.AddSingleton(); CommandLineBuilder.RegisterCommand(VFSIndex.Definition, c => ((VFSIndex)c).Run); services.AddSingleton(); } diff --git a/Wabbajack.CLI/Verbs/ModlistReport.cs b/Wabbajack.CLI/Verbs/ModlistReport.cs index 64f31f96..dd9fdd96 100644 --- a/Wabbajack.CLI/Verbs/ModlistReport.cs +++ b/Wabbajack.CLI/Verbs/ModlistReport.cs @@ -70,7 +70,7 @@ public class ModlistReport string FixupTo(RelativePath path) { - if (path.GetPart(0) != StandardInstaller.BSACreationDir.ToString()) return path.ToString(); + if (path.GetPart(0) != Consts.BSACreationDir.ToString()) return path.ToString(); var bsaId = path.GetPart(1); if (!bsas.TryGetValue(bsaId, out var bsa)) diff --git a/Wabbajack.CLI/Verbs/VerifyModlistInstall.cs b/Wabbajack.CLI/Verbs/VerifyModlistInstall.cs new file mode 100644 index 00000000..7a197fe0 --- /dev/null +++ b/Wabbajack.CLI/Verbs/VerifyModlistInstall.cs @@ -0,0 +1,173 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.CLI.Builder; +using Wabbajack.Common; +using Wabbajack.Compression.BSA; +using Wabbajack.DTOs; +using Wabbajack.DTOs.BSA.FileStates; +using Wabbajack.DTOs.Directives; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Installer; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; +using Wabbajack.RateLimiter; +using Wabbajack.VFS; +using AbsolutePathExtensions = Wabbajack.Common.AbsolutePathExtensions; + +namespace Wabbajack.CLI.Verbs; + +public class VerifyModlistInstall +{ + private readonly DTOSerializer _dtos; + private readonly ILogger _logger; + + public VerifyModlistInstall(ILogger logger, DTOSerializer dtos, IResource limiter) + { + _limiter = limiter; + _logger = logger; + _dtos = dtos; + } + + public static VerbDefinition Definition = new("verify-modlist-install", "Verify a modlist installed correctly", + new[] + { + new OptionDefinition(typeof(AbsolutePath), "m", "modlistLocation", + "The .wabbajack file used to install the modlist"), + new OptionDefinition(typeof(AbsolutePath), "i", "installFolder", "The installation folder of the modlist") + }); + + private readonly IResource _limiter; + + + public async Task Run(AbsolutePath modlistLocation, AbsolutePath installFolder, CancellationToken token) + { + _logger.LogInformation("Loading modlist {ModList}", modlistLocation); + var list = await StandardInstaller.LoadFromFile(_dtos, modlistLocation); + + _logger.LogInformation("Indexing files"); + var byTo = list.Directives.ToDictionary(d => d.To); + + + _logger.LogInformation("Scanning files"); + var errors = await list.Directives.PMapAllBatchedAsync(_limiter, async directive => + { + if (!(directive is CreateBSA || directive.IsDeterministic)) + return null; + + if (directive.To.InFolder(Consts.BSACreationDir)) + return null; + + var dest = directive.To.RelativeTo(installFolder); + if (!dest.FileExists()) + { + return new Result + { + Path = directive.To, + Message = $"File does not exist directive {directive.GetType()}" + }; + } + + if (Consts.KnownModifiedFiles.Contains(directive.To.FileName)) + return null; + + if (directive is CreateBSA bsa) + { + return await VerifyBSA(dest, bsa, byTo, token); + } + + if (dest.Size() != directive.Size) + { + return new Result + { + Path = directive.To, + Message = $"Sizes do not match got {dest.Size()} expected {directive.Size}" + }; + } + + if (directive.Size > (1024 * 1024 * 128)) + { + _logger.LogInformation("Hashing {Size} file at {Path}", directive.Size.ToFileSizeString(), + directive.To); + } + + var hash = await AbsolutePathExtensions.Hash(dest, token); + if (hash != directive.Hash) + { + return new Result + { + Path = directive.To, + Message = $"Hashes do not match, got {hash} expected {directive.Hash}" + }; + } + + return null; + }).Where(r => r != null) + .ToList(); + + _logger.LogInformation("Found {Count} errors", errors.Count); + + + foreach (var error in errors) + { + _logger.LogError("{File} | {Message}", error.Path, error.Message); + } + + + return 0; + } + + private async Task VerifyBSA(AbsolutePath dest, CreateBSA bsa, Dictionary byTo, CancellationToken token) + { + _logger.LogInformation("Verifying Created BSA {To}", bsa.To); + var archive = await BSADispatch.Open(dest); + var filesIndexed = archive.Files.ToDictionary(d => d.Path); + + if (dest.Extension == Ext.Bsa && dest.Size() >= 1024L * 1024 * 1024 * 2) + { + return new Result() + { + Path = bsa.To, + Message = $"BSA is over 2GB in size, this will cause crashes : {bsa.To}" + }; + } + + foreach (var file in bsa.FileStates) + { + if (file is BA2DX10File) continue; + var state = filesIndexed[file.Path]; + var sf = await state.GetStreamFactory(token); + await using var stream = await sf.GetStream(); + var hash = await stream.Hash(token); + + var astate = bsa.FileStates.First(f => f.Path == state.Path); + var srcDirective = byTo[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)]; + + if (!srcDirective.IsDeterministic) + continue; + + if (srcDirective.Hash != hash) + { + return new Result + { + Path = bsa.To, + Message = + $"BSA {bsa.To} contents do not match at {file.Path} got {hash} expected {srcDirective.Hash}" + }; + } + } + + + return null; + } + + public class Result + { + public RelativePath Path { get; set; } + public string Message { get; set; } + } +} \ No newline at end of file diff --git a/Wabbajack.Common/AbsolutePathExtensions.cs b/Wabbajack.Common/AbsolutePathExtensions.cs index c2b4ff36..d2221daa 100644 --- a/Wabbajack.Common/AbsolutePathExtensions.cs +++ b/Wabbajack.Common/AbsolutePathExtensions.cs @@ -21,4 +21,25 @@ public static class AbsolutePathExtensions await using var fs = path.Open(FileMode.Open, FileAccess.Read, FileShare.Read); return await fs.FromJson(dtos); } + + public static async Task WriteAllHashedAsync(this AbsolutePath file, Stream srcStream, CancellationToken token, + bool closeWhenDone = true) + { + try + { + await using var dest = file.Open(FileMode.Create, FileAccess.Write, FileShare.None); + return await srcStream.HashingCopy(dest, token); + } + finally + { + if (closeWhenDone) + srcStream.Close(); + } + } + + public static async Task WriteAllHashedAsync(this AbsolutePath file, byte[] data, CancellationToken token) + { + await using var dest = file.Open(FileMode.Create, FileAccess.Write, FileShare.None); + return await new MemoryStream(data).HashingCopy(dest, token); + } } \ No newline at end of file diff --git a/Wabbajack.DTOs/ModList/Directive.cs b/Wabbajack.DTOs/ModList/Directive.cs index 659b8125..9b13d3ec 100644 --- a/Wabbajack.DTOs/ModList/Directive.cs +++ b/Wabbajack.DTOs/ModList/Directive.cs @@ -1,3 +1,4 @@ +using System.Text.Json.Serialization; using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; @@ -12,4 +13,6 @@ public abstract class Directive /// location the file will be copied to, relative to the install path. /// public RelativePath To { get; set; } + + [JsonIgnore] public virtual bool IsDeterministic => true; } \ No newline at end of file diff --git a/Wabbajack.DTOs/ModList/Directives/ArchiveMeta.cs b/Wabbajack.DTOs/ModList/Directives/ArchiveMeta.cs index fe982eab..6dff60a6 100644 --- a/Wabbajack.DTOs/ModList/Directives/ArchiveMeta.cs +++ b/Wabbajack.DTOs/ModList/Directives/ArchiveMeta.cs @@ -8,4 +8,6 @@ namespace Wabbajack.DTOs.Directives; public class ArchiveMeta : Directive { public RelativePath SourceDataID { get; set; } + + public override bool IsDeterministic => false; } \ No newline at end of file diff --git a/Wabbajack.DTOs/ModList/Directives/CreateBSA.cs b/Wabbajack.DTOs/ModList/Directives/CreateBSA.cs index f1bd6839..3d9dc48d 100644 --- a/Wabbajack.DTOs/ModList/Directives/CreateBSA.cs +++ b/Wabbajack.DTOs/ModList/Directives/CreateBSA.cs @@ -13,4 +13,6 @@ public class CreateBSA : Directive public RelativePath TempID { get; set; } public IArchive State { get; set; } public AFile[] FileStates { get; set; } = Array.Empty(); + + public override bool IsDeterministic => false; } \ No newline at end of file diff --git a/Wabbajack.DTOs/ModList/Directives/RemappedInlineFile.cs b/Wabbajack.DTOs/ModList/Directives/RemappedInlineFile.cs index fbdc1b89..fea843d6 100644 --- a/Wabbajack.DTOs/ModList/Directives/RemappedInlineFile.cs +++ b/Wabbajack.DTOs/ModList/Directives/RemappedInlineFile.cs @@ -9,4 +9,5 @@ namespace Wabbajack.DTOs.Directives; [JsonAlias("RemappedInlineFile, Wabbajack.Lib")] public class RemappedInlineFile : InlineFile { + public override bool IsDeterministic => false; } \ No newline at end of file diff --git a/Wabbajack.DTOs/ModList/Directives/TransformedTexture.cs b/Wabbajack.DTOs/ModList/Directives/TransformedTexture.cs index dcf6eb83..15e9412b 100644 --- a/Wabbajack.DTOs/ModList/Directives/TransformedTexture.cs +++ b/Wabbajack.DTOs/ModList/Directives/TransformedTexture.cs @@ -11,4 +11,5 @@ public class TransformedTexture : FromArchive /// The file to apply to the source file to patch it /// public ImageState ImageState { get; set; } = new(); + public override bool IsDeterministic => false; } \ No newline at end of file diff --git a/Wabbajack.Downloaders.Dispatcher.Test/VerificationCacheTests.cs b/Wabbajack.Downloaders.Dispatcher.Test/VerificationCacheTests.cs index ac202ca5..9627f268 100644 --- a/Wabbajack.Downloaders.Dispatcher.Test/VerificationCacheTests.cs +++ b/Wabbajack.Downloaders.Dispatcher.Test/VerificationCacheTests.cs @@ -9,7 +9,6 @@ namespace Wabbajack.Downloaders.Dispatcher.Test; public class VerificationCacheTests { - private readonly TemporaryFileManager _temp; private readonly ILogger _logger; public VerificationCacheTests(ILogger logger) diff --git a/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs b/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs index 3e7c1851..c65d12ee 100644 --- a/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs +++ b/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs @@ -40,7 +40,8 @@ public class MediaFireDownloader : ADownloader, I public override bool IsAllowed(ServerAllowList allowList, IDownloadState state) { - return true; + var mediaFireState = (DTOs.DownloadStates.MediaFire) state; + return allowList.AllowedPrefixes.Any(p => mediaFireState.Url.ToString().StartsWith(p, StringComparison.OrdinalIgnoreCase)); } public override IDownloadState? Resolve(IReadOnlyDictionary iniData) @@ -105,26 +106,26 @@ public class MediaFireDownloader : ADownloader, I if (!result.IsSuccessStatusCode) return null; - if (job != null) + if (job != null) job.Size = result.Content.Headers.ContentLength ?? 0; if (result.Content.Headers.ContentType!.MediaType!.StartsWith("text/html", StringComparison.OrdinalIgnoreCase)) { var bodyData = await result.Content.ReadAsStringAsync((CancellationToken) token); - if (job != null) + if (job != null) await job.Report((int) (job.Size ?? 0), (CancellationToken) token); var body = new HtmlDocument(); body.LoadHtml(bodyData); var node = body.DocumentNode.DescendantsAndSelf().FirstOrDefault(d => d.HasClass("input") && d.HasClass("popsok") && d.GetAttributeValue("aria-label", "") == "Download file"); - if (node != null) + if (node != null) return new Uri(node.GetAttributeValue("href", "not-found")); - + var startText = "window.location.href = '"; var start = body.DocumentNode.InnerHtml.IndexOf(startText, StringComparison.CurrentCultureIgnoreCase); - + if (start != -1) { var end = body.DocumentNode.InnerHtml.IndexOf("\'", start + startText.Length, @@ -141,4 +142,4 @@ public class MediaFireDownloader : ADownloader, I { return new[] {$"directURL={state.Url}"}; } -} \ No newline at end of file +} diff --git a/Wabbajack.Downloaders.Nexus/NexusDownloader.cs b/Wabbajack.Downloaders.Nexus/NexusDownloader.cs index 883dd5ef..89aa9917 100644 --- a/Wabbajack.Downloaders.Nexus/NexusDownloader.cs +++ b/Wabbajack.Downloaders.Nexus/NexusDownloader.cs @@ -129,8 +129,23 @@ public class NexusDownloader : ADownloader, IUrlDownloader var urls = await _api.DownloadLink(state.Game.MetaData().NexusName!, state.ModID, state.FileID, token); _logger.LogInformation("Downloading Nexus File: {game}|{modid}|{fileid}", state.Game, state.ModID, state.FileID); - var message = new HttpRequestMessage(HttpMethod.Get, urls.info.First().URI); - return await _downloader.Download(message, destination, job, token); + foreach (var link in urls.info) + { + try + { + var message = new HttpRequestMessage(HttpMethod.Get, link.URI); + return await _downloader.Download(message, destination, job, token); + } + catch (Exception ex) + { + if (link.URI == urls.info.Last().URI) + throw; + _logger.LogInformation(ex, "While downloading {URI}, trying another link", link.URI); + } + } + + // Should never be hit + throw new NotImplementedException(); } catch (HttpRequestException ex) { diff --git a/Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs b/Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs index c96543a9..70b146ae 100644 --- a/Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs +++ b/Wabbajack.FileExtractor/ExtractedFiles/IExtractedFile.cs @@ -1,6 +1,8 @@ using System.Threading; using System.Threading.Tasks; +using Wabbajack.Common; using Wabbajack.DTOs.Streams; +using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; namespace Wabbajack.FileExtractor.ExtractedFiles; @@ -16,4 +18,23 @@ public interface IExtractedFile : IStreamFactory /// destination to move the entry to /// public ValueTask Move(AbsolutePath newPath, CancellationToken token); + + +} + +public static class IExtractedFileExtensions +{ + public static async Task MoveHashedAsync(this IExtractedFile file, AbsolutePath destPath, CancellationToken token) + { + if (file.CanMove) + { + await file.Move(destPath, token); + return await destPath.Hash(token); + } + else + { + await using var s = await file.GetStream(); + return await destPath.WriteAllHashedAsync(s, token, false); + } + } } \ No newline at end of file diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index 1d20a863..1fa6b156 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -12,9 +12,11 @@ using Wabbajack.Common; using Wabbajack.Downloaders; using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; +using Wabbajack.DTOs.BSA.FileStates; using Wabbajack.DTOs.Directives; using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.JsonConverters; +using Wabbajack.FileExtractor.ExtractedFiles; using Wabbajack.Hashing.PHash; using Wabbajack.Hashing.xxHash64; using Wabbajack.Installer.Utilities; @@ -39,7 +41,7 @@ public abstract class AInstaller where T : AInstaller { private const int _limitMS = 100; - public static RelativePath BSACreationDir = "TEMP_BSA_FILES".ToRelativePath(); + private static readonly Regex NoDeleteRegex = new(@"(?i)[\\\/]\[NoDelete\]", RegexOptions.Compiled); protected readonly InstallerConfiguration _configuration; @@ -243,7 +245,8 @@ public abstract class AInstaller await using var patchDataStream = await InlinedFileStream(pfa.PatchID); { await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); - await BinaryPatching.ApplyPatch(s, patchDataStream, os); + var hash = await BinaryPatching.ApplyPatch(s, patchDataStream, os); + ThrowOnNonMatchingHash(file, hash); } } break; @@ -263,12 +266,14 @@ public abstract class AInstaller case FromArchive _: if (grouped[vf].Count() == 1) { - await sf.Move(destPath, token); + var hash = await sf.MoveHashedAsync(destPath, token); + ThrowOnNonMatchingHash(file, hash); } else { await using var s = await sf.GetStream(); - await destPath.WriteAllAsync(s, token, false); + var hash = await destPath.WriteAllHashedAsync(s, token, false); + ThrowOnNonMatchingHash(file, hash); } break; @@ -282,6 +287,25 @@ public abstract class AInstaller }, token); } + protected void ThrowOnNonMatchingHash(Directive file, Hash gotHash) + { + if (file.Hash != gotHash) + ThrowNonMatchingError(file, gotHash); + } + private void ThrowNonMatchingError(Directive file, Hash gotHash) + { + _logger.LogError("Hashes for {Path} did not match, expected {Expected} got {Got}", file.To, file.Hash, gotHash); + throw new Exception($"Hashes for {file.To} did not match, expected {file.Hash} got {gotHash}"); + } + + + protected void ThrowOnNonMatchingHash(CreateBSA bsa, Directive directive, AFile state, Hash hash) + { + if (hash == directive.Hash) return; + _logger.LogError("Hashes for BSA don't match after extraction, {BSA}, {Directive}, {ExpectedHash}, {Hash}", bsa.To, directive.To, directive.Hash, hash); + throw new Exception($"Hashes for {bsa.To} file {directive.To} did not match, expected {directive.Hash} got {hash}"); + } + public async Task DownloadArchives(CancellationToken token) { var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList(); @@ -474,8 +498,8 @@ public abstract class AInstaller return d switch { CreateBSA bsa => !bsasToNotBuild.Contains(bsa.TempID), - FromArchive a when a.To.StartsWith($"{BSACreationDir}") => !bsasToNotBuild.Any(b => - a.To.RelativeTo(_configuration.Install).InFolder(_configuration.Install.Combine(BSACreationDir, b))), + FromArchive a when a.To.StartsWith($"{Consts.BSACreationDir}") => !bsasToNotBuild.Any(b => + a.To.RelativeTo(_configuration.Install).InFolder(_configuration.Install.Combine(Consts.BSACreationDir, b))), _ => true }; }).ToDictionary(d => d.To); diff --git a/Wabbajack.Installer/Consts.cs b/Wabbajack.Installer/Consts.cs index a4c81009..db5f690a 100644 --- a/Wabbajack.Installer/Consts.cs +++ b/Wabbajack.Installer/Consts.cs @@ -1,3 +1,5 @@ +using System.Collections.Generic; +using System.Linq; using Wabbajack.Paths; namespace Wabbajack.Installer; @@ -26,4 +28,11 @@ public static class Consts public const string StepDownloading = "Downloading"; public const string StepHashing = "Hashing"; public const string StepFinished = "Finished"; + public static RelativePath BSACreationDir = "TEMP_BSA_FILES".ToRelativePath(); + + public static HashSet KnownModifiedFiles = new[] + { + "modlist.txt", + "SkyrimPrefs.ini" + }.Select(r => r.ToRelativePath()).ToHashSet(); } \ No newline at end of file diff --git a/Wabbajack.Installer/StandardInstaller.cs b/Wabbajack.Installer/StandardInstaller.cs index e6a9d7cc..e1a87938 100644 --- a/Wabbajack.Installer/StandardInstaller.cs +++ b/Wabbajack.Installer/StandardInstaller.cs @@ -19,9 +19,11 @@ using Wabbajack.Compression.Zip; using Wabbajack.Downloaders; using Wabbajack.Downloaders.GameFile; using Wabbajack.DTOs; +using Wabbajack.DTOs.BSA.FileStates; using Wabbajack.DTOs.Directives; using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Hashing.xxHash64; using Wabbajack.Installer.Utilities; using Wabbajack.Networking.WabbajackClientApi; using Wabbajack.Paths; @@ -271,6 +273,8 @@ public class StandardInstaller : AInstaller private async Task BuildBSAs(CancellationToken token) { var bsas = ModList.Directives.OfType().ToList(); + _logger.LogInformation("Generating debug caches"); + var indexedByDestination = UnoptimizedDirectives.ToDictionary(d => d.To); _logger.LogInformation("Building {bsasCount} bsa files", bsas.Count); NextStep("Installing", "Building BSAs", bsas.Count); @@ -278,34 +282,50 @@ public class StandardInstaller : AInstaller { UpdateProgress(1); _logger.LogInformation("Building {bsaTo}", bsa.To.FileName); - var sourceDir = _configuration.Install.Combine(BSACreationDir, bsa.TempID); + var sourceDir = _configuration.Install.Combine(Consts.BSACreationDir, bsa.TempID); await using var a = BSADispatch.CreateBuilder(bsa.State, _manager); - var streams = await bsa.FileStates.PMapAll(async state => + var streams = await bsa.FileStates.PMapAllBatchedAsync(_limiter, async state => { - using var job = await _limiter.Begin($"Adding {state.Path.FileName}", 0, token); var fs = sourceDir.Combine(state.Path).Open(FileMode.Open, FileAccess.Read, FileShare.Read); - var size = fs.Length; - job.Size = size; await a.AddFile(state, fs, token); - await job.Report((int)size, token); return fs; }).ToList(); _logger.LogInformation("Writing {bsaTo}", bsa.To); var outPath = _configuration.Install.Combine(bsa.To); - await using var outStream = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); - await a.Build(outStream, token); + + await using (var outStream = outPath.Open(FileMode.Create, FileAccess.Write, FileShare.None)) + { + await a.Build(outStream, token); + } + streams.Do(s => s.Dispose()); await FileHashCache.FileHashWriteCache(outPath, bsa.Hash); sourceDir.DeleteDirectory(); + + _logger.LogInformation("Verifying {bsaTo}", bsa.To); + var reader = await BSADispatch.Open(outPath); + var results = await reader.Files.PMapAllBatchedAsync(_limiter, async state => + { + var sf = await state.GetStreamFactory(token); + await using var stream = await sf.GetStream(); + var hash = await stream.Hash(token); + + var astate = bsa.FileStates.First(f => f.Path == state.Path); + var srcDirective = indexedByDestination[Consts.BSACreationDir.Combine(bsa.TempID, astate.Path)]; + //DX10Files are lossy + if (astate is not BA2DX10File && srcDirective.IsDeterministic) + ThrowOnNonMatchingHash(bsa, srcDirective, astate, hash); + return (srcDirective, hash); + }).ToHashSet(); } - var bsaDir = _configuration.Install.Combine(BSACreationDir); + var bsaDir = _configuration.Install.Combine(Consts.BSACreationDir); if (bsaDir.DirectoryExists()) { - _logger.LogInformation("Removing temp folder {bsaCreationDir}", BSACreationDir); + _logger.LogInformation("Removing temp folder {bsaCreationDir}", Consts.BSACreationDir); bsaDir.DeleteDirectory(); } } @@ -329,7 +349,10 @@ public class StandardInstaller : AInstaller await FileHashCache.FileHashCachedAsync(outPath, token); break; default: - await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID), token); + var hash = await outPath.WriteAllHashedAsync(await LoadBytesFromPath(directive.SourceDataID), token); + if (!Consts.KnownModifiedFiles.Contains(directive.To.FileName)) + ThrowOnNonMatchingHash(directive, hash); + await FileHashCache.FileHashWriteCache(outPath, directive.Hash); break; } @@ -481,7 +504,8 @@ public class StandardInstaller : AInstaller .Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); try { - await BinaryPatching.ApplyPatch(new MemoryStream(srcData), new MemoryStream(patchData), fs); + var hash = await BinaryPatching.ApplyPatch(new MemoryStream(srcData), new MemoryStream(patchData), fs); + ThrowOnNonMatchingHash(m, hash); } catch (Exception ex) { diff --git a/Wabbajack.Installer/Utilities/BinaryPatching.cs b/Wabbajack.Installer/Utilities/BinaryPatching.cs index 59e65cb9..03cd46a1 100644 --- a/Wabbajack.Installer/Utilities/BinaryPatching.cs +++ b/Wabbajack.Installer/Utilities/BinaryPatching.cs @@ -1,16 +1,19 @@ using System.IO; +using System.Threading; using System.Threading.Tasks; using Octodiff.Core; using Octodiff.Diagnostics; +using Wabbajack.Hashing.xxHash64; namespace Wabbajack.Installer.Utilities; public class BinaryPatching { - public static ValueTask ApplyPatch(Stream input, Stream deltaStream, Stream output) + public static async ValueTask ApplyPatch(Stream input, Stream deltaStream, Stream output, CancellationToken? token = null) { var deltaApplier = new DeltaApplier(); deltaApplier.Apply(input, new BinaryDeltaReader(deltaStream, new NullProgressReporter()), output); - return ValueTask.CompletedTask; + output.Position = 0; + return await output.Hash(token ?? CancellationToken.None); } } \ No newline at end of file diff --git a/Wabbajack.Installer/Wabbajack.Installer.csproj b/Wabbajack.Installer/Wabbajack.Installer.csproj index 23356d4d..7b3417ae 100644 --- a/Wabbajack.Installer/Wabbajack.Installer.csproj +++ b/Wabbajack.Installer/Wabbajack.Installer.csproj @@ -24,7 +24,7 @@ - + diff --git a/Wabbajack.Networking.NexusApi/NexusApi.cs b/Wabbajack.Networking.NexusApi/NexusApi.cs index caa4cbb9..f78c8e94 100644 --- a/Wabbajack.Networking.NexusApi/NexusApi.cs +++ b/Wabbajack.Networking.NexusApi/NexusApi.cs @@ -33,8 +33,7 @@ public class NexusApi private (ValidateInfo info, ResponseMetadata header) _lastValidatedInfo; public NexusApi(ITokenProvider apiKey, ILogger logger, HttpClient client, - IResource limiter, - ApplicationInfo appInfo, JsonSerializerOptions jsonOptions) + IResource limiter, ApplicationInfo appInfo, JsonSerializerOptions jsonOptions) { ApiKey = apiKey; _logger = logger; diff --git a/Wabbajack.Paths.IO/AbsolutePathExtensions.cs b/Wabbajack.Paths.IO/AbsolutePathExtensions.cs index 81bd6848..266dd5ed 100644 --- a/Wabbajack.Paths.IO/AbsolutePathExtensions.cs +++ b/Wabbajack.Paths.IO/AbsolutePathExtensions.cs @@ -55,6 +55,11 @@ public static class AbsolutePathExtensions { return new FileInfo(file.ToNativePath()).LastWriteTimeUtc; } + + public static DateTime CreatedUtc(this AbsolutePath file) + { + return new FileInfo(file.ToNativePath()).CreationTimeUtc; + } public static DateTime LastModified(this AbsolutePath file) { diff --git a/Wabbajack.VFS/FileHashCache.cs b/Wabbajack.VFS/FileHashCache.cs index 64dfd90c..f9391b31 100644 --- a/Wabbajack.VFS/FileHashCache.cs +++ b/Wabbajack.VFS/FileHashCache.cs @@ -91,6 +91,10 @@ public class FileHashCache var result = await Get(file); if (result == default || result.Hash == default) return default; + + // Fix for strange issue where dates are messed up on some systems + if (file.LastModifiedUtc() < file.CreatedUtc()) + file.Touch(); if (result.LastModified == file.LastModifiedUtc().ToFileTimeUtc()) {