diff --git a/Wabbajack.CLI/Verbs/ChangeDownload.cs b/Wabbajack.CLI/Verbs/ChangeDownload.cs index 9e1e5fc1..c353efad 100644 --- a/Wabbajack.CLI/Verbs/ChangeDownload.cs +++ b/Wabbajack.CLI/Verbs/ChangeDownload.cs @@ -139,6 +139,8 @@ namespace Wabbajack.CLI.Verbs CLIUtils.Log($"Hashing {f}"); return (f, await f.FileHashCachedAsync()); })) + .Where(x => x.Item2.HasValue) + .Select(x => (x.f, x.Item2!.Value)) .GroupBy(d => d.Item2) .ToDictionary(d => d.Key, d => d.First().f); diff --git a/Wabbajack.CLI/Verbs/ForceHealing.cs b/Wabbajack.CLI/Verbs/ForceHealing.cs index 574347c9..26526428 100644 --- a/Wabbajack.CLI/Verbs/ForceHealing.cs +++ b/Wabbajack.CLI/Verbs/ForceHealing.cs @@ -28,8 +28,14 @@ namespace Wabbajack.CLI.Verbs var oldHash = await Old.FileHashCachedAsync(); var newHash = await New.FileHashCachedAsync(); - var oldArchive = new Archive(oldState) {Hash = oldHash, Size = Old.Size}; - var newArchive = new Archive(newState) {Hash = newHash, Size = New.Size}; + if (oldHash == null) + return ExitCode.Error; + + if (newHash == null) + return ExitCode.Error; + + var oldArchive = new Archive(oldState) {Hash = oldHash!.Value, Size = Old.Size}; + var newArchive = new Archive(newState) {Hash = newHash!.Value, Size = New.Size}; Utils.Log($"Contacting Server to request patch ({oldHash} -> {newHash}"); Utils.Log($"Response: {await ClientAPI.GetModUpgrade(oldArchive, newArchive, useAuthor: true)}"); diff --git a/Wabbajack.CLI/Verbs/HashFile.cs b/Wabbajack.CLI/Verbs/HashFile.cs index 7e54fa9f..aa58b739 100644 --- a/Wabbajack.CLI/Verbs/HashFile.cs +++ b/Wabbajack.CLI/Verbs/HashFile.cs @@ -16,7 +16,12 @@ namespace Wabbajack.CLI.Verbs { var abs = (AbsolutePath)Input; var hash = await abs.FileHashAsync(); - Console.WriteLine($"{abs} hash: {hash} {hash.ToHex()} {(long)hash}"); + if (hash == null) + { + Console.WriteLine("Hash is null!"); + return ExitCode.Error; + } + Console.WriteLine($"{abs} hash: {hash} {hash.Value.ToHex()} {(long)hash}"); return ExitCode.Ok; } } diff --git a/Wabbajack.CLI/Verbs/HashGameFiles.cs b/Wabbajack.CLI/Verbs/HashGameFiles.cs index 096bd1be..0aa61159 100644 --- a/Wabbajack.CLI/Verbs/HashGameFiles.cs +++ b/Wabbajack.CLI/Verbs/HashGameFiles.cs @@ -31,25 +31,26 @@ namespace Wabbajack.CLI.Verbs Utils.Log($"Hashing files for {_game} {version}"); - var indexed = await gameLocation + var indexed = (await gameLocation .EnumerateFiles() .PMap(queue, async f => { var hash = await f.FileHashCachedAsync(); + if (hash == null) return null; return new Archive(new GameFileSourceDownloader.State { Game = _game, GameFile = f.RelativeTo(gameLocation), - Hash = hash, + Hash = hash.Value, GameVersion = version }) { Name = f.FileName.ToString(), - Hash = hash, + Hash = hash.Value, Size = f.Size }; - }); + })).NotNull().ToArray(); Utils.Log($"Found and hashed {indexed.Length} files"); await indexed.ToJsonAsync(file, prettyPrint: true); diff --git a/Wabbajack.Common/Extensions/EnumerableExt.cs b/Wabbajack.Common/Extensions/EnumerableExt.cs index 50dfb778..dbf63e2a 100644 --- a/Wabbajack.Common/Extensions/EnumerableExt.cs +++ b/Wabbajack.Common/Extensions/EnumerableExt.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; namespace Wabbajack { @@ -37,14 +38,21 @@ namespace Wabbajack /// /// Converts and filters a nullable enumerable to a non-nullable enumerable /// - public static IEnumerable NotNull(this IEnumerable e) + public static IEnumerable NotNull(this IEnumerable enumerable) where T : class { // Filter out nulls - return e.Where(e => e != null) + return enumerable.Where(e => e != null) // Cast to non nullable type .Select(e => e!); } + + public static IEnumerable NotNull(this IEnumerable enumerable) where T : struct + { + return enumerable + .Where(x => x.HasValue) + .Select(x => x!.Value); + } /// /// Selects items that are castable to the desired type diff --git a/Wabbajack.Common/Hash.cs b/Wabbajack.Common/Hash.cs index 05e809d3..0c361099 100644 --- a/Wabbajack.Common/Hash.cs +++ b/Wabbajack.Common/Hash.cs @@ -274,17 +274,17 @@ namespace Wabbajack.Common WriteHashCache(file, hash); } - public static async Task FileHashCachedAsync(this AbsolutePath file, bool nullOnIOError = false) + public static async Task FileHashCachedAsync(this AbsolutePath file) { if (TryGetHashCache(file, out var foundHash)) return foundHash; - var hash = await file.FileHashAsync(nullOnIOError); - if (hash != Hash.Empty) - WriteHashCache(file, hash); + var hash = await file.FileHashAsync(); + if (hash != null && hash != Hash.Empty) + WriteHashCache(file, hash.Value); return hash; } - public static async Task FileHashAsync(this AbsolutePath file, bool nullOnIOError = false) + public static async Task FileHashAsync(this AbsolutePath file) { try { @@ -294,10 +294,10 @@ namespace Wabbajack.Common var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(hs); return new Hash(BitConverter.ToUInt64(value.Hash)); } - catch (IOException) + catch (IOException e) { - if (nullOnIOError) return Hash.Empty; - throw; + Utils.Error(e, $"Unable to hash file {file}"); + return null; } } } diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index 445ab884..16339a32 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -308,7 +308,7 @@ namespace Wabbajack.Lib ModList.ToJson(of); await ModListOutputFolder.Combine("sig") - .WriteAllBytesAsync((await ModListOutputFolder.Combine("modlist").FileHashAsync()).ToArray()); + .WriteAllBytesAsync(((await ModListOutputFolder.Combine("modlist").FileHashAsync()) ?? Hash.Empty).ToArray()); await ClientAPI.SendModListDefinition(ModList); @@ -338,11 +338,18 @@ namespace Wabbajack.Lib } } - Utils.Log("Exporting ModList metadata"); + Utils.Log("Exporting Modlist metadata"); + var outputFileHash = await ModListOutputFile.FileHashAsync(); + if (outputFileHash == null) + { + Utils.Error("Unable to hash Modlist Output File"); + return; + } + var metadata = new DownloadMetadata { Size = ModListOutputFile.Size, - Hash = await ModListOutputFile.FileHashAsync(), + Hash = outputFileHash.Value, NumberOfArchives = ModList.Archives.Count, SizeOfArchives = ModList.Archives.Sum(a => a.Size), NumberOfInstalledFiles = ModList.Directives.Count, diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 6f1c5c13..7fa918b5 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -145,7 +145,7 @@ namespace Wabbajack.Lib await ClientAPI.GetVirusScanResult(toFile) == VirusScanner.Result.Malware) { await toFile.DeleteAsync(); - Utils.ErrorThrow(new Exception($"Virus scan of patched executable reported possible malware: {toFile.ToString()} ({(long)await toFile.FileHashCachedAsync()})")); + Utils.ErrorThrow(new Exception($"Virus scan of patched executable reported possible malware: {toFile.ToString()} ({(long)(await toFile.FileHashCachedAsync())!.Value})")); } } break; @@ -298,7 +298,8 @@ namespace Wabbajack.Lib .OrderByDescending(e => e.Item2.LastModified) .GroupBy(e => e.Item1) .Select(e => e.First()) - .Select(e => new KeyValuePair(e.Item1, e.Item2))); + .Where(x => x.Item1 != null) + .Select(e => new KeyValuePair(e.Item1!.Value, e.Item2))); } /// diff --git a/Wabbajack.Lib/AuthorApi/Client.cs b/Wabbajack.Lib/AuthorApi/Client.cs index bf2da6a0..68705af5 100644 --- a/Wabbajack.Lib/AuthorApi/Client.cs +++ b/Wabbajack.Lib/AuthorApi/Client.cs @@ -58,7 +58,7 @@ namespace Wabbajack.Lib.AuthorApi { OriginalFileName = path.FileName, Size = path.Size, - Hash = await path.FileHashCachedAsync(), + Hash = await path.FileHashCachedAsync() ?? Hash.Empty, Parts = await parts.PMap(queue, async part => { progressFn("Hashing file parts", Percent.FactoryPutInRange(part.Index, parts.Length)); diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index f866f53b..cfc56de2 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -242,8 +242,12 @@ using Wabbajack.Lib.Downloaders; Utils.Log($"Checking virus result for {path}"); var hash = await path.FileHashAsync(); + if (hash == null) + { + throw new Exception("Hash is null!"); + } - using var result = await client.GetAsync($"{Consts.WabbajackBuildServerUri}virus_scan/{hash.ToHex()}", errorsAsExceptions: false); + using var result = await client.GetAsync($"{Consts.WabbajackBuildServerUri}virus_scan/{hash.Value.ToHex()}", errorsAsExceptions: false); if (result.StatusCode == HttpStatusCode.OK) { var data = await result.Content.ReadAsStringAsync(); diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs index 495d2d53..a8c3117f 100644 --- a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs @@ -334,14 +334,15 @@ namespace Wabbajack.Lib.Downloaders if (await newFile.State.Download(newFile, tmp.Path)) { newFile.Size = tmp.Path.Size; - newFile.Hash = await tmp.Path.FileHashAsync(); + var tmpHash = await tmp.Path.FileHashAsync(); + if (tmpHash == null) return default; + newFile.Hash = tmpHash.Value; return (newFile, tmp); } await tmp.DisposeAsync(); } return default; - } public override async Task ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState) diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs index 0753e010..3483f18d 100644 --- a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs @@ -28,12 +28,13 @@ namespace Wabbajack.Lib.Downloaders var fp = filePath.Value; var hash = await fp.FileHashCachedAsync(); + if (hash == null) return null; return new State(game.InstalledVersion) { Game = game.Game, GameFile = (RelativePath)gameFile, - Hash = hash + Hash = hash.Value }; } diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index b554b7a7..27302b91 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -234,7 +234,9 @@ TOP: return default; } - newArchive.Hash = await tmpFile.Path.FileHashAsync(); + var hash = await tmpFile.Path.FileHashAsync(); + if (hash == null) return default; + newArchive.Hash = hash.Value; newArchive.Size = tmpFile.Path.Size; if (newArchive.Hash == a.Hash || a.Size > 2_500_000_000 || newArchive.Size > 2_500_000_000) diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index a52840d8..38a1639f 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -290,7 +290,9 @@ namespace Wabbajack.Lib.Downloaders if (fastPath != default) { newArchive.Size = fastPath.Size; - newArchive.Hash = await fastPath.FileHashAsync(); + var hash = await fastPath.FileHashAsync(); + if (hash == null) return default; + newArchive.Hash = hash.Value; return (newArchive, new TempFile()); } @@ -300,7 +302,9 @@ namespace Wabbajack.Lib.Downloaders await newArchive.State.Download(newArchive, tempFile.Path); newArchive.Size = tempFile.Path.Size; - newArchive.Hash = await tempFile.Path.FileHashAsync(); + var newArchiveHash = await tempFile.Path.FileHashAsync(); + if (newArchiveHash == null) return default; + newArchive.Hash = newArchiveHash.Value; Utils.Log($"Possible upgrade {newArchive.State.PrimaryKeyString} downloaded"); diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index e8be9553..6632fe86 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -382,11 +382,15 @@ namespace Wabbajack.Lib var source = DownloadsPath.Combine(a.Name + Consts.MetaFileExtension); var ini = a.State.GetMetaIniString(); var (id, fullPath) = await IncludeString(ini); + var hash = await fullPath.FileHashAsync(); + + if (hash == null) return; + InstallDirectives.Add(new ArchiveMeta { SourceDataID = id, Size = fullPath.Size, - Hash = await fullPath.FileHashAsync(), + Hash = hash.Value, To = source.FileName }); }); diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index f1bf7a82..668d89d8 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -294,7 +294,7 @@ namespace Wabbajack.Lib var hash = await gameFile.FileHashAsync(); if (hash != esm.SourceESMHash) { - Utils.ErrorThrow(new InvalidGameESMError(esm, hash, gameFile)); + Utils.ErrorThrow(new InvalidGameESMError(esm, hash ?? Hash.Empty, gameFile)); } } } diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index acbe7ef0..7504a877 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -128,7 +128,7 @@ namespace Wabbajack.Lib.ModListRegistry { return true; } - return DownloadMetadata.Hash != await modlistPath.FileHashCachedAsync(true); + return DownloadMetadata.Hash != await modlistPath.FileHashCachedAsync(); } } diff --git a/Wabbajack.Server.Test/ABuildServerSystemTest.cs b/Wabbajack.Server.Test/ABuildServerSystemTest.cs index 92fbf7ee..c3ae1621 100644 --- a/Wabbajack.Server.Test/ABuildServerSystemTest.cs +++ b/Wabbajack.Server.Test/ABuildServerSystemTest.cs @@ -211,9 +211,9 @@ namespace Wabbajack.BuildServer.Test ModListData = new ModList(); ModListData.Archives.Add( - new Archive(new HTTPDownloader.State(MakeURL(modFileName.ToString()))) + new Archive(new HTTPDownloader.State(MakeURL(modFileName))) { - Hash = await test_archive_path.FileHashAsync(), + Hash = await test_archive_path.FileHashAsync() ?? Hash.Empty, Name = "test_archive", Size = test_archive_path.Size, }); @@ -237,7 +237,7 @@ namespace Wabbajack.BuildServer.Test Description = "A test", DownloadMetadata = new DownloadMetadata { - Hash = await modListPath.FileHashAsync(), + Hash = await modListPath.FileHashAsync() ?? Hash.Empty, Size = modListPath.Size }, Links = new ModlistMetadata.LinksObject diff --git a/Wabbajack.Server.Test/ArchiveMaintainerTests.cs b/Wabbajack.Server.Test/ArchiveMaintainerTests.cs index 62423b02..481823ac 100644 --- a/Wabbajack.Server.Test/ArchiveMaintainerTests.cs +++ b/Wabbajack.Server.Test/ArchiveMaintainerTests.cs @@ -26,7 +26,9 @@ namespace Wabbajack.BuildServer.Test var hash = await tf.Path.FileHashAsync(); await maintainer.Ingest(tf.Path); - Assert.True(maintainer.TryGetPath(hash, out var found)); + Assert.NotNull(hash); + + Assert.True(maintainer.TryGetPath(hash!.Value, out var found)); Assert.Equal(await tf2.Path.ReadAllBytesAsync(), await found.ReadAllBytesAsync()); } @@ -40,10 +42,12 @@ namespace Wabbajack.BuildServer.Test await tf.Path.WriteAllBytesAsync(RandomData(1024)); var hash = await tf.Path.FileHashAsync(); - await tf.Path.CopyToAsync(Fixture.ServerArchivesFolder.Combine(hash.ToHex())); + Assert.NotNull(hash); + + await tf.Path.CopyToAsync(Fixture.ServerArchivesFolder.Combine(hash!.Value.ToHex())); maintainer.Start(); - Assert.True(maintainer.TryGetPath(hash, out var found)); + Assert.True(maintainer.TryGetPath(hash!.Value, out var found)); } } } diff --git a/Wabbajack.Server.Test/MirroredFilesTests.cs b/Wabbajack.Server.Test/MirroredFilesTests.cs index 30727abd..2b840c82 100644 --- a/Wabbajack.Server.Test/MirroredFilesTests.cs +++ b/Wabbajack.Server.Test/MirroredFilesTests.cs @@ -24,9 +24,10 @@ namespace Wabbajack.Server.Test var file = new TempFile(); await file.Path.WriteAllBytesAsync(RandomData(1024 * 1024 * 6)); var dataHash = await file.Path.FileHashAsync(); + Assert.NotNull(dataHash); await Fixture.GetService().Ingest(file.Path); - Assert.True(Fixture.GetService().HaveArchive(dataHash)); + Assert.True(Fixture.GetService().HaveArchive(dataHash!.Value)); var sql = Fixture.GetService(); @@ -34,7 +35,7 @@ namespace Wabbajack.Server.Test { Created = DateTime.UtcNow, Rationale = "Test File", - Hash = dataHash + Hash = dataHash!.Value }); var uploader = Fixture.GetService(); @@ -43,7 +44,7 @@ namespace Wabbajack.Server.Test var archive = new Archive(new HTTPDownloader.State(MakeURL(dataHash.ToString()))) { - Hash = dataHash, + Hash = dataHash!.Value, Size = file.Path.Size }; diff --git a/Wabbajack.Server/Controllers/GameFiles.cs b/Wabbajack.Server/Controllers/GameFiles.cs index 27bb7174..13d2060c 100644 --- a/Wabbajack.Server/Controllers/GameFiles.cs +++ b/Wabbajack.Server/Controllers/GameFiles.cs @@ -1,4 +1,4 @@ -using System; +#nullable enable using System.Linq; using System.Threading.Tasks; using Microsoft.AspNetCore.Authorization; @@ -40,17 +40,19 @@ namespace Wabbajack.BuildServer.Controllers _logger.Log(LogLevel.Information, $"Found {files.Count} game files"); using var queue = new WorkQueue(); - var hashed = await files.PMap(queue, async pair => + var hashed = (await files.PMap(queue, async pair => { var hash = await pair.File.FileHashCachedAsync(); + if (hash == null) return null; + return await _sql.GetOrEnqueueArchive(new Archive(new GameFileSourceDownloader.State { Game = pair.Game.Game, GameFile = pair.File.RelativeTo(pair.Game.GameLocation()), GameVersion = pair.Game.InstalledVersion, - Hash = hash - }) {Name = pair.File.FileName.ToString(), Size = pair.File.Size, Hash = hash}); - }); + Hash = hash.Value + }) {Name = pair.File.FileName.ToString(), Size = pair.File.Size, Hash = hash.Value}); + })).NotNull(); await _quickSync.Notify(); return Ok(hashed); diff --git a/Wabbajack.Server/Services/ArchiveDownloader.cs b/Wabbajack.Server/Services/ArchiveDownloader.cs index ccd804df..db2b5fe1 100644 --- a/Wabbajack.Server/Services/ArchiveDownloader.cs +++ b/Wabbajack.Server/Services/ArchiveDownloader.cs @@ -83,7 +83,7 @@ namespace Wabbajack.Server.Services var hash = await tempPath.Path.FileHashAsync(); - if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash) + if (hash == null || (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)) { _logger.Log(LogLevel.Warning, $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}"); @@ -98,7 +98,7 @@ namespace Wabbajack.Server.Services continue; } - nextDownload.Archive.Hash = hash; + nextDownload.Archive.Hash = hash.Value; nextDownload.Archive.Size = tempPath.Path.Size; _logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}"); diff --git a/Wabbajack.Server/Services/ArchiveMaintainer.cs b/Wabbajack.Server/Services/ArchiveMaintainer.cs index 0adf35fd..4fa26a43 100644 --- a/Wabbajack.Server/Services/ArchiveMaintainer.cs +++ b/Wabbajack.Server/Services/ArchiveMaintainer.cs @@ -36,19 +36,20 @@ namespace Wabbajack.Server.Services return _settings.ArchivePath.Combine(hash.ToHex()); } - public async Task Ingest(AbsolutePath file) + public async Task Ingest(AbsolutePath file) { var hash = await file.FileHashAsync(); - var path = ArchivePath(hash); - if (HaveArchive(hash)) + if (hash == null) return; + + var path = ArchivePath(hash.Value); + if (HaveArchive(hash.Value)) { await file.DeleteAsync(); - return path; + return; } - var newPath = _settings.ArchivePath.Combine(hash.ToHex()); + var newPath = _settings.ArchivePath.Combine(hash.Value.ToHex()); await file.MoveToAsync(newPath); - return path; } public bool HaveArchive(Hash hash) diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 8498fbe9..ff92f29e 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -190,10 +190,10 @@ namespace Wabbajack.VirtualFileSystem public static async Task Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile, IPath relPath, int depth = 0) { - Hash hash = default; + Hash hash; if (extractedFile is NativeFileStreamFactory) { - hash = await ((AbsolutePath)extractedFile.Name).FileHashCachedAsync(); + hash = await ((AbsolutePath)extractedFile.Name).FileHashCachedAsync() ?? Hash.Empty; } else {