Merge pull request #1252 from erri120/better-hashing

Better hashing

Wabbajack.CLI/Verbs/ChangeDownload.cs

@@ -139,6 +139,8 @@ namespace Wabbajack.CLI.Verbs
                         CLIUtils.Log($"Hashing {f}");
                         return (f, await f.FileHashCachedAsync());
                     }))
+                    .Where(x => x.Item2.HasValue)
+                    .Select(x => (x.f, x.Item2!.Value))
                     .GroupBy(d => d.Item2)
                     .ToDictionary(d => d.Key, d => d.First().f);
 

Wabbajack.CLI/Verbs/ForceHealing.cs

@@ -28,8 +28,14 @@ namespace Wabbajack.CLI.Verbs
             var oldHash = await Old.FileHashCachedAsync();
             var newHash = await New.FileHashCachedAsync();
 
-            var oldArchive = new Archive(oldState) {Hash = oldHash, Size = Old.Size};
-            var newArchive = new Archive(newState) {Hash = newHash, Size = New.Size};
+            if (oldHash == null)
+                return ExitCode.Error;
+
+            if (newHash == null)
+                return ExitCode.Error;
+            
+            var oldArchive = new Archive(oldState) {Hash = oldHash!.Value, Size = Old.Size};
+            var newArchive = new Archive(newState) {Hash = newHash!.Value, Size = New.Size};
 
             Utils.Log($"Contacting Server to request patch ({oldHash} -> {newHash}");
             Utils.Log($"Response: {await ClientAPI.GetModUpgrade(oldArchive, newArchive, useAuthor: true)}");

Wabbajack.CLI/Verbs/HashFile.cs

@@ -16,7 +16,12 @@ namespace Wabbajack.CLI.Verbs
         {
             var abs = (AbsolutePath)Input;
             var hash = await abs.FileHashAsync();
-            Console.WriteLine($"{abs} hash: {hash} {hash.ToHex()} {(long)hash}");
+            if (hash == null)
+            {
+                Console.WriteLine("Hash is null!");
+                return ExitCode.Error;
+            }
+            Console.WriteLine($"{abs} hash: {hash} {hash.Value.ToHex()} {(long)hash}");
             return ExitCode.Ok;
         }
     }

Wabbajack.CLI/Verbs/HashGameFiles.cs

@@ -31,25 +31,26 @@ namespace Wabbajack.CLI.Verbs
             
             Utils.Log($"Hashing files for {_game} {version}");
             
-            var indexed = await gameLocation
+            var indexed = (await gameLocation
                 .EnumerateFiles()
                 .PMap(queue, async f =>
                 {
                     var hash = await f.FileHashCachedAsync();
+                    if (hash == null) return null;
                     return new Archive(new GameFileSourceDownloader.State
                     {
                         Game = _game, 
                         GameFile = f.RelativeTo(gameLocation), 
-                        Hash = hash, 
+                        Hash = hash.Value, 
                         GameVersion = version
                     })
                     {
                         Name = f.FileName.ToString(),
-                        Hash = hash,
+                        Hash = hash.Value,
                         Size = f.Size
                     };
 
-                });
+                })).NotNull().ToArray();
 
             Utils.Log($"Found and hashed {indexed.Length} files");
             await indexed.ToJsonAsync(file, prettyPrint: true);

Wabbajack.Common/Extensions/EnumerableExt.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 
 namespace Wabbajack
 {
@@ -37,14 +38,21 @@ namespace Wabbajack
         /// <summary>
         /// Converts and filters a nullable enumerable to a non-nullable enumerable
         /// </summary>
-        public static IEnumerable<T> NotNull<T>(this IEnumerable<T?> e)
+        public static IEnumerable<T> NotNull<T>(this IEnumerable<T?> enumerable)
             where T : class
         {
             // Filter out nulls
-            return e.Where(e => e != null)
+            return enumerable.Where(e => e != null)
                 // Cast to non nullable type
                 .Select(e => e!);
         }
+        
+        public static IEnumerable<T> NotNull<T>(this IEnumerable<T?> enumerable) where T : struct
+        {
+            return enumerable
+                .Where(x => x.HasValue)
+                .Select(x => x!.Value);
+        }
 
         /// <summary>
         /// Selects items that are castable to the desired type

Wabbajack.Common/Hash.cs

@@ -274,17 +274,17 @@ namespace Wabbajack.Common
             WriteHashCache(file, hash);
         }
 
-        public static async Task<Hash> FileHashCachedAsync(this AbsolutePath file, bool nullOnIOError = false)
+        public static async Task<Hash?> FileHashCachedAsync(this AbsolutePath file)
         {
             if (TryGetHashCache(file, out var foundHash)) return foundHash;
 
-            var hash = await file.FileHashAsync(nullOnIOError);
-            if (hash != Hash.Empty)
-                WriteHashCache(file, hash);
+            var hash = await file.FileHashAsync();
+            if (hash != null && hash != Hash.Empty)
+                WriteHashCache(file, hash.Value);
             return hash;
         }
 
-        public static async Task<Hash> FileHashAsync(this AbsolutePath file, bool nullOnIOError = false)
+        public static async Task<Hash?> FileHashAsync(this AbsolutePath file)
         {
             try
             {
@@ -294,10 +294,10 @@ namespace Wabbajack.Common
                 var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(hs);
                 return new Hash(BitConverter.ToUInt64(value.Hash));
             }
-            catch (IOException)
+            catch (IOException e)
             {
-                if (nullOnIOError) return Hash.Empty;
-                throw;
+                Utils.Error(e, $"Unable to hash file {file}");
+                return null;
             }
         }
     }

Wabbajack.Lib/ACompiler.cs

@@ -308,7 +308,7 @@ namespace Wabbajack.Lib
                 ModList.ToJson(of);
 
             await ModListOutputFolder.Combine("sig")
-                .WriteAllBytesAsync((await ModListOutputFolder.Combine("modlist").FileHashAsync()).ToArray());
+                .WriteAllBytesAsync(((await ModListOutputFolder.Combine("modlist").FileHashAsync()) ?? Hash.Empty).ToArray());
 
             await ClientAPI.SendModListDefinition(ModList);
 
@@ -338,11 +338,18 @@ namespace Wabbajack.Lib
                 }
             }
 
-            Utils.Log("Exporting ModList metadata");
+            Utils.Log("Exporting Modlist metadata");
+            var outputFileHash = await ModListOutputFile.FileHashAsync();
+            if (outputFileHash == null)
+            {
+                Utils.Error("Unable to hash Modlist Output File");
+                return;
+            }
+            
             var metadata = new DownloadMetadata
             {
                 Size = ModListOutputFile.Size,
-                Hash = await ModListOutputFile.FileHashAsync(),
+                Hash = outputFileHash.Value,
                 NumberOfArchives = ModList.Archives.Count,
                 SizeOfArchives = ModList.Archives.Sum(a => a.Size),
                 NumberOfInstalledFiles = ModList.Directives.Count,

Wabbajack.Lib/AInstaller.cs

@@ -145,7 +145,7 @@ namespace Wabbajack.Lib
                                 await ClientAPI.GetVirusScanResult(toFile) == VirusScanner.Result.Malware)
                             {
                                 await toFile.DeleteAsync();
-                                Utils.ErrorThrow(new Exception($"Virus scan of patched executable reported possible malware: {toFile.ToString()} ({(long)await toFile.FileHashCachedAsync()})"));
+                                Utils.ErrorThrow(new Exception($"Virus scan of patched executable reported possible malware: {toFile.ToString()} ({(long)(await toFile.FileHashCachedAsync())!.Value})"));
                             }
                         }
                             break;
@@ -298,7 +298,8 @@ namespace Wabbajack.Lib
                 .OrderByDescending(e => e.Item2.LastModified)
                 .GroupBy(e => e.Item1)
                 .Select(e => e.First())
-                .Select(e => new KeyValuePair<Hash, AbsolutePath>(e.Item1, e.Item2)));
+                .Where(x => x.Item1 != null)
+                .Select(e => new KeyValuePair<Hash, AbsolutePath>(e.Item1!.Value, e.Item2)));
         }
 
         /// <summary>

Wabbajack.Lib/AuthorApi/Client.cs

@@ -58,7 +58,7 @@ namespace Wabbajack.Lib.AuthorApi
             {
                 OriginalFileName = path.FileName, 
                 Size = path.Size, 
-                Hash = await path.FileHashCachedAsync(),
+                Hash = await path.FileHashCachedAsync() ?? Hash.Empty,
                 Parts = await parts.PMap(queue, async part =>
                 {
                     progressFn("Hashing file parts", Percent.FactoryPutInRange(part.Index, parts.Length));

Wabbajack.Lib/ClientAPI.cs

@@ -242,8 +242,12 @@ using Wabbajack.Lib.Downloaders;
             Utils.Log($"Checking virus result for {path}");
 
             var hash = await path.FileHashAsync();
+            if (hash == null)
+            {
+                throw new Exception("Hash is null!");
+            }
 
-            using var result = await client.GetAsync($"{Consts.WabbajackBuildServerUri}virus_scan/{hash.ToHex()}", errorsAsExceptions: false);
+            using var result = await client.GetAsync($"{Consts.WabbajackBuildServerUri}virus_scan/{hash.Value.ToHex()}", errorsAsExceptions: false);
             if (result.StatusCode == HttpStatusCode.OK)
             {
                 var data = await result.Content.ReadAsStringAsync();

Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs

@@ -334,14 +334,15 @@ namespace Wabbajack.Lib.Downloaders
                     if (await newFile.State.Download(newFile, tmp.Path))
                     {
                         newFile.Size = tmp.Path.Size;
-                        newFile.Hash = await tmp.Path.FileHashAsync();
+                        var tmpHash = await tmp.Path.FileHashAsync();
+                        if (tmpHash == null) return default;
+                        newFile.Hash = tmpHash.Value;
                         return (newFile, tmp);
                     }
 
                     await tmp.DisposeAsync();
                 }
                 return default;
-
             }
 
             public override async Task<bool> ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState)

Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs

@@ -28,12 +28,13 @@ namespace Wabbajack.Lib.Downloaders
 
             var fp = filePath.Value;
             var hash = await fp.FileHashCachedAsync();
+            if (hash == null) return null;
 
             return new State(game.InstalledVersion)
             {
                 Game = game.Game, 
                 GameFile = (RelativePath)gameFile,
-                Hash = hash
+                Hash = hash.Value
             };
         }
 

Wabbajack.Lib/Downloaders/HTTPDownloader.cs

@@ -234,7 +234,9 @@ TOP:
                     return default;
                 }
 
-                newArchive.Hash = await tmpFile.Path.FileHashAsync();
+                var hash = await tmpFile.Path.FileHashAsync();
+                if (hash == null) return default;
+                newArchive.Hash = hash.Value;
                 newArchive.Size = tmpFile.Path.Size;
 
                 if (newArchive.Hash == a.Hash || a.Size > 2_500_000_000 || newArchive.Size > 2_500_000_000)

Wabbajack.Lib/Downloaders/NexusDownloader.cs

@@ -290,7 +290,9 @@ namespace Wabbajack.Lib.Downloaders
                 if (fastPath != default)
                 {
                     newArchive.Size = fastPath.Size;
-                    newArchive.Hash = await fastPath.FileHashAsync();
+                    var hash = await fastPath.FileHashAsync();
+                    if (hash == null) return default;
+                    newArchive.Hash = hash.Value;
                     return (newArchive, new TempFile());
                 }
 
@@ -300,7 +302,9 @@ namespace Wabbajack.Lib.Downloaders
                 await newArchive.State.Download(newArchive, tempFile.Path);
 
                 newArchive.Size = tempFile.Path.Size;
-                newArchive.Hash = await tempFile.Path.FileHashAsync();
+                var newArchiveHash = await tempFile.Path.FileHashAsync();
+                if (newArchiveHash == null) return default;
+                newArchive.Hash = newArchiveHash.Value;
 
                 Utils.Log($"Possible upgrade {newArchive.State.PrimaryKeyString} downloaded");
 

Wabbajack.Lib/MO2Compiler.cs

@@ -382,11 +382,15 @@ namespace Wabbajack.Lib
                 var source = DownloadsPath.Combine(a.Name + Consts.MetaFileExtension);
                 var ini = a.State.GetMetaIniString();
                 var (id, fullPath) = await IncludeString(ini);
+                var hash = await fullPath.FileHashAsync();
+
+                if (hash == null) return;
+                
                 InstallDirectives.Add(new ArchiveMeta
                 {
                     SourceDataID = id,
                     Size = fullPath.Size,
-                    Hash = await fullPath.FileHashAsync(),
+                    Hash = hash.Value,
                     To = source.FileName
                 });
             });

Wabbajack.Lib/MO2Installer.cs

@@ -294,7 +294,7 @@ namespace Wabbajack.Lib
                 var hash = await gameFile.FileHashAsync();
                 if (hash != esm.SourceESMHash)
                 {
-                    Utils.ErrorThrow(new InvalidGameESMError(esm, hash, gameFile));
+                    Utils.ErrorThrow(new InvalidGameESMError(esm, hash ?? Hash.Empty, gameFile));
                 }
             }
         }

Wabbajack.Lib/ModListRegistry/ModListMetadata.cs

@@ -128,7 +128,7 @@ namespace Wabbajack.Lib.ModListRegistry
             {
                 return true;
             }
-            return DownloadMetadata.Hash != await modlistPath.FileHashCachedAsync(true);
+            return DownloadMetadata.Hash != await modlistPath.FileHashCachedAsync();
         }
     }
 

Wabbajack.Server.Test/ABuildServerSystemTest.cs

@@ -211,9 +211,9 @@ namespace Wabbajack.BuildServer.Test
 
             ModListData = new ModList();
             ModListData.Archives.Add(
-                new Archive(new HTTPDownloader.State(MakeURL(modFileName.ToString())))
+                new Archive(new HTTPDownloader.State(MakeURL(modFileName)))
                 {
-                    Hash = await test_archive_path.FileHashAsync(),
+                    Hash = await test_archive_path.FileHashAsync() ?? Hash.Empty,
                     Name = "test_archive",
                     Size = test_archive_path.Size,
                 });
@@ -237,7 +237,7 @@ namespace Wabbajack.BuildServer.Test
                     Description = "A test",
                     DownloadMetadata = new DownloadMetadata
                     {
-                        Hash = await modListPath.FileHashAsync(), 
+                        Hash = await modListPath.FileHashAsync() ?? Hash.Empty, 
                         Size = modListPath.Size
                     },
                     Links = new ModlistMetadata.LinksObject

Wabbajack.Server.Test/ArchiveMaintainerTests.cs

@@ -26,7 +26,9 @@ namespace Wabbajack.BuildServer.Test
             var hash = await tf.Path.FileHashAsync();
             await maintainer.Ingest(tf.Path);
             
-            Assert.True(maintainer.TryGetPath(hash, out var found));
+            Assert.NotNull(hash);
+            
+            Assert.True(maintainer.TryGetPath(hash!.Value, out var found));
             Assert.Equal(await tf2.Path.ReadAllBytesAsync(), await found.ReadAllBytesAsync());
         }
         
@@ -40,10 +42,12 @@ namespace Wabbajack.BuildServer.Test
             await tf.Path.WriteAllBytesAsync(RandomData(1024));
             var hash = await tf.Path.FileHashAsync();
             
-            await tf.Path.CopyToAsync(Fixture.ServerArchivesFolder.Combine(hash.ToHex()));
+            Assert.NotNull(hash);
+            
+            await tf.Path.CopyToAsync(Fixture.ServerArchivesFolder.Combine(hash!.Value.ToHex()));
             maintainer.Start();
 
-            Assert.True(maintainer.TryGetPath(hash, out var found));
+            Assert.True(maintainer.TryGetPath(hash!.Value, out var found));
         }
     }
 }

Wabbajack.Server.Test/MirroredFilesTests.cs

@@ -24,9 +24,10 @@ namespace Wabbajack.Server.Test
             var file = new TempFile();
             await file.Path.WriteAllBytesAsync(RandomData(1024 * 1024 * 6));
             var dataHash = await file.Path.FileHashAsync();
+            Assert.NotNull(dataHash);
 
             await Fixture.GetService<ArchiveMaintainer>().Ingest(file.Path);
-            Assert.True(Fixture.GetService<ArchiveMaintainer>().HaveArchive(dataHash));
+            Assert.True(Fixture.GetService<ArchiveMaintainer>().HaveArchive(dataHash!.Value));
 
             var sql = Fixture.GetService<SqlService>();
             
@@ -34,7 +35,7 @@ namespace Wabbajack.Server.Test
             {
                 Created = DateTime.UtcNow,
                 Rationale = "Test File", 
-                Hash = dataHash
+                Hash = dataHash!.Value
             });
 
             var uploader = Fixture.GetService<MirrorUploader>();
@@ -43,7 +44,7 @@ namespace Wabbajack.Server.Test
             
             var archive = new Archive(new HTTPDownloader.State(MakeURL(dataHash.ToString())))
             {
-                Hash = dataHash,
+                Hash = dataHash!.Value,
                 Size = file.Path.Size
             };
             

Wabbajack.Server/Controllers/GameFiles.cs

@@ -1,4 +1,4 @@
-using System;
+#nullable enable
 using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
@@ -40,17 +40,19 @@ namespace Wabbajack.BuildServer.Controllers
             _logger.Log(LogLevel.Information, $"Found {files.Count} game files");
             
             using var queue = new WorkQueue();
-            var hashed = await files.PMap(queue, async pair =>
+            var hashed = (await files.PMap(queue, async pair =>
             {
                 var hash = await pair.File.FileHashCachedAsync();
+                if (hash == null) return null;
+                
                 return await _sql.GetOrEnqueueArchive(new Archive(new GameFileSourceDownloader.State
                 {
                     Game = pair.Game.Game,
                     GameFile = pair.File.RelativeTo(pair.Game.GameLocation()),
                     GameVersion = pair.Game.InstalledVersion,
-                    Hash = hash
-                }) {Name = pair.File.FileName.ToString(), Size = pair.File.Size, Hash = hash});
-            });
+                    Hash = hash.Value
+                }) {Name = pair.File.FileName.ToString(), Size = pair.File.Size, Hash = hash.Value});
+            })).NotNull();
 
             await _quickSync.Notify<ArchiveDownloader>();
             return Ok(hashed);

Wabbajack.Server/Services/ArchiveDownloader.cs

@@ -83,7 +83,7 @@ namespace Wabbajack.Server.Services
 
                     var hash = await tempPath.Path.FileHashAsync();
 
-                    if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)
+                    if (hash == null || (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash))
                     {
                         _logger.Log(LogLevel.Warning,
                             $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}");
@@ -98,7 +98,7 @@ namespace Wabbajack.Server.Services
                         continue;
                     }
 
-                    nextDownload.Archive.Hash = hash;
+                    nextDownload.Archive.Hash = hash.Value;
                     nextDownload.Archive.Size = tempPath.Path.Size;
 
                     _logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}");

Wabbajack.Server/Services/ArchiveMaintainer.cs

@@ -36,19 +36,20 @@ namespace Wabbajack.Server.Services
             return _settings.ArchivePath.Combine(hash.ToHex());
         }
 
-        public async Task<AbsolutePath> Ingest(AbsolutePath file)
+        public async Task Ingest(AbsolutePath file)
         {
             var hash = await file.FileHashAsync();
-            var path = ArchivePath(hash);
-            if (HaveArchive(hash))
+            if (hash == null) return;
+            
+            var path = ArchivePath(hash.Value);
+            if (HaveArchive(hash.Value))
             {
                 await file.DeleteAsync();
-                return path;
+                return;
             }
             
-            var newPath = _settings.ArchivePath.Combine(hash.ToHex());
+            var newPath = _settings.ArchivePath.Combine(hash.Value.ToHex());
             await file.MoveToAsync(newPath);
-            return path;
         }
 
         public bool HaveArchive(Hash hash)

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -190,10 +190,10 @@ namespace Wabbajack.VirtualFileSystem
         public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile,
             IPath relPath, int depth = 0)
         {
-            Hash hash = default;
+            Hash hash;
             if (extractedFile is NativeFileStreamFactory)
             {
-                hash = await ((AbsolutePath)extractedFile.Name).FileHashCachedAsync();
+                hash = await ((AbsolutePath)extractedFile.Name).FileHashCachedAsync() ?? Hash.Empty;
             } 
             else
             {