WIP, converted Hashes to a Hash struct

Wabbajack.BuildServer/Controllers/IndexedFiles.cs

@@ -38,7 +38,7 @@ namespace Wabbajack.BuildServer.Controllers
         [Route("{xxHashAsBase64}/meta.ini")]
         public async Task<IActionResult> GetFileMeta(string xxHashAsBase64)
         {
-            var id = xxHashAsBase64.FromHex().ToBase64();
+            var id = Hash.FromHex(xxHashAsBase64);
             var state = await Db.DownloadStates.AsQueryable()
                 .Where(d => d.Hash == id && d.IsValid)
                 .OrderByDescending(d => d.LastValidationTime)
@@ -51,48 +51,6 @@ namespace Wabbajack.BuildServer.Controllers
             return Ok(string.Join("\r\n", state.FirstOrDefault().State.GetMetaIni()));
         }
 
-        [Authorize]
-        [HttpDelete]
-        [Route("/indexed_files/nexus/{Game}/mod/{ModId}")]
-        public async Task<IActionResult> PurgeBySHA256(string Game, string ModId)
-        {
-            var files = await Db.DownloadStates.AsQueryable().Where(d => d.State is NexusDownloader.State &&
-                                                                   ((NexusDownloader.State)d.State).GameName == Game &&
-                                                                   ((NexusDownloader.State)d.State).ModID == ModId)
-                .ToListAsync();
-
-            async Task DeleteParentsOf(HashSet<string> acc, string hash)
-            {
-                var parents = await Db.IndexedFiles.AsQueryable().Where(f => f.Children.Any(c => c.Hash == hash))
-                    .ToListAsync();
-
-                foreach (var parent in parents)
-                    await DeleteThisAndAllChildren(acc, parent.Hash);
-            }
-            
-            async Task DeleteThisAndAllChildren(HashSet<string> acc, string hash)
-            {
-                acc.Add(hash);
-                var children = await Db.IndexedFiles.AsQueryable().Where(f => f.Hash == hash).FirstOrDefaultAsync();
-                if (children == null) return;
-                foreach (var child in children.Children)
-                {
-                    await DeleteThisAndAllChildren(acc, child.Hash);
-                }
-
-            }
-            
-            var acc = new HashSet<string>();
-            foreach (var file in files)
-                await DeleteThisAndAllChildren(acc, file.Hash);
-
-            var acclst = acc.ToList();
-            await Db.DownloadStates.DeleteManyAsync(d => acc.Contains(d.Hash));
-            await Db.IndexedFiles.DeleteManyAsync(d => acc.Contains(d.Hash));
-
-            return Ok(acc.ToList());
-        }
-
         [HttpPost]
         [Route("notify")]
         public async Task<IActionResult> Notify()

Wabbajack.BuildServer/Controllers/ModlistUpdater.cs

@@ -43,7 +43,7 @@ namespace Wabbajack.BuildServer.Controllers
         {
             var lists = await Db.ModListStatus.AsQueryable().ToListAsync();
             var archives = lists.SelectMany(list => list.DetailedStatus.Archives)
-                .Select(a => a.Archive.Hash.FromBase64().ToHex())
+                .Select(a => a.Archive.Hash.ToHex())
                 .ToHashSet();
 
             var toDelete = new List<string>();
@@ -89,9 +89,9 @@ namespace Wabbajack.BuildServer.Controllers
         [Route("/alternative/{xxHash}")]
         public async Task<IActionResult> GetAlternative(string xxHash)
         {
-            var startingHash = xxHash.FromHex().ToBase64();
+            var startingHash = Hash.FromHex(xxHash);
             Utils.Log($"Alternative requested for {startingHash}");
-            await Metric("requested_upgrade", startingHash);
+            await Metric("requested_upgrade", startingHash.ToString());
 
             var state = await Db.DownloadStates.AsQueryable()
                 .Where(s => s.Hash == startingHash)
@@ -110,7 +110,7 @@ namespace Wabbajack.BuildServer.Controllers
             if (mod_files.SelectMany(f => f.Data.files)
                 .Any(f => f.category_name != null && f.file_id.ToString() == nexusState.FileID))
             {
-                await Metric("not_required_upgrade", startingHash);
+                await Metric("not_required_upgrade", startingHash.ToString());
                 return BadRequest("Upgrade Not Required");
             }
 
@@ -122,7 +122,7 @@ namespace Wabbajack.BuildServer.Controllers
             }
 
             Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}");
-            if (newArchive.Hash == null)
+            if (newArchive.Hash == Hash.Empty)
             {
                 Db.Jobs.InsertOne(new Job
                 {
@@ -160,7 +160,7 @@ namespace Wabbajack.BuildServer.Controllers
             return Ok(newArchive.ToJSON());
         }
 
-        private async Task<Archive> FindAlternatives(NexusDownloader.State state, string srcHash)
+        private async Task<Archive> FindAlternatives(NexusDownloader.State state, Hash srcHash)
         {
             var origSize = AlphaFile.GetSize(_settings.PathForArchive(srcHash));
             var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());

Wabbajack.BuildServer/Controllers/UploadedFiles.cs

@@ -121,7 +121,7 @@ namespace Wabbajack.BuildServer.Controllers
         [Route("upload_file/{Key}/finish/{xxHashAsHex}")]
         public async Task<IActionResult> UploadFileFinish(string Key, string xxHashAsHex)
         {
-            var expectedHash = xxHashAsHex.FromHex().ToBase64();
+            var expectedHash = Hash.FromHex(xxHashAsHex);
             var user = User.FindFirstValue(ClaimTypes.Name);
             if (!Key.All(a => HexChars.Contains(a)))
                 return BadRequest("NOT A VALID FILENAME");

Wabbajack.BuildServer/Extensions.cs

@@ -47,13 +47,13 @@ namespace Wabbajack.BuildServer
             return authenticationBuilder.AddScheme<ApiKeyAuthenticationOptions, ApiKeyAuthenticationHandler>(ApiKeyAuthenticationOptions.DefaultScheme, options);
         }
         
-        private static ConcurrentDictionary<string, string> PathForArchiveHash = new ConcurrentDictionary<string, string>();
+        private static readonly ConcurrentDictionary<Hash, string> PathForArchiveHash = new ConcurrentDictionary<Hash, string>();
-        public static string PathForArchive(this AppSettings settings, string hash)
+        public static string PathForArchive(this AppSettings settings, Hash hash)
         {
             if (PathForArchiveHash.TryGetValue(hash, out string result))
                 return result;
             
-            var hexHash = hash.FromBase64().ToHex();
+            var hexHash = hash.ToHex();
 
             var ends = "_" + hexHash + "_";
             var file = Directory.EnumerateFiles(settings.ArchiveDir, DirectoryEnumerationOptions.Files,

Wabbajack.BuildServer/Models/DownloadState.cs

@@ -1,9 +1,6 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using MongoDB.Bson.Serialization.Attributes;
+using Wabbajack.Common;
 using Wabbajack.Lib.Downloaders;
 
 namespace Wabbajack.BuildServer.Models
@@ -12,7 +9,7 @@ namespace Wabbajack.BuildServer.Models
     {
         [BsonId]
         public string Key { get; set; }
-        public string Hash { get; set; }
+        public Hash Hash { get; set; }
         
         public AbstractDownloadState State { get; set; }
 

Wabbajack.BuildServer/Models/IndexedFile.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using MongoDB.Bson.Serialization.Attributes;
+using Wabbajack.Common;
 using Wabbajack.VirtualFileSystem;
 
 namespace Wabbajack.BuildServer.Models
@@ -11,7 +12,7 @@ namespace Wabbajack.BuildServer.Models
     public class IndexedFile
     {
         [BsonId]
-        public string Hash { get; set; }
+        public Hash Hash { get; set; }
         public string SHA256 { get; set; }
         public string SHA1 { get; set; }
         public string MD5 { get; set; }
@@ -25,6 +26,6 @@ namespace Wabbajack.BuildServer.Models
     {
         public string Name;
         public string Extension;
-        public string Hash;
+        public Hash Hash;
     }
 }

Wabbajack.BuildServer/Models/Jobs/IndexJob.cs

@@ -56,7 +56,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
                 });
                 
                 var to_path = Path.Combine(settings.ArchiveDir,
-                    $"{Path.GetFileName(fileName)}_{archive.Hash.FromBase64().ToHex()}_{Path.GetExtension(fileName)}");
+                    $"{Path.GetFileName(fileName)}_{archive.Hash.ToHex()}_{Path.GetExtension(fileName)}");
                 if (File.Exists(to_path))
                     File.Delete(downloadDest);
                 else

Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs

@@ -27,7 +27,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
             using (var queue = new WorkQueue())
             {
                             
-                var whitelists = new ValidateModlist(queue);
+                var whitelists = new ValidateModlist();
                 await whitelists.LoadListsFromGithub();
                 
                 foreach (var list in modlists)

Wabbajack.BuildServer/Models/PatchArchive.cs

@@ -18,7 +18,7 @@ namespace Wabbajack.BuildServer.Models
     public class PatchArchive : AJobPayload
     {
         public override string Description => "Create a archive update patch";
-        public string Src { get; set; }
+        public Hash Src { get; set; }
         public string DestPK { get; set; }
         public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
         {
@@ -56,7 +56,7 @@ namespace Wabbajack.BuildServer.Models
                 await client.ConnectAsync();
                 try
                 {
-                    await client.UploadAsync(fs, $"updates/{Src.FromBase64().ToHex()}_{destHash.FromBase64().ToHex()}", progress: new UploadToCDN.Progress(cdnPath));
+                    await client.UploadAsync(fs, $"updates/{Src.ToHex()}_{destHash.ToHex()}", progress: new UploadToCDN.Progress(cdnPath));
                 }
                 catch (Exception ex)
                 {
@@ -72,9 +72,9 @@ namespace Wabbajack.BuildServer.Models
             
         }
 
-        public static string CdnPath(string srcHash, string destHash)
+        public static string CdnPath(Hash srcHash, Hash destHash)
         {
-            return $"updates/{srcHash.FromBase64().ToHex()}_{destHash.FromBase64().ToHex()}";
+            return $"updates/{srcHash.ToHex()}_{destHash.ToHex()}";
         }
     }
 }

Wabbajack.BuildServer/Models/Sql/SqlService.cs

@@ -49,10 +49,9 @@ namespace Wabbajack.BuildServer.Model.Models
         
         private static void IngestFile(VirtualFile root, ICollection<IndexedFile> files, ICollection<ArchiveContent> contents)
         {
-            var hash = BitConverter.ToInt64(root.Hash.FromBase64());
             files.Add(new IndexedFile
             {
-                Hash = hash,
+                Hash = (long)root.Hash,
                 Sha256 = root.ExtendedHashes.SHA256.FromHex(),
                 Sha1 = root.ExtendedHashes.SHA1.FromHex(),
                 Md5 = root.ExtendedHashes.MD5.FromHex(),
@@ -66,22 +65,21 @@ namespace Wabbajack.BuildServer.Model.Models
             {
                 IngestFile(child, files, contents);
 
-                var child_hash = BitConverter.ToInt64(child.Hash.FromBase64());
                 contents.Add(new ArchiveContent
                 {
-                    Parent = hash,
+                    Parent = (long)root.Hash,
-                    Child = child_hash,
+                    Child = (long)child.Hash,
                     Path = child.Name
                 });
             }
 
         }
 
-        public async Task<bool> HaveIndexdFile(string hash)
+        public async Task<bool> HaveIndexdFile(Hash hash)
         {
             await using var conn = await Open();
             var row = await conn.QueryAsync(@"SELECT * FROM IndexedFile WHERE Hash = @Hash",
-                new {Hash = BitConverter.ToInt64(hash.FromBase64())});
+                new {Hash = (long)hash});
             return row.Any();
         }
         
@@ -123,7 +121,7 @@ namespace Wabbajack.BuildServer.Model.Models
                     return children.Select(f => new IndexedVirtualFile
                     {
                         Name = f.Path,
-                        Hash = BitConverter.GetBytes(f.Hash).ToBase64(),
+                        Hash = Hash.FromLong(f.Hash),
                         Size = f.Size,
                         Children = Build(f.Hash)
                     }).ToList();

Wabbajack.BuildServer/Models/UploadedFile.cs

@@ -13,7 +13,7 @@ namespace Wabbajack.BuildServer.Models
         public string Id { get; set; }
         public string Name { get; set; }
         public long Size { get; set; }
-        public string Hash { get; set; }
+        public Hash Hash { get; set; }
         public string Uploader { get; set; }
         public DateTime UploadDate { get; set; } = DateTime.UtcNow;
         

Wabbajack.CLI/Verbs/Validate.cs

@@ -57,7 +57,7 @@ namespace Wabbajack.CLI.Verbs
 
             try
             {
-                ValidateModlist.RunValidation(queue, modlist).RunSynchronously();
+                ValidateModlist.RunValidation(modlist).RunSynchronously();
             }
             catch (Exception e)
             {

Wabbajack.Common/Hash.cs

@@ -0,0 +1,198 @@
+using System;
+using System.Collections.Generic;
+using System.Data.HashFunction.xxHash;
+using System.IO;
+using System.Security.Cryptography;
+using System.Text;
+using System.Threading.Tasks;
+using File = Alphaleonis.Win32.Filesystem.File;
+using Path = Alphaleonis.Win32.Filesystem.Path;
+
+namespace Wabbajack.Common
+{
+    
+    /// <summary>
+    /// Struct representing a xxHash64 value. It's a struct with a ulong in it, but wrapped so we don't confuse
+    /// it with other longs in the system.
+    /// </summary>
+    public struct Hash
+    {
+        private readonly ulong _code;
+        public Hash(ulong code = 0)
+        {
+            _code = code;
+        }
+
+        public override string ToString()
+        {
+            return BitConverter.GetBytes(_code).ToBase64();
+        }
+
+        public override bool Equals(object? obj)
+        {
+            if (obj is Hash h)
+                return h._code == _code;
+            return false;
+        }
+
+        public override int GetHashCode()
+        {
+            return (int)(_code >> 32) ^ (int)_code;
+        }
+
+        public static bool operator ==(Hash a, Hash b)
+        {
+            return a._code == b._code;
+        }
+
+        public static bool operator !=(Hash a, Hash b)
+        {
+            return !(a == b);
+        }
+        
+        public static explicit operator ulong(Hash a)
+        {
+            return a._code;
+        }
+        
+        public static explicit operator long(Hash a)
+        {
+            return BitConverter.ToInt64(BitConverter.GetBytes(a._code));
+        }
+
+        public string ToHex()
+        {
+            return BitConverter.GetBytes(_code).ToHex();
+        }
+
+        public string ToBase64()
+        {
+            return BitConverter.GetBytes(_code).ToBase64();
+        }
+
+        public static Hash FromBase64(string hash)
+        {
+            return new Hash(BitConverter.ToUInt64(hash.FromBase64()));
+        }
+
+        public static Hash Empty = new Hash();
+
+        public static Hash FromLong(in long argHash)
+        {
+            return new Hash(BitConverter.ToUInt64(BitConverter.GetBytes(argHash)));
+        }
+
+        public static Hash FromHex(string xxHashAsHex)
+        {
+            return new Hash(BitConverter.ToUInt64(xxHashAsHex.FromHex()));
+        }
+    }
+    
+    public static partial class Utils
+    {
+        public static Hash ReadHash(this BinaryReader br)
+        {
+            return new Hash(br.ReadUInt64());
+        }
+
+        public static void Write(this BinaryWriter bw, Hash hash)
+        {
+            bw.Write((ulong)hash);
+        }
+
+        public static string StringSha256Hex(this string s)
+        {
+            var sha = new SHA256Managed();
+            using (var o = new CryptoStream(Stream.Null, sha, CryptoStreamMode.Write))
+            {
+                using var i = new MemoryStream(Encoding.UTF8.GetBytes(s));
+                i.CopyTo(o);
+            }
+            return sha.Hash.ToHex();
+        }
+
+        public static Hash FileHash(this string file, bool nullOnIoError = false)
+        {
+            try
+            {
+                using var fs = File.OpenRead(file);
+                var config = new xxHashConfig {HashSizeInBits = 64};
+                using var f = new StatusFileStream(fs, $"Hashing {Path.GetFileName(file)}");
+                return new Hash(BitConverter.ToUInt64(xxHashFactory.Instance.Create(config).ComputeHash(f).Hash));
+            }
+            catch (IOException)
+            {
+                if (nullOnIoError) return Hash.Empty;
+                throw;
+            }
+        }
+        
+        public static Hash FileHashCached(this string file, bool nullOnIoError = false)
+        {
+            if (TryGetHashCache(file, out var foundHash)) return foundHash;
+
+            var hash = file.FileHash(nullOnIoError);
+            if (hash != Hash.Empty) 
+                WriteHashCache(file, hash);
+            return hash;
+        }
+
+        public static bool TryGetHashCache(string file, out Hash hash)
+        {
+            var hashFile = file + Consts.HashFileExtension;
+            hash = Hash.Empty; 
+            if (!File.Exists(hashFile)) return false;
+            
+            if (File.GetSize(hashFile) != 20) return false;
+
+            using var fs = File.OpenRead(hashFile);
+            using var br = new BinaryReader(fs);
+            var version = br.ReadUInt32();
+            if (version != HashCacheVersion) return false;
+
+            var lastModified = br.ReadUInt64();
+            if (lastModified != File.GetLastWriteTimeUtc(file).AsUnixTime()) return false;
+            hash = new Hash(br.ReadUInt64());
+            return true;
+        }
+
+
+        private const uint HashCacheVersion = 0x01;
+        private static void WriteHashCache(string file, Hash hash)
+        {
+            using var fs = File.Create(file + Consts.HashFileExtension);
+            using var bw = new BinaryWriter(fs);
+            bw.Write(HashCacheVersion);
+            var lastModified = File.GetLastWriteTimeUtc(file).AsUnixTime();
+            bw.Write(lastModified);
+            bw.Write((ulong)hash);
+        }
+
+        public static async Task<Hash> FileHashCachedAsync(this string file, bool nullOnIOError = false)
+        {
+            if (TryGetHashCache(file, out var foundHash)) return foundHash;
+
+            var hash = await file.FileHashAsync(nullOnIOError);
+            if (hash != Hash.Empty) 
+                WriteHashCache(file, hash);
+            return hash;
+        }
+
+        public static async Task<Hash> FileHashAsync(this string file, bool nullOnIOError = false)
+        {
+            try
+            {
+                await using var fs = File.OpenRead(file);
+                var config = new xxHashConfig {HashSizeInBits = 64};
+                var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(fs);
+                return new Hash(BitConverter.ToUInt64(value.Hash));
+            }
+            catch (IOException)
+            {
+                if (nullOnIOError) return Hash.Empty;
+                throw;
+            }
+        }
+
+    }
+}

Wabbajack.Common/Serialization/Deserializer.cs

@@ -1,9 +0,0 @@
-using System.IO;
-
-namespace Wabbajack.Common.Serialization
-{
-    public class Deserializer
-    {
-        public BinaryReader Reader { get; }
-    }
-}

Wabbajack.Common/Serialization/IHandler.cs

@@ -1,9 +0,0 @@
-namespace Wabbajack.Common.Serialization
-{
-    public interface IHandler
-    {
-        public void Write<T>(Serializer serializer, T data);
-        public T Read<T>(Deserializer deserialiser);
-
-    }
-}

Wabbajack.Common/Serialization/PrimitiveHandlers.cs

@@ -1,34 +0,0 @@
-using System;
-namespace Wabbajack.Common.Serialization {
-public class UInt32Handler : IHandler {
-
-    public void Write<T>(Serializer serializer, UInt32 data)
-    {
-        serializer.Writer.Write(data);
-    }
-
-    public T Read<T>(Deserializer deserializer)
-    {
-        return deserializer.Reader.ReadUInt32();
-    }
-
-
-}
-
-public class Int32Handler : IHandler {
-
-    public void Write<T>(Serializer serializer, Int32 data)
-    {
-        serializer.Writer.Write(data);
-    }
-
-    public T Read<T>(Deserializer deserializer)
-    {
-        return deserializer.Reader.ReadInt32();
-    }
-
-
-}
-
-
-}

Wabbajack.Common/Serialization/PrimitiveHandlers.tt

@@ -1,34 +0,0 @@
-<#@ template language="C#" #>
-<#@ import namespace="System.Text" #>
-<#@ import namespace="System.Collections.Generic" #>
-using System;
-namespace Wabbajack.Common.Serialization {
-<#
-    var types = new List<(Type, string)>()
-    {
-        (typeof(UInt32), "UInt32"),
-        (typeof(Int32), "Int32")
-    };
-
-
-    foreach (var type in types)
-    {
-#>
-public class <#=type.Item2#>Handler : IHandler {
-
-    public void Write<T>(Serializer serializer, <#=type.Item2#> data)
-    {
-        serializer.Writer.Write(data);
-    }
-
-    public T Read<T>(Deserializer deserializer)
-    {
-        return deserializer.Reader.Read<#=type.Item2#>();
-    }
-
-
-}
-
-<#  } #>
-
-}

Wabbajack.Common/Serialization/Serializer.cs

@@ -1,53 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Threading.Tasks;
-
-namespace Wabbajack.Common.Serialization
-{
-    public class Serializer
-    {
-        private Dictionary<string, int> _internedStrings = new Dictionary<string, int>();
-        private Dictionary<Type, HandlerRecord> _handlers = new Dictionary<Type, HandlerRecord>();
-        public BinaryWriter Writer { get; }
-
-        public Serializer(BinaryWriter bw)
-        {
-            Writer = bw;
-        }
-
-        public void RegisterWriteHandler<T>(string name, IHandler handler)
-        {
-            _handlers.Add(typeof(T), new HandlerRecord
-            {
-                TypeName = name,
-                TypeId = Intern(name),
-                Handler = handler
-            });
-        }
-
-        public async Task Write<T>(BinaryWriter bw, T data)
-        {
-            var handler = _handlers[typeof(T)];
-            handler.Handler.Write<T>(this, data);
-        }
-
-
-        private int Intern(string s)
-        {
-            if (_internedStrings.TryGetValue(s, out var idx))
-                return idx;
-            idx = _internedStrings.Count;
-            _internedStrings[s] = idx;
-            return idx;
-        }
-    }
-
-    class HandlerRecord
-    {
-        public string TypeName;
-        public int TypeId;
-        public IHandler Handler;
-
-    }
-}

Wabbajack.Common/Utils.cs

@@ -32,7 +32,7 @@ using Path = Alphaleonis.Win32.Filesystem.Path;
 
 namespace Wabbajack.Common
 {
-    public static class Utils
+    public static partial class Utils
     {
         public static bool IsMO2Running(string mo2Path)
         {
@@ -206,135 +206,6 @@ namespace Wabbajack.Common
             }
         }
 
-        /// <summary>
-        ///     MurMur3 hashes the file pointed to by this string
-        /// </summary>
-        /// <param name="file"></param>
-        /// <returns></returns>
-        public static string FileSHA256(this string file)
-        {
-            var sha = new SHA256Managed();
-            using (var o = new CryptoStream(Stream.Null, sha, CryptoStreamMode.Write))
-            {
-                using (var i = File.OpenRead(file))
-                {
-                    i.CopyToWithStatus(new FileInfo(file).Length, o, $"Hashing {Path.GetFileName(file)}");
-                }
-            }
-
-            return sha.Hash.ToBase64();
-        }
-
-        public static string StringSHA256Hex(this string s)
-        {
-            var sha = new SHA256Managed();
-            using (var o = new CryptoStream(Stream.Null, sha, CryptoStreamMode.Write))
-            {
-                using var i = new MemoryStream(Encoding.UTF8.GetBytes(s));
-                i.CopyTo(o);
-            }
-
-            return sha.Hash.ToHex();
-        }
-
-        public static string FileHash(this string file, bool nullOnIOError = false)
-        {
-            try
-            {
-                var hash = new xxHashConfig();
-                hash.HashSizeInBits = 64;
-                hash.Seed = 0x42;
-                using (var fs = File.OpenRead(file))
-                {
-                    var config = new xxHashConfig();
-                    config.HashSizeInBits = 64;
-                    using (var f = new StatusFileStream(fs, $"Hashing {Path.GetFileName(file)}"))    
-                    {
-                        var value = xxHashFactory.Instance.Create(config).ComputeHash(f);
-                        return value.AsBase64String();
-                    }
-                }
-            }
-            catch (IOException ex)
-            {
-                if (nullOnIOError) return null;
-                throw ex;
-            }
-        }
-        
-        public static string FileHashCached(this string file, bool nullOnIOError = false)
-        {
-            if (TryGetHashCache(file, out var foundHash)) return foundHash;
-
-            var hash = file.FileHash(nullOnIOError);
-            if (hash != null) 
-                WriteHashCache(file, hash);
-            return hash;
-        }
-
-        public static bool TryGetHashCache(string file, out string hash)
-        {
-            var hashFile = file + Consts.HashFileExtension;
-            hash = null; 
-            if (!File.Exists(hashFile)) return false;
-            
-            if (File.GetSize(hashFile) != 20) return false;
-
-            using var fs = File.OpenRead(hashFile);
-            using var br = new BinaryReader(fs);
-            var version = br.ReadUInt32();
-            if (version != HashCacheVersion) return false;
-
-            var lastModified = br.ReadUInt64();
-            if (lastModified != File.GetLastWriteTimeUtc(file).AsUnixTime()) return false;
-            hash = BitConverter.GetBytes(br.ReadUInt64()).ToBase64();
-            return true;
-        }
-
-
-        private const uint HashCacheVersion = 0x01;
-        private static void WriteHashCache(string file, string hash)
-        {
-            using var fs = File.Create(file + Consts.HashFileExtension);
-            using var bw = new BinaryWriter(fs);
-            bw.Write(HashCacheVersion);
-            var lastModified = File.GetLastWriteTimeUtc(file).AsUnixTime();
-            bw.Write(lastModified);
-            bw.Write(BitConverter.ToUInt64(hash.FromBase64()));
-        }
-
-        public static async Task<string> FileHashCachedAsync(this string file, bool nullOnIOError = false)
-        {
-            if (TryGetHashCache(file, out var foundHash)) return foundHash;
-
-            var hash = await file.FileHashAsync(nullOnIOError);
-            if (hash != null) 
-                WriteHashCache(file, hash);
-            return hash;
-        }
-
-        public static async Task<string> FileHashAsync(this string file, bool nullOnIOError = false)
-        {
-            try
-            {
-                var hash = new xxHashConfig();
-                hash.HashSizeInBits = 64;
-                hash.Seed = 0x42;
-                using (var fs = File.OpenRead(file))
-                {
-                    var config = new xxHashConfig();
-                    config.HashSizeInBits = 64;
-                    var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(fs);
-                    return value.AsBase64String();
-                }
-            }
-            catch (IOException ex)
-            {
-                if (nullOnIOError) return null;
-                throw ex;
-            }
-        }
-
         public static void CopyToWithStatus(this Stream istream, long maxSize, Stream ostream, string status)
         {
             var buffer = new byte[1024 * 64];
@@ -980,7 +851,7 @@ namespace Wabbajack.Common
             }
         }
 
-        public static async Task CreatePatch(FileStream srcStream, string srcHash, FileStream destStream, string destHash,
+        public static async Task CreatePatch(FileStream srcStream, Hash srcHash, FileStream destStream, Hash destHash,
             FileStream patchStream)
         {
             await using var sigFile = new TempStream();
@@ -996,7 +867,7 @@ namespace Wabbajack.Common
             
             try
             {
-                var cacheFile = Path.Combine(Consts.PatchCacheFolder, $"{srcHash.FromBase64().ToHex()}_{srcHash.FromBase64().ToHex()}.patch");
+                var cacheFile = Path.Combine(Consts.PatchCacheFolder, $"{srcHash.ToHex()}_{destHash.ToHex()}.patch");
                 if (!Directory.Exists(Consts.PatchCacheFolder))
                     Directory.CreateDirectory(Consts.PatchCacheFolder);
 
@@ -1009,10 +880,10 @@ namespace Wabbajack.Common
             }
         }
 
-        public static bool TryGetPatch(string foundHash, string fileHash, out byte[] ePatch)
+        public static bool TryGetPatch(Hash foundHash, Hash fileHash, out byte[] ePatch)
         {
             var patchName = Path.Combine(Consts.PatchCacheFolder,
-                $"{foundHash.FromBase64().ToHex()}_{fileHash.FromBase64().ToHex()}.patch");
+                $"{foundHash.ToHex()}_{fileHash.ToHex()}.patch");
             if (File.Exists(patchName))
             {
                 ePatch = File.ReadAllBytes(patchName);

Wabbajack.Common/Wabbajack.Common.csproj

@@ -29,6 +29,7 @@
         <PackageReference Include="Ceras" Version="4.1.7" />
         <PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
         <PackageReference Include="ini-parser-netstandard" Version="2.5.2" />
+        <PackageReference Include="MessagePack" Version="2.1.90" />
         <PackageReference Include="Microsoft.Win32.Registry" Version="4.7.0" />
         <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
         <PackageReference Include="Octodiff" Version="1.2.1" />

Wabbajack.Lib/ACompiler.cs

@@ -46,7 +46,7 @@ namespace Wabbajack.Lib
         public ModList ModList = new ModList();
 
         public List<IndexedArchive> IndexedArchives = new List<IndexedArchive>();
-        public Dictionary<string, IEnumerable<VirtualFile>> IndexedFiles = new Dictionary<string, IEnumerable<VirtualFile>>();
+        public Dictionary<Hash, IEnumerable<VirtualFile>> IndexedFiles = new Dictionary<Hash, IEnumerable<VirtualFile>>();
 
         public static void Info(string msg)
         {
@@ -194,25 +194,25 @@ namespace Wabbajack.Lib
         {
             Info("Building a list of archives based on the files required");
 
-            var shas = InstallDirectives.OfType<FromArchive>()
+            var hashes = InstallDirectives.OfType<FromArchive>()
-                .Select(a => a.ArchiveHashPath[0])
+                .Select(a => Hash.FromBase64(a.ArchiveHashPath[0]))
                 .Distinct();
 
             var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified)
                 .GroupBy(f => f.File.Hash)
                 .ToDictionary(f => f.Key, f => f.First());
 
-            SelectedArchives = await shas.PMap(Queue, sha => ResolveArchive(sha, archives));
+            SelectedArchives = await hashes.PMap(Queue, hash => ResolveArchive(hash, archives));
         }
 
-        public async Task<Archive> ResolveArchive(string sha, IDictionary<string, IndexedArchive> archives)
+        public async Task<Archive> ResolveArchive(Hash hash, IDictionary<Hash, IndexedArchive> archives)
         {
-            if (archives.TryGetValue(sha, out var found))
+            if (archives.TryGetValue(hash, out var found))
             {
                 return await ResolveArchive(found);
             }
 
-            Error($"No match found for Archive sha: {sha} this shouldn't happen");
+            Error($"No match found for Archive sha: {hash.ToBase64()} this shouldn't happen");
             return null;
         }
 

Wabbajack.Lib/AInstaller.cs

@@ -27,7 +27,7 @@ namespace Wabbajack.Lib
 
         public string ModListArchive { get; private set; }
         public ModList ModList { get; private set; }
-        public Dictionary<string, string> HashedArchives { get; set; }
+        public Dictionary<Hash, string> HashedArchives { get; set; }
         
         public SystemParameters SystemParameters { get; set; }
 
@@ -127,7 +127,7 @@ namespace Wabbajack.Lib
             var grouped = ModList.Directives
                 .OfType<FromArchive>()
                 .GroupBy(e => e.ArchiveHashPath[0])
-                .ToDictionary(k => k.Key);
+                .ToDictionary(k => Hash.FromBase64(k.Key));
             var archives = ModList.Archives
                 .Select(a => new { Archive = a, AbsolutePath = HashedArchives.GetOrDefault(a.Hash) })
                 .Where(a => a.AbsolutePath != null)
@@ -264,7 +264,7 @@ namespace Wabbajack.Lib
                         {
                             var orig_name = Path.GetFileNameWithoutExtension(archive.Name);
                             var ext = Path.GetExtension(archive.Name);
-                            var unique_key = archive.State.PrimaryKeyString.StringSHA256Hex();
+                            var unique_key = archive.State.PrimaryKeyString.StringSha256Hex();
                             outputPath = Path.Combine(DownloadFolder, orig_name + "_" + unique_key + "_" + ext);
                             if (outputPath.FileExists())
                                 File.Delete(outputPath);
@@ -413,7 +413,7 @@ namespace Wabbajack.Lib
             Utils.Log($"Optimized {ModList.Directives.Count} directives to {indexed.Count} required");
             var requiredArchives = indexed.Values.OfType<FromArchive>()
                 .GroupBy(d => d.ArchiveHashPath[0])
-                .Select(d => d.Key)
+                .Select(d => Hash.FromBase64(d.Key))
                 .ToHashSet();
             
             ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToList();

Wabbajack.Lib/ClientAPI.cs

@@ -13,10 +13,10 @@ namespace Wabbajack.Lib
             return client;
         }
 
-        public static async Task<Archive> GetModUpgrade(string hash)
+        public static async Task<Archive> GetModUpgrade(Hash hash)
         {
             using var response = await GetClient()
-                .GetAsync($"https://{Consts.WabbajackCacheHostname}/alternative/{hash.FromBase64().ToHex()}");
+                .GetAsync($"https://{Consts.WabbajackCacheHostname}/alternative/{hash.ToHex()}");
             return !response.IsSuccessStatusCode ? null : (await response.Content.ReadAsStringAsync()).FromJSONString<Archive>();
         }
     }

Wabbajack.Lib/CompilationSteps/CompilationErrors/InvalidGameESMError.cs

@@ -4,13 +4,14 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
+using Wabbajack.Common;
 using Wabbajack.Common.StatusFeed;
 
 namespace Wabbajack.Lib.CompilationSteps.CompilationErrors
 {
     public class InvalidGameESMError : AErrorMessage
     {
-        public string Hash { get; }
+        public Hash Hash { get; }
         public string PathToFile { get; }
         private readonly CleanedESM _esm;
         public string GameFileName => Path.GetFileName(_esm.To);
@@ -29,7 +30,7 @@ the modlist expecting a different of the game than you currently have installed,
 the game, and then attempting to re-install this modlist. Also verify that the version of the game you have installed matches the version expected by this modlist.";
         }
 
-        public InvalidGameESMError(CleanedESM esm, string hash, string path)
+        public InvalidGameESMError(CleanedESM esm, Hash hash, string path)
         {
             Hash = hash;
             PathToFile = path;

Wabbajack.Lib/Data.cs

@@ -26,7 +26,7 @@ namespace Wabbajack.Lib
 
         public VirtualFile File { get; }
 
-        public string Hash => File.Hash;
+        public Hash Hash => File.Hash;
 
         public T EvolveTo<T>() where T : Directive, new()
         {
@@ -130,7 +130,7 @@ namespace Wabbajack.Lib
         /// </summary>
         public string To;
         public long Size;
-        public string Hash;
+        public Hash Hash;
     }
 
     public class IgnoredDirectly : Directive
@@ -167,7 +167,7 @@ namespace Wabbajack.Lib
 
     public class CleanedESM : InlineFile
     {
-        public string SourceESMHash;
+        public Hash SourceESMHash;
     }
 
     /// <summary>
@@ -191,23 +191,13 @@ namespace Wabbajack.Lib
     {
         private string _fullPath;
 
-        /// <summary>
-        ///     MurMur3 hash of the archive this file comes from
-        /// </summary>
         public string[] ArchiveHashPath;
 
         [Exclude]
         public VirtualFile FromFile;
 
         [Exclude]
-        public string FullPath
+        public string FullPath => _fullPath ??= string.Join("|", ArchiveHashPath);
-        {
-            get
-            {
-                if (_fullPath == null) _fullPath = string.Join("|", ArchiveHashPath);
-                return _fullPath;
-            }
-        }
     }
 
     public class CreateBSA : Directive
@@ -226,13 +216,13 @@ namespace Wabbajack.Lib
         public string PatchID;
 
         [Exclude]
-        public string FromHash;
+        public Hash FromHash;
     }
 
     public class SourcePatch
     {
         public string RelativePath;
-        public string Hash;
+        public Hash Hash;
     }
 
     public class MergedPatch : Directive
@@ -246,7 +236,7 @@ namespace Wabbajack.Lib
         /// <summary>
         ///     MurMur3 Hash of the archive
         /// </summary>
-        public string Hash { get; set; }
+        public Hash Hash { get; set; }
 
         /// <summary>
         /// Meta INI for the downloaded archive

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -108,7 +108,7 @@ namespace Wabbajack.Lib.Downloaders
             var upgradeResult = await Download(upgrade, upgradePath);
             if (!upgradeResult) return false;
 
-            var patchName = $"{archive.Hash.FromBase64().ToHex()}_{upgrade.Hash.FromBase64().ToHex()}";
+            var patchName = $"{archive.Hash.ToHex()}_{upgrade.Hash.ToHex()}";
             var patchPath = Path.Combine(Path.GetDirectoryName(destination), "_Patch_" + patchName);
 
             var patchState = new Archive

Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs

@@ -49,7 +49,7 @@ namespace Wabbajack.Lib.Downloaders
         {
             public Game Game { get; set; }
             public string GameFile { get; set; }
-            public string Hash { get; set; }
+            public Hash Hash { get; set; }
             
             public string GameVersion { get; set; }
 

Wabbajack.Lib/FileUploader/AuthorAPI.cs

@@ -105,7 +105,7 @@ namespace Wabbajack.Lib.FileUploader
                 if (!tcs.Task.IsFaulted)
                 {
                     progressFn(1.0);
-                    var hash = (await hash_task).FromBase64().ToHex();
+                    var hash = (await hash_task).ToHex();
                     response = await client.PutAsync(UploadURL + $"/{key}/finish/{hash}", new StringContent(""));
                     if (response.IsSuccessStatusCode)
                         tcs.SetResult(await response.Content.ReadAsStringAsync());

Wabbajack.Lib/MO2Compiler.cs

@@ -45,7 +45,7 @@ namespace Wabbajack.Lib
 
         public override string VFSCacheName => Path.Combine(
             Consts.LocalAppDataPath, 
-            $"vfs_compile_cache-{Path.Combine(MO2Folder ?? "Unknown", "ModOrganizer.exe").StringSHA256Hex()}.bin");
+            $"vfs_compile_cache-{Path.Combine(MO2Folder ?? "Unknown", "ModOrganizer.exe").StringSha256Hex()}.bin");
 
         public MO2Compiler(string mo2Folder, string mo2Profile, string outputFile)
         {
@@ -318,7 +318,7 @@ namespace Wabbajack.Lib
 
             UpdateTracker.NextStep("Running Validation");
 
-            await ValidateModlist.RunValidation(Queue, ModList);
+            await ValidateModlist.RunValidation(ModList);
             UpdateTracker.NextStep("Generating Report");
 
             GenerateManifest();
@@ -382,7 +382,7 @@ namespace Wabbajack.Lib
                 var client = new Common.Http.Client();
                 using var response =
                     await client.GetAsync(
-                        $"http://build.wabbajack.org/indexed_files/{vf.Hash.FromBase64().ToHex()}/meta.ini");
+                        $"http://build.wabbajack.org/indexed_files/{vf.Hash.ToHex()}/meta.ini");
 
                 if (!response.IsSuccessStatusCode)
                 {

Wabbajack.Lib/MO2Installer.cs

@@ -82,7 +82,7 @@ namespace Wabbajack.Lib
 
             if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Validating Modlist");
-            await ValidateModlist.RunValidation(Queue, ModList);
+            await ValidateModlist.RunValidation(ModList);
 
             Directory.CreateDirectory(OutputFolder);
             Directory.CreateDirectory(DownloadFolder);

Wabbajack.Lib/ModListRegistry/ModListMetadata.cs

@@ -96,7 +96,7 @@ namespace Wabbajack.Lib.ModListRegistry
 
     public class DownloadMetadata
     {
-        public string Hash { get; set; }
+        public Hash Hash { get; set; }
         public long Size { get; set; }
 
         public long NumberOfArchives { get; set; }

Wabbajack.Lib/Validation/ValidateModlist.cs

@@ -16,21 +16,7 @@ namespace Wabbajack.Lib.Validation
     /// </summary>
     public class ValidateModlist
     {
-        public Dictionary<string, Author> AuthorPermissions { get; set; } = new Dictionary<string, Author>();
+        public ServerWhitelist ServerWhitelist { get; private set; } = new ServerWhitelist();
-
-        private readonly WorkQueue _queue;
-        public ServerWhitelist ServerWhitelist { get; set; } = new ServerWhitelist();
-
-        public ValidateModlist(WorkQueue workQueue)
-        {
-            _queue = workQueue;
-        }
-
-        public void LoadAuthorPermissionsFromString(string s)
-        {
-            AuthorPermissions = s.FromYaml<Dictionary<string, Author>>();
-        }
-
         public void LoadServerWhitelist(string s)
         {
             ServerWhitelist = s.FromYaml<ServerWhitelist>();
@@ -50,9 +36,9 @@ namespace Wabbajack.Lib.Validation
 
         }
 
-        public static async Task RunValidation(WorkQueue queue, ModList modlist)
+        public static async Task RunValidation(ModList modlist)
         {
-            var validator = new ValidateModlist(queue);
+            var validator = new ValidateModlist();
 
             await validator.LoadListsFromGithub();
 
@@ -69,92 +55,9 @@ namespace Wabbajack.Lib.Validation
             }
         }
 
-        /// <summary>
-        /// Takes all the permissions for a given Nexus mods and merges them down to a single permissions record
-        /// the more specific record having precedence in each field.
-        /// </summary>
-        /// <param name="mod"></param>
-        /// <returns></returns>
-        public Permissions FilePermissions(NexusDownloader.State mod)
-        {
-            var author_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Permissions;
-            var game_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Permissions;
-            var mod_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Mods.GetOrDefault(mod.ModID)
-                ?.Permissions;
-            var file_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Mods
-                .GetOrDefault(mod.ModID)?.Files.GetOrDefault(mod.FileID)?.Permissions;
-
-            return new Permissions
-            {
-                CanExtractBSAs = file_permissions?.CanExtractBSAs ?? mod_permissions?.CanExtractBSAs ??
-                                 game_permissions?.CanExtractBSAs ?? author_permissions?.CanExtractBSAs ?? true,
-                CanModifyAssets = file_permissions?.CanModifyAssets ?? mod_permissions?.CanModifyAssets ??
-                                  game_permissions?.CanModifyAssets ?? author_permissions?.CanModifyAssets ?? true,
-                CanModifyESPs = file_permissions?.CanModifyESPs ?? mod_permissions?.CanModifyESPs ??
-                                game_permissions?.CanModifyESPs ?? author_permissions?.CanModifyESPs ?? true,
-                CanUseInOtherGames = file_permissions?.CanUseInOtherGames ?? mod_permissions?.CanUseInOtherGames ??
-                                     game_permissions?.CanUseInOtherGames ?? author_permissions?.CanUseInOtherGames ?? true,
-            };
-        }
-
         public async Task<IEnumerable<string>> Validate(ModList modlist)
         {
             ConcurrentStack<string> ValidationErrors = new ConcurrentStack<string>();
-            
-            var nexus_mod_permissions = (await modlist.Archives
-                .Where(a => a.State is NexusDownloader.State)
-                .PMap(_queue, a => (a.Hash, FilePermissions((NexusDownloader.State)a.State), a)))
-                .ToDictionary(a => a.Hash, a => new { permissions = a.Item2, archive = a.a });
-
-            await modlist.Directives
-                .OfType<PatchedFromArchive>()
-                .PMap(_queue, p =>
-                {
-                    if (nexus_mod_permissions.TryGetValue(p.ArchiveHashPath[0], out var archive))
-                    {
-                        var ext = Path.GetExtension(p.ArchiveHashPath.Last());
-                        var url = (archive.archive.State as NexusDownloader.State).URL;
-                        if (Consts.AssetFileExtensions.Contains(ext) && !(archive.permissions.CanModifyAssets ?? true))
-                        {
-                            ValidationErrors.Push($"{p.To} from {url} is set to disallow asset modification");
-                        }
-                        else if (Consts.ESPFileExtensions.Contains(ext) && !(archive.permissions.CanModifyESPs ?? true))
-                        {
-                            ValidationErrors.Push($"{p.To} from {url} is set to disallow asset ESP modification");
-                        }
-                    }
-                });
-
-            await modlist.Directives
-                .OfType<FromArchive>()
-                .PMap(_queue, p =>
-                {
-                    if (nexus_mod_permissions.TryGetValue(p.ArchiveHashPath[0], out var archive))
-                    {
-                        var url = (archive.archive.State as NexusDownloader.State).URL;
-                        if (!(archive.permissions.CanExtractBSAs ?? true) &&
-                            p.ArchiveHashPath.Skip(1).ButLast().Any(a => Consts.SupportedBSAs.Contains(Path.GetExtension(a).ToLower())))
-                        {
-                            ValidationErrors.Push($"{p.To} from {url} is set to disallow BSA extraction");
-                        }
-                    }
-                });
-
-            var nexus = NexusApi.NexusApiUtils.ConvertGameName(modlist.GameType.MetaData().NexusName);
-
-            modlist.Archives
-                   .Where(a => a.State is NexusDownloader.State)
-                   .Where(m => NexusApi.NexusApiUtils.ConvertGameName(((NexusDownloader.State)m.State).GameName) != nexus)
-                   .Do(m =>
-                   {
-                       var permissions = FilePermissions((NexusDownloader.State)m.State);
-                       if (!(permissions.CanUseInOtherGames ?? true))
-                       {
-                           ValidationErrors.Push(
-                               $"The ModList is for {nexus} but {m.Name} is for game type {((NexusDownloader.State)m.State).GameName} and is not allowed to be converted to other game types");
-                       }
-                   });
-
             modlist.Archives
                 .Where(m => !m.State.IsWhitelisted(ServerWhitelist))
                 .Do(m =>

Wabbajack.Lib/VortexCompiler.cs

@@ -49,7 +49,7 @@ namespace Wabbajack.Lib
 
         public override string VFSCacheName => Path.Combine(
             Consts.LocalAppDataPath, 
-            $"vfs_compile_cache-{StagingFolder?.StringSHA256Hex() ?? "Unknown"}.bin");
+            $"vfs_compile_cache-{StagingFolder?.StringSha256Hex() ?? "Unknown"}.bin");
 
         public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder, string stagingFolder, string outputFile)
         {
@@ -245,7 +245,7 @@ namespace Wabbajack.Lib
             };
             
             UpdateTracker.NextStep("Running Validation");
-            await ValidateModlist.RunValidation(Queue, ModList);
+            await ValidateModlist.RunValidation(ModList);
 
             UpdateTracker.NextStep("Generating Report");
             GenerateManifest();

Wabbajack.Test/ContentRightsManagementTests.cs

@@ -52,8 +52,7 @@ namespace Wabbajack.Test
         public void TestSetup()
         {
             queue = new WorkQueue();
-            validate = new ValidateModlist(queue);
+            validate = new ValidateModlist();
-            validate.LoadAuthorPermissionsFromString(permissions);
             validate.LoadServerWhitelist(server_whitelist);
         }
 
@@ -63,76 +62,6 @@ namespace Wabbajack.Test
             queue?.Dispose();
         }
 
-        [TestMethod]
-        public void TestRightsFallthrough()
-        {
-            var permissions = validate.FilePermissions(new NexusDownloader.State
-            {
-                Author = "bill",
-                GameName = "Skyrim",
-                ModID = "42",
-                FileID = "33"
-            });
-
-            permissions.CanExtractBSAs.AssertIsFalse();
-            permissions.CanModifyESPs.AssertIsFalse();
-            permissions.CanModifyAssets.AssertIsFalse();
-            permissions.CanUseInOtherGames.AssertIsFalse();
-
-            permissions = validate.FilePermissions(new NexusDownloader.State
-            {
-                Author = "bob",
-                GameName = "Skyrim",
-                ModID = "42",
-                FileID = "33"
-            });
-
-            permissions.CanExtractBSAs.AssertIsTrue();
-            permissions.CanModifyESPs.AssertIsTrue();
-            permissions.CanModifyAssets.AssertIsTrue();
-            permissions.CanUseInOtherGames.AssertIsTrue();
-
-            permissions = validate.FilePermissions(new NexusDownloader.State
-            {
-                Author = "bill",
-                GameName = "Fallout4",
-                ModID = "42",
-                FileID = "33"
-            });
-
-            permissions.CanExtractBSAs.AssertIsFalse();
-            permissions.CanModifyESPs.AssertIsTrue();
-            permissions.CanModifyAssets.AssertIsTrue();
-            permissions.CanUseInOtherGames.AssertIsTrue();
-
-            permissions = validate.FilePermissions(new NexusDownloader.State
-            {
-                Author = "bill",
-                GameName = "Skyrim",
-                ModID = "43",
-                FileID = "33"
-            });
-
-            permissions.CanExtractBSAs.AssertIsFalse();
-            permissions.CanModifyESPs.AssertIsFalse();
-            permissions.CanModifyAssets.AssertIsTrue();
-            permissions.CanUseInOtherGames.AssertIsTrue();
-
-            permissions = validate.FilePermissions(new NexusDownloader.State
-            {
-                Author = "bill",
-                GameName = "Skyrim",
-                ModID = "42",
-                FileID = "31"
-            });
-
-            permissions.CanExtractBSAs.AssertIsFalse();
-            permissions.CanModifyESPs.AssertIsFalse();
-            permissions.CanModifyAssets.AssertIsFalse();
-            permissions.CanUseInOtherGames.AssertIsTrue();
-        }
-
-
         [TestMethod]
         public async Task TestModValidation()
         {
@@ -150,9 +79,8 @@ namespace Wabbajack.Test
                             ModID = "42",
                             FileID = "33",
                         },
-                        Hash = "DEADBEEF"
+                        Hash = Hash.FromLong(42)
                     }
-
                 },
                 Directives = new List<Directive>
                 {
@@ -163,59 +91,14 @@ namespace Wabbajack.Test
                     }
                 }
             };
-
-            IEnumerable<string> errors;
-
-            // No errors, simple archive extraction
-            errors = await validate.Validate(modlist);
-            Assert.AreEqual(errors.Count(), 0);
-
-
-            // Error due to patched file
-            modlist.Directives[0] = new PatchedFromArchive
-            {
-                PatchID = Guid.NewGuid().ToString(),
-                ArchiveHashPath = new[] {"DEADBEEF", "foo\\bar\\baz.pex"},
-            };
-
-            errors = await validate.Validate(modlist);
-            Assert.AreEqual(errors.Count(), 1);
-
-            // Error due to extracted BSA file
-            modlist.Directives[0] = new FromArchive
-            {
-                ArchiveHashPath = new[] { "DEADBEEF", "foo.bsa", "foo\\bar\\baz.dds" },
-            };
-
-            errors = await validate.Validate(modlist);
-            Assert.AreEqual(errors.Count(), 1);
-
-            // No error since we're just installing the .bsa, not extracting it
-            modlist.Directives[0] = new FromArchive
-            {
-                ArchiveHashPath = new[] { "DEADBEEF", "foo.bsa"},
-            };
-
-            errors = await validate.Validate(modlist);
-            Assert.AreEqual(0, errors.Count());
-            
-            // Error due to game conversion
-            modlist.GameType = Game.SkyrimSpecialEdition;
-            modlist.Directives[0] = new FromArchive
-            {
-                ArchiveHashPath = new[] { "DEADBEEF", "foo\\bar\\baz.dds" },
-            };
-            errors = await validate.Validate(modlist);
-            Assert.AreEqual(errors.Count(), 1);
-
             // Error due to file downloaded from 3rd party
             modlist.GameType = Game.Skyrim;
             modlist.Archives[0] = new Archive()
             {
                 State = new HTTPDownloader.State() { Url = "https://somebadplace.com" },
-                Hash = "DEADBEEF"
+                Hash = Hash.FromLong(42)
             };
-            errors = await validate.Validate(modlist);
+            var errors = await validate.Validate(modlist);
             Assert.AreEqual(1, errors.Count());
 
             // Ok due to file downloaded from whitelisted 3rd party
@@ -223,7 +106,7 @@ namespace Wabbajack.Test
             modlist.Archives[0] = new Archive
             {
                 State = new HTTPDownloader.State { Url = "https://somegoodplace.com/baz.7z" },
-                Hash = "DEADBEEF"
+                Hash = Hash.FromLong(42)
             };
             errors = await validate.Validate(modlist);
             Assert.AreEqual(0, errors.Count());
@@ -234,7 +117,7 @@ namespace Wabbajack.Test
             modlist.Archives[0] = new Archive
             {
                 State = new GoogleDriveDownloader.State { Id = "bleg"},
-                Hash = "DEADBEEF"
+                Hash = Hash.FromLong(42)
             };
             errors = await validate.Validate(modlist);
             Assert.AreEqual(errors.Count(), 1);
@@ -244,7 +127,7 @@ namespace Wabbajack.Test
             modlist.Archives[0] = new Archive
             {
                 State = new GoogleDriveDownloader.State { Id = "googleDEADBEEF" },
-                Hash = "DEADBEEF"
+                Hash = Hash.FromLong(42)
             };
             errors = await validate.Validate(modlist);
             Assert.AreEqual(0, errors.Count());
@@ -256,7 +139,7 @@ namespace Wabbajack.Test
         {
             using (var workQueue = new WorkQueue())
             {
-                await new ValidateModlist(workQueue).LoadListsFromGithub();
+                await new ValidateModlist().LoadListsFromGithub();
             }
         }
     }

Wabbajack.Test/DownloaderTests.cs

@@ -536,7 +536,7 @@ namespace Wabbajack.Test
             var archive = new Archive
             {
                 Name = "Cori.7z",
-                Hash = "gCRVrvzDNH0=",
+                Hash = Hash.FromBase64("gCRVrvzDNH0="),
                 State = new NexusDownloader.State
                 {
                     GameName = Game.SkyrimSpecialEdition.MetaData().NexusName,

Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs

@@ -82,7 +82,7 @@ namespace Wabbajack.VirtualFileSystem.Test
             await AddTestRoot();
 
 
-            var files = context.Index.ByHash["qX0GZvIaTKM="];
+            var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")];
             Assert.AreEqual(files.Count(), 2);
         }
 
@@ -150,7 +150,7 @@ namespace Wabbajack.VirtualFileSystem.Test
 
             await AddTestRoot();
 
-            var files = context.Index.ByHash["qX0GZvIaTKM="];
+            var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")];
 
             var cleanup = await context.Stage(files);
 
@@ -173,7 +173,7 @@ namespace Wabbajack.VirtualFileSystem.Test
 
             await AddTestRoot();
 
-            var files = context.Index.ByHash["qX0GZvIaTKM="];
+            var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")];
             var archive = context.Index.ByRootPath[Path.Combine(VFS_TEST_DIR_FULL, "test.zip")];
 
             var state = context.GetPortableState(files);
@@ -181,9 +181,9 @@ namespace Wabbajack.VirtualFileSystem.Test
             var new_context = new Context(Queue);
 
             await new_context.IntegrateFromPortable(state,
-                new Dictionary<string, string> {{archive.Hash, archive.FullPath}});
+                new Dictionary<Hash, string> {{archive.Hash, archive.FullPath}});
 
-            var new_files = new_context.Index.ByHash["qX0GZvIaTKM="];
+            var new_files = new_context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")];
 
             var close = await new_context.Stage(new_files);
 

Wabbajack.VirtualFileSystem/Context.cs

@@ -249,16 +249,16 @@ namespace Wabbajack.VirtualFileSystem
                 {
                     Name = f.Parent != null ? f.Name : null,
                     Hash = f.Hash,
-                    ParentHash = f.Parent?.Hash,
+                    ParentHash = f.Parent?.Hash ?? Hash.Empty,
                     Size = f.Size
                 }).ToList();
         }
 
-        public async Task IntegrateFromPortable(List<PortableFile> state, Dictionary<string, string> links)
+        public async Task IntegrateFromPortable(List<PortableFile> state, Dictionary<Hash, string> links)
         {
             var indexedState = state.GroupBy(f => f.ParentHash)
-                .ToDictionary(f => f.Key ?? "", f => (IEnumerable<PortableFile>) f);
+                .ToDictionary(f => f.Key, f => (IEnumerable<PortableFile>) f);
-            var parents = await indexedState[""]
+            var parents = await indexedState[Hash.Empty]
                 .PMap(Queue,f => VirtualFile.CreateFromPortable(this, indexedState, links, f));
 
             var newIndex = await Index.Integrate(parents);
@@ -297,7 +297,7 @@ namespace Wabbajack.VirtualFileSystem
 
             void BackFillOne(KnownFile file)
             {
-                var parent = newFiles[file.Paths[0]];
+                var parent = newFiles[Hash.FromBase64(file.Paths[0])];
                 foreach (var path in file.Paths.Skip(1))
                 {
                     if (parentchild.TryGetValue((parent, path), out var foundParent))
@@ -331,7 +331,7 @@ namespace Wabbajack.VirtualFileSystem
     public class KnownFile
     {
         public string[] Paths { get; set; }
-        public string Hash { get; set; }
+        public Hash Hash { get; set; }
     }
 
     public class DisposableList<T> : List<T>, IDisposable
@@ -355,7 +355,7 @@ namespace Wabbajack.VirtualFileSystem
 
         public IndexRoot(ImmutableList<VirtualFile> aFiles,
             ImmutableDictionary<string, VirtualFile> byFullPath,
-            ImmutableDictionary<string, ImmutableStack<VirtualFile>> byHash,
+            ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> byHash,
             ImmutableDictionary<string, VirtualFile> byRoot,
             ImmutableDictionary<string, ImmutableStack<VirtualFile>> byName)
         {
@@ -370,7 +370,7 @@ namespace Wabbajack.VirtualFileSystem
         {
             AllFiles = ImmutableList<VirtualFile>.Empty;
             ByFullPath = ImmutableDictionary<string, VirtualFile>.Empty;
-            ByHash = ImmutableDictionary<string, ImmutableStack<VirtualFile>>.Empty;
+            ByHash = ImmutableDictionary<Hash, ImmutableStack<VirtualFile>>.Empty;
             ByRootPath = ImmutableDictionary<string, VirtualFile>.Empty;
             ByName = ImmutableDictionary<string, ImmutableStack<VirtualFile>>.Empty;
         }
@@ -378,7 +378,7 @@ namespace Wabbajack.VirtualFileSystem
 
         public ImmutableList<VirtualFile> AllFiles { get; }
         public ImmutableDictionary<string, VirtualFile> ByFullPath { get; }
-        public ImmutableDictionary<string, ImmutableStack<VirtualFile>> ByHash { get; }
+        public ImmutableDictionary<Hash, ImmutableStack<VirtualFile>> ByHash { get; }
         public ImmutableDictionary<string, ImmutableStack<VirtualFile>> ByName { get; set; }
         public ImmutableDictionary<string, VirtualFile> ByRootPath { get; }
 
@@ -391,7 +391,7 @@ namespace Wabbajack.VirtualFileSystem
                                      .ToImmutableDictionary(f => f.FullPath));
 
             var byHash = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
-                                 .Where(f => f.Hash != null)
+                                 .Where(f => f.Hash != Hash.Empty)
                                  .ToGroupedImmutableDictionary(f => f.Hash));
 
             var byName = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren)
@@ -410,7 +410,7 @@ namespace Wabbajack.VirtualFileSystem
 
         public VirtualFile FileForArchiveHashPath(string[] argArchiveHashPath)
         {
-            var cur = ByHash[argArchiveHashPath[0]].First(f => f.Parent == null);
+            var cur = ByHash[Hash.FromBase64(argArchiveHashPath[0])].First(f => f.Parent == null);
             return argArchiveHashPath.Skip(1).Aggregate(cur, (current, itm) => ByName[itm].First(f => f.Parent == current));
         }
     }

Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using Wabbajack.Common;
 
 namespace Wabbajack.VirtualFileSystem
 {
@@ -8,7 +9,7 @@ namespace Wabbajack.VirtualFileSystem
     public class IndexedVirtualFile
     {
         public string Name { get; set; }
-        public string Hash { get; set; }
+        public Hash Hash { get; set; }
         public long Size { get; set; }
         public List<IndexedVirtualFile> Children { get; set; } = new List<IndexedVirtualFile>();
     }

Wabbajack.VirtualFileSystem/PortableFile.cs

@@ -1,10 +1,12 @@
-namespace Wabbajack.VirtualFileSystem
+using Wabbajack.Common;
+
+namespace Wabbajack.VirtualFileSystem
 {
     public class PortableFile
     {
         public string Name { get; set; }
-        public string Hash { get; set; }
+        public Hash Hash { get; set; }
-        public string ParentHash { get; set; }
+        public Hash ParentHash { get; set; }
         public long Size { get; set; }
     }
 }

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -42,7 +42,7 @@ namespace Wabbajack.VirtualFileSystem
             }
         }
 
-        public string Hash { get; internal set; }
+        public Hash Hash { get; internal set; }
         public ExtendedHashes ExtendedHashes { get; set; }
         public long Size { get; internal set; }
 
@@ -214,12 +214,12 @@ namespace Wabbajack.VirtualFileSystem
             return self;
         }
 
-        private static async Task<IndexedVirtualFile> TryGetContentsFromServer(string hash)
+        private static async Task<IndexedVirtualFile> TryGetContentsFromServer(Hash hash)
         {
             try
             {
                 var client = new HttpClient();
-                var response = await client.GetAsync($"http://{Consts.WabbajackCacheHostname}/indexed_files/{hash.FromBase64().ToHex()}");
+                var response = await client.GetAsync($"http://{Consts.WabbajackCacheHostname}/indexed_files/{hash.ToHex()}");
                 if (!response.IsSuccessStatusCode)
                     return null;
 
@@ -274,7 +274,7 @@ namespace Wabbajack.VirtualFileSystem
                 Parent = parent,
                 Name = br.ReadString(),
                 _fullPath = br.ReadString(),
-                Hash = br.ReadString(),
+                Hash = br.ReadHash(),
                 Size = br.ReadInt64(),
                 LastModified = br.ReadInt64(),
                 LastAnalyzed = br.ReadInt64(),
@@ -288,7 +288,7 @@ namespace Wabbajack.VirtualFileSystem
         }
 
         public static VirtualFile CreateFromPortable(Context context,
-            Dictionary<string, IEnumerable<PortableFile>> state, Dictionary<string, string> links,
+            Dictionary<Hash, IEnumerable<PortableFile>> state, Dictionary<Hash, string> links,
             PortableFile portableFile)
         {
             var vf = new VirtualFile
@@ -305,7 +305,7 @@ namespace Wabbajack.VirtualFileSystem
         }
 
         public static VirtualFile CreateFromPortable(Context context, VirtualFile parent,
-            Dictionary<string, IEnumerable<PortableFile>> state, PortableFile portableFile)
+            Dictionary<Hash, IEnumerable<PortableFile>> state, PortableFile portableFile)
         {
             var vf = new VirtualFile
             {
@@ -323,7 +323,7 @@ namespace Wabbajack.VirtualFileSystem
         public string[] MakeRelativePaths()
         {
             var path = new string[NestingFactor];
-            path[0] = FilesInFullPath.First().Hash;
+            path[0] = FilesInFullPath.First().Hash.ToBase64();
             
             var idx = 1;
 

Wabbajack/View Models/Gallery/ModListGalleryVM.cs

@@ -65,19 +65,19 @@ namespace Wabbajack
                         return list
                             // Sort randomly initially, just to give each list a fair shake
                             .Shuffle(random)
-                            .AsObservableChangeSet(x => x.DownloadMetadata?.Hash ?? $"Fallback{missingHashFallbackCounter++}");
+                            .AsObservableChangeSet(x => x.DownloadMetadata?.Hash ?? Hash.Empty);
                     }
                     catch (Exception ex)
                     {
                         Utils.Error(ex);
                         Error = ErrorResponse.Fail(ex);
-                        return Observable.Empty<IChangeSet<ModlistMetadata, string>>();
+                        return Observable.Empty<IChangeSet<ModlistMetadata, Hash>>();
                     }
                 })
                 // Unsubscribe and release when not active
                 .FlowSwitch(
                     this.WhenAny(x => x.IsActive),
-                    valueWhenOff: Observable.Return(ChangeSet<ModlistMetadata, string>.Empty))
+                    valueWhenOff: Observable.Return(ChangeSet<ModlistMetadata, Hash>.Empty))
                 .Switch()
                 .RefCount();
 

Wabbajack/View Models/ManifestVM.cs

@@ -92,7 +92,7 @@ namespace Wabbajack
                         return Order(Archives.Where(x =>
                         {
                             if (term.StartsWith("hash:"))
-                                return x.Hash.StartsWith(term.Replace("hash:", ""));
+                                return x.Hash.ToString().StartsWith(term.Replace("hash:", ""));
                             return x.Name.StartsWith(term);
                         }));
                     })