diff --git a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs index 376092ae..9391a4a5 100644 --- a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs @@ -38,7 +38,7 @@ namespace Wabbajack.BuildServer.Controllers [Route("{xxHashAsBase64}/meta.ini")] public async Task GetFileMeta(string xxHashAsBase64) { - var id = xxHashAsBase64.FromHex().ToBase64(); + var id = Hash.FromHex(xxHashAsBase64); var state = await Db.DownloadStates.AsQueryable() .Where(d => d.Hash == id && d.IsValid) .OrderByDescending(d => d.LastValidationTime) @@ -51,48 +51,6 @@ namespace Wabbajack.BuildServer.Controllers return Ok(string.Join("\r\n", state.FirstOrDefault().State.GetMetaIni())); } - [Authorize] - [HttpDelete] - [Route("/indexed_files/nexus/{Game}/mod/{ModId}")] - public async Task PurgeBySHA256(string Game, string ModId) - { - var files = await Db.DownloadStates.AsQueryable().Where(d => d.State is NexusDownloader.State && - ((NexusDownloader.State)d.State).GameName == Game && - ((NexusDownloader.State)d.State).ModID == ModId) - .ToListAsync(); - - async Task DeleteParentsOf(HashSet acc, string hash) - { - var parents = await Db.IndexedFiles.AsQueryable().Where(f => f.Children.Any(c => c.Hash == hash)) - .ToListAsync(); - - foreach (var parent in parents) - await DeleteThisAndAllChildren(acc, parent.Hash); - } - - async Task DeleteThisAndAllChildren(HashSet acc, string hash) - { - acc.Add(hash); - var children = await Db.IndexedFiles.AsQueryable().Where(f => f.Hash == hash).FirstOrDefaultAsync(); - if (children == null) return; - foreach (var child in children.Children) - { - await DeleteThisAndAllChildren(acc, child.Hash); - } - - } - - var acc = new HashSet(); - foreach (var file in files) - await DeleteThisAndAllChildren(acc, file.Hash); - - var acclst = acc.ToList(); - await Db.DownloadStates.DeleteManyAsync(d => acc.Contains(d.Hash)); - await Db.IndexedFiles.DeleteManyAsync(d => acc.Contains(d.Hash)); - - return Ok(acc.ToList()); - } - [HttpPost] [Route("notify")] public async Task Notify() diff --git a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs index d93cf2be..4e8ab3ab 100644 --- a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs +++ b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs @@ -43,7 +43,7 @@ namespace Wabbajack.BuildServer.Controllers { var lists = await Db.ModListStatus.AsQueryable().ToListAsync(); var archives = lists.SelectMany(list => list.DetailedStatus.Archives) - .Select(a => a.Archive.Hash.FromBase64().ToHex()) + .Select(a => a.Archive.Hash.ToHex()) .ToHashSet(); var toDelete = new List(); @@ -89,9 +89,9 @@ namespace Wabbajack.BuildServer.Controllers [Route("/alternative/{xxHash}")] public async Task GetAlternative(string xxHash) { - var startingHash = xxHash.FromHex().ToBase64(); + var startingHash = Hash.FromHex(xxHash); Utils.Log($"Alternative requested for {startingHash}"); - await Metric("requested_upgrade", startingHash); + await Metric("requested_upgrade", startingHash.ToString()); var state = await Db.DownloadStates.AsQueryable() .Where(s => s.Hash == startingHash) @@ -110,7 +110,7 @@ namespace Wabbajack.BuildServer.Controllers if (mod_files.SelectMany(f => f.Data.files) .Any(f => f.category_name != null && f.file_id.ToString() == nexusState.FileID)) { - await Metric("not_required_upgrade", startingHash); + await Metric("not_required_upgrade", startingHash.ToString()); return BadRequest("Upgrade Not Required"); } @@ -122,7 +122,7 @@ namespace Wabbajack.BuildServer.Controllers } Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}"); - if (newArchive.Hash == null) + if (newArchive.Hash == Hash.Empty) { Db.Jobs.InsertOne(new Job { @@ -160,7 +160,7 @@ namespace Wabbajack.BuildServer.Controllers return Ok(newArchive.ToJSON()); } - private async Task FindAlternatives(NexusDownloader.State state, string srcHash) + private async Task FindAlternatives(NexusDownloader.State state, Hash srcHash) { var origSize = AlphaFile.GetSize(_settings.PathForArchive(srcHash)); var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); diff --git a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs index 9dbb40c5..9218d184 100644 --- a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs @@ -121,7 +121,7 @@ namespace Wabbajack.BuildServer.Controllers [Route("upload_file/{Key}/finish/{xxHashAsHex}")] public async Task UploadFileFinish(string Key, string xxHashAsHex) { - var expectedHash = xxHashAsHex.FromHex().ToBase64(); + var expectedHash = Hash.FromHex(xxHashAsHex); var user = User.FindFirstValue(ClaimTypes.Name); if (!Key.All(a => HexChars.Contains(a))) return BadRequest("NOT A VALID FILENAME"); diff --git a/Wabbajack.BuildServer/Extensions.cs b/Wabbajack.BuildServer/Extensions.cs index 9264795f..34c8a060 100644 --- a/Wabbajack.BuildServer/Extensions.cs +++ b/Wabbajack.BuildServer/Extensions.cs @@ -47,13 +47,13 @@ namespace Wabbajack.BuildServer return authenticationBuilder.AddScheme(ApiKeyAuthenticationOptions.DefaultScheme, options); } - private static ConcurrentDictionary PathForArchiveHash = new ConcurrentDictionary(); - public static string PathForArchive(this AppSettings settings, string hash) + private static readonly ConcurrentDictionary PathForArchiveHash = new ConcurrentDictionary(); + public static string PathForArchive(this AppSettings settings, Hash hash) { if (PathForArchiveHash.TryGetValue(hash, out string result)) return result; - var hexHash = hash.FromBase64().ToHex(); + var hexHash = hash.ToHex(); var ends = "_" + hexHash + "_"; var file = Directory.EnumerateFiles(settings.ArchiveDir, DirectoryEnumerationOptions.Files, diff --git a/Wabbajack.BuildServer/Models/DownloadState.cs b/Wabbajack.BuildServer/Models/DownloadState.cs index b7f6ab24..bfd5e183 100644 --- a/Wabbajack.BuildServer/Models/DownloadState.cs +++ b/Wabbajack.BuildServer/Models/DownloadState.cs @@ -1,9 +1,6 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using MongoDB.Bson.Serialization.Attributes; +using Wabbajack.Common; using Wabbajack.Lib.Downloaders; namespace Wabbajack.BuildServer.Models @@ -12,7 +9,7 @@ namespace Wabbajack.BuildServer.Models { [BsonId] public string Key { get; set; } - public string Hash { get; set; } + public Hash Hash { get; set; } public AbstractDownloadState State { get; set; } diff --git a/Wabbajack.BuildServer/Models/IndexedFile.cs b/Wabbajack.BuildServer/Models/IndexedFile.cs index 6ef4d224..241b619c 100644 --- a/Wabbajack.BuildServer/Models/IndexedFile.cs +++ b/Wabbajack.BuildServer/Models/IndexedFile.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; using MongoDB.Bson.Serialization.Attributes; +using Wabbajack.Common; using Wabbajack.VirtualFileSystem; namespace Wabbajack.BuildServer.Models @@ -11,7 +12,7 @@ namespace Wabbajack.BuildServer.Models public class IndexedFile { [BsonId] - public string Hash { get; set; } + public Hash Hash { get; set; } public string SHA256 { get; set; } public string SHA1 { get; set; } public string MD5 { get; set; } @@ -25,6 +26,6 @@ namespace Wabbajack.BuildServer.Models { public string Name; public string Extension; - public string Hash; + public Hash Hash; } } diff --git a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs index 20039e5f..c8c8db18 100644 --- a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs @@ -56,7 +56,7 @@ namespace Wabbajack.BuildServer.Models.Jobs }); var to_path = Path.Combine(settings.ArchiveDir, - $"{Path.GetFileName(fileName)}_{archive.Hash.FromBase64().ToHex()}_{Path.GetExtension(fileName)}"); + $"{Path.GetFileName(fileName)}_{archive.Hash.ToHex()}_{Path.GetExtension(fileName)}"); if (File.Exists(to_path)) File.Delete(downloadDest); else diff --git a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs index 136d3456..d67d5535 100644 --- a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs +++ b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs @@ -27,7 +27,7 @@ namespace Wabbajack.BuildServer.Models.Jobs using (var queue = new WorkQueue()) { - var whitelists = new ValidateModlist(queue); + var whitelists = new ValidateModlist(); await whitelists.LoadListsFromGithub(); foreach (var list in modlists) diff --git a/Wabbajack.BuildServer/Models/PatchArchive.cs b/Wabbajack.BuildServer/Models/PatchArchive.cs index 46ae68ea..62be9f84 100644 --- a/Wabbajack.BuildServer/Models/PatchArchive.cs +++ b/Wabbajack.BuildServer/Models/PatchArchive.cs @@ -18,7 +18,7 @@ namespace Wabbajack.BuildServer.Models public class PatchArchive : AJobPayload { public override string Description => "Create a archive update patch"; - public string Src { get; set; } + public Hash Src { get; set; } public string DestPK { get; set; } public override async Task Execute(DBContext db, SqlService sql, AppSettings settings) { @@ -56,7 +56,7 @@ namespace Wabbajack.BuildServer.Models await client.ConnectAsync(); try { - await client.UploadAsync(fs, $"updates/{Src.FromBase64().ToHex()}_{destHash.FromBase64().ToHex()}", progress: new UploadToCDN.Progress(cdnPath)); + await client.UploadAsync(fs, $"updates/{Src.ToHex()}_{destHash.ToHex()}", progress: new UploadToCDN.Progress(cdnPath)); } catch (Exception ex) { @@ -72,9 +72,9 @@ namespace Wabbajack.BuildServer.Models } - public static string CdnPath(string srcHash, string destHash) + public static string CdnPath(Hash srcHash, Hash destHash) { - return $"updates/{srcHash.FromBase64().ToHex()}_{destHash.FromBase64().ToHex()}"; + return $"updates/{srcHash.ToHex()}_{destHash.ToHex()}"; } } } diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index 0d2907c9..2798aaf6 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -49,10 +49,9 @@ namespace Wabbajack.BuildServer.Model.Models private static void IngestFile(VirtualFile root, ICollection files, ICollection contents) { - var hash = BitConverter.ToInt64(root.Hash.FromBase64()); files.Add(new IndexedFile { - Hash = hash, + Hash = (long)root.Hash, Sha256 = root.ExtendedHashes.SHA256.FromHex(), Sha1 = root.ExtendedHashes.SHA1.FromHex(), Md5 = root.ExtendedHashes.MD5.FromHex(), @@ -66,22 +65,21 @@ namespace Wabbajack.BuildServer.Model.Models { IngestFile(child, files, contents); - var child_hash = BitConverter.ToInt64(child.Hash.FromBase64()); contents.Add(new ArchiveContent { - Parent = hash, - Child = child_hash, + Parent = (long)root.Hash, + Child = (long)child.Hash, Path = child.Name }); } } - public async Task HaveIndexdFile(string hash) + public async Task HaveIndexdFile(Hash hash) { await using var conn = await Open(); var row = await conn.QueryAsync(@"SELECT * FROM IndexedFile WHERE Hash = @Hash", - new {Hash = BitConverter.ToInt64(hash.FromBase64())}); + new {Hash = (long)hash}); return row.Any(); } @@ -123,7 +121,7 @@ namespace Wabbajack.BuildServer.Model.Models return children.Select(f => new IndexedVirtualFile { Name = f.Path, - Hash = BitConverter.GetBytes(f.Hash).ToBase64(), + Hash = Hash.FromLong(f.Hash), Size = f.Size, Children = Build(f.Hash) }).ToList(); diff --git a/Wabbajack.BuildServer/Models/UploadedFile.cs b/Wabbajack.BuildServer/Models/UploadedFile.cs index 8c7d9503..af4b9aec 100644 --- a/Wabbajack.BuildServer/Models/UploadedFile.cs +++ b/Wabbajack.BuildServer/Models/UploadedFile.cs @@ -13,7 +13,7 @@ namespace Wabbajack.BuildServer.Models public string Id { get; set; } public string Name { get; set; } public long Size { get; set; } - public string Hash { get; set; } + public Hash Hash { get; set; } public string Uploader { get; set; } public DateTime UploadDate { get; set; } = DateTime.UtcNow; diff --git a/Wabbajack.CLI/Verbs/Validate.cs b/Wabbajack.CLI/Verbs/Validate.cs index 5a21e0af..3f4c8503 100644 --- a/Wabbajack.CLI/Verbs/Validate.cs +++ b/Wabbajack.CLI/Verbs/Validate.cs @@ -57,7 +57,7 @@ namespace Wabbajack.CLI.Verbs try { - ValidateModlist.RunValidation(queue, modlist).RunSynchronously(); + ValidateModlist.RunValidation(modlist).RunSynchronously(); } catch (Exception e) { diff --git a/Wabbajack.Common/Hash.cs b/Wabbajack.Common/Hash.cs new file mode 100644 index 00000000..87b3d712 --- /dev/null +++ b/Wabbajack.Common/Hash.cs @@ -0,0 +1,198 @@ +using System; +using System.Collections.Generic; +using System.Data.HashFunction.xxHash; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using File = Alphaleonis.Win32.Filesystem.File; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.Common +{ + + /// + /// Struct representing a xxHash64 value. It's a struct with a ulong in it, but wrapped so we don't confuse + /// it with other longs in the system. + /// + public struct Hash + { + private readonly ulong _code; + public Hash(ulong code = 0) + { + _code = code; + } + + public override string ToString() + { + return BitConverter.GetBytes(_code).ToBase64(); + } + + public override bool Equals(object? obj) + { + if (obj is Hash h) + return h._code == _code; + return false; + } + + public override int GetHashCode() + { + return (int)(_code >> 32) ^ (int)_code; + } + + public static bool operator ==(Hash a, Hash b) + { + return a._code == b._code; + } + + public static bool operator !=(Hash a, Hash b) + { + return !(a == b); + } + + public static explicit operator ulong(Hash a) + { + return a._code; + } + + public static explicit operator long(Hash a) + { + return BitConverter.ToInt64(BitConverter.GetBytes(a._code)); + } + + public string ToHex() + { + return BitConverter.GetBytes(_code).ToHex(); + } + + public string ToBase64() + { + return BitConverter.GetBytes(_code).ToBase64(); + } + + public static Hash FromBase64(string hash) + { + return new Hash(BitConverter.ToUInt64(hash.FromBase64())); + } + + public static Hash Empty = new Hash(); + + public static Hash FromLong(in long argHash) + { + return new Hash(BitConverter.ToUInt64(BitConverter.GetBytes(argHash))); + } + + public static Hash FromHex(string xxHashAsHex) + { + return new Hash(BitConverter.ToUInt64(xxHashAsHex.FromHex())); + } + } + + public static partial class Utils + { + public static Hash ReadHash(this BinaryReader br) + { + return new Hash(br.ReadUInt64()); + } + + public static void Write(this BinaryWriter bw, Hash hash) + { + bw.Write((ulong)hash); + } + + public static string StringSha256Hex(this string s) + { + var sha = new SHA256Managed(); + using (var o = new CryptoStream(Stream.Null, sha, CryptoStreamMode.Write)) + { + using var i = new MemoryStream(Encoding.UTF8.GetBytes(s)); + i.CopyTo(o); + } + return sha.Hash.ToHex(); + } + + public static Hash FileHash(this string file, bool nullOnIoError = false) + { + try + { + using var fs = File.OpenRead(file); + var config = new xxHashConfig {HashSizeInBits = 64}; + using var f = new StatusFileStream(fs, $"Hashing {Path.GetFileName(file)}"); + return new Hash(BitConverter.ToUInt64(xxHashFactory.Instance.Create(config).ComputeHash(f).Hash)); + } + catch (IOException) + { + if (nullOnIoError) return Hash.Empty; + throw; + } + } + + public static Hash FileHashCached(this string file, bool nullOnIoError = false) + { + if (TryGetHashCache(file, out var foundHash)) return foundHash; + + var hash = file.FileHash(nullOnIoError); + if (hash != Hash.Empty) + WriteHashCache(file, hash); + return hash; + } + + public static bool TryGetHashCache(string file, out Hash hash) + { + var hashFile = file + Consts.HashFileExtension; + hash = Hash.Empty; + if (!File.Exists(hashFile)) return false; + + if (File.GetSize(hashFile) != 20) return false; + + using var fs = File.OpenRead(hashFile); + using var br = new BinaryReader(fs); + var version = br.ReadUInt32(); + if (version != HashCacheVersion) return false; + + var lastModified = br.ReadUInt64(); + if (lastModified != File.GetLastWriteTimeUtc(file).AsUnixTime()) return false; + hash = new Hash(br.ReadUInt64()); + return true; + } + + + private const uint HashCacheVersion = 0x01; + private static void WriteHashCache(string file, Hash hash) + { + using var fs = File.Create(file + Consts.HashFileExtension); + using var bw = new BinaryWriter(fs); + bw.Write(HashCacheVersion); + var lastModified = File.GetLastWriteTimeUtc(file).AsUnixTime(); + bw.Write(lastModified); + bw.Write((ulong)hash); + } + + public static async Task FileHashCachedAsync(this string file, bool nullOnIOError = false) + { + if (TryGetHashCache(file, out var foundHash)) return foundHash; + + var hash = await file.FileHashAsync(nullOnIOError); + if (hash != Hash.Empty) + WriteHashCache(file, hash); + return hash; + } + + public static async Task FileHashAsync(this string file, bool nullOnIOError = false) + { + try + { + await using var fs = File.OpenRead(file); + var config = new xxHashConfig {HashSizeInBits = 64}; + var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(fs); + return new Hash(BitConverter.ToUInt64(value.Hash)); + } + catch (IOException) + { + if (nullOnIOError) return Hash.Empty; + throw; + } + } + + } +} diff --git a/Wabbajack.Common/Serialization/Deserializer.cs b/Wabbajack.Common/Serialization/Deserializer.cs deleted file mode 100644 index 25b03468..00000000 --- a/Wabbajack.Common/Serialization/Deserializer.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System.IO; - -namespace Wabbajack.Common.Serialization -{ - public class Deserializer - { - public BinaryReader Reader { get; } - } -} diff --git a/Wabbajack.Common/Serialization/IHandler.cs b/Wabbajack.Common/Serialization/IHandler.cs deleted file mode 100644 index b7f988eb..00000000 --- a/Wabbajack.Common/Serialization/IHandler.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Wabbajack.Common.Serialization -{ - public interface IHandler - { - public void Write(Serializer serializer, T data); - public T Read(Deserializer deserialiser); - - } -} diff --git a/Wabbajack.Common/Serialization/PrimitiveHandlers.cs b/Wabbajack.Common/Serialization/PrimitiveHandlers.cs deleted file mode 100644 index 258b5c11..00000000 --- a/Wabbajack.Common/Serialization/PrimitiveHandlers.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System; -namespace Wabbajack.Common.Serialization { -public class UInt32Handler : IHandler { - - public void Write(Serializer serializer, UInt32 data) - { - serializer.Writer.Write(data); - } - - public T Read(Deserializer deserializer) - { - return deserializer.Reader.ReadUInt32(); - } - - -} - -public class Int32Handler : IHandler { - - public void Write(Serializer serializer, Int32 data) - { - serializer.Writer.Write(data); - } - - public T Read(Deserializer deserializer) - { - return deserializer.Reader.ReadInt32(); - } - - -} - - -} \ No newline at end of file diff --git a/Wabbajack.Common/Serialization/PrimitiveHandlers.tt b/Wabbajack.Common/Serialization/PrimitiveHandlers.tt deleted file mode 100644 index fc69aae7..00000000 --- a/Wabbajack.Common/Serialization/PrimitiveHandlers.tt +++ /dev/null @@ -1,34 +0,0 @@ -<#@ template language="C#" #> -<#@ import namespace="System.Text" #> -<#@ import namespace="System.Collections.Generic" #> -using System; -namespace Wabbajack.Common.Serialization { -<# - var types = new List<(Type, string)>() - { - (typeof(UInt32), "UInt32"), - (typeof(Int32), "Int32") - }; - - - foreach (var type in types) - { -#> -public class <#=type.Item2#>Handler : IHandler { - - public void Write(Serializer serializer, <#=type.Item2#> data) - { - serializer.Writer.Write(data); - } - - public T Read(Deserializer deserializer) - { - return deserializer.Reader.Read<#=type.Item2#>(); - } - - -} - -<# } #> - -} \ No newline at end of file diff --git a/Wabbajack.Common/Serialization/Serializer.cs b/Wabbajack.Common/Serialization/Serializer.cs deleted file mode 100644 index 759478e2..00000000 --- a/Wabbajack.Common/Serialization/Serializer.cs +++ /dev/null @@ -1,53 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading.Tasks; - -namespace Wabbajack.Common.Serialization -{ - public class Serializer - { - private Dictionary _internedStrings = new Dictionary(); - private Dictionary _handlers = new Dictionary(); - public BinaryWriter Writer { get; } - - public Serializer(BinaryWriter bw) - { - Writer = bw; - } - - public void RegisterWriteHandler(string name, IHandler handler) - { - _handlers.Add(typeof(T), new HandlerRecord - { - TypeName = name, - TypeId = Intern(name), - Handler = handler - }); - } - - public async Task Write(BinaryWriter bw, T data) - { - var handler = _handlers[typeof(T)]; - handler.Handler.Write(this, data); - } - - - private int Intern(string s) - { - if (_internedStrings.TryGetValue(s, out var idx)) - return idx; - idx = _internedStrings.Count; - _internedStrings[s] = idx; - return idx; - } - } - - class HandlerRecord - { - public string TypeName; - public int TypeId; - public IHandler Handler; - - } -} diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 2494701a..c8887062 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -32,7 +32,7 @@ using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Common { - public static class Utils + public static partial class Utils { public static bool IsMO2Running(string mo2Path) { @@ -206,135 +206,6 @@ namespace Wabbajack.Common } } - /// - /// MurMur3 hashes the file pointed to by this string - /// - /// - /// - public static string FileSHA256(this string file) - { - var sha = new SHA256Managed(); - using (var o = new CryptoStream(Stream.Null, sha, CryptoStreamMode.Write)) - { - using (var i = File.OpenRead(file)) - { - i.CopyToWithStatus(new FileInfo(file).Length, o, $"Hashing {Path.GetFileName(file)}"); - } - } - - return sha.Hash.ToBase64(); - } - - public static string StringSHA256Hex(this string s) - { - var sha = new SHA256Managed(); - using (var o = new CryptoStream(Stream.Null, sha, CryptoStreamMode.Write)) - { - using var i = new MemoryStream(Encoding.UTF8.GetBytes(s)); - i.CopyTo(o); - } - - return sha.Hash.ToHex(); - } - - public static string FileHash(this string file, bool nullOnIOError = false) - { - try - { - var hash = new xxHashConfig(); - hash.HashSizeInBits = 64; - hash.Seed = 0x42; - using (var fs = File.OpenRead(file)) - { - var config = new xxHashConfig(); - config.HashSizeInBits = 64; - using (var f = new StatusFileStream(fs, $"Hashing {Path.GetFileName(file)}")) - { - var value = xxHashFactory.Instance.Create(config).ComputeHash(f); - return value.AsBase64String(); - } - } - } - catch (IOException ex) - { - if (nullOnIOError) return null; - throw ex; - } - } - - public static string FileHashCached(this string file, bool nullOnIOError = false) - { - if (TryGetHashCache(file, out var foundHash)) return foundHash; - - var hash = file.FileHash(nullOnIOError); - if (hash != null) - WriteHashCache(file, hash); - return hash; - } - - public static bool TryGetHashCache(string file, out string hash) - { - var hashFile = file + Consts.HashFileExtension; - hash = null; - if (!File.Exists(hashFile)) return false; - - if (File.GetSize(hashFile) != 20) return false; - - using var fs = File.OpenRead(hashFile); - using var br = new BinaryReader(fs); - var version = br.ReadUInt32(); - if (version != HashCacheVersion) return false; - - var lastModified = br.ReadUInt64(); - if (lastModified != File.GetLastWriteTimeUtc(file).AsUnixTime()) return false; - hash = BitConverter.GetBytes(br.ReadUInt64()).ToBase64(); - return true; - } - - - private const uint HashCacheVersion = 0x01; - private static void WriteHashCache(string file, string hash) - { - using var fs = File.Create(file + Consts.HashFileExtension); - using var bw = new BinaryWriter(fs); - bw.Write(HashCacheVersion); - var lastModified = File.GetLastWriteTimeUtc(file).AsUnixTime(); - bw.Write(lastModified); - bw.Write(BitConverter.ToUInt64(hash.FromBase64())); - } - - public static async Task FileHashCachedAsync(this string file, bool nullOnIOError = false) - { - if (TryGetHashCache(file, out var foundHash)) return foundHash; - - var hash = await file.FileHashAsync(nullOnIOError); - if (hash != null) - WriteHashCache(file, hash); - return hash; - } - - public static async Task FileHashAsync(this string file, bool nullOnIOError = false) - { - try - { - var hash = new xxHashConfig(); - hash.HashSizeInBits = 64; - hash.Seed = 0x42; - using (var fs = File.OpenRead(file)) - { - var config = new xxHashConfig(); - config.HashSizeInBits = 64; - var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(fs); - return value.AsBase64String(); - } - } - catch (IOException ex) - { - if (nullOnIOError) return null; - throw ex; - } - } - public static void CopyToWithStatus(this Stream istream, long maxSize, Stream ostream, string status) { var buffer = new byte[1024 * 64]; @@ -980,7 +851,7 @@ namespace Wabbajack.Common } } - public static async Task CreatePatch(FileStream srcStream, string srcHash, FileStream destStream, string destHash, + public static async Task CreatePatch(FileStream srcStream, Hash srcHash, FileStream destStream, Hash destHash, FileStream patchStream) { await using var sigFile = new TempStream(); @@ -996,7 +867,7 @@ namespace Wabbajack.Common try { - var cacheFile = Path.Combine(Consts.PatchCacheFolder, $"{srcHash.FromBase64().ToHex()}_{srcHash.FromBase64().ToHex()}.patch"); + var cacheFile = Path.Combine(Consts.PatchCacheFolder, $"{srcHash.ToHex()}_{destHash.ToHex()}.patch"); if (!Directory.Exists(Consts.PatchCacheFolder)) Directory.CreateDirectory(Consts.PatchCacheFolder); @@ -1009,10 +880,10 @@ namespace Wabbajack.Common } } - public static bool TryGetPatch(string foundHash, string fileHash, out byte[] ePatch) + public static bool TryGetPatch(Hash foundHash, Hash fileHash, out byte[] ePatch) { var patchName = Path.Combine(Consts.PatchCacheFolder, - $"{foundHash.FromBase64().ToHex()}_{fileHash.FromBase64().ToHex()}.patch"); + $"{foundHash.ToHex()}_{fileHash.ToHex()}.patch"); if (File.Exists(patchName)) { ePatch = File.ReadAllBytes(patchName); diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 157f60eb..39f626dd 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -29,6 +29,7 @@ + diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index f389ec69..cec70db9 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -46,7 +46,7 @@ namespace Wabbajack.Lib public ModList ModList = new ModList(); public List IndexedArchives = new List(); - public Dictionary> IndexedFiles = new Dictionary>(); + public Dictionary> IndexedFiles = new Dictionary>(); public static void Info(string msg) { @@ -194,25 +194,25 @@ namespace Wabbajack.Lib { Info("Building a list of archives based on the files required"); - var shas = InstallDirectives.OfType() - .Select(a => a.ArchiveHashPath[0]) + var hashes = InstallDirectives.OfType() + .Select(a => Hash.FromBase64(a.ArchiveHashPath[0])) .Distinct(); var archives = IndexedArchives.OrderByDescending(f => f.File.LastModified) .GroupBy(f => f.File.Hash) .ToDictionary(f => f.Key, f => f.First()); - SelectedArchives = await shas.PMap(Queue, sha => ResolveArchive(sha, archives)); + SelectedArchives = await hashes.PMap(Queue, hash => ResolveArchive(hash, archives)); } - public async Task ResolveArchive(string sha, IDictionary archives) + public async Task ResolveArchive(Hash hash, IDictionary archives) { - if (archives.TryGetValue(sha, out var found)) + if (archives.TryGetValue(hash, out var found)) { return await ResolveArchive(found); } - Error($"No match found for Archive sha: {sha} this shouldn't happen"); + Error($"No match found for Archive sha: {hash.ToBase64()} this shouldn't happen"); return null; } diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index cf9f1163..efa6a42b 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -27,7 +27,7 @@ namespace Wabbajack.Lib public string ModListArchive { get; private set; } public ModList ModList { get; private set; } - public Dictionary HashedArchives { get; set; } + public Dictionary HashedArchives { get; set; } public SystemParameters SystemParameters { get; set; } @@ -127,7 +127,7 @@ namespace Wabbajack.Lib var grouped = ModList.Directives .OfType() .GroupBy(e => e.ArchiveHashPath[0]) - .ToDictionary(k => k.Key); + .ToDictionary(k => Hash.FromBase64(k.Key)); var archives = ModList.Archives .Select(a => new { Archive = a, AbsolutePath = HashedArchives.GetOrDefault(a.Hash) }) .Where(a => a.AbsolutePath != null) @@ -264,7 +264,7 @@ namespace Wabbajack.Lib { var orig_name = Path.GetFileNameWithoutExtension(archive.Name); var ext = Path.GetExtension(archive.Name); - var unique_key = archive.State.PrimaryKeyString.StringSHA256Hex(); + var unique_key = archive.State.PrimaryKeyString.StringSha256Hex(); outputPath = Path.Combine(DownloadFolder, orig_name + "_" + unique_key + "_" + ext); if (outputPath.FileExists()) File.Delete(outputPath); @@ -413,7 +413,7 @@ namespace Wabbajack.Lib Utils.Log($"Optimized {ModList.Directives.Count} directives to {indexed.Count} required"); var requiredArchives = indexed.Values.OfType() .GroupBy(d => d.ArchiveHashPath[0]) - .Select(d => d.Key) + .Select(d => Hash.FromBase64(d.Key)) .ToHashSet(); ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToList(); diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index 1cd031c6..b26063f8 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -13,10 +13,10 @@ namespace Wabbajack.Lib return client; } - public static async Task GetModUpgrade(string hash) + public static async Task GetModUpgrade(Hash hash) { using var response = await GetClient() - .GetAsync($"https://{Consts.WabbajackCacheHostname}/alternative/{hash.FromBase64().ToHex()}"); + .GetAsync($"https://{Consts.WabbajackCacheHostname}/alternative/{hash.ToHex()}"); return !response.IsSuccessStatusCode ? null : (await response.Content.ReadAsStringAsync()).FromJSONString(); } } diff --git a/Wabbajack.Lib/CompilationSteps/CompilationErrors/InvalidGameESMError.cs b/Wabbajack.Lib/CompilationSteps/CompilationErrors/InvalidGameESMError.cs index 7b43ce72..4bb367c3 100644 --- a/Wabbajack.Lib/CompilationSteps/CompilationErrors/InvalidGameESMError.cs +++ b/Wabbajack.Lib/CompilationSteps/CompilationErrors/InvalidGameESMError.cs @@ -4,13 +4,14 @@ using System.Linq; using System.Text; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; +using Wabbajack.Common; using Wabbajack.Common.StatusFeed; namespace Wabbajack.Lib.CompilationSteps.CompilationErrors { public class InvalidGameESMError : AErrorMessage { - public string Hash { get; } + public Hash Hash { get; } public string PathToFile { get; } private readonly CleanedESM _esm; public string GameFileName => Path.GetFileName(_esm.To); @@ -29,7 +30,7 @@ the modlist expecting a different of the game than you currently have installed, the game, and then attempting to re-install this modlist. Also verify that the version of the game you have installed matches the version expected by this modlist."; } - public InvalidGameESMError(CleanedESM esm, string hash, string path) + public InvalidGameESMError(CleanedESM esm, Hash hash, string path) { Hash = hash; PathToFile = path; diff --git a/Wabbajack.Lib/Data.cs b/Wabbajack.Lib/Data.cs index a7f6bdda..fbc42d68 100644 --- a/Wabbajack.Lib/Data.cs +++ b/Wabbajack.Lib/Data.cs @@ -26,7 +26,7 @@ namespace Wabbajack.Lib public VirtualFile File { get; } - public string Hash => File.Hash; + public Hash Hash => File.Hash; public T EvolveTo() where T : Directive, new() { @@ -130,7 +130,7 @@ namespace Wabbajack.Lib /// public string To; public long Size; - public string Hash; + public Hash Hash; } public class IgnoredDirectly : Directive @@ -167,7 +167,7 @@ namespace Wabbajack.Lib public class CleanedESM : InlineFile { - public string SourceESMHash; + public Hash SourceESMHash; } /// @@ -191,23 +191,13 @@ namespace Wabbajack.Lib { private string _fullPath; - /// - /// MurMur3 hash of the archive this file comes from - /// public string[] ArchiveHashPath; [Exclude] public VirtualFile FromFile; [Exclude] - public string FullPath - { - get - { - if (_fullPath == null) _fullPath = string.Join("|", ArchiveHashPath); - return _fullPath; - } - } + public string FullPath => _fullPath ??= string.Join("|", ArchiveHashPath); } public class CreateBSA : Directive @@ -226,13 +216,13 @@ namespace Wabbajack.Lib public string PatchID; [Exclude] - public string FromHash; + public Hash FromHash; } public class SourcePatch { public string RelativePath; - public string Hash; + public Hash Hash; } public class MergedPatch : Directive @@ -246,7 +236,7 @@ namespace Wabbajack.Lib /// /// MurMur3 Hash of the archive /// - public string Hash { get; set; } + public Hash Hash { get; set; } /// /// Meta INI for the downloaded archive diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index 31b499cb..e8e3ecb3 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -108,7 +108,7 @@ namespace Wabbajack.Lib.Downloaders var upgradeResult = await Download(upgrade, upgradePath); if (!upgradeResult) return false; - var patchName = $"{archive.Hash.FromBase64().ToHex()}_{upgrade.Hash.FromBase64().ToHex()}"; + var patchName = $"{archive.Hash.ToHex()}_{upgrade.Hash.ToHex()}"; var patchPath = Path.Combine(Path.GetDirectoryName(destination), "_Patch_" + patchName); var patchState = new Archive diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs index fb412251..5df9ac94 100644 --- a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs @@ -49,7 +49,7 @@ namespace Wabbajack.Lib.Downloaders { public Game Game { get; set; } public string GameFile { get; set; } - public string Hash { get; set; } + public Hash Hash { get; set; } public string GameVersion { get; set; } diff --git a/Wabbajack.Lib/FileUploader/AuthorAPI.cs b/Wabbajack.Lib/FileUploader/AuthorAPI.cs index 98b6c4e3..05703d5b 100644 --- a/Wabbajack.Lib/FileUploader/AuthorAPI.cs +++ b/Wabbajack.Lib/FileUploader/AuthorAPI.cs @@ -105,7 +105,7 @@ namespace Wabbajack.Lib.FileUploader if (!tcs.Task.IsFaulted) { progressFn(1.0); - var hash = (await hash_task).FromBase64().ToHex(); + var hash = (await hash_task).ToHex(); response = await client.PutAsync(UploadURL + $"/{key}/finish/{hash}", new StringContent("")); if (response.IsSuccessStatusCode) tcs.SetResult(await response.Content.ReadAsStringAsync()); diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index f625c561..8ff47a1a 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -45,7 +45,7 @@ namespace Wabbajack.Lib public override string VFSCacheName => Path.Combine( Consts.LocalAppDataPath, - $"vfs_compile_cache-{Path.Combine(MO2Folder ?? "Unknown", "ModOrganizer.exe").StringSHA256Hex()}.bin"); + $"vfs_compile_cache-{Path.Combine(MO2Folder ?? "Unknown", "ModOrganizer.exe").StringSha256Hex()}.bin"); public MO2Compiler(string mo2Folder, string mo2Profile, string outputFile) { @@ -318,7 +318,7 @@ namespace Wabbajack.Lib UpdateTracker.NextStep("Running Validation"); - await ValidateModlist.RunValidation(Queue, ModList); + await ValidateModlist.RunValidation(ModList); UpdateTracker.NextStep("Generating Report"); GenerateManifest(); @@ -382,7 +382,7 @@ namespace Wabbajack.Lib var client = new Common.Http.Client(); using var response = await client.GetAsync( - $"http://build.wabbajack.org/indexed_files/{vf.Hash.FromBase64().ToHex()}/meta.ini"); + $"http://build.wabbajack.org/indexed_files/{vf.Hash.ToHex()}/meta.ini"); if (!response.IsSuccessStatusCode) { diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index 0f856f46..7a1fa17c 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -82,7 +82,7 @@ namespace Wabbajack.Lib if (cancel.IsCancellationRequested) return false; UpdateTracker.NextStep("Validating Modlist"); - await ValidateModlist.RunValidation(Queue, ModList); + await ValidateModlist.RunValidation(ModList); Directory.CreateDirectory(OutputFolder); Directory.CreateDirectory(DownloadFolder); diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index 3657efff..2e495a90 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -96,7 +96,7 @@ namespace Wabbajack.Lib.ModListRegistry public class DownloadMetadata { - public string Hash { get; set; } + public Hash Hash { get; set; } public long Size { get; set; } public long NumberOfArchives { get; set; } diff --git a/Wabbajack.Lib/Validation/ValidateModlist.cs b/Wabbajack.Lib/Validation/ValidateModlist.cs index 88cce7ad..bd01808d 100644 --- a/Wabbajack.Lib/Validation/ValidateModlist.cs +++ b/Wabbajack.Lib/Validation/ValidateModlist.cs @@ -16,21 +16,7 @@ namespace Wabbajack.Lib.Validation /// public class ValidateModlist { - public Dictionary AuthorPermissions { get; set; } = new Dictionary(); - - private readonly WorkQueue _queue; - public ServerWhitelist ServerWhitelist { get; set; } = new ServerWhitelist(); - - public ValidateModlist(WorkQueue workQueue) - { - _queue = workQueue; - } - - public void LoadAuthorPermissionsFromString(string s) - { - AuthorPermissions = s.FromYaml>(); - } - + public ServerWhitelist ServerWhitelist { get; private set; } = new ServerWhitelist(); public void LoadServerWhitelist(string s) { ServerWhitelist = s.FromYaml(); @@ -50,9 +36,9 @@ namespace Wabbajack.Lib.Validation } - public static async Task RunValidation(WorkQueue queue, ModList modlist) + public static async Task RunValidation(ModList modlist) { - var validator = new ValidateModlist(queue); + var validator = new ValidateModlist(); await validator.LoadListsFromGithub(); @@ -69,92 +55,9 @@ namespace Wabbajack.Lib.Validation } } - /// - /// Takes all the permissions for a given Nexus mods and merges them down to a single permissions record - /// the more specific record having precedence in each field. - /// - /// - /// - public Permissions FilePermissions(NexusDownloader.State mod) - { - var author_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Permissions; - var game_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Permissions; - var mod_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Mods.GetOrDefault(mod.ModID) - ?.Permissions; - var file_permissions = AuthorPermissions.GetOrDefault(mod.Author)?.Games.GetOrDefault(mod.GameName)?.Mods - .GetOrDefault(mod.ModID)?.Files.GetOrDefault(mod.FileID)?.Permissions; - - return new Permissions - { - CanExtractBSAs = file_permissions?.CanExtractBSAs ?? mod_permissions?.CanExtractBSAs ?? - game_permissions?.CanExtractBSAs ?? author_permissions?.CanExtractBSAs ?? true, - CanModifyAssets = file_permissions?.CanModifyAssets ?? mod_permissions?.CanModifyAssets ?? - game_permissions?.CanModifyAssets ?? author_permissions?.CanModifyAssets ?? true, - CanModifyESPs = file_permissions?.CanModifyESPs ?? mod_permissions?.CanModifyESPs ?? - game_permissions?.CanModifyESPs ?? author_permissions?.CanModifyESPs ?? true, - CanUseInOtherGames = file_permissions?.CanUseInOtherGames ?? mod_permissions?.CanUseInOtherGames ?? - game_permissions?.CanUseInOtherGames ?? author_permissions?.CanUseInOtherGames ?? true, - }; - } - public async Task> Validate(ModList modlist) { ConcurrentStack ValidationErrors = new ConcurrentStack(); - - var nexus_mod_permissions = (await modlist.Archives - .Where(a => a.State is NexusDownloader.State) - .PMap(_queue, a => (a.Hash, FilePermissions((NexusDownloader.State)a.State), a))) - .ToDictionary(a => a.Hash, a => new { permissions = a.Item2, archive = a.a }); - - await modlist.Directives - .OfType() - .PMap(_queue, p => - { - if (nexus_mod_permissions.TryGetValue(p.ArchiveHashPath[0], out var archive)) - { - var ext = Path.GetExtension(p.ArchiveHashPath.Last()); - var url = (archive.archive.State as NexusDownloader.State).URL; - if (Consts.AssetFileExtensions.Contains(ext) && !(archive.permissions.CanModifyAssets ?? true)) - { - ValidationErrors.Push($"{p.To} from {url} is set to disallow asset modification"); - } - else if (Consts.ESPFileExtensions.Contains(ext) && !(archive.permissions.CanModifyESPs ?? true)) - { - ValidationErrors.Push($"{p.To} from {url} is set to disallow asset ESP modification"); - } - } - }); - - await modlist.Directives - .OfType() - .PMap(_queue, p => - { - if (nexus_mod_permissions.TryGetValue(p.ArchiveHashPath[0], out var archive)) - { - var url = (archive.archive.State as NexusDownloader.State).URL; - if (!(archive.permissions.CanExtractBSAs ?? true) && - p.ArchiveHashPath.Skip(1).ButLast().Any(a => Consts.SupportedBSAs.Contains(Path.GetExtension(a).ToLower()))) - { - ValidationErrors.Push($"{p.To} from {url} is set to disallow BSA extraction"); - } - } - }); - - var nexus = NexusApi.NexusApiUtils.ConvertGameName(modlist.GameType.MetaData().NexusName); - - modlist.Archives - .Where(a => a.State is NexusDownloader.State) - .Where(m => NexusApi.NexusApiUtils.ConvertGameName(((NexusDownloader.State)m.State).GameName) != nexus) - .Do(m => - { - var permissions = FilePermissions((NexusDownloader.State)m.State); - if (!(permissions.CanUseInOtherGames ?? true)) - { - ValidationErrors.Push( - $"The ModList is for {nexus} but {m.Name} is for game type {((NexusDownloader.State)m.State).GameName} and is not allowed to be converted to other game types"); - } - }); - modlist.Archives .Where(m => !m.State.IsWhitelisted(ServerWhitelist)) .Do(m => diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs index d625e938..b029542f 100644 --- a/Wabbajack.Lib/VortexCompiler.cs +++ b/Wabbajack.Lib/VortexCompiler.cs @@ -49,7 +49,7 @@ namespace Wabbajack.Lib public override string VFSCacheName => Path.Combine( Consts.LocalAppDataPath, - $"vfs_compile_cache-{StagingFolder?.StringSHA256Hex() ?? "Unknown"}.bin"); + $"vfs_compile_cache-{StagingFolder?.StringSha256Hex() ?? "Unknown"}.bin"); public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder, string stagingFolder, string outputFile) { @@ -245,7 +245,7 @@ namespace Wabbajack.Lib }; UpdateTracker.NextStep("Running Validation"); - await ValidateModlist.RunValidation(Queue, ModList); + await ValidateModlist.RunValidation(ModList); UpdateTracker.NextStep("Generating Report"); GenerateManifest(); diff --git a/Wabbajack.Test/ContentRightsManagementTests.cs b/Wabbajack.Test/ContentRightsManagementTests.cs index 80a4c91e..c00dc8ec 100644 --- a/Wabbajack.Test/ContentRightsManagementTests.cs +++ b/Wabbajack.Test/ContentRightsManagementTests.cs @@ -52,8 +52,7 @@ namespace Wabbajack.Test public void TestSetup() { queue = new WorkQueue(); - validate = new ValidateModlist(queue); - validate.LoadAuthorPermissionsFromString(permissions); + validate = new ValidateModlist(); validate.LoadServerWhitelist(server_whitelist); } @@ -63,76 +62,6 @@ namespace Wabbajack.Test queue?.Dispose(); } - [TestMethod] - public void TestRightsFallthrough() - { - var permissions = validate.FilePermissions(new NexusDownloader.State - { - Author = "bill", - GameName = "Skyrim", - ModID = "42", - FileID = "33" - }); - - permissions.CanExtractBSAs.AssertIsFalse(); - permissions.CanModifyESPs.AssertIsFalse(); - permissions.CanModifyAssets.AssertIsFalse(); - permissions.CanUseInOtherGames.AssertIsFalse(); - - permissions = validate.FilePermissions(new NexusDownloader.State - { - Author = "bob", - GameName = "Skyrim", - ModID = "42", - FileID = "33" - }); - - permissions.CanExtractBSAs.AssertIsTrue(); - permissions.CanModifyESPs.AssertIsTrue(); - permissions.CanModifyAssets.AssertIsTrue(); - permissions.CanUseInOtherGames.AssertIsTrue(); - - permissions = validate.FilePermissions(new NexusDownloader.State - { - Author = "bill", - GameName = "Fallout4", - ModID = "42", - FileID = "33" - }); - - permissions.CanExtractBSAs.AssertIsFalse(); - permissions.CanModifyESPs.AssertIsTrue(); - permissions.CanModifyAssets.AssertIsTrue(); - permissions.CanUseInOtherGames.AssertIsTrue(); - - permissions = validate.FilePermissions(new NexusDownloader.State - { - Author = "bill", - GameName = "Skyrim", - ModID = "43", - FileID = "33" - }); - - permissions.CanExtractBSAs.AssertIsFalse(); - permissions.CanModifyESPs.AssertIsFalse(); - permissions.CanModifyAssets.AssertIsTrue(); - permissions.CanUseInOtherGames.AssertIsTrue(); - - permissions = validate.FilePermissions(new NexusDownloader.State - { - Author = "bill", - GameName = "Skyrim", - ModID = "42", - FileID = "31" - }); - - permissions.CanExtractBSAs.AssertIsFalse(); - permissions.CanModifyESPs.AssertIsFalse(); - permissions.CanModifyAssets.AssertIsFalse(); - permissions.CanUseInOtherGames.AssertIsTrue(); - } - - [TestMethod] public async Task TestModValidation() { @@ -150,9 +79,8 @@ namespace Wabbajack.Test ModID = "42", FileID = "33", }, - Hash = "DEADBEEF" + Hash = Hash.FromLong(42) } - }, Directives = new List { @@ -163,59 +91,14 @@ namespace Wabbajack.Test } } }; - - IEnumerable errors; - - // No errors, simple archive extraction - errors = await validate.Validate(modlist); - Assert.AreEqual(errors.Count(), 0); - - - // Error due to patched file - modlist.Directives[0] = new PatchedFromArchive - { - PatchID = Guid.NewGuid().ToString(), - ArchiveHashPath = new[] {"DEADBEEF", "foo\\bar\\baz.pex"}, - }; - - errors = await validate.Validate(modlist); - Assert.AreEqual(errors.Count(), 1); - - // Error due to extracted BSA file - modlist.Directives[0] = new FromArchive - { - ArchiveHashPath = new[] { "DEADBEEF", "foo.bsa", "foo\\bar\\baz.dds" }, - }; - - errors = await validate.Validate(modlist); - Assert.AreEqual(errors.Count(), 1); - - // No error since we're just installing the .bsa, not extracting it - modlist.Directives[0] = new FromArchive - { - ArchiveHashPath = new[] { "DEADBEEF", "foo.bsa"}, - }; - - errors = await validate.Validate(modlist); - Assert.AreEqual(0, errors.Count()); - - // Error due to game conversion - modlist.GameType = Game.SkyrimSpecialEdition; - modlist.Directives[0] = new FromArchive - { - ArchiveHashPath = new[] { "DEADBEEF", "foo\\bar\\baz.dds" }, - }; - errors = await validate.Validate(modlist); - Assert.AreEqual(errors.Count(), 1); - // Error due to file downloaded from 3rd party modlist.GameType = Game.Skyrim; modlist.Archives[0] = new Archive() { State = new HTTPDownloader.State() { Url = "https://somebadplace.com" }, - Hash = "DEADBEEF" + Hash = Hash.FromLong(42) }; - errors = await validate.Validate(modlist); + var errors = await validate.Validate(modlist); Assert.AreEqual(1, errors.Count()); // Ok due to file downloaded from whitelisted 3rd party @@ -223,7 +106,7 @@ namespace Wabbajack.Test modlist.Archives[0] = new Archive { State = new HTTPDownloader.State { Url = "https://somegoodplace.com/baz.7z" }, - Hash = "DEADBEEF" + Hash = Hash.FromLong(42) }; errors = await validate.Validate(modlist); Assert.AreEqual(0, errors.Count()); @@ -234,7 +117,7 @@ namespace Wabbajack.Test modlist.Archives[0] = new Archive { State = new GoogleDriveDownloader.State { Id = "bleg"}, - Hash = "DEADBEEF" + Hash = Hash.FromLong(42) }; errors = await validate.Validate(modlist); Assert.AreEqual(errors.Count(), 1); @@ -244,7 +127,7 @@ namespace Wabbajack.Test modlist.Archives[0] = new Archive { State = new GoogleDriveDownloader.State { Id = "googleDEADBEEF" }, - Hash = "DEADBEEF" + Hash = Hash.FromLong(42) }; errors = await validate.Validate(modlist); Assert.AreEqual(0, errors.Count()); @@ -256,7 +139,7 @@ namespace Wabbajack.Test { using (var workQueue = new WorkQueue()) { - await new ValidateModlist(workQueue).LoadListsFromGithub(); + await new ValidateModlist().LoadListsFromGithub(); } } } diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 0a1c0c08..a6047e30 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -536,7 +536,7 @@ namespace Wabbajack.Test var archive = new Archive { Name = "Cori.7z", - Hash = "gCRVrvzDNH0=", + Hash = Hash.FromBase64("gCRVrvzDNH0="), State = new NexusDownloader.State { GameName = Game.SkyrimSpecialEdition.MetaData().NexusName, diff --git a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs index a6669f2e..235e8777 100644 --- a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs @@ -82,7 +82,7 @@ namespace Wabbajack.VirtualFileSystem.Test await AddTestRoot(); - var files = context.Index.ByHash["qX0GZvIaTKM="]; + var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")]; Assert.AreEqual(files.Count(), 2); } @@ -150,7 +150,7 @@ namespace Wabbajack.VirtualFileSystem.Test await AddTestRoot(); - var files = context.Index.ByHash["qX0GZvIaTKM="]; + var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")]; var cleanup = await context.Stage(files); @@ -173,7 +173,7 @@ namespace Wabbajack.VirtualFileSystem.Test await AddTestRoot(); - var files = context.Index.ByHash["qX0GZvIaTKM="]; + var files = context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")]; var archive = context.Index.ByRootPath[Path.Combine(VFS_TEST_DIR_FULL, "test.zip")]; var state = context.GetPortableState(files); @@ -181,9 +181,9 @@ namespace Wabbajack.VirtualFileSystem.Test var new_context = new Context(Queue); await new_context.IntegrateFromPortable(state, - new Dictionary {{archive.Hash, archive.FullPath}}); + new Dictionary {{archive.Hash, archive.FullPath}}); - var new_files = new_context.Index.ByHash["qX0GZvIaTKM="]; + var new_files = new_context.Index.ByHash[Hash.FromBase64("qX0GZvIaTKM=")]; var close = await new_context.Stage(new_files); diff --git a/Wabbajack.VirtualFileSystem/Context.cs b/Wabbajack.VirtualFileSystem/Context.cs index df0eb818..d48ea697 100644 --- a/Wabbajack.VirtualFileSystem/Context.cs +++ b/Wabbajack.VirtualFileSystem/Context.cs @@ -249,16 +249,16 @@ namespace Wabbajack.VirtualFileSystem { Name = f.Parent != null ? f.Name : null, Hash = f.Hash, - ParentHash = f.Parent?.Hash, + ParentHash = f.Parent?.Hash ?? Hash.Empty, Size = f.Size }).ToList(); } - public async Task IntegrateFromPortable(List state, Dictionary links) + public async Task IntegrateFromPortable(List state, Dictionary links) { var indexedState = state.GroupBy(f => f.ParentHash) - .ToDictionary(f => f.Key ?? "", f => (IEnumerable) f); - var parents = await indexedState[""] + .ToDictionary(f => f.Key, f => (IEnumerable) f); + var parents = await indexedState[Hash.Empty] .PMap(Queue,f => VirtualFile.CreateFromPortable(this, indexedState, links, f)); var newIndex = await Index.Integrate(parents); @@ -297,7 +297,7 @@ namespace Wabbajack.VirtualFileSystem void BackFillOne(KnownFile file) { - var parent = newFiles[file.Paths[0]]; + var parent = newFiles[Hash.FromBase64(file.Paths[0])]; foreach (var path in file.Paths.Skip(1)) { if (parentchild.TryGetValue((parent, path), out var foundParent)) @@ -331,7 +331,7 @@ namespace Wabbajack.VirtualFileSystem public class KnownFile { public string[] Paths { get; set; } - public string Hash { get; set; } + public Hash Hash { get; set; } } public class DisposableList : List, IDisposable @@ -355,7 +355,7 @@ namespace Wabbajack.VirtualFileSystem public IndexRoot(ImmutableList aFiles, ImmutableDictionary byFullPath, - ImmutableDictionary> byHash, + ImmutableDictionary> byHash, ImmutableDictionary byRoot, ImmutableDictionary> byName) { @@ -370,7 +370,7 @@ namespace Wabbajack.VirtualFileSystem { AllFiles = ImmutableList.Empty; ByFullPath = ImmutableDictionary.Empty; - ByHash = ImmutableDictionary>.Empty; + ByHash = ImmutableDictionary>.Empty; ByRootPath = ImmutableDictionary.Empty; ByName = ImmutableDictionary>.Empty; } @@ -378,7 +378,7 @@ namespace Wabbajack.VirtualFileSystem public ImmutableList AllFiles { get; } public ImmutableDictionary ByFullPath { get; } - public ImmutableDictionary> ByHash { get; } + public ImmutableDictionary> ByHash { get; } public ImmutableDictionary> ByName { get; set; } public ImmutableDictionary ByRootPath { get; } @@ -391,7 +391,7 @@ namespace Wabbajack.VirtualFileSystem .ToImmutableDictionary(f => f.FullPath)); var byHash = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) - .Where(f => f.Hash != null) + .Where(f => f.Hash != Hash.Empty) .ToGroupedImmutableDictionary(f => f.Hash)); var byName = Task.Run(() => allFiles.SelectMany(f => f.ThisAndAllChildren) @@ -410,7 +410,7 @@ namespace Wabbajack.VirtualFileSystem public VirtualFile FileForArchiveHashPath(string[] argArchiveHashPath) { - var cur = ByHash[argArchiveHashPath[0]].First(f => f.Parent == null); + var cur = ByHash[Hash.FromBase64(argArchiveHashPath[0])].First(f => f.Parent == null); return argArchiveHashPath.Skip(1).Aggregate(cur, (current, itm) => ByName[itm].First(f => f.Parent == current)); } } diff --git a/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs b/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs index 28049661..0b5bf3fc 100644 --- a/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using Wabbajack.Common; namespace Wabbajack.VirtualFileSystem { @@ -8,7 +9,7 @@ namespace Wabbajack.VirtualFileSystem public class IndexedVirtualFile { public string Name { get; set; } - public string Hash { get; set; } + public Hash Hash { get; set; } public long Size { get; set; } public List Children { get; set; } = new List(); } diff --git a/Wabbajack.VirtualFileSystem/PortableFile.cs b/Wabbajack.VirtualFileSystem/PortableFile.cs index 8b3a7e78..d2b9a09f 100644 --- a/Wabbajack.VirtualFileSystem/PortableFile.cs +++ b/Wabbajack.VirtualFileSystem/PortableFile.cs @@ -1,10 +1,12 @@ -namespace Wabbajack.VirtualFileSystem +using Wabbajack.Common; + +namespace Wabbajack.VirtualFileSystem { public class PortableFile { public string Name { get; set; } - public string Hash { get; set; } - public string ParentHash { get; set; } + public Hash Hash { get; set; } + public Hash ParentHash { get; set; } public long Size { get; set; } } -} \ No newline at end of file +} diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index 17a98bb6..79db8940 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -42,7 +42,7 @@ namespace Wabbajack.VirtualFileSystem } } - public string Hash { get; internal set; } + public Hash Hash { get; internal set; } public ExtendedHashes ExtendedHashes { get; set; } public long Size { get; internal set; } @@ -214,12 +214,12 @@ namespace Wabbajack.VirtualFileSystem return self; } - private static async Task TryGetContentsFromServer(string hash) + private static async Task TryGetContentsFromServer(Hash hash) { try { var client = new HttpClient(); - var response = await client.GetAsync($"http://{Consts.WabbajackCacheHostname}/indexed_files/{hash.FromBase64().ToHex()}"); + var response = await client.GetAsync($"http://{Consts.WabbajackCacheHostname}/indexed_files/{hash.ToHex()}"); if (!response.IsSuccessStatusCode) return null; @@ -274,7 +274,7 @@ namespace Wabbajack.VirtualFileSystem Parent = parent, Name = br.ReadString(), _fullPath = br.ReadString(), - Hash = br.ReadString(), + Hash = br.ReadHash(), Size = br.ReadInt64(), LastModified = br.ReadInt64(), LastAnalyzed = br.ReadInt64(), @@ -288,7 +288,7 @@ namespace Wabbajack.VirtualFileSystem } public static VirtualFile CreateFromPortable(Context context, - Dictionary> state, Dictionary links, + Dictionary> state, Dictionary links, PortableFile portableFile) { var vf = new VirtualFile @@ -305,7 +305,7 @@ namespace Wabbajack.VirtualFileSystem } public static VirtualFile CreateFromPortable(Context context, VirtualFile parent, - Dictionary> state, PortableFile portableFile) + Dictionary> state, PortableFile portableFile) { var vf = new VirtualFile { @@ -323,7 +323,7 @@ namespace Wabbajack.VirtualFileSystem public string[] MakeRelativePaths() { var path = new string[NestingFactor]; - path[0] = FilesInFullPath.First().Hash; + path[0] = FilesInFullPath.First().Hash.ToBase64(); var idx = 1; diff --git a/Wabbajack/View Models/Gallery/ModListGalleryVM.cs b/Wabbajack/View Models/Gallery/ModListGalleryVM.cs index b1160b72..5b3d4148 100644 --- a/Wabbajack/View Models/Gallery/ModListGalleryVM.cs +++ b/Wabbajack/View Models/Gallery/ModListGalleryVM.cs @@ -65,19 +65,19 @@ namespace Wabbajack return list // Sort randomly initially, just to give each list a fair shake .Shuffle(random) - .AsObservableChangeSet(x => x.DownloadMetadata?.Hash ?? $"Fallback{missingHashFallbackCounter++}"); + .AsObservableChangeSet(x => x.DownloadMetadata?.Hash ?? Hash.Empty); } catch (Exception ex) { Utils.Error(ex); Error = ErrorResponse.Fail(ex); - return Observable.Empty>(); + return Observable.Empty>(); } }) // Unsubscribe and release when not active .FlowSwitch( this.WhenAny(x => x.IsActive), - valueWhenOff: Observable.Return(ChangeSet.Empty)) + valueWhenOff: Observable.Return(ChangeSet.Empty)) .Switch() .RefCount(); diff --git a/Wabbajack/View Models/ManifestVM.cs b/Wabbajack/View Models/ManifestVM.cs index 8287e81f..dd52745a 100644 --- a/Wabbajack/View Models/ManifestVM.cs +++ b/Wabbajack/View Models/ManifestVM.cs @@ -92,7 +92,7 @@ namespace Wabbajack return Order(Archives.Where(x => { if (term.StartsWith("hash:")) - return x.Hash.StartsWith(term.Replace("hash:", "")); + return x.Hash.ToString().StartsWith(term.Replace("hash:", "")); return x.Name.StartsWith(term); })); })