diff --git a/Wabbajack.Common.Test/MiscTests.cs b/Wabbajack.Common.Test/MiscTests.cs index 47d73577..339219f5 100644 --- a/Wabbajack.Common.Test/MiscTests.cs +++ b/Wabbajack.Common.Test/MiscTests.cs @@ -23,7 +23,7 @@ namespace Wabbajack.Common.Test await testFile.WriteAllTextAsync(data); File.WriteAllText("test.data", data); Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await testFile.FileHashCachedAsync()); - Assert.True(Utils.TryGetHashCache(testFile, out var fileHash)); + Assert.True(testFile.TryGetHashCache(out var fileHash)); Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), fileHash); } diff --git a/Wabbajack.Common/Hash.cs b/Wabbajack.Common/Hash.cs index 8935b56e..131f88a3 100644 --- a/Wabbajack.Common/Hash.cs +++ b/Wabbajack.Common/Hash.cs @@ -114,11 +114,20 @@ namespace Wabbajack.Common }; } } - - public static partial class Utils + + public static class HashCache { + private const uint HashCacheVersion = 0x01; + + // Keep rock DB out of Utils, as it causes lock problems for users of Wabbajack.Common that aren't interested in it, otherwise private static RocksDb _hashCache; + static HashCache() + { + var options = new DbOptions().SetCreateIfMissing(true); + _hashCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalHashCache.rocksDb")); + } + public static Hash ReadHash(this BinaryReader br) { return new Hash(br.ReadUInt64()); @@ -146,36 +155,37 @@ namespace Wabbajack.Common hash.HashSizeInBits = 64; hash.Seed = 0x42; using var fs = new MemoryStream(data); - var config = new xxHashConfig {HashSizeInBits = 64}; + var config = new xxHashConfig { HashSizeInBits = 64 }; using var f = new StatusFileStream(fs, $"Hashing memory stream"); var value = xxHashFactory.Instance.Create(config).ComputeHash(f); return Hash.FromULong(BitConverter.ToUInt64(value.Hash)); } - + public static Hash xxHash(this Stream stream) { var hash = new xxHashConfig(); hash.HashSizeInBits = 64; hash.Seed = 0x42; - var config = new xxHashConfig {HashSizeInBits = 64}; + var config = new xxHashConfig { HashSizeInBits = 64 }; using var f = new StatusFileStream(stream, $"Hashing memory stream"); var value = xxHashFactory.Instance.Create(config).ComputeHash(f); return Hash.FromULong(BitConverter.ToUInt64(value.Hash)); } - + public static async Task xxHashAsync(this Stream stream) { - var config = new xxHashConfig {HashSizeInBits = 64}; + var config = new xxHashConfig { HashSizeInBits = 64 }; await using var f = new StatusFileStream(stream, $"Hashing memory stream"); var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(f); return Hash.FromULong(BitConverter.ToUInt64(value.Hash)); } - public static bool TryGetHashCache(AbsolutePath file, out Hash hash) + + public static bool TryGetHashCache(this AbsolutePath file, out Hash hash) { var normPath = Encoding.UTF8.GetBytes(file.Normalize()); var value = _hashCache.Get(normPath); hash = default; - + if (value == null) return false; if (value.Length != 20) return false; @@ -190,9 +200,7 @@ namespace Wabbajack.Common return true; } - - private const uint HashCacheVersion = 0x01; - private static void WriteHashCache(AbsolutePath file, Hash hash) + private static void WriteHashCache(this AbsolutePath file, Hash hash) { using var ms = new MemoryStream(20); using var bw = new BinaryWriter(ms); @@ -202,6 +210,7 @@ namespace Wabbajack.Common bw.Write((ulong)hash); _hashCache.Put(Encoding.UTF8.GetBytes(file.Normalize()), ms.ToArray()); } + public static void FileHashWriteCache(this AbsolutePath file, Hash hash) { WriteHashCache(file, hash); @@ -212,7 +221,7 @@ namespace Wabbajack.Common if (TryGetHashCache(file, out var foundHash)) return foundHash; var hash = await file.FileHashAsync(nullOnIOError); - if (hash != Hash.Empty) + if (hash != Hash.Empty) WriteHashCache(file, hash); return hash; } @@ -222,7 +231,7 @@ namespace Wabbajack.Common try { await using var fs = await file.OpenRead(); - var config = new xxHashConfig {HashSizeInBits = 64}; + var config = new xxHashConfig { HashSizeInBits = 64 }; await using var hs = new StatusFileStream(fs, $"Hashing {file}"); var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(hs); return new Hash(BitConverter.ToUInt64(value.Hash)); @@ -233,6 +242,5 @@ namespace Wabbajack.Common throw; } } - } } diff --git a/Wabbajack.Common/Patches.cs b/Wabbajack.Common/Patches.cs index 1b21da54..6e9f800f 100644 --- a/Wabbajack.Common/Patches.cs +++ b/Wabbajack.Common/Patches.cs @@ -7,17 +7,17 @@ using RocksDbSharp; namespace Wabbajack.Common { - public static partial class Utils + public static class PatchCache { - + // Keep rock DB out of Utils, as it causes lock problems for users of Wabbajack.Common that aren't interested in it, otherwise private static RocksDb? _patchCache; - private static void InitPatches() + + static PatchCache() { - var options = new DbOptions().SetCreateIfMissing(true); + var options = new DbOptions().SetCreateIfMissing(true); _patchCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("PatchCache.rocksDb")); } - private static byte[] PatchKey(Hash src, Hash dest) { var arr = new byte[16]; @@ -25,6 +25,7 @@ namespace Wabbajack.Common Array.Copy(BitConverter.GetBytes((ulong)dest), 0, arr, 8, 8); return arr; } + public static async Task CreatePatchCached(byte[] a, byte[] b, Stream output) { var dataA = a.xxHash(); @@ -40,9 +41,9 @@ namespace Wabbajack.Common await using var patch = new MemoryStream(); - Status("Creating Patch"); + Utils.Status("Creating Patch"); OctoDiff.Create(a, b, patch); - + _patchCache.Put(key, patch.ToArray()); patch.Position = 0; @@ -57,19 +58,19 @@ namespace Wabbajack.Common if (patch != null) { if (patchOutStream == null) return patch.Length; - + await patchOutStream.WriteAsync(patch); return patch.Length; } - - Status("Creating Patch"); + + Utils.Status("Creating Patch"); await using var sigStream = new MemoryStream(); await using var patchStream = new MemoryStream(); OctoDiff.Create(srcStream, destStream, sigStream, patchStream); _patchCache.Put(key, patchStream.ToArray()); if (patchOutStream == null) return patchStream.Position; - + patchStream.Position = 0; await patchStream.CopyToAsync(patchOutStream); @@ -117,4 +118,20 @@ namespace Wabbajack.Common } } } + + // Convenience hook ins to offer the API from Utils, without having the init fire until they're actually called + public static partial class Utils + { + public static void ApplyPatch(Stream input, Func openPatchStream, Stream output) => + PatchCache.ApplyPatch(input, openPatchStream, output); + + public static Task CreatePatchCached(byte[] a, byte[] b, Stream output) => + PatchCache.CreatePatchCached(a, b, output); + + public static Task CreatePatchCached(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash, Stream? patchOutStream = null) => + PatchCache.CreatePatchCached(srcStream, srcHash, destStream, destHash, patchOutStream); + + public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch) => + PatchCache.TryGetPatch(foundHash, fileHash, out ePatch); + } } diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 09e1a7ec..4388eef2 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -24,7 +24,6 @@ using IniParser.Model.Configuration; using IniParser.Parser; using Microsoft.Win32; using Newtonsoft.Json; -using RocksDbSharp; using Wabbajack.Common.StatusFeed; using Wabbajack.Common.StatusFeed.Errors; using YamlDotNet.Serialization; @@ -60,9 +59,6 @@ namespace Wabbajack.Common LogFile = Consts.LogFile; Consts.LocalAppDataPath.CreateDirectory(); Consts.LogsFolder.CreateDirectory(); - - var options = new DbOptions().SetCreateIfMissing(true); - _hashCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalHashCache.rocksDb")); _startTime = DateTime.Now; @@ -109,7 +105,6 @@ namespace Wabbajack.Common Observable.FromEventPattern(h => watcher.Deleted += h, h => watcher.Deleted -= h).Select(e => (FileEventType.Deleted, e.EventArgs))) .ObserveOn(Scheduler.Default); watcher.EnableRaisingEvents = true; - InitPatches(); } private static readonly Subject LoggerSubj = new Subject(); diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index d7a9f3a7..d19ca85e 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -471,7 +471,7 @@ namespace Wabbajack.Test await converted.Download(new Archive(state: null!) { Name = "Update.esm" }, filename.Path); - Assert.Equal(Hash.FromBase64("/DLG/LjdGXI="), await Utils.FileHashAsync(filename.Path)); + Assert.Equal(Hash.FromBase64("/DLG/LjdGXI="), await filename.Path.FileHashAsync()); Assert.Equal(await filename.Path.ReadAllBytesAsync(), await Game.SkyrimSpecialEdition.MetaData().GameLocation().Combine("Data/Update.esm").ReadAllBytesAsync()); Consts.TestMode = true; }