Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Made RocksDB initialization lazier

Browse Source 
Was causing problems, as it's a singleton across the entire computer, so eagerly initializing them was causing problems for users of Wabbajack.Common that weren't interested in RocksDB.  Only one could run at a time.
This commit is contained in:
Justin Swanson 2020-08-14 07:16:09 -05:00
parent 167b81d586
commit b77fa3d0c7
5 changed files with 53 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.Common.Test/MiscTests.cs
View File

@ -23,7 +23,7 @@ namespace Wabbajack.Common.Test
await testFile.WriteAllTextAsync(data);
File.WriteAllText("test.data", data);
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await testFile.FileHashCachedAsync());
Assert.True(Utils.TryGetHashCache(testFile, out var fileHash));
Assert.True(testFile.TryGetHashCache(out var fileHash));
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), fileHash);
}

38
Wabbajack.Common/Hash.cs
View File

@ -114,11 +114,20 @@ namespace Wabbajack.Common
};
}
}
public static partial class Utils
public static class HashCache
{
private const uint HashCacheVersion = 0x01;
// Keep rock DB out of Utils, as it causes lock problems for users of Wabbajack.Common that aren't interested in it, otherwise
private static RocksDb _hashCache;
static HashCache()
{
var options = new DbOptions().SetCreateIfMissing(true);
_hashCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalHashCache.rocksDb"));
}
public static Hash ReadHash(this BinaryReader br)
{
return new Hash(br.ReadUInt64());
@ -146,36 +155,37 @@ namespace Wabbajack.Common
hash.HashSizeInBits = 64;
hash.Seed = 0x42;
using var fs = new MemoryStream(data);
var config = new xxHashConfig {HashSizeInBits = 64};
var config = new xxHashConfig { HashSizeInBits = 64 };
using var f = new StatusFileStream(fs, $"Hashing memory stream");
var value = xxHashFactory.Instance.Create(config).ComputeHash(f);
return Hash.FromULong(BitConverter.ToUInt64(value.Hash));
}
public static Hash xxHash(this Stream stream)
{
var hash = new xxHashConfig();
hash.HashSizeInBits = 64;
hash.Seed = 0x42;
var config = new xxHashConfig {HashSizeInBits = 64};
var config = new xxHashConfig { HashSizeInBits = 64 };
using var f = new StatusFileStream(stream, $"Hashing memory stream");
var value = xxHashFactory.Instance.Create(config).ComputeHash(f);
return Hash.FromULong(BitConverter.ToUInt64(value.Hash));
}
public static async Task<Hash> xxHashAsync(this Stream stream)
{
var config = new xxHashConfig {HashSizeInBits = 64};
var config = new xxHashConfig { HashSizeInBits = 64 };
await using var f = new StatusFileStream(stream, $"Hashing memory stream");
var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(f);
return Hash.FromULong(BitConverter.ToUInt64(value.Hash));
}
public static bool TryGetHashCache(AbsolutePath file, out Hash hash)
public static bool TryGetHashCache(this AbsolutePath file, out Hash hash)
{
var normPath = Encoding.UTF8.GetBytes(file.Normalize());
var value = _hashCache.Get(normPath);
hash = default;
if (value == null) return false;
if (value.Length != 20) return false;
@ -190,9 +200,7 @@ namespace Wabbajack.Common
return true;
}
private const uint HashCacheVersion = 0x01;
private static void WriteHashCache(AbsolutePath file, Hash hash)
private static void WriteHashCache(this AbsolutePath file, Hash hash)
{
using var ms = new MemoryStream(20);
using var bw = new BinaryWriter(ms);
@ -202,6 +210,7 @@ namespace Wabbajack.Common
bw.Write((ulong)hash);
_hashCache.Put(Encoding.UTF8.GetBytes(file.Normalize()), ms.ToArray());
}
public static void FileHashWriteCache(this AbsolutePath file, Hash hash)
{
WriteHashCache(file, hash);
@ -212,7 +221,7 @@ namespace Wabbajack.Common
if (TryGetHashCache(file, out var foundHash)) return foundHash;
var hash = await file.FileHashAsync(nullOnIOError);
if (hash != Hash.Empty)
if (hash != Hash.Empty)
WriteHashCache(file, hash);
return hash;
}
@ -222,7 +231,7 @@ namespace Wabbajack.Common
try
{
await using var fs = await file.OpenRead();
var config = new xxHashConfig {HashSizeInBits = 64};
var config = new xxHashConfig { HashSizeInBits = 64 };
await using var hs = new StatusFileStream(fs, $"Hashing {file}");
var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(hs);
return new Hash(BitConverter.ToUInt64(value.Hash));
@ -233,6 +242,5 @@ namespace Wabbajack.Common
throw;
}
}
}
}

39
Wabbajack.Common/Patches.cs
View File

@ -7,17 +7,17 @@ using RocksDbSharp;
namespace Wabbajack.Common
{
public static partial class Utils
public static class PatchCache
{
// Keep rock DB out of Utils, as it causes lock problems for users of Wabbajack.Common that aren't interested in it, otherwise
private static RocksDb? _patchCache;
private static void InitPatches()
static PatchCache()
{
var options = new DbOptions().SetCreateIfMissing(true);
var options = new DbOptions().SetCreateIfMissing(true);
_patchCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("PatchCache.rocksDb"));
}
private static byte[] PatchKey(Hash src, Hash dest)
{
var arr = new byte[16];
@ -25,6 +25,7 @@ namespace Wabbajack.Common
Array.Copy(BitConverter.GetBytes((ulong)dest), 0, arr, 8, 8);
return arr;
}
public static async Task CreatePatchCached(byte[] a, byte[] b, Stream output)
{
var dataA = a.xxHash();
@ -40,9 +41,9 @@ namespace Wabbajack.Common
await using var patch = new MemoryStream();
Status("Creating Patch");
Utils.Status("Creating Patch");
OctoDiff.Create(a, b, patch);
_patchCache.Put(key, patch.ToArray());
patch.Position = 0;
@ -57,19 +58,19 @@ namespace Wabbajack.Common
if (patch != null)
{
if (patchOutStream == null) return patch.Length;
await patchOutStream.WriteAsync(patch);
return patch.Length;
}
Status("Creating Patch");
Utils.Status("Creating Patch");
await using var sigStream = new MemoryStream();
await using var patchStream = new MemoryStream();
OctoDiff.Create(srcStream, destStream, sigStream, patchStream);
_patchCache.Put(key, patchStream.ToArray());
if (patchOutStream == null) return patchStream.Position;
patchStream.Position = 0;
await patchStream.CopyToAsync(patchOutStream);
@ -117,4 +118,20 @@ namespace Wabbajack.Common
}
}
}
// Convenience hook ins to offer the API from Utils, without having the init fire until they're actually called
public static partial class Utils
{
public static void ApplyPatch(Stream input, Func<Stream> openPatchStream, Stream output) =>
PatchCache.ApplyPatch(input, openPatchStream, output);
public static Task CreatePatchCached(byte[] a, byte[] b, Stream output) =>
PatchCache.CreatePatchCached(a, b, output);
public static Task<long> CreatePatchCached(Stream srcStream, Hash srcHash, FileStream destStream, Hash destHash, Stream? patchOutStream = null) =>
PatchCache.CreatePatchCached(srcStream, srcHash, destStream, destHash, patchOutStream);
public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch) =>
PatchCache.TryGetPatch(foundHash, fileHash, out ePatch);
}
}

5
Wabbajack.Common/Utils.cs
View File

@ -24,7 +24,6 @@ using IniParser.Model.Configuration;
using IniParser.Parser;
using Microsoft.Win32;
using Newtonsoft.Json;
using RocksDbSharp;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Common.StatusFeed.Errors;
using YamlDotNet.Serialization;
@ -60,9 +59,6 @@ namespace Wabbajack.Common
LogFile = Consts.LogFile;
Consts.LocalAppDataPath.CreateDirectory();
Consts.LogsFolder.CreateDirectory();
var options = new DbOptions().SetCreateIfMissing(true);
_hashCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalHashCache.rocksDb"));
_startTime = DateTime.Now;
@ -109,7 +105,6 @@ namespace Wabbajack.Common
Observable.FromEventPattern<FileSystemEventHandler, FileSystemEventArgs>(h => watcher.Deleted += h, h => watcher.Deleted -= h).Select(e => (FileEventType.Deleted, e.EventArgs)))
.ObserveOn(Scheduler.Default);
watcher.EnableRaisingEvents = true;
InitPatches();
}
private static readonly Subject<IStatusMessage> LoggerSubj = new Subject<IStatusMessage>();

2
Wabbajack.Test/DownloaderTests.cs
View File

@ -471,7 +471,7 @@ namespace Wabbajack.Test
await converted.Download(new Archive(state: null!) { Name = "Update.esm" }, filename.Path);
Assert.Equal(Hash.FromBase64("/DLG/LjdGXI="), await Utils.FileHashAsync(filename.Path));
Assert.Equal(Hash.FromBase64("/DLG/LjdGXI="), await filename.Path.FileHashAsync());
Assert.Equal(await filename.Path.ReadAllBytesAsync(), await Game.SkyrimSpecialEdition.MetaData().GameLocation().Combine("Data/Update.esm").ReadAllBytesAsync());
Consts.TestMode = true;
}