From 13eef5c695058c21362be4ac43c4ffe2582ec11f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 5 Jan 2021 15:09:32 -0700 Subject: [PATCH] Remove RocksDB (replaced with SQLite) --- CHANGELOG.md | 1 + Compression.BSA/BSA/Reader/FolderRecord.cs | 3 - Wabbajack.Common/Hash.cs | 98 +++++++++++++----- Wabbajack.Common/Patches.cs | 99 +++++++++++-------- Wabbajack.Common/Wabbajack.Common.csproj | 3 +- Wabbajack.Server/Services/ListValidator.cs | 5 - .../IndexedVirtualFile.cs | 3 +- Wabbajack.VirtualFileSystem/VirtualFile.cs | 65 +++++++++--- 8 files changed, 183 insertions(+), 94 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09002955..e363207c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * Wabbajack is now based on .NET 5.0 (does not require a runtime download by users) * Origin is now supported as a game source * Basic (mostly untested) support for Dragon Age : Origins +* Replace RocksDB with SQLite should result in less process contention when running the UI and the CLI at the same time #### Version - 2.3.6.2 - 12/31/2020 * HOTFIX: Also apply the IPS4 changes to LL Meta lookups diff --git a/Compression.BSA/BSA/Reader/FolderRecord.cs b/Compression.BSA/BSA/Reader/FolderRecord.cs index ce9bb18d..8e29a58a 100644 --- a/Compression.BSA/BSA/Reader/FolderRecord.cs +++ b/Compression.BSA/BSA/Reader/FolderRecord.cs @@ -2,10 +2,7 @@ using System.Buffers.Binary; using System.Collections.Generic; using System.IO; -using System.Text; -using NativeImport; using Wabbajack.Common; -using File = Alphaleonis.Win32.Filesystem.File; namespace Compression.BSA { diff --git a/Wabbajack.Common/Hash.cs b/Wabbajack.Common/Hash.cs index 6286eca9..59487bee 100644 --- a/Wabbajack.Common/Hash.cs +++ b/Wabbajack.Common/Hash.cs @@ -1,14 +1,15 @@ using System; using System.Collections.Generic; +using System.Data; using System.Data.HashFunction.xxHash; using System.IO; using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; using Newtonsoft.Json; -using RocksDbSharp; using File = Alphaleonis.Win32.Filesystem.File; using Path = Alphaleonis.Win32.Filesystem.Path; +using System.Data.SQLite; namespace Wabbajack.Common { @@ -117,20 +118,73 @@ namespace Wabbajack.Common public static class HashCache { - private const uint HashCacheVersion = 0x01; + private static AbsolutePath DBLocation = Consts.LocalAppDataPath.Combine("GlobalHashCache.sqlite"); + private static string _connectionString; + private static SQLiteConnection _conn; + // Keep rock DB out of Utils, as it causes lock problems for users of Wabbajack.Common that aren't interested in it, otherwise - private static RocksDb _hashCache; static HashCache() { - var options = new DbOptions().SetCreateIfMissing(true); - _hashCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalHashCache.rocksDb")); + _connectionString = String.Intern($"URI=file:{DBLocation};Pooling=True;Max Pool Size=100;"); + _conn = new SQLiteConnection(_connectionString); + _conn.Open(); + + using var cmd = new SQLiteCommand(_conn); + cmd.CommandText = @"CREATE TABLE IF NOT EXISTS HashCache ( + Path TEXT PRIMARY KEY, + LastModified BIGINT, + Hash BIGINT)"; + cmd.ExecuteNonQuery(); + + + + + } + + private static (AbsolutePath Path, long LastModified, Hash Hash) GetFromCache(AbsolutePath path) + { + using var cmd = new SQLiteCommand(_conn); + cmd.CommandText = "SELECT LastModified, Hash FROM HashCache WHERE Path = @path"; + cmd.Parameters.AddWithValue("@path", path.ToString()); + cmd.PrepareAsync(); + + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + return (path, reader.GetInt64(0), Hash.FromLong(reader.GetInt64(1))); + } + + return default; + } + + private static void PurgeCacheEntry(AbsolutePath path) + { + using var cmd = new SQLiteCommand(_conn); + cmd.CommandText = "DELETE FROM HashCache WHERE Path = @path"; + cmd.Parameters.AddWithValue("@path", path.ToString()); + cmd.PrepareAsync(); + + cmd.ExecuteNonQuery(); + } + + private static void UpsertCacheEntry(AbsolutePath path, long lastModified, Hash hash) + { + using var cmd = new SQLiteCommand(_conn); + cmd.CommandText = @"INSERT INTO HashCache (Path, LastModified, Hash) VALUES (@path, @lastModified, @hash) + ON CONFLICT(Path) DO UPDATE SET LastModified = @lastModified, Hash = @hash"; + cmd.Parameters.AddWithValue("@path", path.ToString()); + cmd.Parameters.AddWithValue("@lastModified", lastModified); + cmd.Parameters.AddWithValue("@hash", (long)hash); + cmd.PrepareAsync(); + + cmd.ExecuteNonQuery(); } public static Hash ReadHash(this BinaryReader br) { - return new Hash(br.ReadUInt64()); + return new(br.ReadUInt64()); } public static void Write(this BinaryWriter bw, Hash hash) @@ -182,33 +236,27 @@ namespace Wabbajack.Common public static bool TryGetHashCache(this AbsolutePath file, out Hash hash) { - var normPath = Encoding.UTF8.GetBytes(file.Normalize()); - var value = _hashCache.Get(normPath); hash = default; + if (!file.Exists) return false; - if (value == null) return false; - if (value.Length != 20) return false; + var result = GetFromCache(file); + if (result == default) + return false; - using var ms = new MemoryStream(value); - using var br = new BinaryReader(ms); - var version = br.ReadUInt32(); - if (version != HashCacheVersion) return false; + if (result.LastModified == file.LastModifiedUtc.ToFileTimeUtc()) + { + hash = result.Hash; + return true; + } - var lastModified = br.ReadUInt64(); - if (lastModified != file.LastModifiedUtc.AsUnixTime()) return false; - hash = new Hash(br.ReadUInt64()); - return true; + PurgeCacheEntry(file); + return false; } private static void WriteHashCache(this AbsolutePath file, Hash hash) { - using var ms = new MemoryStream(20); - using var bw = new BinaryWriter(ms); - bw.Write(HashCacheVersion); - var lastModified = file.LastModifiedUtc.AsUnixTime(); - bw.Write(lastModified); - bw.Write((ulong)hash); - _hashCache.Put(Encoding.UTF8.GetBytes(file.Normalize()), ms.ToArray()); + if (!file.Exists) return; + UpsertCacheEntry(file, file.LastModifiedUtc.ToFileTimeUtc(), hash); } public static void FileHashWriteCache(this AbsolutePath file, Hash hash) diff --git a/Wabbajack.Common/Patches.cs b/Wabbajack.Common/Patches.cs index 2791e0dd..cb5b925f 100644 --- a/Wabbajack.Common/Patches.cs +++ b/Wabbajack.Common/Patches.cs @@ -1,73 +1,84 @@ using System; +using System.Data.SQLite; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Text; using System.Threading.Tasks; -using RocksDbSharp; namespace Wabbajack.Common { public static class PatchCache { - // Keep rock DB out of Utils, as it causes lock problems for users of Wabbajack.Common that aren't interested in it, otherwise - private static RocksDb? _patchCache; + private static AbsolutePath DBLocation = Consts.LocalAppDataPath.Combine("GlobalPatchCache.sqlite"); + private static string _connectionString; static PatchCache() { - var options = new DbOptions().SetCreateIfMissing(true); - _patchCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("PatchCache.rocksDb")); - } + _connectionString = String.Intern($"URI=file:{DBLocation};Pooling=True;Max Pool Size=100;"); + using var conn = new SQLiteConnection(_connectionString); + conn.Open(); + + using var cmd = new SQLiteCommand(conn); + cmd.CommandText = @"CREATE TABLE IF NOT EXISTS PatchCache ( + FromHash BIGINT, + ToHash BIGINT, + PatchSize BLOB, + Patch BLOB, + PRIMARY KEY (FromHash, ToHash))"; + cmd.ExecuteNonQuery(); - private static byte[] PatchKey(Hash src, Hash dest) - { - var arr = new byte[16]; - Array.Copy(BitConverter.GetBytes((ulong)src), 0, arr, 0, 8); - Array.Copy(BitConverter.GetBytes((ulong)dest), 0, arr, 8, 8); - return arr; } public static async Task CreatePatchCached(byte[] a, byte[] b, Stream output) { + await using var conn = new SQLiteConnection(_connectionString); + await conn.OpenAsync(); + + await using var cmd = new SQLiteCommand(conn); + cmd.CommandText = @"INSERT INTO PatchCache (FromHash, ToHash, PatchSize, Patch) + VALUES (@fromHash, @toHash, @patchSize, @patch)"; + var dataA = a.xxHash(); var dataB = b.xxHash(); - var key = PatchKey(dataA, dataB); - var found = _patchCache!.Get(key); - - if (found != null) - { - await output.WriteAsync(found); - return; - } + cmd.Parameters.AddWithValue("@fromHash", (long)dataA); + cmd.Parameters.AddWithValue("@toHash", (long)dataB); + await using var patch = new MemoryStream(); Utils.Status("Creating Patch"); OctoDiff.Create(a, b, patch); - - _patchCache.Put(key, patch.ToArray()); patch.Position = 0; + cmd.Parameters.AddWithValue("@patchSize", patch.Length); + cmd.Parameters.AddWithValue("@patch", patch.ToArray()); + await cmd.ExecuteNonQueryAsync(); + + await patch.CopyToAsync(output); } public static async Task CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash, Stream? patchOutStream = null) { - var key = PatchKey(srcHash, destHash); - var patch = _patchCache!.Get(key); - if (patch != null) - { - if (patchOutStream == null) return patch.Length; + await using var conn = new SQLiteConnection(_connectionString); + await conn.OpenAsync(); - await patchOutStream.WriteAsync(patch); - return patch.Length; - } + await using var cmd = new SQLiteCommand(conn); + cmd.CommandText = @"INSERT INTO PatchCache (FromHash, ToHash, PatchSize, Patch) + VALUES (@fromHash, @toHash, @patchSize, @patch)"; + + cmd.Parameters.AddWithValue("@fromHash", (long)srcHash); + cmd.Parameters.AddWithValue("@toHash", (long)destHash); Utils.Status("Creating Patch"); await using var sigStream = new MemoryStream(); await using var patchStream = new MemoryStream(); OctoDiff.Create(srcStream, destStream, sigStream, patchStream); - _patchCache.Put(key, patchStream.ToArray()); + + cmd.Parameters.AddWithValue("@patchSize", patchStream.Length); + cmd.Parameters.AddWithValue("@patch", patchStream.ToArray()); + await cmd.ExecuteNonQueryAsync(); if (patchOutStream == null) return patchStream.Position; @@ -77,26 +88,30 @@ namespace Wabbajack.Common return patchStream.Position; } - public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch) + public static bool TryGetPatch(Hash fromHash, Hash toHash, [MaybeNullWhen(false)] out byte[] array) { - var key = PatchKey(foundHash, fileHash); - var patch = _patchCache!.Get(key); + using var conn = new SQLiteConnection(_connectionString); + conn.Open(); - if (patch != null) + using var cmd = new SQLiteCommand(conn); + cmd.CommandText = @"SELECT PatchSize, Patch FROM PatchCache WHERE FromHash = @fromHash AND ToHash = @toHash"; + cmd.Parameters.AddWithValue("@fromHash", (long)fromHash); + cmd.Parameters.AddWithValue("@toHash", (long)toHash); + + using var rdr = cmd.ExecuteReader(); + while (rdr.Read()) { - ePatch = patch; + array = new byte[rdr.GetInt64(0)]; + rdr.GetBytes(1, 0, array, 0, array.Length); return true; } - ePatch = null; + array = Array.Empty(); return false; - } - public static bool HavePatch(Hash foundHash, Hash fileHash) - { - var key = PatchKey(foundHash, fileHash); - return _patchCache!.Get(key) != null; + + } public static void ApplyPatch(Stream input, Func openPatchStream, Stream output) diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index af0d5ce5..86b25d5e 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -54,10 +54,9 @@ - - + diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 8621495b..9b63978b 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -3,13 +3,8 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using System.Text.RegularExpressions; using System.Threading.Tasks; -using Microsoft.AspNetCore.Mvc.Filters; -using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; -using Org.BouncyCastle.Crypto.Digests; -using RocksDbSharp; using Wabbajack.BuildServer; using Wabbajack.Common; using Wabbajack.Lib; diff --git a/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs b/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs index 3755d9c0..cca6a44f 100644 --- a/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Common.Serialization.Json; @@ -30,7 +31,7 @@ namespace Wabbajack.VirtualFileSystem public void Write(Stream s) { - using var bw = new BinaryWriter(s); + using var bw = new BinaryWriter(s, Encoding.UTF8, true); bw.Write(Size); bw.Write(Children.Count); foreach (var file in Children) diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index a038f099..18ef3fb0 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -1,26 +1,37 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.Data; +using System.Data.SQLite; using System.IO; using System.Linq; using System.Net.Http; using System.Threading.Tasks; using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using K4os.Hash.Crc; -using RocksDbSharp; using Wabbajack.Common; namespace Wabbajack.VirtualFileSystem { public class VirtualFile { - private static RocksDb _vfsCache; + private static AbsolutePath DBLocation = Consts.LocalAppDataPath.Combine("GlobalVFSCache.sqlite"); + private static string _connectionString; + private static SQLiteConnection _conn; static VirtualFile() { - var options = new DbOptions().SetCreateIfMissing(true); - _vfsCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalVFSCache2.rocksDb")); + _connectionString = String.Intern($"URI=file:{DBLocation};Pooling=True;Max Pool Size=100;"); + _conn = new SQLiteConnection(_connectionString); + _conn.Open(); + + using var cmd = new SQLiteCommand(_conn); + cmd.CommandText = @"CREATE TABLE IF NOT EXISTS VFSCache ( + Hash BIGINT PRIMARY KEY , + Contents BLOB)"; + cmd.ExecuteNonQuery(); + } private IEnumerable _thisAndAllChildren; @@ -145,19 +156,22 @@ namespace Wabbajack.VirtualFileSystem private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IStreamFactory extractedFile, Hash hash, out VirtualFile found) { - var result = _vfsCache.Get(hash.ToArray()); - if (result == null) + using var cmd = new SQLiteCommand(_conn); + cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash"; + cmd.Parameters.AddWithValue("@hash", (long)hash); + + using var rdr = cmd.ExecuteReader(); + while (rdr.Read()) { - found = null; - return false; + var data = IndexedVirtualFile.Read(rdr.GetStream(0)); + found = ConvertFromIndexedFile(context, data, path, parent, extractedFile); + found.Name = path; + found.Hash = hash; + return true; } - var data = IndexedVirtualFile.Read(new MemoryStream(result)); - found = ConvertFromIndexedFile(context, data, path, parent, extractedFile); - found.Name = path; - found.Hash = hash; - return true; - + found = default; + return false; } private IndexedVirtualFile ToIndexedVirtualFile() @@ -236,11 +250,30 @@ namespace Wabbajack.VirtualFileSystem await using var ms = new MemoryStream(); self.ToIndexedVirtualFile().Write(ms); - _vfsCache.Put(self.Hash.ToArray(), ms.ToArray()); - + ms.Position = 0; + await InsertIntoVFSCache(self.Hash, ms); return self; } + private static async Task InsertIntoVFSCache(Hash hash, MemoryStream data) + { + await using var cmd = new SQLiteCommand(_conn); + cmd.CommandText = @"INSERT INTO VFSCache (Hash, Contents) VALUES (@hash, @contents)"; + cmd.Parameters.AddWithValue("@hash", (long)hash); + var val = new SQLiteParameter("@contents", DbType.Binary) {Value = data.ToArray()}; + cmd.Parameters.Add(val); + try + { + await cmd.ExecuteNonQueryAsync(); + } + catch (SQLiteException ex) + { + if (ex.Message.StartsWith("constraint failed")) + return; + throw; + } + } + internal void FillFullPath() { int depth = 0;