Remove RocksDB (replaced with SQLite)

CHANGELOG.md

@@ -4,6 +4,7 @@
 * Wabbajack is now based on .NET 5.0 (does not require a runtime download by users)
 * Origin is now supported as a game source
 * Basic (mostly untested) support for Dragon Age : Origins
+* Replace RocksDB with SQLite should result in less process contention when running the UI and the CLI at the same time
 
 #### Version - 2.3.6.2 - 12/31/2020
 * HOTFIX: Also apply the IPS4 changes to LL Meta lookups

Compression.BSA/BSA/Reader/FolderRecord.cs

@@ -2,10 +2,7 @@
 using System.Buffers.Binary;
 using System.Collections.Generic;
 using System.IO;
-using System.Text;
-using NativeImport;
 using Wabbajack.Common;
-using File = Alphaleonis.Win32.Filesystem.File;
 
 namespace Compression.BSA
 {

Wabbajack.Common/Hash.cs

@@ -1,14 +1,15 @@
 using System;
 using System.Collections.Generic;
+using System.Data;
 using System.Data.HashFunction.xxHash;
 using System.IO;
 using System.Security.Cryptography;
 using System.Text;
 using System.Threading.Tasks;
 using Newtonsoft.Json;
-using RocksDbSharp;
 using File = Alphaleonis.Win32.Filesystem.File;
 using Path = Alphaleonis.Win32.Filesystem.Path;
+using System.Data.SQLite;
 
 namespace Wabbajack.Common
 {
@@ -117,20 +118,73 @@ namespace Wabbajack.Common
 
     public static class HashCache
     {
-        private const uint HashCacheVersion = 0x01;
+        private static AbsolutePath DBLocation = Consts.LocalAppDataPath.Combine("GlobalHashCache.sqlite");
+        private static string _connectionString;
+        private static SQLiteConnection _conn;
+
 
         // Keep rock DB out of Utils, as it causes lock problems for users of Wabbajack.Common that aren't interested in it, otherwise
-        private static RocksDb _hashCache;
 
         static HashCache()
         {
-            var options = new DbOptions().SetCreateIfMissing(true);
+            _connectionString = String.Intern($"URI=file:{DBLocation};Pooling=True;Max Pool Size=100;");
-            _hashCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalHashCache.rocksDb"));
+            _conn = new SQLiteConnection(_connectionString);
+            _conn.Open();
+
+            using var cmd = new SQLiteCommand(_conn);
+            cmd.CommandText = @"CREATE TABLE IF NOT EXISTS HashCache (
+            Path TEXT PRIMARY KEY,
+            LastModified BIGINT,
+            Hash BIGINT)";
+            cmd.ExecuteNonQuery();
+
+
+
+
+        }
+
+        private static (AbsolutePath Path, long LastModified, Hash Hash) GetFromCache(AbsolutePath path)
+        {
+            using var cmd = new SQLiteCommand(_conn);
+            cmd.CommandText = "SELECT LastModified, Hash FROM HashCache WHERE Path = @path";
+            cmd.Parameters.AddWithValue("@path", path.ToString());
+            cmd.PrepareAsync();
+            
+            using var reader = cmd.ExecuteReader();
+            while (reader.Read())
+            {
+                return (path, reader.GetInt64(0), Hash.FromLong(reader.GetInt64(1)));
+            }
+
+            return default;
+        }
+        
+        private static void PurgeCacheEntry(AbsolutePath path)
+        {
+            using var cmd = new SQLiteCommand(_conn);
+            cmd.CommandText = "DELETE FROM HashCache WHERE Path = @path";
+            cmd.Parameters.AddWithValue("@path", path.ToString());
+            cmd.PrepareAsync();
+            
+            cmd.ExecuteNonQuery();
+        }
+
+        private static void UpsertCacheEntry(AbsolutePath path, long lastModified, Hash hash)
+        {
+            using var cmd = new SQLiteCommand(_conn);
+            cmd.CommandText = @"INSERT INTO HashCache (Path, LastModified, Hash) VALUES (@path, @lastModified, @hash)
+            ON CONFLICT(Path) DO UPDATE SET LastModified = @lastModified, Hash = @hash";
+            cmd.Parameters.AddWithValue("@path", path.ToString());
+            cmd.Parameters.AddWithValue("@lastModified", lastModified);
+            cmd.Parameters.AddWithValue("@hash", (long)hash);
+            cmd.PrepareAsync();
+            
+            cmd.ExecuteNonQuery();
         }
 
         public static Hash ReadHash(this BinaryReader br)
         {
-            return new Hash(br.ReadUInt64());
+            return new(br.ReadUInt64());
         }
 
         public static void Write(this BinaryWriter bw, Hash hash)
@@ -182,33 +236,27 @@ namespace Wabbajack.Common
 
         public static bool TryGetHashCache(this AbsolutePath file, out Hash hash)
         {
-            var normPath = Encoding.UTF8.GetBytes(file.Normalize());
-            var value = _hashCache.Get(normPath);
             hash = default;
+            if (!file.Exists) return false;
 
-            if (value == null) return false;
+            var result = GetFromCache(file);
-            if (value.Length != 20) return false;
+            if (result == default)
+                return false;
 
-            using var ms = new MemoryStream(value);
+            if (result.LastModified == file.LastModifiedUtc.ToFileTimeUtc())
-            using var br = new BinaryReader(ms);
+            {
-            var version = br.ReadUInt32();
+                hash = result.Hash;
-            if (version != HashCacheVersion) return false;
-
-            var lastModified = br.ReadUInt64();
-            if (lastModified != file.LastModifiedUtc.AsUnixTime()) return false;
-            hash = new Hash(br.ReadUInt64());
                 return true;
             }
 
+            PurgeCacheEntry(file);
+            return false;
+        }
+
         private static void WriteHashCache(this AbsolutePath file, Hash hash)
         {
-            using var ms = new MemoryStream(20);
+            if (!file.Exists) return;
-            using var bw = new BinaryWriter(ms);
+            UpsertCacheEntry(file, file.LastModifiedUtc.ToFileTimeUtc(), hash);
-            bw.Write(HashCacheVersion);
-            var lastModified = file.LastModifiedUtc.AsUnixTime();
-            bw.Write(lastModified);
-            bw.Write((ulong)hash);
-            _hashCache.Put(Encoding.UTF8.GetBytes(file.Normalize()), ms.ToArray());
         }
 
         public static void FileHashWriteCache(this AbsolutePath file, Hash hash)

Wabbajack.Common/Patches.cs

@@ -1,73 +1,84 @@
 using System;
+using System.Data.SQLite;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Text;
 using System.Threading.Tasks;
-using RocksDbSharp;
 
 namespace Wabbajack.Common
 {
     public static class PatchCache
     {
-        // Keep rock DB out of Utils, as it causes lock problems for users of Wabbajack.Common that aren't interested in it, otherwise
+        private static AbsolutePath DBLocation = Consts.LocalAppDataPath.Combine("GlobalPatchCache.sqlite");
-        private static RocksDb? _patchCache;
+        private static string _connectionString;
 
         static PatchCache()
         {
-            var options = new DbOptions().SetCreateIfMissing(true);
+            _connectionString = String.Intern($"URI=file:{DBLocation};Pooling=True;Max Pool Size=100;");
-            _patchCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("PatchCache.rocksDb"));
+            using var conn = new SQLiteConnection(_connectionString);
-        }
+            conn.Open();
+
+            using var cmd = new SQLiteCommand(conn);
+            cmd.CommandText = @"CREATE TABLE IF NOT EXISTS PatchCache (
+            FromHash BIGINT,
+            ToHash BIGINT,
+            PatchSize BLOB,
+            Patch BLOB,
+            PRIMARY KEY (FromHash, ToHash))";
+            cmd.ExecuteNonQuery();
 
-        private static byte[] PatchKey(Hash src, Hash dest)
-        {
-            var arr = new byte[16];
-            Array.Copy(BitConverter.GetBytes((ulong)src), 0, arr, 0, 8);
-            Array.Copy(BitConverter.GetBytes((ulong)dest), 0, arr, 8, 8);
-            return arr;
         }
 
         public static async Task CreatePatchCached(byte[] a, byte[] b, Stream output)
         {
+            await using var conn = new SQLiteConnection(_connectionString);
+            await conn.OpenAsync();
+
+            await using var cmd = new SQLiteCommand(conn);
+            cmd.CommandText = @"INSERT INTO PatchCache (FromHash, ToHash, PatchSize, Patch) 
+                  VALUES (@fromHash, @toHash, @patchSize, @patch)";
+
             var dataA = a.xxHash();
             var dataB = b.xxHash();
-            var key = PatchKey(dataA, dataB);
-            var found = _patchCache!.Get(key);
 
-            if (found != null)
+            cmd.Parameters.AddWithValue("@fromHash", (long)dataA);
-            {
+            cmd.Parameters.AddWithValue("@toHash", (long)dataB);
-                await output.WriteAsync(found);
-                return;
-            }
             
             await using var patch = new MemoryStream();
 
             Utils.Status("Creating Patch");
             OctoDiff.Create(a, b, patch);
-
-            _patchCache.Put(key, patch.ToArray());
             patch.Position = 0;
 
+            cmd.Parameters.AddWithValue("@patchSize", patch.Length);
+            cmd.Parameters.AddWithValue("@patch", patch.ToArray());
+            await cmd.ExecuteNonQueryAsync();
+
+
             await patch.CopyToAsync(output);
         }
 
         public static async Task<long> CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash,
             Stream? patchOutStream = null)
         {
-            var key = PatchKey(srcHash, destHash);
+            await using var conn = new SQLiteConnection(_connectionString);
-            var patch = _patchCache!.Get(key);
+            await conn.OpenAsync();
-            if (patch != null)
-            {
-                if (patchOutStream == null) return patch.Length;
 
-                await patchOutStream.WriteAsync(patch);
+            await using var cmd = new SQLiteCommand(conn);
-                return patch.Length;
+            cmd.CommandText = @"INSERT INTO PatchCache (FromHash, ToHash, PatchSize, Patch) 
-            }
+                  VALUES (@fromHash, @toHash, @patchSize, @patch)";
+
+            cmd.Parameters.AddWithValue("@fromHash", (long)srcHash);
+            cmd.Parameters.AddWithValue("@toHash", (long)destHash);
 
             Utils.Status("Creating Patch");
             await using var sigStream = new MemoryStream();
             await using var patchStream = new MemoryStream();
             OctoDiff.Create(srcStream, destStream, sigStream, patchStream);
-            _patchCache.Put(key, patchStream.ToArray());
+
+            cmd.Parameters.AddWithValue("@patchSize", patchStream.Length);
+            cmd.Parameters.AddWithValue("@patch", patchStream.ToArray());
+            await cmd.ExecuteNonQueryAsync();
 
             if (patchOutStream == null) return patchStream.Position;
 
@@ -77,26 +88,30 @@ namespace Wabbajack.Common
             return patchStream.Position;
         }
 
-        public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch)
+        public static bool TryGetPatch(Hash fromHash, Hash toHash, [MaybeNullWhen(false)] out byte[] array)
         {
-            var key = PatchKey(foundHash, fileHash);
+            using var conn = new SQLiteConnection(_connectionString);
-            var patch = _patchCache!.Get(key);
+            conn.Open();
 
-            if (patch != null)
+            using var cmd = new SQLiteCommand(conn);
+            cmd.CommandText = @"SELECT PatchSize, Patch FROM PatchCache WHERE FromHash = @fromHash AND ToHash = @toHash";
+            cmd.Parameters.AddWithValue("@fromHash", (long)fromHash);
+            cmd.Parameters.AddWithValue("@toHash", (long)toHash);
+
+            using var rdr = cmd.ExecuteReader();
+            while (rdr.Read())
             {
-                ePatch = patch;
+                array = new byte[rdr.GetInt64(0)];
+                rdr.GetBytes(1, 0, array, 0, array.Length);
                 return true;
             }
 
-            ePatch = null;
+            array = Array.Empty<byte>();
             return false;
 
-        }
 
-        public static bool HavePatch(Hash foundHash, Hash fileHash)
+
-        {
+
-            var key = PatchKey(foundHash, fileHash);
-            return _patchCache!.Get(key) != null;
         }
 
         public static void ApplyPatch(Stream input, Func<Stream> openPatchStream, Stream output)

Wabbajack.Common/Wabbajack.Common.csproj

@@ -54,10 +54,9 @@
         <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
         <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
         <PackageReference Include="Octodiff" Version="1.2.1" />
-        <PackageReference Include="RocksDbNative" Version="6.2.2" />
-        <PackageReference Include="RocksDbSharp" Version="6.2.2" />
         <PackageReference Include="SharpZipLib" Version="1.3.1" />
         <PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
+        <PackageReference Include="System.Data.SQLite.Core" Version="1.0.113.7" />
         <PackageReference Include="System.Net.Http" Version="4.3.4" />
         <PackageReference Include="System.Reactive" Version="5.0.0" />
         <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="5.0.0" />

Wabbajack.Server/Services/ListValidator.cs

@@ -3,13 +3,8 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
-using System.Text.RegularExpressions;
 using System.Threading.Tasks;
-using Microsoft.AspNetCore.Mvc.Filters;
-using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
-using Org.BouncyCastle.Crypto.Digests;
-using RocksDbSharp;
 using Wabbajack.BuildServer;
 using Wabbajack.Common;
 using Wabbajack.Lib;

Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs

@@ -1,6 +1,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading.Tasks;
 using Wabbajack.Common;
 using Wabbajack.Common.Serialization.Json;
@@ -30,7 +31,7 @@ namespace Wabbajack.VirtualFileSystem
 
         public void Write(Stream s)
         {
-            using var bw = new BinaryWriter(s);
+            using var bw = new BinaryWriter(s, Encoding.UTF8, true);
             bw.Write(Size);
             bw.Write(Children.Count);
             foreach (var file in Children)

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -1,26 +1,37 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Data;
+using System.Data.SQLite;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Threading.Tasks;
 using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
 using K4os.Hash.Crc;
-using RocksDbSharp;
 using Wabbajack.Common;
 
 namespace Wabbajack.VirtualFileSystem
 {
     public class VirtualFile
     {
-        private static RocksDb _vfsCache;
+        private static AbsolutePath DBLocation = Consts.LocalAppDataPath.Combine("GlobalVFSCache.sqlite");
+        private static string _connectionString;
+        private static SQLiteConnection _conn;
 
 
         static VirtualFile()
         {
-            var options = new DbOptions().SetCreateIfMissing(true);
+            _connectionString = String.Intern($"URI=file:{DBLocation};Pooling=True;Max Pool Size=100;");
-            _vfsCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalVFSCache2.rocksDb"));
+            _conn = new SQLiteConnection(_connectionString);
+            _conn.Open();
+
+            using var cmd = new SQLiteCommand(_conn);
+            cmd.CommandText = @"CREATE TABLE IF NOT EXISTS VFSCache (
+            Hash BIGINT PRIMARY KEY ,
+            Contents BLOB)";
+            cmd.ExecuteNonQuery();
+
         }
         
         private IEnumerable<VirtualFile> _thisAndAllChildren;
@@ -145,19 +156,22 @@ namespace Wabbajack.VirtualFileSystem
 
         private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IStreamFactory extractedFile, Hash hash, out VirtualFile found)
         {
-            var result = _vfsCache.Get(hash.ToArray());
+            using var cmd = new SQLiteCommand(_conn);
-            if (result == null)
+            cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash";
-            {
+            cmd.Parameters.AddWithValue("@hash", (long)hash);
-                found = null;
-                return false;
-            }
 
-            var data = IndexedVirtualFile.Read(new MemoryStream(result));
+            using var rdr = cmd.ExecuteReader();
+            while (rdr.Read())
+            {
+                var data = IndexedVirtualFile.Read(rdr.GetStream(0));
                 found = ConvertFromIndexedFile(context, data, path, parent, extractedFile);
                 found.Name = path;
                 found.Hash = hash;
                 return true;
+            }
 
+            found = default;
+            return false;
         }
 
         private IndexedVirtualFile ToIndexedVirtualFile()
@@ -236,11 +250,30 @@ namespace Wabbajack.VirtualFileSystem
 
             await using var ms = new MemoryStream();
             self.ToIndexedVirtualFile().Write(ms);
-            _vfsCache.Put(self.Hash.ToArray(), ms.ToArray());
+            ms.Position = 0;
-            
+            await InsertIntoVFSCache(self.Hash, ms);
             return self;
         }
 
+        private static async Task InsertIntoVFSCache(Hash hash, MemoryStream data)
+        {
+            await using var cmd = new SQLiteCommand(_conn);
+            cmd.CommandText = @"INSERT INTO VFSCache (Hash, Contents) VALUES (@hash, @contents)";
+            cmd.Parameters.AddWithValue("@hash", (long)hash);
+            var val = new SQLiteParameter("@contents", DbType.Binary) {Value = data.ToArray()};
+            cmd.Parameters.Add(val);
+            try
+            {
+                await cmd.ExecuteNonQueryAsync();
+            }
+            catch (SQLiteException ex)
+            {
+                if (ex.Message.StartsWith("constraint failed"))
+                    return;
+                throw;
+            }
+        }
+
         internal void FillFullPath()
         {
             int depth = 0;