Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1243 from wabbajack-tools/rocksdb-sqlite

Browse Source 
Remove RocksDB (replaced with SQLite)
This commit is contained in:
Timothy Baldridge 2021-01-05 18:25:15 -07:00 committed by GitHub
commit cb3af67f57
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 217 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/tests.yml vendored
View File

@ -33,3 +33,9 @@ jobs:
run: dotnet test --verbosity normal --configuration Release /p:Platform=x64 Wabbajack.App.Test
- name: Test Wabbajack.Server.Test
run: dotnet test --verbosity normal --configuration Release /p:Platform=x64 Wabbajack.Server.Test
- name: Publish Unit Test Results
uses: EnricoMi/publish-unit-test-result-action@v1.6
if: always()
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
files: test-results/**/*.xml

1
CHANGELOG.md
View File

@ -4,6 +4,7 @@
* Wabbajack is now based on .NET 5.0 (does not require a runtime download by users)
* Origin is now supported as a game source
* Basic (mostly untested) support for Dragon Age : Origins
* Replace RocksDB with SQLite should result in less process contention when running the UI and the CLI at the same time
#### Version - 2.3.6.2 - 12/31/2020
* HOTFIX: Also apply the IPS4 changes to LL Meta lookups

3
Compression.BSA/BSA/Reader/FolderRecord.cs
View File

@ -2,10 +2,7 @@
using System.Buffers.Binary;
using System.Collections.Generic;
using System.IO;
using System.Text;
using NativeImport;
using Wabbajack.Common;
using File = Alphaleonis.Win32.Filesystem.File;
namespace Compression.BSA
{

98
Wabbajack.Common/Hash.cs
View File

@ -1,14 +1,15 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.HashFunction.xxHash;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json;
using RocksDbSharp;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
using System.Data.SQLite;
namespace Wabbajack.Common
{
@ -117,20 +118,73 @@ namespace Wabbajack.Common
public static class HashCache
{
private const uint HashCacheVersion = 0x01;
private static AbsolutePath DBLocation = Consts.LocalAppDataPath.Combine("GlobalHashCache.sqlite");
private static string _connectionString;
private static SQLiteConnection _conn;
// Keep rock DB out of Utils, as it causes lock problems for users of Wabbajack.Common that aren't interested in it, otherwise
private static RocksDb _hashCache;
static HashCache()
{
var options = new DbOptions().SetCreateIfMissing(true);
_hashCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalHashCache.rocksDb"));
_connectionString = String.Intern($"URI=file:{DBLocation};Pooling=True;Max Pool Size=100;");
_conn = new SQLiteConnection(_connectionString);
_conn.Open();
using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"CREATE TABLE IF NOT EXISTS HashCache (
Path TEXT PRIMARY KEY,
LastModified BIGINT,
Hash BIGINT)";
cmd.ExecuteNonQuery();
}
private static (AbsolutePath Path, long LastModified, Hash Hash) GetFromCache(AbsolutePath path)
{
using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = "SELECT LastModified, Hash FROM HashCache WHERE Path = @path";
cmd.Parameters.AddWithValue("@path", path.ToString());
cmd.PrepareAsync();
using var reader = cmd.ExecuteReader();
while (reader.Read())
{
return (path, reader.GetInt64(0), Hash.FromLong(reader.GetInt64(1)));
}
return default;
}
private static void PurgeCacheEntry(AbsolutePath path)
{
using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = "DELETE FROM HashCache WHERE Path = @path";
cmd.Parameters.AddWithValue("@path", path.ToString());
cmd.PrepareAsync();
cmd.ExecuteNonQuery();
}
private static void UpsertCacheEntry(AbsolutePath path, long lastModified, Hash hash)
{
using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"INSERT INTO HashCache (Path, LastModified, Hash) VALUES (@path, @lastModified, @hash)
ON CONFLICT(Path) DO UPDATE SET LastModified = @lastModified, Hash = @hash";
cmd.Parameters.AddWithValue("@path", path.ToString());
cmd.Parameters.AddWithValue("@lastModified", lastModified);
cmd.Parameters.AddWithValue("@hash", (long)hash);
cmd.PrepareAsync();
cmd.ExecuteNonQuery();
}
public static Hash ReadHash(this BinaryReader br)
{
return new Hash(br.ReadUInt64());
return new(br.ReadUInt64());
}
public static void Write(this BinaryWriter bw, Hash hash)
@ -182,33 +236,27 @@ namespace Wabbajack.Common
public static bool TryGetHashCache(this AbsolutePath file, out Hash hash)
{
var normPath = Encoding.UTF8.GetBytes(file.Normalize());
var value = _hashCache.Get(normPath);
hash = default;
if (!file.Exists) return false;
if (value == null) return false;
if (value.Length != 20) return false;
var result = GetFromCache(file);
if (result == default)
return false;
using var ms = new MemoryStream(value);
using var br = new BinaryReader(ms);
var version = br.ReadUInt32();
if (version != HashCacheVersion) return false;
if (result.LastModified == file.LastModifiedUtc.ToFileTimeUtc())
{
hash = result.Hash;
return true;
}
var lastModified = br.ReadUInt64();
if (lastModified != file.LastModifiedUtc.AsUnixTime()) return false;
hash = new Hash(br.ReadUInt64());
return true;
PurgeCacheEntry(file);
return false;
}
private static void WriteHashCache(this AbsolutePath file, Hash hash)
{
using var ms = new MemoryStream(20);
using var bw = new BinaryWriter(ms);
bw.Write(HashCacheVersion);
var lastModified = file.LastModifiedUtc.AsUnixTime();
bw.Write(lastModified);
bw.Write((ulong)hash);
_hashCache.Put(Encoding.UTF8.GetBytes(file.Normalize()), ms.ToArray());
if (!file.Exists) return;
UpsertCacheEntry(file, file.LastModifiedUtc.ToFileTimeUtc(), hash);
}
public static void FileHashWriteCache(this AbsolutePath file, Hash hash)

125
Wabbajack.Common/Patches.cs
View File

@ -1,73 +1,117 @@
using System;
using System.Data.SQLite;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using RocksDbSharp;
namespace Wabbajack.Common
{
public static class PatchCache
{
// Keep rock DB out of Utils, as it causes lock problems for users of Wabbajack.Common that aren't interested in it, otherwise
private static RocksDb? _patchCache;
private static AbsolutePath DBLocation = Consts.LocalAppDataPath.Combine("GlobalPatchCache.sqlite");
private static string _connectionString;
private static SQLiteConnection _conn;
static PatchCache()
{
var options = new DbOptions().SetCreateIfMissing(true);
_patchCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("PatchCache.rocksDb"));
}
_connectionString = String.Intern($"URI=file:{DBLocation};Pooling=True;Max Pool Size=100;");
_conn = new SQLiteConnection(_connectionString);
_conn.Open();
using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"CREATE TABLE IF NOT EXISTS PatchCache (
FromHash BIGINT,
ToHash BIGINT,
PatchSize BLOB,
Patch BLOB,
PRIMARY KEY (FromHash, ToHash))";
cmd.ExecuteNonQuery();
private static byte[] PatchKey(Hash src, Hash dest)
{
var arr = new byte[16];
Array.Copy(BitConverter.GetBytes((ulong)src), 0, arr, 0, 8);
Array.Copy(BitConverter.GetBytes((ulong)dest), 0, arr, 8, 8);
return arr;
}
public static async Task CreatePatchCached(byte[] a, byte[] b, Stream output)
{
await using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"INSERT INTO PatchCache (FromHash, ToHash, PatchSize, Patch)
VALUES (@fromHash, @toHash, @patchSize, @patch)";
var dataA = a.xxHash();
var dataB = b.xxHash();
var key = PatchKey(dataA, dataB);
var found = _patchCache!.Get(key);
if (found != null)
{
await output.WriteAsync(found);
return;
}
cmd.Parameters.AddWithValue("@fromHash", (long)dataA);
cmd.Parameters.AddWithValue("@toHash", (long)dataB);
await using var patch = new MemoryStream();
Utils.Status("Creating Patch");
OctoDiff.Create(a, b, patch);
_patchCache.Put(key, patch.ToArray());
patch.Position = 0;
cmd.Parameters.AddWithValue("@patchSize", patch.Length);
cmd.Parameters.AddWithValue("@patch", patch.ToArray());
try
{
await cmd.ExecuteNonQueryAsync();
}
catch (SQLiteException ex)
{
if (!ex.Message.StartsWith("constraint exception"))
throw;
}
await patch.CopyToAsync(output);
}
public static async Task<long> CreatePatchCached(Stream srcStream, Hash srcHash, Stream destStream, Hash destHash,
Stream? patchOutStream = null)
{
var key = PatchKey(srcHash, destHash);
var patch = _patchCache!.Get(key);
if (patch != null)
if (patchOutStream == null)
{
if (patchOutStream == null) return patch.Length;
await using var rcmd = new SQLiteCommand(_conn);
rcmd.CommandText = "SELECT PatchSize FROM PatchCache WHERE FromHash = @fromHash AND ToHash = @toHash";
rcmd.Parameters.AddWithValue("@fromHash", (long)srcHash);
rcmd.Parameters.AddWithValue("@toHash", (long)destHash);
await patchOutStream.WriteAsync(patch);
return patch.Length;
await using var rdr = await rcmd.ExecuteReaderAsync();
while (await rdr.ReadAsync())
{
return rdr.GetInt64(0);
}
}
else
{
if (TryGetPatch(srcHash, destHash, out var array))
{
await patchOutStream!.WriteAsync(array);
return array.Length;
}
}
await using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"INSERT INTO PatchCache (FromHash, ToHash, PatchSize, Patch)
VALUES (@fromHash, @toHash, @patchSize, @patch)";
cmd.Parameters.AddWithValue("@fromHash", (long)srcHash);
cmd.Parameters.AddWithValue("@toHash", (long)destHash);
Utils.Status("Creating Patch");
await using var sigStream = new MemoryStream();
await using var patchStream = new MemoryStream();
OctoDiff.Create(srcStream, destStream, sigStream, patchStream);
_patchCache.Put(key, patchStream.ToArray());
cmd.Parameters.AddWithValue("@patchSize", patchStream.Length);
cmd.Parameters.AddWithValue("@patch", patchStream.ToArray());
try
{
await cmd.ExecuteNonQueryAsync();
}
catch (SQLiteException ex)
{
if (!ex.Message.StartsWith("constraint exception"))
throw;
}
if (patchOutStream == null) return patchStream.Position;
@ -77,26 +121,27 @@ namespace Wabbajack.Common
return patchStream.Position;
}
public static bool TryGetPatch(Hash foundHash, Hash fileHash, [MaybeNullWhen(false)] out byte[] ePatch)
public static bool TryGetPatch(Hash fromHash, Hash toHash, [MaybeNullWhen(false)] out byte[] array)
{
var key = PatchKey(foundHash, fileHash);
var patch = _patchCache!.Get(key);
using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"SELECT PatchSize, Patch FROM PatchCache WHERE FromHash = @fromHash AND ToHash = @toHash";
cmd.Parameters.AddWithValue("@fromHash", (long)fromHash);
cmd.Parameters.AddWithValue("@toHash", (long)toHash);
if (patch != null)
using var rdr = cmd.ExecuteReader();
while (rdr.Read())
{
ePatch = patch;
array = new byte[rdr.GetInt64(0)];
rdr.GetBytes(1, 0, array, 0, array.Length);
return true;
}
ePatch = null;
array = Array.Empty<byte>();
return false;
}
public static bool HavePatch(Hash foundHash, Hash fileHash)
{
var key = PatchKey(foundHash, fileHash);
return _patchCache!.Get(key) != null;
}
public static void ApplyPatch(Stream input, Func<Stream> openPatchStream, Stream output)

3
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -54,10 +54,9 @@
<PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="Octodiff" Version="1.2.1" />
<PackageReference Include="RocksDbNative" Version="6.2.2" />
<PackageReference Include="RocksDbSharp" Version="6.2.2" />
<PackageReference Include="SharpZipLib" Version="1.3.1" />
<PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
<PackageReference Include="System.Data.SQLite.Core" Version="1.0.113.7" />
<PackageReference Include="System.Net.Http" Version="4.3.4" />
<PackageReference Include="System.Reactive" Version="5.0.0" />
<PackageReference Include="System.Security.Cryptography.ProtectedData" Version="5.0.0" />

5
Wabbajack.Server/Services/ListValidator.cs
View File

@ -3,13 +3,8 @@ using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc.Filters;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Org.BouncyCastle.Crypto.Digests;
using RocksDbSharp;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib;

3
Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs
View File

@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
@ -30,7 +31,7 @@ namespace Wabbajack.VirtualFileSystem
public void Write(Stream s)
{
using var bw = new BinaryWriter(s);
using var bw = new BinaryWriter(s, Encoding.UTF8, true);
bw.Write(Size);
bw.Write(Children.Count);
foreach (var file in Children)

65
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -1,26 +1,37 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Data;
using System.Data.SQLite;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using K4os.Hash.Crc;
using RocksDbSharp;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public class VirtualFile
{
private static RocksDb _vfsCache;
private static AbsolutePath DBLocation = Consts.LocalAppDataPath.Combine("GlobalVFSCache.sqlite");
private static string _connectionString;
private static SQLiteConnection _conn;
static VirtualFile()
{
var options = new DbOptions().SetCreateIfMissing(true);
_vfsCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalVFSCache2.rocksDb"));
_connectionString = String.Intern($"URI=file:{DBLocation};Pooling=True;Max Pool Size=100;");
_conn = new SQLiteConnection(_connectionString);
_conn.Open();
using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"CREATE TABLE IF NOT EXISTS VFSCache (
Hash BIGINT PRIMARY KEY ,
Contents BLOB)";
cmd.ExecuteNonQuery();
}
private IEnumerable<VirtualFile> _thisAndAllChildren;
@ -145,19 +156,22 @@ namespace Wabbajack.VirtualFileSystem
private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IStreamFactory extractedFile, Hash hash, out VirtualFile found)
{
var result = _vfsCache.Get(hash.ToArray());
if (result == null)
using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash";
cmd.Parameters.AddWithValue("@hash", (long)hash);
using var rdr = cmd.ExecuteReader();
while (rdr.Read())
{
found = null;
return false;
var data = IndexedVirtualFile.Read(rdr.GetStream(0));
found = ConvertFromIndexedFile(context, data, path, parent, extractedFile);
found.Name = path;
found.Hash = hash;
return true;
}
var data = IndexedVirtualFile.Read(new MemoryStream(result));
found = ConvertFromIndexedFile(context, data, path, parent, extractedFile);
found.Name = path;
found.Hash = hash;
return true;
found = default;
return false;
}
private IndexedVirtualFile ToIndexedVirtualFile()
@ -236,11 +250,30 @@ namespace Wabbajack.VirtualFileSystem
await using var ms = new MemoryStream();
self.ToIndexedVirtualFile().Write(ms);
_vfsCache.Put(self.Hash.ToArray(), ms.ToArray());
ms.Position = 0;
await InsertIntoVFSCache(self.Hash, ms);
return self;
}
private static async Task InsertIntoVFSCache(Hash hash, MemoryStream data)
{
await using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"INSERT INTO VFSCache (Hash, Contents) VALUES (@hash, @contents)";
cmd.Parameters.AddWithValue("@hash", (long)hash);
var val = new SQLiteParameter("@contents", DbType.Binary) {Value = data.ToArray()};
cmd.Parameters.Add(val);
try
{
await cmd.ExecuteNonQueryAsync();
}
catch (SQLiteException ex)
{
if (ex.Message.StartsWith("constraint failed"))
return;
throw;
}
}
internal void FillFullPath()
{
int depth = 0;