Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #827 from wabbajack-tools/rocksdb-caching

Browse Source 
Rocksdb caching
This commit is contained in:
Timothy Baldridge 2020-05-12 16:20:31 -07:00 committed by GitHub
commit f2b1ec5ed7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 99 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Wabbajack.Common.Test/MiscTests.cs
View File

@ -22,10 +22,9 @@ namespace Wabbajack.Common.Test
const string data = "Cheese for Everyone!";
await testFile.WriteAllTextAsync(data);
File.WriteAllText("test.data", data);
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), testFile.FileHashCached());
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await testFile.FileHashCachedAsync());
Assert.True(Utils.TryGetHashCache(testFile, out var fileHash));
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), fileHash);
Assert.NotEqual("eSIyd+KOG3s=", await testFile.WithExtension(Consts.HashFileExtension).ReadAllTextAsync());
}
}
}

26
Wabbajack.Common/Hash.cs
View File

@ -6,6 +6,7 @@ using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json;
using RocksDbSharp;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
@ -96,10 +97,17 @@ namespace Wabbajack.Common
{
return new Hash(BitConverter.ToUInt64(xxHashAsHex.FromHex()));
}
public byte[] ToArray()
{
return BitConverter.GetBytes(_code);
}
}
public static partial class Utils
{
private static RocksDb _hashCache;
public static Hash ReadHash(this BinaryReader br)
{
return new Hash(br.ReadUInt64());
@ -170,14 +178,15 @@ namespace Wabbajack.Common
public static bool TryGetHashCache(AbsolutePath file, out Hash hash)
{
var hashFile = file.WithExtension(Consts.HashFileExtension);
hash = Hash.Empty;
if (!hashFile.IsFile) return false;
var normPath = Encoding.UTF8.GetBytes(file.Normalize());
var value = _hashCache.Get(normPath);
hash = default;
if (hashFile.Size != 20) return false;
if (value == null) return false;
if (value.Length != 20) return false;
using var fs = hashFile.OpenRead();
using var br = new BinaryReader(fs);
using var ms = new MemoryStream(value);
using var br = new BinaryReader(ms);
var version = br.ReadUInt32();
if (version != HashCacheVersion) return false;
@ -191,12 +200,13 @@ namespace Wabbajack.Common
private const uint HashCacheVersion = 0x01;
private static void WriteHashCache(AbsolutePath file, Hash hash)
{
using var fs = file.WithExtension(Consts.HashFileExtension).Create();
using var bw = new BinaryWriter(fs);
using var ms = new MemoryStream(20);
using var bw = new BinaryWriter(ms);
bw.Write(HashCacheVersion);
var lastModified = file.LastModifiedUtc.AsUnixTime();
bw.Write(lastModified);
bw.Write((ulong)hash);
_hashCache.Put(Encoding.UTF8.GetBytes(file.Normalize()), ms.ToArray());
}
public static async Task<Hash> FileHashCachedAsync(this AbsolutePath file, bool nullOnIOError = false)

5
Wabbajack.Common/Paths.cs
View File

@ -80,6 +80,11 @@ namespace Wabbajack.Common
throw new InvalidDataException("Absolute path must be absolute");
}
public string Normalize()
{
return _path.Replace("/", "\\").TrimEnd('\\');
}
public Extension Extension => Extension.FromPath(_path);
public FileStream OpenRead()

4
Wabbajack.Common/Utils.cs
View File

@ -23,6 +23,7 @@ using IniParser.Model.Configuration;
using IniParser.Parser;
using Newtonsoft.Json;
using ReactiveUI;
using RocksDbSharp;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Common.StatusFeed.Errors;
using YamlDotNet.Serialization;
@ -58,6 +59,9 @@ namespace Wabbajack.Common
LogFile = Consts.LogFile;
Consts.LocalAppDataPath.CreateDirectory();
Consts.LogsFolder.CreateDirectory();
var options = new DbOptions().SetCreateIfMissing(true);
_hashCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalHashCache.rocksDb"));
_startTime = DateTime.Now;

2
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -33,6 +33,8 @@
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="Octodiff" Version="1.2.1" />
<PackageReference Include="ReactiveUI" Version="11.3.8" />
<PackageReference Include="RocksDbNative" Version="6.2.2" />
<PackageReference Include="RocksDbSharp" Version="6.2.2" />
<PackageReference Include="SharpZipLib" Version="1.2.0" />
<PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
<PackageReference Include="System.Net.Http" Version="4.3.4" />

2
Wabbajack.Lib/AInstaller.cs
View File

@ -402,7 +402,7 @@ namespace Wabbajack.Lib
if (path.Size != d.Size) return null;
return await path.FileHashAsync() == d.Hash ? d : null;
return await path.FileHashCachedAsync() == d.Hash ? d : null;
}))
.Do(d =>
{

2
Wabbajack.Lib/MO2Installer.cs
View File

@ -316,7 +316,7 @@ namespace Wabbajack.Lib
Info($"Generating cleaned ESM for {filename}");
if (!gameFile.Exists) throw new InvalidDataException($"Missing {filename} at {gameFile}");
Status($"Hashing game version of {filename}");
var sha = await gameFile.FileHashAsync();
var sha = await gameFile.FileHashCachedAsync();
if (sha != directive.SourceESMHash)
throw new InvalidDataException(
$"Cannot patch {filename} from the game folder because the hashes do not match. Have you already cleaned the file?");

2
Wabbajack.Test/DownloaderTests.cs
View File

@ -513,9 +513,7 @@ namespace Wabbajack.Test
Assert.Equal(new[]
{
(RelativePath)@"Download.esm",
(RelativePath)@"Download.esm.xxHash",
(RelativePath)@"Download_ed33cbb256e5328361da8d9227df9cab1bb43a79a87dca2f223b2e2762ccaad1_.esm",
(RelativePath)@"Download_ed33cbb256e5328361da8d9227df9cab1bb43a79a87dca2f223b2e2762ccaad1_.esm.xxHash"
}.OrderBy(a => a).ToArray(),
folder.EnumerateFiles().Select(f => f.FileName).OrderBy(a => a).ToArray());

2
Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs
View File

@ -1,11 +1,13 @@
using System.Collections.Generic;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
namespace Wabbajack.VirtualFileSystem
{
/// <summary>
/// Response from the Build server for a indexed file
/// </summary>
[JsonName("IndexedVirtualFile")]
public class IndexedVirtualFile
{
public IPath Name { get; set; }

85
Wabbajack.VirtualFileSystem/VirtualFile.cs
View File

@ -6,12 +6,22 @@ using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using K4os.Hash.Crc;
using RocksDbSharp;
using Wabbajack.Common;
namespace Wabbajack.VirtualFileSystem
{
public class VirtualFile
{
private static RocksDb _vfsCache;
static VirtualFile()
{
var options = new DbOptions().SetCreateIfMissing(true);
_vfsCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalVFSCache.rocksDb"));
}
private AbsolutePath _stagedPath;
private IEnumerable<VirtualFile> _thisAndAllChildren;
@ -127,6 +137,53 @@ namespace Wabbajack.VirtualFileSystem
foreach (var itm in Children)
itm.ThisAndAllChildrenReduced(fn);
}
private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, VirtualFile vparent, IExtractedFile extractedFile)
{
var vself = new VirtualFile
{
Context = context,
Name = path,
Parent = vparent,
Size = file.Size,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = file.Hash
};
vself.FillFullPath();
vself.Children = file.Children.Select(f => ConvertFromIndexedFile(context, f, f.Name, vself, extractedFile)).ToImmutableList();
return vself;
}
private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IExtractedFile extractedFile, Hash hash, out VirtualFile found)
{
var result = _vfsCache.Get(hash.ToArray());
if (result == null)
{
found = null;
return false;
}
var data = new MemoryStream(result).FromJson<IndexedVirtualFile>();
found = ConvertFromIndexedFile(context, data, path, parent, extractedFile);
return true;
}
private IndexedVirtualFile ToIndexedVirtualFile()
{
return new IndexedVirtualFile
{
Hash = Hash,
Name = Name,
Children = Children.Select(c => c.ToIndexedVirtualFile()).ToList(),
Size = Size
};
}
public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile,
IPath relPath, int depth = 0)
@ -141,30 +198,14 @@ namespace Wabbajack.VirtualFileSystem
{
Utils.Log($"Downloaded VFS data for {relPath.FileName}");
VirtualFile Convert(IndexedVirtualFile file, IPath path, VirtualFile vparent)
{
var vself = new VirtualFile
{
Context = context,
Name = path,
Parent = vparent,
Size = file.Size,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = file.Hash
};
vself.FillFullPath();
vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList();
return vself;
}
return Convert(result, relPath, parent);
return ConvertFromIndexedFile(context, result, relPath, parent, extractedFile);
}
}
if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
return vself;
var self = new VirtualFile
{
Context = context,
@ -200,6 +241,10 @@ namespace Wabbajack.VirtualFileSystem
throw;
}
await using var ms = new MemoryStream();
self.ToIndexedVirtualFile().ToJson(ms);
_vfsCache.Put(self.Hash.ToArray(), ms.ToArray());
return self;
}