diff --git a/Wabbajack.Common.Test/MiscTests.cs b/Wabbajack.Common.Test/MiscTests.cs index df0f159d..251fcd71 100644 --- a/Wabbajack.Common.Test/MiscTests.cs +++ b/Wabbajack.Common.Test/MiscTests.cs @@ -22,10 +22,9 @@ namespace Wabbajack.Common.Test const string data = "Cheese for Everyone!"; await testFile.WriteAllTextAsync(data); File.WriteAllText("test.data", data); - Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), testFile.FileHashCached()); + Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await testFile.FileHashCachedAsync()); Assert.True(Utils.TryGetHashCache(testFile, out var fileHash)); Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), fileHash); - Assert.NotEqual("eSIyd+KOG3s=", await testFile.WithExtension(Consts.HashFileExtension).ReadAllTextAsync()); } } } diff --git a/Wabbajack.Common/Hash.cs b/Wabbajack.Common/Hash.cs index fc047914..cd23e1d3 100644 --- a/Wabbajack.Common/Hash.cs +++ b/Wabbajack.Common/Hash.cs @@ -6,6 +6,7 @@ using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; using Newtonsoft.Json; +using RocksDbSharp; using File = Alphaleonis.Win32.Filesystem.File; using Path = Alphaleonis.Win32.Filesystem.Path; @@ -96,10 +97,17 @@ namespace Wabbajack.Common { return new Hash(BitConverter.ToUInt64(xxHashAsHex.FromHex())); } + + public byte[] ToArray() + { + return BitConverter.GetBytes(_code); + } } public static partial class Utils { + private static RocksDb _hashCache; + public static Hash ReadHash(this BinaryReader br) { return new Hash(br.ReadUInt64()); @@ -170,14 +178,15 @@ namespace Wabbajack.Common public static bool TryGetHashCache(AbsolutePath file, out Hash hash) { - var hashFile = file.WithExtension(Consts.HashFileExtension); - hash = Hash.Empty; - if (!hashFile.IsFile) return false; + var normPath = Encoding.UTF8.GetBytes(file.Normalize()); + var value = _hashCache.Get(normPath); + hash = default; - if (hashFile.Size != 20) return false; + if (value == null) return false; + if (value.Length != 20) return false; - using var fs = hashFile.OpenRead(); - using var br = new BinaryReader(fs); + using var ms = new MemoryStream(value); + using var br = new BinaryReader(ms); var version = br.ReadUInt32(); if (version != HashCacheVersion) return false; @@ -191,12 +200,13 @@ namespace Wabbajack.Common private const uint HashCacheVersion = 0x01; private static void WriteHashCache(AbsolutePath file, Hash hash) { - using var fs = file.WithExtension(Consts.HashFileExtension).Create(); - using var bw = new BinaryWriter(fs); + using var ms = new MemoryStream(20); + using var bw = new BinaryWriter(ms); bw.Write(HashCacheVersion); var lastModified = file.LastModifiedUtc.AsUnixTime(); bw.Write(lastModified); bw.Write((ulong)hash); + _hashCache.Put(Encoding.UTF8.GetBytes(file.Normalize()), ms.ToArray()); } public static async Task FileHashCachedAsync(this AbsolutePath file, bool nullOnIOError = false) diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index e2dd9f3b..297dd029 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -80,6 +80,11 @@ namespace Wabbajack.Common throw new InvalidDataException("Absolute path must be absolute"); } + public string Normalize() + { + return _path.Replace("/", "\\").TrimEnd('\\'); + } + public Extension Extension => Extension.FromPath(_path); public FileStream OpenRead() diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 56f048a9..6a2f929c 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -23,6 +23,7 @@ using IniParser.Model.Configuration; using IniParser.Parser; using Newtonsoft.Json; using ReactiveUI; +using RocksDbSharp; using Wabbajack.Common.StatusFeed; using Wabbajack.Common.StatusFeed.Errors; using YamlDotNet.Serialization; @@ -58,6 +59,9 @@ namespace Wabbajack.Common LogFile = Consts.LogFile; Consts.LocalAppDataPath.CreateDirectory(); Consts.LogsFolder.CreateDirectory(); + + var options = new DbOptions().SetCreateIfMissing(true); + _hashCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalHashCache.rocksDb")); _startTime = DateTime.Now; diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index ac2bc5e4..92282cd6 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -33,6 +33,8 @@ + + diff --git a/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs b/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs index 7664314e..f602a3eb 100644 --- a/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs @@ -1,11 +1,13 @@ using System.Collections.Generic; using Wabbajack.Common; +using Wabbajack.Common.Serialization.Json; namespace Wabbajack.VirtualFileSystem { /// /// Response from the Build server for a indexed file /// + [JsonName("IndexedVirtualFile")] public class IndexedVirtualFile { public IPath Name { get; set; } diff --git a/Wabbajack.VirtualFileSystem/VirtualFile.cs b/Wabbajack.VirtualFileSystem/VirtualFile.cs index d01dabaa..3b0ce3a1 100644 --- a/Wabbajack.VirtualFileSystem/VirtualFile.cs +++ b/Wabbajack.VirtualFileSystem/VirtualFile.cs @@ -6,12 +6,22 @@ using System.Linq; using System.Net.Http; using System.Threading.Tasks; using K4os.Hash.Crc; +using RocksDbSharp; using Wabbajack.Common; namespace Wabbajack.VirtualFileSystem { public class VirtualFile { + private static RocksDb _vfsCache; + + + static VirtualFile() + { + var options = new DbOptions().SetCreateIfMissing(true); + _vfsCache = RocksDb.Open(options, (string)Consts.LocalAppDataPath.Combine("GlobalVFSCache.rocksDb")); + } + private AbsolutePath _stagedPath; private IEnumerable _thisAndAllChildren; @@ -127,6 +137,53 @@ namespace Wabbajack.VirtualFileSystem foreach (var itm in Children) itm.ThisAndAllChildrenReduced(fn); } + + private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, VirtualFile vparent, IExtractedFile extractedFile) + { + var vself = new VirtualFile + { + Context = context, + Name = path, + Parent = vparent, + Size = file.Size, + LastModified = extractedFile.LastModifiedUtc.AsUnixTime(), + LastAnalyzed = DateTime.Now.AsUnixTime(), + Hash = file.Hash + }; + + vself.FillFullPath(); + + vself.Children = file.Children.Select(f => ConvertFromIndexedFile(context, f, f.Name, vself, extractedFile)).ToImmutableList(); + + return vself; + } + + private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IExtractedFile extractedFile, Hash hash, out VirtualFile found) + { + var result = _vfsCache.Get(hash.ToArray()); + if (result == null) + { + found = null; + return false; + } + + var data = new MemoryStream(result).FromJson(); + found = ConvertFromIndexedFile(context, data, path, parent, extractedFile); + return true; + + } + + private IndexedVirtualFile ToIndexedVirtualFile() + { + return new IndexedVirtualFile + { + Hash = Hash, + Name = Name, + Children = Children.Select(c => c.ToIndexedVirtualFile()).ToList(), + Size = Size + }; + } + public static async Task Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile, IPath relPath, int depth = 0) @@ -141,30 +198,14 @@ namespace Wabbajack.VirtualFileSystem { Utils.Log($"Downloaded VFS data for {relPath.FileName}"); - VirtualFile Convert(IndexedVirtualFile file, IPath path, VirtualFile vparent) - { - var vself = new VirtualFile - { - Context = context, - Name = path, - Parent = vparent, - Size = file.Size, - LastModified = extractedFile.LastModifiedUtc.AsUnixTime(), - LastAnalyzed = DateTime.Now.AsUnixTime(), - Hash = file.Hash - }; - - vself.FillFullPath(); - vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList(); - - return vself; - } - - return Convert(result, relPath, parent); + return ConvertFromIndexedFile(context, result, relPath, parent, extractedFile); } } + if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself)) + return vself; + var self = new VirtualFile { Context = context, @@ -200,6 +241,10 @@ namespace Wabbajack.VirtualFileSystem throw; } + await using var ms = new MemoryStream(); + self.ToIndexedVirtualFile().ToJson(ms); + _vfsCache.Put(self.Hash.ToArray(), ms.ToArray()); + return self; }