Add Cesi cache

This commit is contained in:
Timothy Baldridge 2022-06-21 19:38:42 -06:00
parent 6724d4ba6e
commit 3c28cdf7b9
22 changed files with 216 additions and 82 deletions

View File

@ -47,7 +47,6 @@ internal class Program
services.AddSingleton<CommandLineBuilder, CommandLineBuilder>(); services.AddSingleton<CommandLineBuilder, CommandLineBuilder>();
services.AddSingleton<TemporaryFileManager>(); services.AddSingleton<TemporaryFileManager>();
services.AddSingleton<FileExtractor.FileExtractor>(); services.AddSingleton<FileExtractor.FileExtractor>();
services.AddSingleton(new VFSCache(KnownFolders.EntryPoint.Combine("vfscache.sqlite")));
services.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount}); services.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount});
services.AddSingleton<Client>(); services.AddSingleton<Client>();
services.AddSingleton<Networking.WabbajackClientApi.Client>(); services.AddSingleton<Networking.WabbajackClientApi.Client>();

View File

@ -0,0 +1,16 @@
using System.Collections.Generic;
using Wabbajack.DTOs.Texture;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
namespace Wabbajack.DTOs.Vfs;
public class IndexedVirtualFile
{
public IPath Name { get; set; }
public Hash Hash { get; set; }
public ImageState? ImageState { get; set; }
public long Size { get; set; }
public List<IndexedVirtualFile> Children { get; set; } = new();
}

View File

@ -0,0 +1,39 @@
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.DTOs.Vfs;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Networking.Http;
using Wabbajack.VFS.Interfaces;
namespace Wabbajack.Networking.WabbajackClientApi;
public class CesiVFSCache : IVfsCache
{
private readonly Client _client;
private readonly ILogger<CesiVFSCache> _logger;
public CesiVFSCache(ILogger<CesiVFSCache> logger, Client client)
{
_logger = logger;
_client = client;
}
public async Task<IndexedVirtualFile?> Get(Hash hash, CancellationToken token)
{
_logger.LogInformation("Requesting CESI Information for: {Hash}", hash.ToHex());
try
{
return await _client.GetCesiVfsEntry(hash, token);
}
catch (HttpException exception)
{
return null;
}
}
public async Task Put(IndexedVirtualFile file, CancellationToken token)
{
return;
}
}

View File

@ -19,13 +19,13 @@ using Wabbajack.DTOs.JsonConverters;
using Wabbajack.DTOs.Logins; using Wabbajack.DTOs.Logins;
using Wabbajack.DTOs.ModListValidation; using Wabbajack.DTOs.ModListValidation;
using Wabbajack.DTOs.Validation; using Wabbajack.DTOs.Validation;
using Wabbajack.DTOs.Vfs;
using Wabbajack.Hashing.xxHash64; using Wabbajack.Hashing.xxHash64;
using Wabbajack.Networking.Http; using Wabbajack.Networking.Http;
using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Networking.Http.Interfaces;
using Wabbajack.Paths; using Wabbajack.Paths;
using Wabbajack.Paths.IO; using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter; using Wabbajack.RateLimiter;
using Wabbajack.VFS;
using YamlDotNet.Serialization; using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions; using YamlDotNet.Serialization.NamingConventions;
@ -38,7 +38,7 @@ public class Client
private readonly HttpClient _client; private readonly HttpClient _client;
private readonly Configuration _configuration; private readonly Configuration _configuration;
private readonly DTOSerializer _dtos; private readonly DTOSerializer _dtos;
private readonly IResource<FileHashCache> _hashLimiter; private readonly IResource<Client> _hashLimiter;
private readonly IResource<HttpClient> _limiter; private readonly IResource<HttpClient> _limiter;
private readonly ILogger<Client> _logger; private readonly ILogger<Client> _logger;
private readonly ParallelOptions _parallelOptions; private readonly ParallelOptions _parallelOptions;
@ -48,7 +48,7 @@ public class Client
public Client(ILogger<Client> logger, HttpClient client, ITokenProvider<WabbajackApiState> token, public Client(ILogger<Client> logger, HttpClient client, ITokenProvider<WabbajackApiState> token,
DTOSerializer dtos, DTOSerializer dtos,
IResource<HttpClient> limiter, IResource<FileHashCache> hashLimiter, Configuration configuration) IResource<HttpClient> limiter, IResource<Client> hashLimiter, Configuration configuration)
{ {
_configuration = configuration; _configuration = configuration;
_token = token; _token = token;
@ -387,4 +387,12 @@ public class Client
return new Uri($"{_configuration.BuildServerUrl}proxy?name={archive.Name}&hash={archive.Hash.ToHex()}&uri={HttpUtility.UrlEncode(uri.ToString())}"); return new Uri($"{_configuration.BuildServerUrl}proxy?name={archive.Name}&hash={archive.Hash.ToHex()}&uri={HttpUtility.UrlEncode(uri.ToString())}");
} }
public async Task<IndexedVirtualFile?> GetCesiVfsEntry(Hash hash, CancellationToken token)
{
var msg = await MakeMessage(HttpMethod.Get, new Uri($"{_configuration.BuildServerUrl}cesi/vfs/{hash.ToHex()}"));
using var response = await _client.SendAsync(msg, token);
HttpException.ThrowOnFailure(response);
return await _dtos.DeserializeAsync<IndexedVirtualFile>(await response.Content.ReadAsStreamAsync(token), token);
}
} }

View File

@ -16,7 +16,7 @@
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" /> <ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
<ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" /> <ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
<ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" /> <ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
<ProjectReference Include="..\Wabbajack.VFS\Wabbajack.VFS.csproj" /> <ProjectReference Include="..\Wabbajack.VFS.Interfaces\Wabbajack.VFS.Interfaces.csproj" />
</ItemGroup> </ItemGroup>
</Project> </Project>

View File

@ -12,7 +12,7 @@ public static class KnownFolders
get get
{ {
var result = Process.GetCurrentProcess().MainModule!.FileName!.ToAbsolutePath().Parent; var result = Process.GetCurrentProcess().MainModule!.FileName!.ToAbsolutePath().Parent;
if (result.FileName == "dotnet".ToRelativePath()) if (result.FileName == "dotnet".ToRelativePath() || Assembly.GetEntryAssembly() != null)
{ {
return Assembly.GetExecutingAssembly().Location.ToAbsolutePath().Parent; return Assembly.GetExecutingAssembly().Location.ToAbsolutePath().Parent;
} }

View File

@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.DTOs.JsonConverters; using Wabbajack.DTOs.JsonConverters;
using Wabbajack.DTOs.Texture; using Wabbajack.DTOs.Texture;
using Wabbajack.DTOs.Vfs;
using Wabbajack.Hashing.xxHash64; using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths; using Wabbajack.Paths;
using Wabbajack.VFS; using Wabbajack.VFS;

View File

@ -1,6 +1,7 @@
using System.Reflection; using System.Reflection;
using System.Text.Json; using System.Text.Json;
using Chronic.Core; using Chronic.Core;
using CouchDB.Driver;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Nettle; using Nettle;
@ -35,13 +36,15 @@ public class MetricsController : ControllerBase
private readonly AppSettings _settings; private readonly AppSettings _settings;
private ILogger<MetricsController> _logger; private ILogger<MetricsController> _logger;
private readonly Metrics _metricsStore; private readonly Metrics _metricsStore;
private readonly ICouchDatabase<Metric> _db;
public MetricsController(ILogger<MetricsController> logger, Metrics metricsStore, public MetricsController(ILogger<MetricsController> logger, Metrics metricsStore,
AppSettings settings) AppSettings settings, ICouchDatabase<Metric> db)
{ {
_logger = logger; _logger = logger;
_settings = settings; _settings = settings;
_metricsStore = metricsStore; _metricsStore = metricsStore;
_db = db;
} }
@ -88,7 +91,7 @@ public class MetricsController : ControllerBase
private static byte[] LBRACKET = {(byte)'['}; private static byte[] LBRACKET = {(byte)'['};
private static byte[] RBRACKET = {(byte)']'}; private static byte[] RBRACKET = {(byte)']'};
private static byte[] COMMA = {(byte) ','}; private static byte[] COMMA = {(byte) ','};
[HttpGet] [HttpGet]
[Route("dump")] [Route("dump")]
public async Task GetMetrics([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to, [FromQuery] string? subject) public async Task GetMetrics([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to, [FromQuery] string? subject)

View File

@ -1,9 +1,10 @@
using System; using System;
using CouchDB.Driver.Types;
using Microsoft.Extensions.Primitives; using Microsoft.Extensions.Primitives;
namespace Wabbajack.Server.DTOs; namespace Wabbajack.Server.DTOs;
public class Metric public class Metric : CouchDocument
{ {
public DateTime Timestamp { get; set; } public DateTime Timestamp { get; set; }
public string Action { get; set; } public string Action { get; set; }

View File

@ -38,6 +38,7 @@ using Wabbajack.Server.Services;
using Wabbajack.Services.OSIntegrated.TokenProviders; using Wabbajack.Services.OSIntegrated.TokenProviders;
using Wabbajack.Networking.WabbajackClientApi; using Wabbajack.Networking.WabbajackClientApi;
using Wabbajack.Paths.IO; using Wabbajack.Paths.IO;
using Wabbajack.Server.DTOs;
using Wabbajack.VFS; using Wabbajack.VFS;
using YamlDotNet.Serialization.NamingConventions; using YamlDotNet.Serialization.NamingConventions;
using Client = Wabbajack.Networking.GitHub.Client; using Client = Wabbajack.Networking.GitHub.Client;
@ -154,6 +155,18 @@ public class Startup
}); });
return client.GetDatabase<Analyzed>("cesi"); return client.GetDatabase<Analyzed>("cesi");
}); });
services.AddSingleton(s =>
{
var settings = s.GetRequiredService<AppSettings>();
var client = new CouchClient(settings.CesiDB.Endpoint, b =>
{
b.UseBasicAuthentication("wabbajack", "password");
b.SetPropertyCase(PropertyCaseType.None);
b.SetJsonNullValueHandling(NullValueHandling.Ignore);
});
return client.GetDatabase<Metric>("cesi");
});
services.AddMvc(); services.AddMvc();
services services

View File

@ -21,6 +21,12 @@
"Database": "cesi", "Database": "cesi",
"Username": "cesi", "Username": "cesi",
"Password": "password" "Password": "password"
},
"MetricsDB": {
"Endpoint": "http://localhost:15984",
"Database": "metrics",
"Username": "wabbajack",
"Password": "password"
} }
}, },
"AllowedHosts": "*" "AllowedHosts": "*"

View File

@ -26,6 +26,7 @@ using Wabbajack.RateLimiter;
using Wabbajack.Services.OSIntegrated.Services; using Wabbajack.Services.OSIntegrated.Services;
using Wabbajack.Services.OSIntegrated.TokenProviders; using Wabbajack.Services.OSIntegrated.TokenProviders;
using Wabbajack.VFS; using Wabbajack.VFS;
using Wabbajack.VFS.Interfaces;
using Client = Wabbajack.Networking.WabbajackClientApi.Client; using Client = Wabbajack.Networking.WabbajackClientApi.Client;
namespace Wabbajack.Services.OSIntegrated; namespace Wabbajack.Services.OSIntegrated;
@ -55,9 +56,9 @@ public static class ServiceExtensions
: new FileHashCache(KnownFolders.AppDataLocal.Combine("Wabbajack", "GlobalHashCache.sqlite"), : new FileHashCache(KnownFolders.AppDataLocal.Combine("Wabbajack", "GlobalHashCache.sqlite"),
s.GetService<IResource<FileHashCache>>()!)); s.GetService<IResource<FileHashCache>>()!));
service.AddSingleton(s => options.UseLocalCache service.AddAllSingleton<IVfsCache, VFSDiskCache>(s => options.UseLocalCache
? new VFSCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path) ? new VFSDiskCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path)
: new VFSCache(KnownFolders.EntryPoint.Combine("GlobalVFSCache3.sqlite"))); : new VFSDiskCache(KnownFolders.EntryPoint.Combine("GlobalVFSCache3.sqlite")));
service.AddSingleton<IBinaryPatchCache>(s => options.UseLocalCache service.AddSingleton<IBinaryPatchCache>(s => options.UseLocalCache
? new BinaryPatchCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path) ? new BinaryPatchCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path)
@ -97,6 +98,8 @@ public static class ServiceExtensions
service.AddAllSingleton<IResource, IResource<Context>>(s => new Resource<Context>("VFS", GetSettings(s, "VFS"))); service.AddAllSingleton<IResource, IResource<Context>>(s => new Resource<Context>("VFS", GetSettings(s, "VFS")));
service.AddAllSingleton<IResource, IResource<FileHashCache>>(s => service.AddAllSingleton<IResource, IResource<FileHashCache>>(s =>
new Resource<FileHashCache>("File Hashing", GetSettings(s, "File Hashing"))); new Resource<FileHashCache>("File Hashing", GetSettings(s, "File Hashing")));
service.AddAllSingleton<IResource, IResource<Client>>(s =>
new Resource<Client>("Wabbajack Client", GetSettings(s, "Wabbajack Client")));
service.AddAllSingleton<IResource, IResource<FileExtractor.FileExtractor>>(s => service.AddAllSingleton<IResource, IResource<FileExtractor.FileExtractor>>(s =>
new Resource<FileExtractor.FileExtractor>("File Extractor", GetSettings(s, "File Extractor"))); new Resource<FileExtractor.FileExtractor>("File Extractor", GetSettings(s, "File Extractor")));

View File

@ -0,0 +1,10 @@
using Wabbajack.DTOs.Vfs;
using Wabbajack.Hashing.xxHash64;
namespace Wabbajack.VFS.Interfaces;
public interface IVfsCache
{
public Task<IndexedVirtualFile?> Get(Hash hash, CancellationToken token);
public Task Put(IndexedVirtualFile file, CancellationToken token);
}

View File

@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
<ProjectReference Include="..\Wabbajack.Hashing.xxHash64\Wabbajack.Hashing.xxHash64.csproj" />
<ProjectReference Include="..\Wabbajack.Paths\Wabbajack.Paths.csproj" />
</ItemGroup>
</Project>

View File

@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging;
using Wabbajack.DTOs; using Wabbajack.DTOs;
using Wabbajack.Paths.IO; using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter; using Wabbajack.RateLimiter;
using Wabbajack.VFS.Interfaces;
using Xunit.DependencyInjection; using Xunit.DependencyInjection;
using Xunit.DependencyInjection.Logging; using Xunit.DependencyInjection.Logging;
@ -32,7 +33,7 @@ public class Startup
service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = 2}); service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = 2});
service.AddSingleton(new FileHashCache(KnownFolders.EntryPoint.Combine("hashcache.sqlite"), service.AddSingleton(new FileHashCache(KnownFolders.EntryPoint.Combine("hashcache.sqlite"),
new Resource<FileHashCache>("File Hashing", 10))); new Resource<FileHashCache>("File Hashing", 10)));
service.AddSingleton(new VFSCache(KnownFolders.EntryPoint.Combine("vfscache.sqlite"))); service.AddAllSingleton<IVfsCache, VFSDiskCache>(x => new VFSDiskCache(KnownFolders.EntryPoint.Combine("vfscache.sqlite")));
service.AddTransient<Context>(); service.AddTransient<Context>();
service.AddSingleton<FileExtractor.FileExtractor>(); service.AddSingleton<FileExtractor.FileExtractor>();
} }

View File

@ -11,6 +11,7 @@ using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths; using Wabbajack.Paths;
using Wabbajack.Paths.IO; using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter; using Wabbajack.RateLimiter;
using Wabbajack.VFS.Interfaces;
namespace Wabbajack.VFS; namespace Wabbajack.VFS;
@ -27,10 +28,10 @@ public class Context
public readonly IResource<Context> Limiter; public readonly IResource<Context> Limiter;
public readonly IResource<FileHashCache> HashLimiter; public readonly IResource<FileHashCache> HashLimiter;
public readonly ILogger<Context> Logger; public readonly ILogger<Context> Logger;
public readonly VFSCache VfsCache; public readonly IVfsCache VfsCache;
public Context(ILogger<Context> logger, ParallelOptions parallelOptions, TemporaryFileManager manager, public Context(ILogger<Context> logger, ParallelOptions parallelOptions, TemporaryFileManager manager,
VFSCache vfsCache, IVfsCache vfsCache,
FileHashCache hashCache, IResource<Context> limiter, IResource<FileHashCache> hashLimiter, FileExtractor.FileExtractor extractor) FileHashCache hashCache, IResource<Context> limiter, IResource<FileHashCache> hashLimiter, FileExtractor.FileExtractor extractor)
{ {
Limiter = limiter; Limiter = limiter;

View File

@ -0,0 +1,37 @@
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.DTOs.Vfs;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.VFS.Interfaces;
namespace Wabbajack.VFS;
public class FallthroughVFSCache : IVfsCache
{
private readonly IVfsCache[] _caches;
public FallthroughVFSCache(IVfsCache[] caches)
{
_caches = caches;
}
public async Task<IndexedVirtualFile?> Get(Hash hash, CancellationToken token)
{
foreach (var cache in _caches)
{
var result = await cache.Get(hash, token);
if (result == null) continue;
return result;
}
return default;
}
public async Task Put(IndexedVirtualFile file, CancellationToken token)
{
foreach (var cache in _caches)
{
await cache.Put(file, token);
}
}
}

View File

@ -3,42 +3,36 @@ using System.IO;
using System.IO.Compression; using System.IO.Compression;
using System.Text; using System.Text;
using Wabbajack.DTOs.Texture; using Wabbajack.DTOs.Texture;
using Wabbajack.DTOs.Vfs;
using Wabbajack.Hashing.xxHash64; using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths; using Wabbajack.Paths;
namespace Wabbajack.VFS; namespace Wabbajack.VFS;
public class IndexedVirtualFile public static class IndexedVirtualFileExtensions
{ {
public IPath Name { get; set; } public static void Write(this IndexedVirtualFile ivf, BinaryWriter bw)
public Hash Hash { get; set; }
public ImageState? ImageState { get; set; }
public long Size { get; set; }
public List<IndexedVirtualFile> Children { get; set; } = new();
private void Write(BinaryWriter bw)
{ {
bw.Write(Name.ToString()!); bw.Write(ivf.Name.ToString()!);
bw.Write((ulong) Hash); bw.Write((ulong) ivf.Hash);
if (ImageState == null) if (ivf.ImageState == null)
{ {
bw.Write(false); bw.Write(false);
} }
else else
{ {
bw.Write(true); bw.Write(true);
WriteImageState(bw, ImageState); WriteImageState(bw, ivf.ImageState);
} }
bw.Write(Size); bw.Write(ivf.Size);
bw.Write(Children.Count); bw.Write(ivf.Children.Count);
foreach (var file in Children) foreach (var file in ivf.Children)
file.Write(bw); file.Write(bw);
} }
private void WriteImageState(BinaryWriter bw, ImageState state) private static void WriteImageState(BinaryWriter bw, ImageState state)
{ {
bw.Write((ushort) state.Width); bw.Write((ushort) state.Width);
bw.Write((ushort) state.Height); bw.Write((ushort) state.Height);
@ -46,7 +40,7 @@ public class IndexedVirtualFile
state.PerceptualHash.Write(bw); state.PerceptualHash.Write(bw);
} }
public static ImageState ReadImageState(BinaryReader br) static ImageState ReadImageState(BinaryReader br)
{ {
return new ImageState return new ImageState
{ {
@ -58,14 +52,14 @@ public class IndexedVirtualFile
} }
public void Write(Stream s) public static void Write(this IndexedVirtualFile ivf, Stream s)
{ {
using var cs = new GZipStream(s, CompressionLevel.Optimal, true); using var cs = new GZipStream(s, CompressionLevel.Optimal, true);
using var bw = new BinaryWriter(cs, Encoding.UTF8, true); using var bw = new BinaryWriter(cs, Encoding.UTF8, true);
Write(bw); ivf.Write(bw);
} }
private static IndexedVirtualFile Read(BinaryReader br) public static IndexedVirtualFile Read(BinaryReader br)
{ {
var ivf = new IndexedVirtualFile var ivf = new IndexedVirtualFile
{ {

View File

@ -4,22 +4,25 @@ using System.Data;
using System.Data.SQLite; using System.Data.SQLite;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.DTOs.Streams; using Wabbajack.DTOs.Streams;
using Wabbajack.DTOs.Vfs;
using Wabbajack.Hashing.xxHash64; using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths; using Wabbajack.Paths;
using Wabbajack.Paths.IO; using Wabbajack.Paths.IO;
using Wabbajack.VFS.Interfaces;
namespace Wabbajack.VFS; namespace Wabbajack.VFS;
public class VFSCache public class VFSDiskCache : IVfsCache
{ {
private readonly SQLiteConnection _conn; private readonly SQLiteConnection _conn;
private readonly string _connectionString; private readonly string _connectionString;
private readonly AbsolutePath _path; private readonly AbsolutePath _path;
public VFSCache(AbsolutePath path) public VFSDiskCache(AbsolutePath path)
{ {
_path = path; _path = path;
@ -38,63 +41,34 @@ public class VFSCache
cmd.ExecuteNonQuery(); cmd.ExecuteNonQuery();
} }
public bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IStreamFactory extractedFile, public async Task<IndexedVirtualFile?> Get(Hash hash, CancellationToken token)
Hash hash, out VirtualFile found)
{ {
if (hash == default) if (hash == default)
throw new ArgumentException("Cannot cache default hashes"); throw new ArgumentException("Cannot cache default hashes");
using var cmd = new SQLiteCommand(_conn); await using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash"; cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash";
cmd.Parameters.AddWithValue("@hash", (long) hash); cmd.Parameters.AddWithValue("@hash", (long) hash);
using var rdr = cmd.ExecuteReader(); await using var rdr = cmd.ExecuteReader();
while (rdr.Read()) while (rdr.Read())
{ {
var data = IndexedVirtualFile.Read(rdr.GetStream(0)); var data = IndexedVirtualFileExtensions.Read(rdr.GetStream(0));
found = ConvertFromIndexedFile(context, data, path, parent, extractedFile); return data;
found.Name = path;
found.Hash = hash;
return true;
} }
found = default; return null;
return false;
} }
private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, public async Task Put(IndexedVirtualFile ivf, CancellationToken token)
VirtualFile vparent, IStreamFactory extractedFile)
{
var vself = new VirtualFile
{
Context = context,
Name = path,
Parent = vparent,
Size = file.Size,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = file.Hash,
ImageState = file.ImageState
};
vself.FillFullPath();
vself.Children = file.Children.Select(f => ConvertFromIndexedFile(context, f, f.Name, vself, extractedFile))
.ToImmutableList();
return vself;
}
public async Task WriteToCache(VirtualFile self)
{ {
await using var ms = new MemoryStream(); await using var ms = new MemoryStream();
var ivf = self.ToIndexedVirtualFile();
// Top level path gets renamed when read, we don't want the absolute path // Top level path gets renamed when read, we don't want the absolute path
// here else the reader will blow up when it tries to convert the value // here else the reader will blow up when it tries to convert the value
ivf.Name = (RelativePath) "not/applicable"; ivf.Name = (RelativePath) "not/applicable";
ivf.Write(ms); ivf.Write(ms);
ms.Position = 0; ms.Position = 0;
await InsertIntoVFSCache(self.Hash, ms); await InsertIntoVFSCache(ivf.Hash, ms);
} }
private async Task InsertIntoVFSCache(Hash hash, MemoryStream data) private async Task InsertIntoVFSCache(Hash hash, MemoryStream data)

View File

@ -10,6 +10,7 @@ using Wabbajack.Common;
using Wabbajack.Common.FileSignatures; using Wabbajack.Common.FileSignatures;
using Wabbajack.DTOs.Streams; using Wabbajack.DTOs.Streams;
using Wabbajack.DTOs.Texture; using Wabbajack.DTOs.Texture;
using Wabbajack.DTOs.Vfs;
using Wabbajack.Hashing.PHash; using Wabbajack.Hashing.PHash;
using Wabbajack.Hashing.xxHash64; using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths; using Wabbajack.Paths;
@ -176,9 +177,13 @@ public class VirtualFile
hash = await hstream.HashingCopy(Stream.Null, token, job); hash = await hstream.HashingCopy(Stream.Null, token, job);
} }
if (context.VfsCache.TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself)) var found = await context.VfsCache.Get(hash, token);
return vself; if (found != null)
{
var file = ConvertFromIndexedFile(context, found!, relPath, parent!, extractedFile);
file.Name = relPath;
return file;
}
await using var stream = await extractedFile.GetStream(); await using var stream = await extractedFile.GetStream();
var sig = await FileExtractor.FileExtractor.ArchiveSigs.MatchesAsync(stream); var sig = await FileExtractor.FileExtractor.ArchiveSigs.MatchesAsync(stream);
@ -219,7 +224,7 @@ public class VirtualFile
if (!sig.HasValue || if (!sig.HasValue ||
!FileExtractor.FileExtractor.ExtractableExtensions.Contains(relPath.FileName.Extension)) !FileExtractor.FileExtractor.ExtractableExtensions.Contains(relPath.FileName.Extension))
{ {
await context.VfsCache.WriteToCache(self); await context.VfsCache.Put(self.ToIndexedVirtualFile(), token);
return self; return self;
} }
@ -242,7 +247,7 @@ public class VirtualFile
throw; throw;
} }
await context.VfsCache.WriteToCache(self); await context.VfsCache.Put(self.ToIndexedVirtualFile(), token);
return self; return self;
} }

View File

@ -19,6 +19,7 @@
<ProjectReference Include="..\Wabbajack.Hashing.xxHash64\Wabbajack.Hashing.xxHash64.csproj" /> <ProjectReference Include="..\Wabbajack.Hashing.xxHash64\Wabbajack.Hashing.xxHash64.csproj" />
<ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" /> <ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
<ProjectReference Include="..\Wabbajack.Paths\Wabbajack.Paths.csproj" /> <ProjectReference Include="..\Wabbajack.Paths\Wabbajack.Paths.csproj" />
<ProjectReference Include="..\Wabbajack.VFS.Interfaces\Wabbajack.VFS.Interfaces.csproj" />
</ItemGroup> </ItemGroup>
</Project> </Project>

View File

@ -141,6 +141,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Downloaders.Manua
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.App.Wpf", "Wabbajack.App.Wpf\Wabbajack.App.Wpf.csproj", "{1196560A-9FFD-4290-9418-470508E984C9}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.App.Wpf", "Wabbajack.App.Wpf\Wabbajack.App.Wpf.csproj", "{1196560A-9FFD-4290-9418-470508E984C9}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VFS.Interfaces", "Wabbajack.VFS.Interfaces\Wabbajack.VFS.Interfaces.csproj", "{E4BDB22D-11A4-452F-8D10-D9CA9777EA22}"
EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU Debug|Any CPU = Debug|Any CPU
@ -387,6 +389,10 @@ Global
{1196560A-9FFD-4290-9418-470508E984C9}.Debug|Any CPU.Build.0 = Debug|x64 {1196560A-9FFD-4290-9418-470508E984C9}.Debug|Any CPU.Build.0 = Debug|x64
{1196560A-9FFD-4290-9418-470508E984C9}.Release|Any CPU.ActiveCfg = Release|x64 {1196560A-9FFD-4290-9418-470508E984C9}.Release|Any CPU.ActiveCfg = Release|x64
{1196560A-9FFD-4290-9418-470508E984C9}.Release|Any CPU.Build.0 = Release|x64 {1196560A-9FFD-4290-9418-470508E984C9}.Release|Any CPU.Build.0 = Release|x64
{E4BDB22D-11A4-452F-8D10-D9CA9777EA22}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E4BDB22D-11A4-452F-8D10-D9CA9777EA22}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E4BDB22D-11A4-452F-8D10-D9CA9777EA22}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E4BDB22D-11A4-452F-8D10-D9CA9777EA22}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE
@ -435,6 +441,7 @@ Global
{A3813D73-9A8E-4CE7-861A-C59043DFFC14} = {F01F8595-5FD7-4506-8469-F4A5522DACC1} {A3813D73-9A8E-4CE7-861A-C59043DFFC14} = {F01F8595-5FD7-4506-8469-F4A5522DACC1}
{B10BB6D6-B3FC-4A76-8A07-6A0A0ADDE198} = {98B731EE-4FC0-4482-A069-BCBA25497871} {B10BB6D6-B3FC-4A76-8A07-6A0A0ADDE198} = {98B731EE-4FC0-4482-A069-BCBA25497871}
{7FC4F129-F0FA-46B7-B7C4-532E371A6326} = {98B731EE-4FC0-4482-A069-BCBA25497871} {7FC4F129-F0FA-46B7-B7C4-532E371A6326} = {98B731EE-4FC0-4482-A069-BCBA25497871}
{E4BDB22D-11A4-452F-8D10-D9CA9777EA22} = {F677890D-5109-43BC-97C7-C4CD47C8EE0C}
EndGlobalSection EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {0AA30275-0F38-4A7D-B645-F5505178DDE8} SolutionGuid = {0AA30275-0F38-4A7D-B645-F5505178DDE8}