mirror of
https://github.com/wabbajack-tools/wabbajack.git
synced 2024-08-30 18:42:17 +00:00
Add Cesi cache
This commit is contained in:
parent
6724d4ba6e
commit
3c28cdf7b9
@ -47,7 +47,6 @@ internal class Program
|
|||||||
services.AddSingleton<CommandLineBuilder, CommandLineBuilder>();
|
services.AddSingleton<CommandLineBuilder, CommandLineBuilder>();
|
||||||
services.AddSingleton<TemporaryFileManager>();
|
services.AddSingleton<TemporaryFileManager>();
|
||||||
services.AddSingleton<FileExtractor.FileExtractor>();
|
services.AddSingleton<FileExtractor.FileExtractor>();
|
||||||
services.AddSingleton(new VFSCache(KnownFolders.EntryPoint.Combine("vfscache.sqlite")));
|
|
||||||
services.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount});
|
services.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount});
|
||||||
services.AddSingleton<Client>();
|
services.AddSingleton<Client>();
|
||||||
services.AddSingleton<Networking.WabbajackClientApi.Client>();
|
services.AddSingleton<Networking.WabbajackClientApi.Client>();
|
||||||
|
16
Wabbajack.DTOs/Vfs/IndexedVirtualFile.cs
Normal file
16
Wabbajack.DTOs/Vfs/IndexedVirtualFile.cs
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
using System.Collections.Generic;
|
||||||
|
using Wabbajack.DTOs.Texture;
|
||||||
|
using Wabbajack.Hashing.xxHash64;
|
||||||
|
using Wabbajack.Paths;
|
||||||
|
|
||||||
|
namespace Wabbajack.DTOs.Vfs;
|
||||||
|
|
||||||
|
public class IndexedVirtualFile
|
||||||
|
{
|
||||||
|
public IPath Name { get; set; }
|
||||||
|
public Hash Hash { get; set; }
|
||||||
|
|
||||||
|
public ImageState? ImageState { get; set; }
|
||||||
|
public long Size { get; set; }
|
||||||
|
public List<IndexedVirtualFile> Children { get; set; } = new();
|
||||||
|
}
|
39
Wabbajack.Networking.WabbajackClientApi/CesiVFSCache.cs
Normal file
39
Wabbajack.Networking.WabbajackClientApi/CesiVFSCache.cs
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
using System.Threading;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Wabbajack.DTOs.Vfs;
|
||||||
|
using Wabbajack.Hashing.xxHash64;
|
||||||
|
using Wabbajack.Networking.Http;
|
||||||
|
using Wabbajack.VFS.Interfaces;
|
||||||
|
|
||||||
|
namespace Wabbajack.Networking.WabbajackClientApi;
|
||||||
|
|
||||||
|
public class CesiVFSCache : IVfsCache
|
||||||
|
{
|
||||||
|
private readonly Client _client;
|
||||||
|
private readonly ILogger<CesiVFSCache> _logger;
|
||||||
|
|
||||||
|
public CesiVFSCache(ILogger<CesiVFSCache> logger, Client client)
|
||||||
|
{
|
||||||
|
_logger = logger;
|
||||||
|
_client = client;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<IndexedVirtualFile?> Get(Hash hash, CancellationToken token)
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Requesting CESI Information for: {Hash}", hash.ToHex());
|
||||||
|
try
|
||||||
|
{
|
||||||
|
return await _client.GetCesiVfsEntry(hash, token);
|
||||||
|
}
|
||||||
|
catch (HttpException exception)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task Put(IndexedVirtualFile file, CancellationToken token)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
@ -19,13 +19,13 @@ using Wabbajack.DTOs.JsonConverters;
|
|||||||
using Wabbajack.DTOs.Logins;
|
using Wabbajack.DTOs.Logins;
|
||||||
using Wabbajack.DTOs.ModListValidation;
|
using Wabbajack.DTOs.ModListValidation;
|
||||||
using Wabbajack.DTOs.Validation;
|
using Wabbajack.DTOs.Validation;
|
||||||
|
using Wabbajack.DTOs.Vfs;
|
||||||
using Wabbajack.Hashing.xxHash64;
|
using Wabbajack.Hashing.xxHash64;
|
||||||
using Wabbajack.Networking.Http;
|
using Wabbajack.Networking.Http;
|
||||||
using Wabbajack.Networking.Http.Interfaces;
|
using Wabbajack.Networking.Http.Interfaces;
|
||||||
using Wabbajack.Paths;
|
using Wabbajack.Paths;
|
||||||
using Wabbajack.Paths.IO;
|
using Wabbajack.Paths.IO;
|
||||||
using Wabbajack.RateLimiter;
|
using Wabbajack.RateLimiter;
|
||||||
using Wabbajack.VFS;
|
|
||||||
using YamlDotNet.Serialization;
|
using YamlDotNet.Serialization;
|
||||||
using YamlDotNet.Serialization.NamingConventions;
|
using YamlDotNet.Serialization.NamingConventions;
|
||||||
|
|
||||||
@ -38,7 +38,7 @@ public class Client
|
|||||||
private readonly HttpClient _client;
|
private readonly HttpClient _client;
|
||||||
private readonly Configuration _configuration;
|
private readonly Configuration _configuration;
|
||||||
private readonly DTOSerializer _dtos;
|
private readonly DTOSerializer _dtos;
|
||||||
private readonly IResource<FileHashCache> _hashLimiter;
|
private readonly IResource<Client> _hashLimiter;
|
||||||
private readonly IResource<HttpClient> _limiter;
|
private readonly IResource<HttpClient> _limiter;
|
||||||
private readonly ILogger<Client> _logger;
|
private readonly ILogger<Client> _logger;
|
||||||
private readonly ParallelOptions _parallelOptions;
|
private readonly ParallelOptions _parallelOptions;
|
||||||
@ -48,7 +48,7 @@ public class Client
|
|||||||
|
|
||||||
public Client(ILogger<Client> logger, HttpClient client, ITokenProvider<WabbajackApiState> token,
|
public Client(ILogger<Client> logger, HttpClient client, ITokenProvider<WabbajackApiState> token,
|
||||||
DTOSerializer dtos,
|
DTOSerializer dtos,
|
||||||
IResource<HttpClient> limiter, IResource<FileHashCache> hashLimiter, Configuration configuration)
|
IResource<HttpClient> limiter, IResource<Client> hashLimiter, Configuration configuration)
|
||||||
{
|
{
|
||||||
_configuration = configuration;
|
_configuration = configuration;
|
||||||
_token = token;
|
_token = token;
|
||||||
@ -387,4 +387,12 @@ public class Client
|
|||||||
|
|
||||||
return new Uri($"{_configuration.BuildServerUrl}proxy?name={archive.Name}&hash={archive.Hash.ToHex()}&uri={HttpUtility.UrlEncode(uri.ToString())}");
|
return new Uri($"{_configuration.BuildServerUrl}proxy?name={archive.Name}&hash={archive.Hash.ToHex()}&uri={HttpUtility.UrlEncode(uri.ToString())}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async Task<IndexedVirtualFile?> GetCesiVfsEntry(Hash hash, CancellationToken token)
|
||||||
|
{
|
||||||
|
var msg = await MakeMessage(HttpMethod.Get, new Uri($"{_configuration.BuildServerUrl}cesi/vfs/{hash.ToHex()}"));
|
||||||
|
using var response = await _client.SendAsync(msg, token);
|
||||||
|
HttpException.ThrowOnFailure(response);
|
||||||
|
return await _dtos.DeserializeAsync<IndexedVirtualFile>(await response.Content.ReadAsStreamAsync(token), token);
|
||||||
|
}
|
||||||
}
|
}
|
@ -16,7 +16,7 @@
|
|||||||
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
|
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
|
||||||
<ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
|
<ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
|
||||||
<ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
|
<ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
|
||||||
<ProjectReference Include="..\Wabbajack.VFS\Wabbajack.VFS.csproj" />
|
<ProjectReference Include="..\Wabbajack.VFS.Interfaces\Wabbajack.VFS.Interfaces.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
@ -12,7 +12,7 @@ public static class KnownFolders
|
|||||||
get
|
get
|
||||||
{
|
{
|
||||||
var result = Process.GetCurrentProcess().MainModule!.FileName!.ToAbsolutePath().Parent;
|
var result = Process.GetCurrentProcess().MainModule!.FileName!.ToAbsolutePath().Parent;
|
||||||
if (result.FileName == "dotnet".ToRelativePath())
|
if (result.FileName == "dotnet".ToRelativePath() || Assembly.GetEntryAssembly() != null)
|
||||||
{
|
{
|
||||||
return Assembly.GetExecutingAssembly().Location.ToAbsolutePath().Parent;
|
return Assembly.GetExecutingAssembly().Location.ToAbsolutePath().Parent;
|
||||||
}
|
}
|
||||||
|
@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging;
|
|||||||
using Wabbajack.Common;
|
using Wabbajack.Common;
|
||||||
using Wabbajack.DTOs.JsonConverters;
|
using Wabbajack.DTOs.JsonConverters;
|
||||||
using Wabbajack.DTOs.Texture;
|
using Wabbajack.DTOs.Texture;
|
||||||
|
using Wabbajack.DTOs.Vfs;
|
||||||
using Wabbajack.Hashing.xxHash64;
|
using Wabbajack.Hashing.xxHash64;
|
||||||
using Wabbajack.Paths;
|
using Wabbajack.Paths;
|
||||||
using Wabbajack.VFS;
|
using Wabbajack.VFS;
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using Chronic.Core;
|
using Chronic.Core;
|
||||||
|
using CouchDB.Driver;
|
||||||
using Microsoft.AspNetCore.Mvc;
|
using Microsoft.AspNetCore.Mvc;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using Nettle;
|
using Nettle;
|
||||||
@ -35,13 +36,15 @@ public class MetricsController : ControllerBase
|
|||||||
private readonly AppSettings _settings;
|
private readonly AppSettings _settings;
|
||||||
private ILogger<MetricsController> _logger;
|
private ILogger<MetricsController> _logger;
|
||||||
private readonly Metrics _metricsStore;
|
private readonly Metrics _metricsStore;
|
||||||
|
private readonly ICouchDatabase<Metric> _db;
|
||||||
|
|
||||||
public MetricsController(ILogger<MetricsController> logger, Metrics metricsStore,
|
public MetricsController(ILogger<MetricsController> logger, Metrics metricsStore,
|
||||||
AppSettings settings)
|
AppSettings settings, ICouchDatabase<Metric> db)
|
||||||
{
|
{
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
_settings = settings;
|
_settings = settings;
|
||||||
_metricsStore = metricsStore;
|
_metricsStore = metricsStore;
|
||||||
|
_db = db;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -88,7 +91,7 @@ public class MetricsController : ControllerBase
|
|||||||
private static byte[] LBRACKET = {(byte)'['};
|
private static byte[] LBRACKET = {(byte)'['};
|
||||||
private static byte[] RBRACKET = {(byte)']'};
|
private static byte[] RBRACKET = {(byte)']'};
|
||||||
private static byte[] COMMA = {(byte) ','};
|
private static byte[] COMMA = {(byte) ','};
|
||||||
|
|
||||||
[HttpGet]
|
[HttpGet]
|
||||||
[Route("dump")]
|
[Route("dump")]
|
||||||
public async Task GetMetrics([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to, [FromQuery] string? subject)
|
public async Task GetMetrics([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to, [FromQuery] string? subject)
|
||||||
|
@ -1,9 +1,10 @@
|
|||||||
using System;
|
using System;
|
||||||
|
using CouchDB.Driver.Types;
|
||||||
using Microsoft.Extensions.Primitives;
|
using Microsoft.Extensions.Primitives;
|
||||||
|
|
||||||
namespace Wabbajack.Server.DTOs;
|
namespace Wabbajack.Server.DTOs;
|
||||||
|
|
||||||
public class Metric
|
public class Metric : CouchDocument
|
||||||
{
|
{
|
||||||
public DateTime Timestamp { get; set; }
|
public DateTime Timestamp { get; set; }
|
||||||
public string Action { get; set; }
|
public string Action { get; set; }
|
||||||
|
@ -38,6 +38,7 @@ using Wabbajack.Server.Services;
|
|||||||
using Wabbajack.Services.OSIntegrated.TokenProviders;
|
using Wabbajack.Services.OSIntegrated.TokenProviders;
|
||||||
using Wabbajack.Networking.WabbajackClientApi;
|
using Wabbajack.Networking.WabbajackClientApi;
|
||||||
using Wabbajack.Paths.IO;
|
using Wabbajack.Paths.IO;
|
||||||
|
using Wabbajack.Server.DTOs;
|
||||||
using Wabbajack.VFS;
|
using Wabbajack.VFS;
|
||||||
using YamlDotNet.Serialization.NamingConventions;
|
using YamlDotNet.Serialization.NamingConventions;
|
||||||
using Client = Wabbajack.Networking.GitHub.Client;
|
using Client = Wabbajack.Networking.GitHub.Client;
|
||||||
@ -154,6 +155,18 @@ public class Startup
|
|||||||
});
|
});
|
||||||
return client.GetDatabase<Analyzed>("cesi");
|
return client.GetDatabase<Analyzed>("cesi");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
services.AddSingleton(s =>
|
||||||
|
{
|
||||||
|
var settings = s.GetRequiredService<AppSettings>();
|
||||||
|
var client = new CouchClient(settings.CesiDB.Endpoint, b =>
|
||||||
|
{
|
||||||
|
b.UseBasicAuthentication("wabbajack", "password");
|
||||||
|
b.SetPropertyCase(PropertyCaseType.None);
|
||||||
|
b.SetJsonNullValueHandling(NullValueHandling.Ignore);
|
||||||
|
});
|
||||||
|
return client.GetDatabase<Metric>("cesi");
|
||||||
|
});
|
||||||
|
|
||||||
services.AddMvc();
|
services.AddMvc();
|
||||||
services
|
services
|
||||||
|
@ -21,6 +21,12 @@
|
|||||||
"Database": "cesi",
|
"Database": "cesi",
|
||||||
"Username": "cesi",
|
"Username": "cesi",
|
||||||
"Password": "password"
|
"Password": "password"
|
||||||
|
},
|
||||||
|
"MetricsDB": {
|
||||||
|
"Endpoint": "http://localhost:15984",
|
||||||
|
"Database": "metrics",
|
||||||
|
"Username": "wabbajack",
|
||||||
|
"Password": "password"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"AllowedHosts": "*"
|
"AllowedHosts": "*"
|
||||||
|
@ -26,6 +26,7 @@ using Wabbajack.RateLimiter;
|
|||||||
using Wabbajack.Services.OSIntegrated.Services;
|
using Wabbajack.Services.OSIntegrated.Services;
|
||||||
using Wabbajack.Services.OSIntegrated.TokenProviders;
|
using Wabbajack.Services.OSIntegrated.TokenProviders;
|
||||||
using Wabbajack.VFS;
|
using Wabbajack.VFS;
|
||||||
|
using Wabbajack.VFS.Interfaces;
|
||||||
using Client = Wabbajack.Networking.WabbajackClientApi.Client;
|
using Client = Wabbajack.Networking.WabbajackClientApi.Client;
|
||||||
|
|
||||||
namespace Wabbajack.Services.OSIntegrated;
|
namespace Wabbajack.Services.OSIntegrated;
|
||||||
@ -55,9 +56,9 @@ public static class ServiceExtensions
|
|||||||
: new FileHashCache(KnownFolders.AppDataLocal.Combine("Wabbajack", "GlobalHashCache.sqlite"),
|
: new FileHashCache(KnownFolders.AppDataLocal.Combine("Wabbajack", "GlobalHashCache.sqlite"),
|
||||||
s.GetService<IResource<FileHashCache>>()!));
|
s.GetService<IResource<FileHashCache>>()!));
|
||||||
|
|
||||||
service.AddSingleton(s => options.UseLocalCache
|
service.AddAllSingleton<IVfsCache, VFSDiskCache>(s => options.UseLocalCache
|
||||||
? new VFSCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path)
|
? new VFSDiskCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path)
|
||||||
: new VFSCache(KnownFolders.EntryPoint.Combine("GlobalVFSCache3.sqlite")));
|
: new VFSDiskCache(KnownFolders.EntryPoint.Combine("GlobalVFSCache3.sqlite")));
|
||||||
|
|
||||||
service.AddSingleton<IBinaryPatchCache>(s => options.UseLocalCache
|
service.AddSingleton<IBinaryPatchCache>(s => options.UseLocalCache
|
||||||
? new BinaryPatchCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path)
|
? new BinaryPatchCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path)
|
||||||
@ -97,6 +98,8 @@ public static class ServiceExtensions
|
|||||||
service.AddAllSingleton<IResource, IResource<Context>>(s => new Resource<Context>("VFS", GetSettings(s, "VFS")));
|
service.AddAllSingleton<IResource, IResource<Context>>(s => new Resource<Context>("VFS", GetSettings(s, "VFS")));
|
||||||
service.AddAllSingleton<IResource, IResource<FileHashCache>>(s =>
|
service.AddAllSingleton<IResource, IResource<FileHashCache>>(s =>
|
||||||
new Resource<FileHashCache>("File Hashing", GetSettings(s, "File Hashing")));
|
new Resource<FileHashCache>("File Hashing", GetSettings(s, "File Hashing")));
|
||||||
|
service.AddAllSingleton<IResource, IResource<Client>>(s =>
|
||||||
|
new Resource<Client>("Wabbajack Client", GetSettings(s, "Wabbajack Client")));
|
||||||
service.AddAllSingleton<IResource, IResource<FileExtractor.FileExtractor>>(s =>
|
service.AddAllSingleton<IResource, IResource<FileExtractor.FileExtractor>>(s =>
|
||||||
new Resource<FileExtractor.FileExtractor>("File Extractor", GetSettings(s, "File Extractor")));
|
new Resource<FileExtractor.FileExtractor>("File Extractor", GetSettings(s, "File Extractor")));
|
||||||
|
|
||||||
|
10
Wabbajack.VFS.Interfaces/IVfsCache.cs
Normal file
10
Wabbajack.VFS.Interfaces/IVfsCache.cs
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
using Wabbajack.DTOs.Vfs;
|
||||||
|
using Wabbajack.Hashing.xxHash64;
|
||||||
|
|
||||||
|
namespace Wabbajack.VFS.Interfaces;
|
||||||
|
|
||||||
|
public interface IVfsCache
|
||||||
|
{
|
||||||
|
public Task<IndexedVirtualFile?> Get(Hash hash, CancellationToken token);
|
||||||
|
public Task Put(IndexedVirtualFile file, CancellationToken token);
|
||||||
|
}
|
15
Wabbajack.VFS.Interfaces/Wabbajack.VFS.Interfaces.csproj
Normal file
15
Wabbajack.VFS.Interfaces/Wabbajack.VFS.Interfaces.csproj
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<TargetFramework>net6.0</TargetFramework>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
|
||||||
|
<ProjectReference Include="..\Wabbajack.Hashing.xxHash64\Wabbajack.Hashing.xxHash64.csproj" />
|
||||||
|
<ProjectReference Include="..\Wabbajack.Paths\Wabbajack.Paths.csproj" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging;
|
|||||||
using Wabbajack.DTOs;
|
using Wabbajack.DTOs;
|
||||||
using Wabbajack.Paths.IO;
|
using Wabbajack.Paths.IO;
|
||||||
using Wabbajack.RateLimiter;
|
using Wabbajack.RateLimiter;
|
||||||
|
using Wabbajack.VFS.Interfaces;
|
||||||
using Xunit.DependencyInjection;
|
using Xunit.DependencyInjection;
|
||||||
using Xunit.DependencyInjection.Logging;
|
using Xunit.DependencyInjection.Logging;
|
||||||
|
|
||||||
@ -32,7 +33,7 @@ public class Startup
|
|||||||
service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = 2});
|
service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = 2});
|
||||||
service.AddSingleton(new FileHashCache(KnownFolders.EntryPoint.Combine("hashcache.sqlite"),
|
service.AddSingleton(new FileHashCache(KnownFolders.EntryPoint.Combine("hashcache.sqlite"),
|
||||||
new Resource<FileHashCache>("File Hashing", 10)));
|
new Resource<FileHashCache>("File Hashing", 10)));
|
||||||
service.AddSingleton(new VFSCache(KnownFolders.EntryPoint.Combine("vfscache.sqlite")));
|
service.AddAllSingleton<IVfsCache, VFSDiskCache>(x => new VFSDiskCache(KnownFolders.EntryPoint.Combine("vfscache.sqlite")));
|
||||||
service.AddTransient<Context>();
|
service.AddTransient<Context>();
|
||||||
service.AddSingleton<FileExtractor.FileExtractor>();
|
service.AddSingleton<FileExtractor.FileExtractor>();
|
||||||
}
|
}
|
||||||
|
@ -11,6 +11,7 @@ using Wabbajack.Hashing.xxHash64;
|
|||||||
using Wabbajack.Paths;
|
using Wabbajack.Paths;
|
||||||
using Wabbajack.Paths.IO;
|
using Wabbajack.Paths.IO;
|
||||||
using Wabbajack.RateLimiter;
|
using Wabbajack.RateLimiter;
|
||||||
|
using Wabbajack.VFS.Interfaces;
|
||||||
|
|
||||||
namespace Wabbajack.VFS;
|
namespace Wabbajack.VFS;
|
||||||
|
|
||||||
@ -27,10 +28,10 @@ public class Context
|
|||||||
public readonly IResource<Context> Limiter;
|
public readonly IResource<Context> Limiter;
|
||||||
public readonly IResource<FileHashCache> HashLimiter;
|
public readonly IResource<FileHashCache> HashLimiter;
|
||||||
public readonly ILogger<Context> Logger;
|
public readonly ILogger<Context> Logger;
|
||||||
public readonly VFSCache VfsCache;
|
public readonly IVfsCache VfsCache;
|
||||||
|
|
||||||
public Context(ILogger<Context> logger, ParallelOptions parallelOptions, TemporaryFileManager manager,
|
public Context(ILogger<Context> logger, ParallelOptions parallelOptions, TemporaryFileManager manager,
|
||||||
VFSCache vfsCache,
|
IVfsCache vfsCache,
|
||||||
FileHashCache hashCache, IResource<Context> limiter, IResource<FileHashCache> hashLimiter, FileExtractor.FileExtractor extractor)
|
FileHashCache hashCache, IResource<Context> limiter, IResource<FileHashCache> hashLimiter, FileExtractor.FileExtractor extractor)
|
||||||
{
|
{
|
||||||
Limiter = limiter;
|
Limiter = limiter;
|
||||||
|
37
Wabbajack.VFS/FallthroughVFSCache.cs
Normal file
37
Wabbajack.VFS/FallthroughVFSCache.cs
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
using System.Threading;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Wabbajack.DTOs.Vfs;
|
||||||
|
using Wabbajack.Hashing.xxHash64;
|
||||||
|
using Wabbajack.VFS.Interfaces;
|
||||||
|
|
||||||
|
namespace Wabbajack.VFS;
|
||||||
|
|
||||||
|
public class FallthroughVFSCache : IVfsCache
|
||||||
|
{
|
||||||
|
private readonly IVfsCache[] _caches;
|
||||||
|
|
||||||
|
public FallthroughVFSCache(IVfsCache[] caches)
|
||||||
|
{
|
||||||
|
_caches = caches;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<IndexedVirtualFile?> Get(Hash hash, CancellationToken token)
|
||||||
|
{
|
||||||
|
foreach (var cache in _caches)
|
||||||
|
{
|
||||||
|
var result = await cache.Get(hash, token);
|
||||||
|
if (result == null) continue;
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
return default;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task Put(IndexedVirtualFile file, CancellationToken token)
|
||||||
|
{
|
||||||
|
foreach (var cache in _caches)
|
||||||
|
{
|
||||||
|
await cache.Put(file, token);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -3,42 +3,36 @@ using System.IO;
|
|||||||
using System.IO.Compression;
|
using System.IO.Compression;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using Wabbajack.DTOs.Texture;
|
using Wabbajack.DTOs.Texture;
|
||||||
|
using Wabbajack.DTOs.Vfs;
|
||||||
using Wabbajack.Hashing.xxHash64;
|
using Wabbajack.Hashing.xxHash64;
|
||||||
using Wabbajack.Paths;
|
using Wabbajack.Paths;
|
||||||
|
|
||||||
namespace Wabbajack.VFS;
|
namespace Wabbajack.VFS;
|
||||||
|
|
||||||
public class IndexedVirtualFile
|
public static class IndexedVirtualFileExtensions
|
||||||
{
|
{
|
||||||
public IPath Name { get; set; }
|
public static void Write(this IndexedVirtualFile ivf, BinaryWriter bw)
|
||||||
public Hash Hash { get; set; }
|
|
||||||
|
|
||||||
public ImageState? ImageState { get; set; }
|
|
||||||
public long Size { get; set; }
|
|
||||||
public List<IndexedVirtualFile> Children { get; set; } = new();
|
|
||||||
|
|
||||||
private void Write(BinaryWriter bw)
|
|
||||||
{
|
{
|
||||||
bw.Write(Name.ToString()!);
|
bw.Write(ivf.Name.ToString()!);
|
||||||
bw.Write((ulong) Hash);
|
bw.Write((ulong) ivf.Hash);
|
||||||
|
|
||||||
if (ImageState == null)
|
if (ivf.ImageState == null)
|
||||||
{
|
{
|
||||||
bw.Write(false);
|
bw.Write(false);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
bw.Write(true);
|
bw.Write(true);
|
||||||
WriteImageState(bw, ImageState);
|
WriteImageState(bw, ivf.ImageState);
|
||||||
}
|
}
|
||||||
|
|
||||||
bw.Write(Size);
|
bw.Write(ivf.Size);
|
||||||
bw.Write(Children.Count);
|
bw.Write(ivf.Children.Count);
|
||||||
foreach (var file in Children)
|
foreach (var file in ivf.Children)
|
||||||
file.Write(bw);
|
file.Write(bw);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void WriteImageState(BinaryWriter bw, ImageState state)
|
private static void WriteImageState(BinaryWriter bw, ImageState state)
|
||||||
{
|
{
|
||||||
bw.Write((ushort) state.Width);
|
bw.Write((ushort) state.Width);
|
||||||
bw.Write((ushort) state.Height);
|
bw.Write((ushort) state.Height);
|
||||||
@ -46,7 +40,7 @@ public class IndexedVirtualFile
|
|||||||
state.PerceptualHash.Write(bw);
|
state.PerceptualHash.Write(bw);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static ImageState ReadImageState(BinaryReader br)
|
static ImageState ReadImageState(BinaryReader br)
|
||||||
{
|
{
|
||||||
return new ImageState
|
return new ImageState
|
||||||
{
|
{
|
||||||
@ -58,14 +52,14 @@ public class IndexedVirtualFile
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public void Write(Stream s)
|
public static void Write(this IndexedVirtualFile ivf, Stream s)
|
||||||
{
|
{
|
||||||
using var cs = new GZipStream(s, CompressionLevel.Optimal, true);
|
using var cs = new GZipStream(s, CompressionLevel.Optimal, true);
|
||||||
using var bw = new BinaryWriter(cs, Encoding.UTF8, true);
|
using var bw = new BinaryWriter(cs, Encoding.UTF8, true);
|
||||||
Write(bw);
|
ivf.Write(bw);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static IndexedVirtualFile Read(BinaryReader br)
|
public static IndexedVirtualFile Read(BinaryReader br)
|
||||||
{
|
{
|
||||||
var ivf = new IndexedVirtualFile
|
var ivf = new IndexedVirtualFile
|
||||||
{
|
{
|
@ -4,22 +4,25 @@ using System.Data;
|
|||||||
using System.Data.SQLite;
|
using System.Data.SQLite;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
using System.Threading;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
using Wabbajack.Common;
|
using Wabbajack.Common;
|
||||||
using Wabbajack.DTOs.Streams;
|
using Wabbajack.DTOs.Streams;
|
||||||
|
using Wabbajack.DTOs.Vfs;
|
||||||
using Wabbajack.Hashing.xxHash64;
|
using Wabbajack.Hashing.xxHash64;
|
||||||
using Wabbajack.Paths;
|
using Wabbajack.Paths;
|
||||||
using Wabbajack.Paths.IO;
|
using Wabbajack.Paths.IO;
|
||||||
|
using Wabbajack.VFS.Interfaces;
|
||||||
|
|
||||||
namespace Wabbajack.VFS;
|
namespace Wabbajack.VFS;
|
||||||
|
|
||||||
public class VFSCache
|
public class VFSDiskCache : IVfsCache
|
||||||
{
|
{
|
||||||
private readonly SQLiteConnection _conn;
|
private readonly SQLiteConnection _conn;
|
||||||
private readonly string _connectionString;
|
private readonly string _connectionString;
|
||||||
private readonly AbsolutePath _path;
|
private readonly AbsolutePath _path;
|
||||||
|
|
||||||
public VFSCache(AbsolutePath path)
|
public VFSDiskCache(AbsolutePath path)
|
||||||
{
|
{
|
||||||
_path = path;
|
_path = path;
|
||||||
|
|
||||||
@ -38,63 +41,34 @@ public class VFSCache
|
|||||||
cmd.ExecuteNonQuery();
|
cmd.ExecuteNonQuery();
|
||||||
}
|
}
|
||||||
|
|
||||||
public bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IStreamFactory extractedFile,
|
public async Task<IndexedVirtualFile?> Get(Hash hash, CancellationToken token)
|
||||||
Hash hash, out VirtualFile found)
|
|
||||||
{
|
{
|
||||||
if (hash == default)
|
if (hash == default)
|
||||||
throw new ArgumentException("Cannot cache default hashes");
|
throw new ArgumentException("Cannot cache default hashes");
|
||||||
|
|
||||||
using var cmd = new SQLiteCommand(_conn);
|
await using var cmd = new SQLiteCommand(_conn);
|
||||||
cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash";
|
cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash";
|
||||||
cmd.Parameters.AddWithValue("@hash", (long) hash);
|
cmd.Parameters.AddWithValue("@hash", (long) hash);
|
||||||
|
|
||||||
using var rdr = cmd.ExecuteReader();
|
await using var rdr = cmd.ExecuteReader();
|
||||||
while (rdr.Read())
|
while (rdr.Read())
|
||||||
{
|
{
|
||||||
var data = IndexedVirtualFile.Read(rdr.GetStream(0));
|
var data = IndexedVirtualFileExtensions.Read(rdr.GetStream(0));
|
||||||
found = ConvertFromIndexedFile(context, data, path, parent, extractedFile);
|
return data;
|
||||||
found.Name = path;
|
|
||||||
found.Hash = hash;
|
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
found = default;
|
return null;
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path,
|
public async Task Put(IndexedVirtualFile ivf, CancellationToken token)
|
||||||
VirtualFile vparent, IStreamFactory extractedFile)
|
|
||||||
{
|
|
||||||
var vself = new VirtualFile
|
|
||||||
{
|
|
||||||
Context = context,
|
|
||||||
Name = path,
|
|
||||||
Parent = vparent,
|
|
||||||
Size = file.Size,
|
|
||||||
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
|
|
||||||
LastAnalyzed = DateTime.Now.AsUnixTime(),
|
|
||||||
Hash = file.Hash,
|
|
||||||
ImageState = file.ImageState
|
|
||||||
};
|
|
||||||
|
|
||||||
vself.FillFullPath();
|
|
||||||
|
|
||||||
vself.Children = file.Children.Select(f => ConvertFromIndexedFile(context, f, f.Name, vself, extractedFile))
|
|
||||||
.ToImmutableList();
|
|
||||||
|
|
||||||
return vself;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task WriteToCache(VirtualFile self)
|
|
||||||
{
|
{
|
||||||
await using var ms = new MemoryStream();
|
await using var ms = new MemoryStream();
|
||||||
var ivf = self.ToIndexedVirtualFile();
|
|
||||||
// Top level path gets renamed when read, we don't want the absolute path
|
// Top level path gets renamed when read, we don't want the absolute path
|
||||||
// here else the reader will blow up when it tries to convert the value
|
// here else the reader will blow up when it tries to convert the value
|
||||||
ivf.Name = (RelativePath) "not/applicable";
|
ivf.Name = (RelativePath) "not/applicable";
|
||||||
ivf.Write(ms);
|
ivf.Write(ms);
|
||||||
ms.Position = 0;
|
ms.Position = 0;
|
||||||
await InsertIntoVFSCache(self.Hash, ms);
|
await InsertIntoVFSCache(ivf.Hash, ms);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task InsertIntoVFSCache(Hash hash, MemoryStream data)
|
private async Task InsertIntoVFSCache(Hash hash, MemoryStream data)
|
||||||
|
@ -10,6 +10,7 @@ using Wabbajack.Common;
|
|||||||
using Wabbajack.Common.FileSignatures;
|
using Wabbajack.Common.FileSignatures;
|
||||||
using Wabbajack.DTOs.Streams;
|
using Wabbajack.DTOs.Streams;
|
||||||
using Wabbajack.DTOs.Texture;
|
using Wabbajack.DTOs.Texture;
|
||||||
|
using Wabbajack.DTOs.Vfs;
|
||||||
using Wabbajack.Hashing.PHash;
|
using Wabbajack.Hashing.PHash;
|
||||||
using Wabbajack.Hashing.xxHash64;
|
using Wabbajack.Hashing.xxHash64;
|
||||||
using Wabbajack.Paths;
|
using Wabbajack.Paths;
|
||||||
@ -176,9 +177,13 @@ public class VirtualFile
|
|||||||
hash = await hstream.HashingCopy(Stream.Null, token, job);
|
hash = await hstream.HashingCopy(Stream.Null, token, job);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (context.VfsCache.TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
|
var found = await context.VfsCache.Get(hash, token);
|
||||||
return vself;
|
if (found != null)
|
||||||
|
{
|
||||||
|
var file = ConvertFromIndexedFile(context, found!, relPath, parent!, extractedFile);
|
||||||
|
file.Name = relPath;
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
|
||||||
await using var stream = await extractedFile.GetStream();
|
await using var stream = await extractedFile.GetStream();
|
||||||
var sig = await FileExtractor.FileExtractor.ArchiveSigs.MatchesAsync(stream);
|
var sig = await FileExtractor.FileExtractor.ArchiveSigs.MatchesAsync(stream);
|
||||||
@ -219,7 +224,7 @@ public class VirtualFile
|
|||||||
if (!sig.HasValue ||
|
if (!sig.HasValue ||
|
||||||
!FileExtractor.FileExtractor.ExtractableExtensions.Contains(relPath.FileName.Extension))
|
!FileExtractor.FileExtractor.ExtractableExtensions.Contains(relPath.FileName.Extension))
|
||||||
{
|
{
|
||||||
await context.VfsCache.WriteToCache(self);
|
await context.VfsCache.Put(self.ToIndexedVirtualFile(), token);
|
||||||
return self;
|
return self;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -242,7 +247,7 @@ public class VirtualFile
|
|||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
|
|
||||||
await context.VfsCache.WriteToCache(self);
|
await context.VfsCache.Put(self.ToIndexedVirtualFile(), token);
|
||||||
return self;
|
return self;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -19,6 +19,7 @@
|
|||||||
<ProjectReference Include="..\Wabbajack.Hashing.xxHash64\Wabbajack.Hashing.xxHash64.csproj" />
|
<ProjectReference Include="..\Wabbajack.Hashing.xxHash64\Wabbajack.Hashing.xxHash64.csproj" />
|
||||||
<ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
|
<ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
|
||||||
<ProjectReference Include="..\Wabbajack.Paths\Wabbajack.Paths.csproj" />
|
<ProjectReference Include="..\Wabbajack.Paths\Wabbajack.Paths.csproj" />
|
||||||
|
<ProjectReference Include="..\Wabbajack.VFS.Interfaces\Wabbajack.VFS.Interfaces.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
@ -141,6 +141,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Downloaders.Manua
|
|||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.App.Wpf", "Wabbajack.App.Wpf\Wabbajack.App.Wpf.csproj", "{1196560A-9FFD-4290-9418-470508E984C9}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.App.Wpf", "Wabbajack.App.Wpf\Wabbajack.App.Wpf.csproj", "{1196560A-9FFD-4290-9418-470508E984C9}"
|
||||||
EndProject
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VFS.Interfaces", "Wabbajack.VFS.Interfaces\Wabbajack.VFS.Interfaces.csproj", "{E4BDB22D-11A4-452F-8D10-D9CA9777EA22}"
|
||||||
|
EndProject
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|Any CPU = Debug|Any CPU
|
Debug|Any CPU = Debug|Any CPU
|
||||||
@ -387,6 +389,10 @@ Global
|
|||||||
{1196560A-9FFD-4290-9418-470508E984C9}.Debug|Any CPU.Build.0 = Debug|x64
|
{1196560A-9FFD-4290-9418-470508E984C9}.Debug|Any CPU.Build.0 = Debug|x64
|
||||||
{1196560A-9FFD-4290-9418-470508E984C9}.Release|Any CPU.ActiveCfg = Release|x64
|
{1196560A-9FFD-4290-9418-470508E984C9}.Release|Any CPU.ActiveCfg = Release|x64
|
||||||
{1196560A-9FFD-4290-9418-470508E984C9}.Release|Any CPU.Build.0 = Release|x64
|
{1196560A-9FFD-4290-9418-470508E984C9}.Release|Any CPU.Build.0 = Release|x64
|
||||||
|
{E4BDB22D-11A4-452F-8D10-D9CA9777EA22}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{E4BDB22D-11A4-452F-8D10-D9CA9777EA22}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{E4BDB22D-11A4-452F-8D10-D9CA9777EA22}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{E4BDB22D-11A4-452F-8D10-D9CA9777EA22}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
@ -435,6 +441,7 @@ Global
|
|||||||
{A3813D73-9A8E-4CE7-861A-C59043DFFC14} = {F01F8595-5FD7-4506-8469-F4A5522DACC1}
|
{A3813D73-9A8E-4CE7-861A-C59043DFFC14} = {F01F8595-5FD7-4506-8469-F4A5522DACC1}
|
||||||
{B10BB6D6-B3FC-4A76-8A07-6A0A0ADDE198} = {98B731EE-4FC0-4482-A069-BCBA25497871}
|
{B10BB6D6-B3FC-4A76-8A07-6A0A0ADDE198} = {98B731EE-4FC0-4482-A069-BCBA25497871}
|
||||||
{7FC4F129-F0FA-46B7-B7C4-532E371A6326} = {98B731EE-4FC0-4482-A069-BCBA25497871}
|
{7FC4F129-F0FA-46B7-B7C4-532E371A6326} = {98B731EE-4FC0-4482-A069-BCBA25497871}
|
||||||
|
{E4BDB22D-11A4-452F-8D10-D9CA9777EA22} = {F677890D-5109-43BC-97C7-C4CD47C8EE0C}
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||||
SolutionGuid = {0AA30275-0F38-4A7D-B645-F5505178DDE8}
|
SolutionGuid = {0AA30275-0F38-4A7D-B645-F5505178DDE8}
|
||||||
|
Loading…
Reference in New Issue
Block a user