From 6724d4ba6e349b3d09d5b82107c3c018e243692d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 20 Jun 2022 17:21:04 -0600 Subject: [PATCH 1/2] Add CESI support, start integration into compiler --- .../ManualBlobDownloadHandler.cs | 2 +- .../View Models/BrowserWindowViewModel.cs | 3 + Wabbajack.CLI/Program.cs | 1 + Wabbajack.CLI/Verbs/DownloadAll.cs | 128 ++++++++++++++++++ Wabbajack.Common/AsyncParallelExtensions.cs | 1 + Wabbajack.DTOs/JsonConverters/DIExtensions.cs | 2 + .../JsonConverters/IPathConverter.cs | 45 ++++++ .../DownloadDispatcher.cs | 48 +++++-- .../ManualDownloader.cs | 17 ++- .../NexusDownloader.cs | 2 +- Wabbajack.Installer/AInstaller.cs | 35 +++-- Wabbajack.Installer/IniExtensions.cs | 2 +- .../SingleThreadedDownloader.cs | 103 +++++++++++++- .../Client.cs | 26 +++- Wabbajack.Server/AppSettings.cs | 11 ++ Wabbajack.Server/Controllers/Cesi.cs | 105 ++++++++++++++ .../Services/NexusCacheManager.cs | 2 + Wabbajack.Server/Startup.cs | 28 +++- Wabbajack.Server/Wabbajack.Server.csproj | 1 + Wabbajack.Server/appsettings.json | 8 +- 20 files changed, 533 insertions(+), 37 deletions(-) create mode 100644 Wabbajack.CLI/Verbs/DownloadAll.cs create mode 100644 Wabbajack.DTOs/JsonConverters/IPathConverter.cs create mode 100644 Wabbajack.Server/Controllers/Cesi.cs diff --git a/Wabbajack.App.Wpf/UserIntervention/ManualBlobDownloadHandler.cs b/Wabbajack.App.Wpf/UserIntervention/ManualBlobDownloadHandler.cs index dc5ff730..2c99cc23 100644 --- a/Wabbajack.App.Wpf/UserIntervention/ManualBlobDownloadHandler.cs +++ b/Wabbajack.App.Wpf/UserIntervention/ManualBlobDownloadHandler.cs @@ -11,7 +11,7 @@ public class ManualBlobDownloadHandler : BrowserWindowViewModel protected override async Task Run(CancellationToken token) { - await WaitForReady(); + //await WaitForReady(); var archive = Intervention.Archive; var md = Intervention.Archive.State as Manual; diff --git a/Wabbajack.App.Wpf/View Models/BrowserWindowViewModel.cs b/Wabbajack.App.Wpf/View Models/BrowserWindowViewModel.cs index 235ee650..6dfb8911 100644 --- a/Wabbajack.App.Wpf/View Models/BrowserWindowViewModel.cs +++ b/Wabbajack.App.Wpf/View Models/BrowserWindowViewModel.cs @@ -135,6 +135,9 @@ public abstract class BrowserWindowViewModel : ViewModel { var source = new TaskCompletionSource(); var referer = _browser.Source; + while (_browser.CoreWebView2 == null) + await Task.Delay(10, token); + _browser.CoreWebView2.DownloadStarting += (sender, args) => { try diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index 5050ff1c..3635b704 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -79,6 +79,7 @@ internal class Program services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); }).Build(); diff --git a/Wabbajack.CLI/Verbs/DownloadAll.cs b/Wabbajack.CLI/Verbs/DownloadAll.cs new file mode 100644 index 00000000..461928c7 --- /dev/null +++ b/Wabbajack.CLI/Verbs/DownloadAll.cs @@ -0,0 +1,128 @@ +using System; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.Common; +using Wabbajack.Downloaders; +using Wabbajack.DTOs; +using Wabbajack.DTOs.DownloadStates; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Installer; +using Wabbajack.Networking.WabbajackClientApi; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; +using Wabbajack.RateLimiter; +using Wabbajack.VFS; + +namespace Wabbajack.CLI.Verbs; + +public class DownloadAll : IVerb +{ + private readonly DownloadDispatcher _dispatcher; + private readonly ILogger _logger; + private readonly Client _wjClient; + private readonly DTOSerializer _dtos; + private readonly Resource _limiter; + private readonly FileHashCache _cache; + + public const int MaxDownload = 6000; + + public DownloadAll(ILogger logger, DownloadDispatcher dispatcher, Client wjClient, DTOSerializer dtos, FileHashCache cache) + { + _logger = logger; + _dispatcher = dispatcher; + _wjClient = wjClient; + _dtos = dtos; + _limiter = new Resource("Download All", 16); + _cache = cache; + } + + public Command MakeCommand() + { + var command = new Command("download-all"); + command.Add(new Option(new[] {"-o", "-output"}, "Output folder")); + command.Description = "Downloads all files for all modlists in the gallery"; + command.Handler = CommandHandler.Create(Run); + return command; + } + + private async Task Run(AbsolutePath output, CancellationToken token) + { + _logger.LogInformation("Downloading modlists"); + + var existing = await output.EnumerateFiles() + .Where(f => f.Extension != Ext.Meta) + .PMapAll(_limiter, async f => + { + _logger.LogInformation("Hashing {File}", f.FileName); + return await _cache.FileHashCachedAsync(f, token); + }) + .ToHashSet(); + + var archives = (await (await _wjClient.LoadLists()) + .PMapAll(_limiter, async m => + { + try + { + return await StandardInstaller.Load(_dtos, _dispatcher, m, token); + } + catch (Exception ex) + { + _logger.LogError(ex, "While downloading list"); + return default; + } + }) + .Where(d => d != default) + .SelectMany(m => m!.Archives) + .ToList()) + .DistinctBy(d => d.Hash) + .Where(d => d.State is Nexus) + .Where(d => !existing.Contains(d.Hash)) + .ToList(); + + + + _logger.LogInformation("Found {Count} Archives totaling {Size}", archives.Count, archives.Sum(a => a.Size).ToFileSizeString()); + + await archives + .OrderBy(a => a.Size) + .Take(MaxDownload) + .PDoAll(_limiter, async file => { + var outputFile = output.Combine(file.Name); + if (outputFile.FileExists()) + { + outputFile = output.Combine(outputFile.FileName.WithoutExtension() + "_" + file.Hash.ToHex() + + outputFile.WithExtension(outputFile.Extension)); + } + + _logger.LogInformation("Downloading {File}", file.Name); + + try + { + var result = await _dispatcher.DownloadWithPossibleUpgrade(file, outputFile, token); + if (result.Item1 == DownloadResult.Failure) + { + if (outputFile.FileExists()) + outputFile.Delete(); + return; + } + + _cache.FileHashWriteCache(output, result.Item2); + + var metaFile = outputFile.WithExtension(Ext.Meta); + await metaFile.WriteAllTextAsync(_dispatcher.MetaIniSection(file), token: token); + } + catch (Exception ex) + { + _logger.LogError(ex, "While downloading {Name}, Ignoring", file.Name); + } + + }); + + + return 0; + } +} \ No newline at end of file diff --git a/Wabbajack.Common/AsyncParallelExtensions.cs b/Wabbajack.Common/AsyncParallelExtensions.cs index e4af8882..10d96c27 100644 --- a/Wabbajack.Common/AsyncParallelExtensions.cs +++ b/Wabbajack.Common/AsyncParallelExtensions.cs @@ -30,6 +30,7 @@ public static class AsyncParallelExtensions foreach (var itm in tasks) yield return await itm; } + // Like PMapAll but don't keep defaults public static async IAsyncEnumerable PKeepAll(this IEnumerable coll, Func> mapFn) diff --git a/Wabbajack.DTOs/JsonConverters/DIExtensions.cs b/Wabbajack.DTOs/JsonConverters/DIExtensions.cs index 30813821..86f89c2c 100644 --- a/Wabbajack.DTOs/JsonConverters/DIExtensions.cs +++ b/Wabbajack.DTOs/JsonConverters/DIExtensions.cs @@ -1,5 +1,6 @@ using System.Text.Json.Serialization; using Microsoft.Extensions.DependencyInjection; +using Wabbajack.Paths; namespace Wabbajack.DTOs.JsonConverters; @@ -18,6 +19,7 @@ public static class DIExtensions services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); return services; } diff --git a/Wabbajack.DTOs/JsonConverters/IPathConverter.cs b/Wabbajack.DTOs/JsonConverters/IPathConverter.cs new file mode 100644 index 00000000..686fd321 --- /dev/null +++ b/Wabbajack.DTOs/JsonConverters/IPathConverter.cs @@ -0,0 +1,45 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using Wabbajack.Paths; + +namespace Wabbajack.DTOs.JsonConverters; + +public class IPathConverter : JsonConverter +{ + public override IPath? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType != JsonTokenType.StartObject) + throw new JsonException("Invalid format, expected StartObject"); + + reader.Read(); + var type = reader.GetString(); + reader.Read(); + var value = reader.GetString(); + reader.Read(); + + + if (type == "Absolute") + return value!.ToAbsolutePath(); + else + return value!.ToRelativePath(); + } + + public override void Write(Utf8JsonWriter writer, IPath value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + switch (value) + { + case AbsolutePath a: + writer.WriteString("Absolute", a.ToString()); + break; + case RelativePath r: + writer.WriteString("Relative", r.ToString()); + break; + default: + throw new NotImplementedException(); + } + + writer.WriteEndObject(); + } +} \ No newline at end of file diff --git a/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs b/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs index e482867f..473727e9 100644 --- a/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs +++ b/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs @@ -44,16 +44,14 @@ public class DownloadDispatcher return await Download(a, dest, job, token); } - public async Task Download(Archive a, AbsolutePath dest, Job job, CancellationToken token) + public async Task MaybeProxy(Archive a, CancellationToken token) { - if (!dest.Parent.DirectoryExists()) - dest.Parent.CreateDirectory(); - - var downloader = Downloader(a); - if (_useProxyCache && downloader is IProxyable p) + if (a.State is not IProxyable p) return a; + + var uri = p.UnParse(a.State); + var newUri = await _wjClient.MakeProxyUrl(a, uri); + if (newUri != null) { - var uri = p.UnParse(a.State); - var newUri = _wjClient.MakeProxyUrl(a, uri); a = new Archive { Name = a.Name, @@ -64,8 +62,36 @@ public class DownloadDispatcher Url = newUri } }; - downloader = Downloader(a); - _logger.LogInformation("Downloading Proxy ({Hash}) {Uri}", (await uri.ToString().Hash()).ToHex(), uri); + } + + return a; + } + + public async Task Download(Archive a, AbsolutePath dest, Job job, CancellationToken token) + { + if (!dest.Parent.DirectoryExists()) + dest.Parent.CreateDirectory(); + + var downloader = Downloader(a); + if (_useProxyCache && downloader is IProxyable p) + { + var uri = p.UnParse(a.State); + var newUri = await _wjClient.MakeProxyUrl(a, uri); + if (newUri != null) + { + a = new Archive + { + Name = a.Name, + Size = a.Size, + Hash = a.Hash, + State = new DTOs.DownloadStates.Http() + { + Url = newUri + } + }; + downloader = Downloader(a); + _logger.LogInformation("Downloading Proxy ({Hash}) {Uri}", (await uri.ToString().Hash()).ToHex(), uri); + } } var hash = await downloader.Download(a, dest, job, token); @@ -160,7 +186,7 @@ public class DownloadDispatcher return DownloadResult.Update; */ } - + private async Task DownloadFromMirror(Archive archive, AbsolutePath destination, CancellationToken token) { try diff --git a/Wabbajack.Downloaders.Manual/ManualDownloader.cs b/Wabbajack.Downloaders.Manual/ManualDownloader.cs index 14d99ecb..13ce4551 100644 --- a/Wabbajack.Downloaders.Manual/ManualDownloader.cs +++ b/Wabbajack.Downloaders.Manual/ManualDownloader.cs @@ -12,7 +12,7 @@ using Wabbajack.RateLimiter; namespace Wabbajack.Downloaders.Manual; -public class ManualDownloader : ADownloader +public class ManualDownloader : ADownloader, IProxyable { private readonly ILogger _logger; private readonly IUserInterventionHandler _interventionHandler; @@ -98,4 +98,19 @@ public class ManualDownloader : ADownloader return new[] {$"manualURL={state.Url}", $"prompt={state.Prompt}"}; } + + public IDownloadState? Parse(Uri uri) + { + return new DTOs.DownloadStates.Manual() {Url = uri}; + } + + public Uri UnParse(IDownloadState state) + { + return (state as DTOs.DownloadStates.Manual)!.Url; + } + + public Task DownloadStream(Archive archive, Func> fn, CancellationToken token) + { + throw new NotImplementedException(); + } } \ No newline at end of file diff --git a/Wabbajack.Downloaders.Nexus/NexusDownloader.cs b/Wabbajack.Downloaders.Nexus/NexusDownloader.cs index b6d65c39..3a010c4d 100644 --- a/Wabbajack.Downloaders.Nexus/NexusDownloader.cs +++ b/Wabbajack.Downloaders.Nexus/NexusDownloader.cs @@ -166,7 +166,7 @@ public class NexusDownloader : ADownloader, IUrlDownloader var msg = browserState.ToHttpRequestMessage(); - using var response = await _client.SendAsync(msg, HttpCompletionOption.ResponseHeadersRead, token); + using var response = await _client.SendAsync(msg, HttpCompletionOption.ResponseHeadersRead, token); if (!response.IsSuccessStatusCode) throw new HttpRequestException(response.ReasonPhrase, null, statusCode:response.StatusCode); diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index 70d82f2d..e2c416a0 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -106,6 +106,7 @@ public abstract class AInstaller _updateStopWatch.Restart(); MaxStepProgress = maxStepProgress; _currentStep += 1; + _currentStepProgress = 0; _statusText = statusText; _statusCategory = statusCategory; _statusFormatter = formatter ?? (x => x.ToString()); @@ -313,6 +314,10 @@ public abstract class AInstaller _logger.LogInformation("Downloading {Count} archives", missing.Count.ToString()); NextStep(Consts.StepDownloading, "Downloading files", missing.Count); + missing = await missing + .SelectAsync(async m => await _downloadDispatcher.MaybeProxy(m, token)) + .ToList(); + if (download) { var result = SendDownloadMetrics(missing); @@ -479,25 +484,24 @@ public abstract class AInstaller var savePath = (RelativePath) "saves"; NextStep(Consts.StepPreparing, "Looking for files to delete", 0); - await _configuration.Install.EnumerateFiles() - .PDoAll(async f => - { - var relativeTo = f.RelativeTo(_configuration.Install); - if (indexed.ContainsKey(relativeTo) || f.InFolder(_configuration.Downloads)) - return; + foreach (var f in _configuration.Install.EnumerateFiles()) + { + var relativeTo = f.RelativeTo(_configuration.Install); + if (indexed.ContainsKey(relativeTo) || f.InFolder(_configuration.Downloads)) + return; - if (f.InFolder(profileFolder) && f.Parent.FileName == savePath) return; + if (f.InFolder(profileFolder) && f.Parent.FileName == savePath) return; - if (NoDeleteRegex.IsMatch(f.ToString())) - return; + if (NoDeleteRegex.IsMatch(f.ToString())) + return; - if (bsaPathsToNotBuild.Contains(f)) - return; - - _logger.LogInformation("Deleting {RelativePath} it's not part of this ModList", relativeTo); - f.Delete(); - }); + if (bsaPathsToNotBuild.Contains(f)) + return; + _logger.LogInformation("Deleting {RelativePath} it's not part of this ModList", relativeTo); + f.Delete(); + } + _logger.LogInformation("Cleaning empty folders"); var expectedFolders = indexed.Keys .Select(f => f.RelativeTo(_configuration.Install)) @@ -540,6 +544,7 @@ public abstract class AInstaller // Bit backwards, but we want to return null for // all files we *want* installed. We return the files // to remove from the install list. + using var job = await _limiter.Begin($"Hashing File {d.To}", 0, token); var path = _configuration.Install.Combine(d.To); if (!existingfiles.Contains(path)) return null; diff --git a/Wabbajack.Installer/IniExtensions.cs b/Wabbajack.Installer/IniExtensions.cs index d41a6c8c..b2e0b838 100644 --- a/Wabbajack.Installer/IniExtensions.cs +++ b/Wabbajack.Installer/IniExtensions.cs @@ -28,7 +28,7 @@ public static class IniExtensions /// /// /// - public static IniData LoadIniFile(this AbsolutePath file) + public static IniData LoadIniFile(this AbsolutePath file) { return new FileIniDataParser(IniParser()).ReadFile(file.ToString()); } diff --git a/Wabbajack.Networking.Http/SingleThreadedDownloader.cs b/Wabbajack.Networking.Http/SingleThreadedDownloader.cs index 414b60ae..b8bc1a89 100644 --- a/Wabbajack.Networking.Http/SingleThreadedDownloader.cs +++ b/Wabbajack.Networking.Http/SingleThreadedDownloader.cs @@ -1,5 +1,9 @@ +using System; +using System.Buffers; using System.IO; +using System.Linq; using System.Net.Http; +using System.Net.Http.Headers; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -28,12 +32,109 @@ public class SingleThreadedDownloader : IHttpDownloader using var response = await _client.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, token); if (!response.IsSuccessStatusCode) throw new HttpException(response); - + if (job.Size == 0) job.Size = response.Content.Headers.ContentLength ?? 0; + /* Need to make this mulitthreaded to be much use + if ((response.Content.Headers.ContentLength ?? 0) != 0 && + response.Headers.AcceptRanges.FirstOrDefault() == "bytes") + { + return await ResettingDownloader(response, message, outputPath, job, token); + } + */ + await using var stream = await response.Content.ReadAsStreamAsync(token); await using var outputStream = outputPath.Open(FileMode.Create, FileAccess.Write); return await stream.HashingCopy(outputStream, token, job); } + + private const int CHUNK_SIZE = 1024 * 1024 * 8; + + private async Task ResettingDownloader(HttpResponseMessage response, HttpRequestMessage message, AbsolutePath outputPath, IJob job, CancellationToken token) + { + + using var rented = MemoryPool.Shared.Rent(CHUNK_SIZE); + var buffer = rented.Memory; + + var hasher = new xxHashAlgorithm(0); + + var running = true; + ulong finalHash = 0; + + var inputStream = await response.Content.ReadAsStreamAsync(token); + await using var outputStream = outputPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); + long writePosition = 0; + + while (running && !token.IsCancellationRequested) + { + var totalRead = 0; + + while (totalRead != buffer.Length) + { + var read = await inputStream.ReadAsync(buffer.Slice(totalRead, buffer.Length - totalRead), + token); + + + if (read == 0) + { + running = false; + break; + } + + if (job != null) + await job.Report(read, token); + + totalRead += read; + } + + var pendingWrite = outputStream.WriteAsync(buffer[..totalRead], token); + if (running) + { + hasher.TransformByteGroupsInternal(buffer.Span); + await pendingWrite; + } + else + { + var preSize = (totalRead >> 5) << 5; + if (preSize > 0) + { + hasher.TransformByteGroupsInternal(buffer[..preSize].Span); + finalHash = hasher.FinalizeHashValueInternal(buffer[preSize..totalRead].Span); + await pendingWrite; + break; + } + + finalHash = hasher.FinalizeHashValueInternal(buffer[..totalRead].Span); + await pendingWrite; + break; + } + + { + writePosition += totalRead; + await job.Report(totalRead, token); + message = CloneMessage(message); + message.Headers.Range = new RangeHeaderValue(writePosition, writePosition + CHUNK_SIZE); + await inputStream.DisposeAsync(); + response.Dispose(); + response = await _client.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, token); + HttpException.ThrowOnFailure(response); + inputStream = await response.Content.ReadAsStreamAsync(token); + } + } + + await outputStream.FlushAsync(token); + + return new Hash(finalHash); + } + + private HttpRequestMessage CloneMessage(HttpRequestMessage message) + { + var newMsg = new HttpRequestMessage(message.Method, message.RequestUri); + foreach (var header in message.Headers) + { + newMsg.Headers.Add(header.Key, header.Value); + } + return newMsg; + } } \ No newline at end of file diff --git a/Wabbajack.Networking.WabbajackClientApi/Client.cs b/Wabbajack.Networking.WabbajackClientApi/Client.cs index d634d80b..214638df 100644 --- a/Wabbajack.Networking.WabbajackClientApi/Client.cs +++ b/Wabbajack.Networking.WabbajackClientApi/Client.cs @@ -8,11 +8,13 @@ using System.Net.Http.Json; using System.Text; using System.Threading; using System.Threading.Tasks; +using System.Web; using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.DTOs; using Wabbajack.DTOs.CDN; using Wabbajack.DTOs.Configs; +using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.JsonConverters; using Wabbajack.DTOs.Logins; using Wabbajack.DTOs.ModListValidation; @@ -362,9 +364,27 @@ public class Client var url = $"https://raw.githubusercontent.com/wabbajack-tools/indexed-game-files/master/{game}/{version}_steam_manifests.json"; return await _client.GetFromJsonAsync(url, _dtos.Options) ?? Array.Empty(); } - - public Uri MakeProxyUrl(Archive archive, Uri uri) + + public async Task ProxyHas(Uri uri) { - return new Uri($"{_configuration.BuildServerUrl}proxy?name={archive.Name}&hash={archive.Hash.ToHex()}&uri={uri}"); + var newUri = new Uri($"{_configuration.BuildServerUrl}proxy?uri={HttpUtility.UrlEncode(uri.ToString())}"); + var msg = new HttpRequestMessage(HttpMethod.Head, newUri); + try + { + var result = await _client.SendAsync(msg); + return result.IsSuccessStatusCode; + } + catch (Exception ex) + { + return false; + } + } + + public async ValueTask MakeProxyUrl(Archive archive, Uri uri) + { + if (archive.State is Manual && !await ProxyHas(uri)) + return null; + + return new Uri($"{_configuration.BuildServerUrl}proxy?name={archive.Name}&hash={archive.Hash.ToHex()}&uri={HttpUtility.UrlEncode(uri.ToString())}"); } } \ No newline at end of file diff --git a/Wabbajack.Server/AppSettings.cs b/Wabbajack.Server/AppSettings.cs index b4e3e4cb..a779e614 100644 --- a/Wabbajack.Server/AppSettings.cs +++ b/Wabbajack.Server/AppSettings.cs @@ -33,4 +33,15 @@ public class AppSettings public string MetricsFolder { get; set; } = ""; public string TarLogPath { get; set; } public string GitHubKey { get; set; } = ""; + + public CouchDBSetting CesiDB { get; set; } + public CouchDBSetting MetricsDB { get; set; } +} + +public class CouchDBSetting +{ + public Uri Endpoint { get; set; } + public string Database { get; set; } + public string Username { get; set; } + public string Password { get; set; } } \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Cesi.cs b/Wabbajack.Server/Controllers/Cesi.cs new file mode 100644 index 00000000..07f7658e --- /dev/null +++ b/Wabbajack.Server/Controllers/Cesi.cs @@ -0,0 +1,105 @@ +using cesi.DTOs; +using CouchDB.Driver; +using CouchDB.Driver.Views; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Wabbajack.Common; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.DTOs.Texture; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Paths; +using Wabbajack.VFS; + +namespace Wabbajack.Server.Controllers; + +[Route("/cesi")] +public class Cesi : ControllerBase +{ + private readonly ILogger _logger; + private readonly ICouchDatabase _db; + private readonly DTOSerializer _dtos; + + public Cesi(ILogger logger, ICouchDatabase db, DTOSerializer serializer) + { + _logger = logger; + _db = db; + _dtos = serializer; + } + + [HttpGet("entry/{hash}")] + public async Task Entry(string hash) + { + return Ok(await _db.FindAsync(hash)); + } + + [HttpGet("vfs/{hash}")] + public async Task Vfs(string hash) + { + var entry = await _db.FindAsync(ReverseHash(hash)); + if (entry == null) return NotFound(new {Message = "Entry not found", Hash = hash, ReverseHash = ReverseHash(hash)}); + + + var indexed = new IndexedVirtualFile + { + Hash = Hash.FromHex(ReverseHash(entry.xxHash64)), + Size = entry.Size, + ImageState = GetImageState(entry), + Children = await GetChildrenState(entry), + }; + + + return Ok(_dtos.Serialize(indexed, true)); + } + + private async Task> GetChildrenState(Analyzed entry) + { + if (entry.Archive == null) return new List(); + + var children = await _db.GetViewAsync("Indexes", "ArchiveContents", new CouchViewOptions + { + IncludeDocs = true, + Key = entry.xxHash64 + }); + + var indexed = children.ToLookup(d => d.Document.xxHash64, v => v.Document); + + return await entry.Archive.Entries.SelectAsync(async e => + { + var found = indexed[e.Value].First(); + return new IndexedVirtualFile + { + Name = e.Key.ToRelativePath(), + Size = found.Size, + Hash = Hash.FromHex(ReverseHash(found.xxHash64)), + ImageState = GetImageState(found), + Children = await GetChildrenState(found), + }; + + }).ToList(); + } + + private ImageState? GetImageState(Analyzed entry) + { + if (entry.DDS == null) return null; + return new ImageState + { + Width = entry.DDS.Width, + Height = entry.DDS.Height, + Format = Enum.Parse(entry.DDS.Format), + PerceptualHash = new PHash(entry.DDS.PHash.FromHex()) + }; + } + + + private Hash ReverseHash(Hash hash) + { + return Hash.FromHex(hash.ToArray().Reverse().ToArray().ToHex()); + } + private string ReverseHash(string hash) + { + return hash.FromHex().Reverse().ToArray().ToHex(); + } + + +} \ No newline at end of file diff --git a/Wabbajack.Server/Services/NexusCacheManager.cs b/Wabbajack.Server/Services/NexusCacheManager.cs index 36b76b42..210055ae 100644 --- a/Wabbajack.Server/Services/NexusCacheManager.cs +++ b/Wabbajack.Server/Services/NexusCacheManager.cs @@ -34,8 +34,10 @@ public class NexusCacheManager _nexusAPI = nexusApi; _discord = discord; + /* TODO - uncomment me! _timer = new Timer(_ => UpdateNexusCacheAPI().FireAndForget(), null, TimeSpan.FromSeconds(2), TimeSpan.FromHours(4)); + */ } diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 510555e4..205141f8 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -2,7 +2,12 @@ using System.IO; using System.Net.Http; using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.Json.Serialization; using System.Threading.Tasks; +using cesi.DTOs; +using CouchDB.Driver; +using CouchDB.Driver.Options; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http.Features; @@ -34,6 +39,7 @@ using Wabbajack.Services.OSIntegrated.TokenProviders; using Wabbajack.Networking.WabbajackClientApi; using Wabbajack.Paths.IO; using Wabbajack.VFS; +using YamlDotNet.Serialization.NamingConventions; using Client = Wabbajack.Networking.GitHub.Client; namespace Wabbajack.Server; @@ -135,10 +141,28 @@ public class Startup options.Providers.Add(); options.MimeTypes = new[] {"application/json"}; }); + + // CouchDB + services.AddSingleton(s => + { + var settings = s.GetRequiredService(); + var client = new CouchClient(settings.CesiDB.Endpoint, b => + { + b.UseBasicAuthentication("cesi", "password"); + b.SetPropertyCase(PropertyCaseType.None); + b.SetJsonNullValueHandling(NullValueHandling.Ignore); + }); + return client.GetDatabase("cesi"); + }); services.AddMvc(); - services.AddControllers() - .AddNewtonsoftJson(o => { o.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; }); + services + .AddControllers() + .AddJsonOptions(j => + { + j.JsonSerializerOptions.PropertyNamingPolicy = null; + j.JsonSerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault; + }); NettleEngine.GetCompiler().RegisterWJFunctions(); } diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index af4542a0..4e49dd26 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -8,6 +8,7 @@ + diff --git a/Wabbajack.Server/appsettings.json b/Wabbajack.Server/appsettings.json index 1a115752..69dbbdc5 100644 --- a/Wabbajack.Server/appsettings.json +++ b/Wabbajack.Server/appsettings.json @@ -15,7 +15,13 @@ "MirrorFilesFolder": "c:\\tmp\\mirrors", "NexusCacheFolder": "c:\\tmp\\nexus-cache", "ProxyFolder": "c:\\tmp\\proxy", - "GitHubKey": "" + "GitHubKey": "", + "CesiDB": { + "Endpoint": "http://localhost:15984", + "Database": "cesi", + "Username": "cesi", + "Password": "password" + } }, "AllowedHosts": "*" } From 3c28cdf7b9acd3e51fd7f1b3c7b53c1d4809770b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 21 Jun 2022 19:38:42 -0600 Subject: [PATCH 2/2] Add Cesi cache --- Wabbajack.CLI/Program.cs | 1 - Wabbajack.DTOs/Vfs/IndexedVirtualFile.cs | 16 ++++++ .../CesiVFSCache.cs | 39 +++++++++++++ .../Client.cs | 14 ++++- ...ajack.Networking.WabbajackClientApi.csproj | 2 +- Wabbajack.Paths.IO/KnownFolders.cs | 2 +- Wabbajack.Server/Controllers/Cesi.cs | 1 + Wabbajack.Server/Controllers/Metrics.cs | 7 ++- Wabbajack.Server/DTOs/Metric.cs | 3 +- Wabbajack.Server/Startup.cs | 13 +++++ Wabbajack.Server/appsettings.json | 6 ++ .../ServiceExtensions.cs | 9 ++- Wabbajack.VFS.Interfaces/IVfsCache.cs | 10 ++++ .../Wabbajack.VFS.Interfaces.csproj | 15 +++++ Wabbajack.VFS.Test/Startup.cs | 3 +- Wabbajack.VFS/Context.cs | 5 +- Wabbajack.VFS/FallthroughVFSCache.cs | 37 ++++++++++++ ...ile.cs => IndexedVirtualFileExtensions.cs} | 36 +++++------- Wabbajack.VFS/VFSCache.cs | 56 +++++-------------- Wabbajack.VFS/VirtualFile.cs | 15 +++-- Wabbajack.VFS/Wabbajack.VFS.csproj | 1 + Wabbajack.sln | 7 +++ 22 files changed, 216 insertions(+), 82 deletions(-) create mode 100644 Wabbajack.DTOs/Vfs/IndexedVirtualFile.cs create mode 100644 Wabbajack.Networking.WabbajackClientApi/CesiVFSCache.cs create mode 100644 Wabbajack.VFS.Interfaces/IVfsCache.cs create mode 100644 Wabbajack.VFS.Interfaces/Wabbajack.VFS.Interfaces.csproj create mode 100644 Wabbajack.VFS/FallthroughVFSCache.cs rename Wabbajack.VFS/{IndexedVirtualFile.cs => IndexedVirtualFileExtensions.cs} (67%) diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index 3635b704..0557510e 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -47,7 +47,6 @@ internal class Program services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(new VFSCache(KnownFolders.EntryPoint.Combine("vfscache.sqlite"))); services.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount}); services.AddSingleton(); services.AddSingleton(); diff --git a/Wabbajack.DTOs/Vfs/IndexedVirtualFile.cs b/Wabbajack.DTOs/Vfs/IndexedVirtualFile.cs new file mode 100644 index 00000000..4e14e7f6 --- /dev/null +++ b/Wabbajack.DTOs/Vfs/IndexedVirtualFile.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; +using Wabbajack.DTOs.Texture; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Paths; + +namespace Wabbajack.DTOs.Vfs; + +public class IndexedVirtualFile +{ + public IPath Name { get; set; } + public Hash Hash { get; set; } + + public ImageState? ImageState { get; set; } + public long Size { get; set; } + public List Children { get; set; } = new(); +} \ No newline at end of file diff --git a/Wabbajack.Networking.WabbajackClientApi/CesiVFSCache.cs b/Wabbajack.Networking.WabbajackClientApi/CesiVFSCache.cs new file mode 100644 index 00000000..d45e8522 --- /dev/null +++ b/Wabbajack.Networking.WabbajackClientApi/CesiVFSCache.cs @@ -0,0 +1,39 @@ +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.DTOs.Vfs; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Networking.Http; +using Wabbajack.VFS.Interfaces; + +namespace Wabbajack.Networking.WabbajackClientApi; + +public class CesiVFSCache : IVfsCache +{ + private readonly Client _client; + private readonly ILogger _logger; + + public CesiVFSCache(ILogger logger, Client client) + { + _logger = logger; + _client = client; + } + + public async Task Get(Hash hash, CancellationToken token) + { + _logger.LogInformation("Requesting CESI Information for: {Hash}", hash.ToHex()); + try + { + return await _client.GetCesiVfsEntry(hash, token); + } + catch (HttpException exception) + { + return null; + } + } + + public async Task Put(IndexedVirtualFile file, CancellationToken token) + { + return; + } +} \ No newline at end of file diff --git a/Wabbajack.Networking.WabbajackClientApi/Client.cs b/Wabbajack.Networking.WabbajackClientApi/Client.cs index 214638df..ce2b18d4 100644 --- a/Wabbajack.Networking.WabbajackClientApi/Client.cs +++ b/Wabbajack.Networking.WabbajackClientApi/Client.cs @@ -19,13 +19,13 @@ using Wabbajack.DTOs.JsonConverters; using Wabbajack.DTOs.Logins; using Wabbajack.DTOs.ModListValidation; using Wabbajack.DTOs.Validation; +using Wabbajack.DTOs.Vfs; using Wabbajack.Hashing.xxHash64; using Wabbajack.Networking.Http; using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Paths; using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; -using Wabbajack.VFS; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; @@ -38,7 +38,7 @@ public class Client private readonly HttpClient _client; private readonly Configuration _configuration; private readonly DTOSerializer _dtos; - private readonly IResource _hashLimiter; + private readonly IResource _hashLimiter; private readonly IResource _limiter; private readonly ILogger _logger; private readonly ParallelOptions _parallelOptions; @@ -48,7 +48,7 @@ public class Client public Client(ILogger logger, HttpClient client, ITokenProvider token, DTOSerializer dtos, - IResource limiter, IResource hashLimiter, Configuration configuration) + IResource limiter, IResource hashLimiter, Configuration configuration) { _configuration = configuration; _token = token; @@ -387,4 +387,12 @@ public class Client return new Uri($"{_configuration.BuildServerUrl}proxy?name={archive.Name}&hash={archive.Hash.ToHex()}&uri={HttpUtility.UrlEncode(uri.ToString())}"); } + + public async Task GetCesiVfsEntry(Hash hash, CancellationToken token) + { + var msg = await MakeMessage(HttpMethod.Get, new Uri($"{_configuration.BuildServerUrl}cesi/vfs/{hash.ToHex()}")); + using var response = await _client.SendAsync(msg, token); + HttpException.ThrowOnFailure(response); + return await _dtos.DeserializeAsync(await response.Content.ReadAsStreamAsync(token), token); + } } \ No newline at end of file diff --git a/Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj b/Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj index 9e8aee7d..1e625c0f 100644 --- a/Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj +++ b/Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj @@ -16,7 +16,7 @@ - + diff --git a/Wabbajack.Paths.IO/KnownFolders.cs b/Wabbajack.Paths.IO/KnownFolders.cs index e1ea4cfe..2c45f5a1 100644 --- a/Wabbajack.Paths.IO/KnownFolders.cs +++ b/Wabbajack.Paths.IO/KnownFolders.cs @@ -12,7 +12,7 @@ public static class KnownFolders get { var result = Process.GetCurrentProcess().MainModule!.FileName!.ToAbsolutePath().Parent; - if (result.FileName == "dotnet".ToRelativePath()) + if (result.FileName == "dotnet".ToRelativePath() || Assembly.GetEntryAssembly() != null) { return Assembly.GetExecutingAssembly().Location.ToAbsolutePath().Parent; } diff --git a/Wabbajack.Server/Controllers/Cesi.cs b/Wabbajack.Server/Controllers/Cesi.cs index 07f7658e..cc8fa19f 100644 --- a/Wabbajack.Server/Controllers/Cesi.cs +++ b/Wabbajack.Server/Controllers/Cesi.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.DTOs.JsonConverters; using Wabbajack.DTOs.Texture; +using Wabbajack.DTOs.Vfs; using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; using Wabbajack.VFS; diff --git a/Wabbajack.Server/Controllers/Metrics.cs b/Wabbajack.Server/Controllers/Metrics.cs index b25b604f..ed48e5f4 100644 --- a/Wabbajack.Server/Controllers/Metrics.cs +++ b/Wabbajack.Server/Controllers/Metrics.cs @@ -1,6 +1,7 @@ using System.Reflection; using System.Text.Json; using Chronic.Core; +using CouchDB.Driver; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Nettle; @@ -35,13 +36,15 @@ public class MetricsController : ControllerBase private readonly AppSettings _settings; private ILogger _logger; private readonly Metrics _metricsStore; + private readonly ICouchDatabase _db; public MetricsController(ILogger logger, Metrics metricsStore, - AppSettings settings) + AppSettings settings, ICouchDatabase db) { _logger = logger; _settings = settings; _metricsStore = metricsStore; + _db = db; } @@ -88,7 +91,7 @@ public class MetricsController : ControllerBase private static byte[] LBRACKET = {(byte)'['}; private static byte[] RBRACKET = {(byte)']'}; private static byte[] COMMA = {(byte) ','}; - + [HttpGet] [Route("dump")] public async Task GetMetrics([FromQuery] string action, [FromQuery] string from, [FromQuery] string? to, [FromQuery] string? subject) diff --git a/Wabbajack.Server/DTOs/Metric.cs b/Wabbajack.Server/DTOs/Metric.cs index 27d56fe8..164564a9 100644 --- a/Wabbajack.Server/DTOs/Metric.cs +++ b/Wabbajack.Server/DTOs/Metric.cs @@ -1,9 +1,10 @@ using System; +using CouchDB.Driver.Types; using Microsoft.Extensions.Primitives; namespace Wabbajack.Server.DTOs; -public class Metric +public class Metric : CouchDocument { public DateTime Timestamp { get; set; } public string Action { get; set; } diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 205141f8..60fdda22 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -38,6 +38,7 @@ using Wabbajack.Server.Services; using Wabbajack.Services.OSIntegrated.TokenProviders; using Wabbajack.Networking.WabbajackClientApi; using Wabbajack.Paths.IO; +using Wabbajack.Server.DTOs; using Wabbajack.VFS; using YamlDotNet.Serialization.NamingConventions; using Client = Wabbajack.Networking.GitHub.Client; @@ -154,6 +155,18 @@ public class Startup }); return client.GetDatabase("cesi"); }); + + services.AddSingleton(s => + { + var settings = s.GetRequiredService(); + var client = new CouchClient(settings.CesiDB.Endpoint, b => + { + b.UseBasicAuthentication("wabbajack", "password"); + b.SetPropertyCase(PropertyCaseType.None); + b.SetJsonNullValueHandling(NullValueHandling.Ignore); + }); + return client.GetDatabase("cesi"); + }); services.AddMvc(); services diff --git a/Wabbajack.Server/appsettings.json b/Wabbajack.Server/appsettings.json index 69dbbdc5..5547a56c 100644 --- a/Wabbajack.Server/appsettings.json +++ b/Wabbajack.Server/appsettings.json @@ -21,6 +21,12 @@ "Database": "cesi", "Username": "cesi", "Password": "password" + }, + "MetricsDB": { + "Endpoint": "http://localhost:15984", + "Database": "metrics", + "Username": "wabbajack", + "Password": "password" } }, "AllowedHosts": "*" diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index b50dbee2..e7e424cc 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -26,6 +26,7 @@ using Wabbajack.RateLimiter; using Wabbajack.Services.OSIntegrated.Services; using Wabbajack.Services.OSIntegrated.TokenProviders; using Wabbajack.VFS; +using Wabbajack.VFS.Interfaces; using Client = Wabbajack.Networking.WabbajackClientApi.Client; namespace Wabbajack.Services.OSIntegrated; @@ -55,9 +56,9 @@ public static class ServiceExtensions : new FileHashCache(KnownFolders.AppDataLocal.Combine("Wabbajack", "GlobalHashCache.sqlite"), s.GetService>()!)); - service.AddSingleton(s => options.UseLocalCache - ? new VFSCache(s.GetService()!.CreateFile().Path) - : new VFSCache(KnownFolders.EntryPoint.Combine("GlobalVFSCache3.sqlite"))); + service.AddAllSingleton(s => options.UseLocalCache + ? new VFSDiskCache(s.GetService()!.CreateFile().Path) + : new VFSDiskCache(KnownFolders.EntryPoint.Combine("GlobalVFSCache3.sqlite"))); service.AddSingleton(s => options.UseLocalCache ? new BinaryPatchCache(s.GetService()!.CreateFile().Path) @@ -97,6 +98,8 @@ public static class ServiceExtensions service.AddAllSingleton>(s => new Resource("VFS", GetSettings(s, "VFS"))); service.AddAllSingleton>(s => new Resource("File Hashing", GetSettings(s, "File Hashing"))); + service.AddAllSingleton>(s => + new Resource("Wabbajack Client", GetSettings(s, "Wabbajack Client"))); service.AddAllSingleton>(s => new Resource("File Extractor", GetSettings(s, "File Extractor"))); diff --git a/Wabbajack.VFS.Interfaces/IVfsCache.cs b/Wabbajack.VFS.Interfaces/IVfsCache.cs new file mode 100644 index 00000000..a25c0f3d --- /dev/null +++ b/Wabbajack.VFS.Interfaces/IVfsCache.cs @@ -0,0 +1,10 @@ +using Wabbajack.DTOs.Vfs; +using Wabbajack.Hashing.xxHash64; + +namespace Wabbajack.VFS.Interfaces; + +public interface IVfsCache +{ + public Task Get(Hash hash, CancellationToken token); + public Task Put(IndexedVirtualFile file, CancellationToken token); +} \ No newline at end of file diff --git a/Wabbajack.VFS.Interfaces/Wabbajack.VFS.Interfaces.csproj b/Wabbajack.VFS.Interfaces/Wabbajack.VFS.Interfaces.csproj new file mode 100644 index 00000000..bf68d60e --- /dev/null +++ b/Wabbajack.VFS.Interfaces/Wabbajack.VFS.Interfaces.csproj @@ -0,0 +1,15 @@ + + + + net6.0 + enable + enable + + + + + + + + + diff --git a/Wabbajack.VFS.Test/Startup.cs b/Wabbajack.VFS.Test/Startup.cs index f3984751..7549fe80 100644 --- a/Wabbajack.VFS.Test/Startup.cs +++ b/Wabbajack.VFS.Test/Startup.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging; using Wabbajack.DTOs; using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; +using Wabbajack.VFS.Interfaces; using Xunit.DependencyInjection; using Xunit.DependencyInjection.Logging; @@ -32,7 +33,7 @@ public class Startup service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = 2}); service.AddSingleton(new FileHashCache(KnownFolders.EntryPoint.Combine("hashcache.sqlite"), new Resource("File Hashing", 10))); - service.AddSingleton(new VFSCache(KnownFolders.EntryPoint.Combine("vfscache.sqlite"))); + service.AddAllSingleton(x => new VFSDiskCache(KnownFolders.EntryPoint.Combine("vfscache.sqlite"))); service.AddTransient(); service.AddSingleton(); } diff --git a/Wabbajack.VFS/Context.cs b/Wabbajack.VFS/Context.cs index 3d57bd31..273b6b5f 100644 --- a/Wabbajack.VFS/Context.cs +++ b/Wabbajack.VFS/Context.cs @@ -11,6 +11,7 @@ using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; +using Wabbajack.VFS.Interfaces; namespace Wabbajack.VFS; @@ -27,10 +28,10 @@ public class Context public readonly IResource Limiter; public readonly IResource HashLimiter; public readonly ILogger Logger; - public readonly VFSCache VfsCache; + public readonly IVfsCache VfsCache; public Context(ILogger logger, ParallelOptions parallelOptions, TemporaryFileManager manager, - VFSCache vfsCache, + IVfsCache vfsCache, FileHashCache hashCache, IResource limiter, IResource hashLimiter, FileExtractor.FileExtractor extractor) { Limiter = limiter; diff --git a/Wabbajack.VFS/FallthroughVFSCache.cs b/Wabbajack.VFS/FallthroughVFSCache.cs new file mode 100644 index 00000000..fb601ca4 --- /dev/null +++ b/Wabbajack.VFS/FallthroughVFSCache.cs @@ -0,0 +1,37 @@ +using System.Threading; +using System.Threading.Tasks; +using Wabbajack.DTOs.Vfs; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.VFS.Interfaces; + +namespace Wabbajack.VFS; + +public class FallthroughVFSCache : IVfsCache +{ + private readonly IVfsCache[] _caches; + + public FallthroughVFSCache(IVfsCache[] caches) + { + _caches = caches; + } + + public async Task Get(Hash hash, CancellationToken token) + { + foreach (var cache in _caches) + { + var result = await cache.Get(hash, token); + if (result == null) continue; + return result; + } + + return default; + } + + public async Task Put(IndexedVirtualFile file, CancellationToken token) + { + foreach (var cache in _caches) + { + await cache.Put(file, token); + } + } +} \ No newline at end of file diff --git a/Wabbajack.VFS/IndexedVirtualFile.cs b/Wabbajack.VFS/IndexedVirtualFileExtensions.cs similarity index 67% rename from Wabbajack.VFS/IndexedVirtualFile.cs rename to Wabbajack.VFS/IndexedVirtualFileExtensions.cs index b937bd9f..adf9ca84 100644 --- a/Wabbajack.VFS/IndexedVirtualFile.cs +++ b/Wabbajack.VFS/IndexedVirtualFileExtensions.cs @@ -3,42 +3,36 @@ using System.IO; using System.IO.Compression; using System.Text; using Wabbajack.DTOs.Texture; +using Wabbajack.DTOs.Vfs; using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; namespace Wabbajack.VFS; -public class IndexedVirtualFile +public static class IndexedVirtualFileExtensions { - public IPath Name { get; set; } - public Hash Hash { get; set; } - - public ImageState? ImageState { get; set; } - public long Size { get; set; } - public List Children { get; set; } = new(); - - private void Write(BinaryWriter bw) + public static void Write(this IndexedVirtualFile ivf, BinaryWriter bw) { - bw.Write(Name.ToString()!); - bw.Write((ulong) Hash); + bw.Write(ivf.Name.ToString()!); + bw.Write((ulong) ivf.Hash); - if (ImageState == null) + if (ivf.ImageState == null) { bw.Write(false); } else { bw.Write(true); - WriteImageState(bw, ImageState); + WriteImageState(bw, ivf.ImageState); } - bw.Write(Size); - bw.Write(Children.Count); - foreach (var file in Children) + bw.Write(ivf.Size); + bw.Write(ivf.Children.Count); + foreach (var file in ivf.Children) file.Write(bw); } - private void WriteImageState(BinaryWriter bw, ImageState state) + private static void WriteImageState(BinaryWriter bw, ImageState state) { bw.Write((ushort) state.Width); bw.Write((ushort) state.Height); @@ -46,7 +40,7 @@ public class IndexedVirtualFile state.PerceptualHash.Write(bw); } - public static ImageState ReadImageState(BinaryReader br) + static ImageState ReadImageState(BinaryReader br) { return new ImageState { @@ -58,14 +52,14 @@ public class IndexedVirtualFile } - public void Write(Stream s) + public static void Write(this IndexedVirtualFile ivf, Stream s) { using var cs = new GZipStream(s, CompressionLevel.Optimal, true); using var bw = new BinaryWriter(cs, Encoding.UTF8, true); - Write(bw); + ivf.Write(bw); } - private static IndexedVirtualFile Read(BinaryReader br) + public static IndexedVirtualFile Read(BinaryReader br) { var ivf = new IndexedVirtualFile { diff --git a/Wabbajack.VFS/VFSCache.cs b/Wabbajack.VFS/VFSCache.cs index 59359f4a..1a9fe2e3 100644 --- a/Wabbajack.VFS/VFSCache.cs +++ b/Wabbajack.VFS/VFSCache.cs @@ -4,22 +4,25 @@ using System.Data; using System.Data.SQLite; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.DTOs.Streams; +using Wabbajack.DTOs.Vfs; using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; using Wabbajack.Paths.IO; +using Wabbajack.VFS.Interfaces; namespace Wabbajack.VFS; -public class VFSCache +public class VFSDiskCache : IVfsCache { private readonly SQLiteConnection _conn; private readonly string _connectionString; private readonly AbsolutePath _path; - public VFSCache(AbsolutePath path) + public VFSDiskCache(AbsolutePath path) { _path = path; @@ -38,63 +41,34 @@ public class VFSCache cmd.ExecuteNonQuery(); } - public bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IStreamFactory extractedFile, - Hash hash, out VirtualFile found) + public async Task Get(Hash hash, CancellationToken token) { if (hash == default) throw new ArgumentException("Cannot cache default hashes"); - - using var cmd = new SQLiteCommand(_conn); + + await using var cmd = new SQLiteCommand(_conn); cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash"; cmd.Parameters.AddWithValue("@hash", (long) hash); - using var rdr = cmd.ExecuteReader(); + await using var rdr = cmd.ExecuteReader(); while (rdr.Read()) { - var data = IndexedVirtualFile.Read(rdr.GetStream(0)); - found = ConvertFromIndexedFile(context, data, path, parent, extractedFile); - found.Name = path; - found.Hash = hash; - return true; + var data = IndexedVirtualFileExtensions.Read(rdr.GetStream(0)); + return data; } - found = default; - return false; + return null; } - - private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, - VirtualFile vparent, IStreamFactory extractedFile) - { - var vself = new VirtualFile - { - Context = context, - Name = path, - Parent = vparent, - Size = file.Size, - LastModified = extractedFile.LastModifiedUtc.AsUnixTime(), - LastAnalyzed = DateTime.Now.AsUnixTime(), - Hash = file.Hash, - ImageState = file.ImageState - }; - - vself.FillFullPath(); - - vself.Children = file.Children.Select(f => ConvertFromIndexedFile(context, f, f.Name, vself, extractedFile)) - .ToImmutableList(); - - return vself; - } - - public async Task WriteToCache(VirtualFile self) + + public async Task Put(IndexedVirtualFile ivf, CancellationToken token) { await using var ms = new MemoryStream(); - var ivf = self.ToIndexedVirtualFile(); // Top level path gets renamed when read, we don't want the absolute path // here else the reader will blow up when it tries to convert the value ivf.Name = (RelativePath) "not/applicable"; ivf.Write(ms); ms.Position = 0; - await InsertIntoVFSCache(self.Hash, ms); + await InsertIntoVFSCache(ivf.Hash, ms); } private async Task InsertIntoVFSCache(Hash hash, MemoryStream data) diff --git a/Wabbajack.VFS/VirtualFile.cs b/Wabbajack.VFS/VirtualFile.cs index 5a8c217a..67af6d2a 100644 --- a/Wabbajack.VFS/VirtualFile.cs +++ b/Wabbajack.VFS/VirtualFile.cs @@ -10,6 +10,7 @@ using Wabbajack.Common; using Wabbajack.Common.FileSignatures; using Wabbajack.DTOs.Streams; using Wabbajack.DTOs.Texture; +using Wabbajack.DTOs.Vfs; using Wabbajack.Hashing.PHash; using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; @@ -176,9 +177,13 @@ public class VirtualFile hash = await hstream.HashingCopy(Stream.Null, token, job); } - if (context.VfsCache.TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself)) - return vself; - + var found = await context.VfsCache.Get(hash, token); + if (found != null) + { + var file = ConvertFromIndexedFile(context, found!, relPath, parent!, extractedFile); + file.Name = relPath; + return file; + } await using var stream = await extractedFile.GetStream(); var sig = await FileExtractor.FileExtractor.ArchiveSigs.MatchesAsync(stream); @@ -219,7 +224,7 @@ public class VirtualFile if (!sig.HasValue || !FileExtractor.FileExtractor.ExtractableExtensions.Contains(relPath.FileName.Extension)) { - await context.VfsCache.WriteToCache(self); + await context.VfsCache.Put(self.ToIndexedVirtualFile(), token); return self; } @@ -242,7 +247,7 @@ public class VirtualFile throw; } - await context.VfsCache.WriteToCache(self); + await context.VfsCache.Put(self.ToIndexedVirtualFile(), token); return self; } diff --git a/Wabbajack.VFS/Wabbajack.VFS.csproj b/Wabbajack.VFS/Wabbajack.VFS.csproj index 0d8f4da6..31348691 100644 --- a/Wabbajack.VFS/Wabbajack.VFS.csproj +++ b/Wabbajack.VFS/Wabbajack.VFS.csproj @@ -19,6 +19,7 @@ + diff --git a/Wabbajack.sln b/Wabbajack.sln index b26a8dc7..a52c3c6f 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -141,6 +141,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Downloaders.Manua EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.App.Wpf", "Wabbajack.App.Wpf\Wabbajack.App.Wpf.csproj", "{1196560A-9FFD-4290-9418-470508E984C9}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VFS.Interfaces", "Wabbajack.VFS.Interfaces\Wabbajack.VFS.Interfaces.csproj", "{E4BDB22D-11A4-452F-8D10-D9CA9777EA22}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -387,6 +389,10 @@ Global {1196560A-9FFD-4290-9418-470508E984C9}.Debug|Any CPU.Build.0 = Debug|x64 {1196560A-9FFD-4290-9418-470508E984C9}.Release|Any CPU.ActiveCfg = Release|x64 {1196560A-9FFD-4290-9418-470508E984C9}.Release|Any CPU.Build.0 = Release|x64 + {E4BDB22D-11A4-452F-8D10-D9CA9777EA22}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E4BDB22D-11A4-452F-8D10-D9CA9777EA22}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E4BDB22D-11A4-452F-8D10-D9CA9777EA22}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E4BDB22D-11A4-452F-8D10-D9CA9777EA22}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -435,6 +441,7 @@ Global {A3813D73-9A8E-4CE7-861A-C59043DFFC14} = {F01F8595-5FD7-4506-8469-F4A5522DACC1} {B10BB6D6-B3FC-4A76-8A07-6A0A0ADDE198} = {98B731EE-4FC0-4482-A069-BCBA25497871} {7FC4F129-F0FA-46B7-B7C4-532E371A6326} = {98B731EE-4FC0-4482-A069-BCBA25497871} + {E4BDB22D-11A4-452F-8D10-D9CA9777EA22} = {F677890D-5109-43BC-97C7-C4CD47C8EE0C} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {0AA30275-0F38-4A7D-B645-F5505178DDE8}