From 4210234fe5f9da79684d7b90be95c9c83aaed71e Mon Sep 17 00:00:00 2001 From: Halgari Date: Mon, 17 Oct 2022 22:48:49 -0600 Subject: [PATCH] Implement an archive verification cache --- .../VerificationCacheTests.cs | 43 +++++++++ ...bbajack.Downloaders.Dispatcher.Test.csproj | 1 + .../DownloadDispatcher.cs | 15 ++- .../ServiceExtensions.cs | 12 ++- .../Wabbajack.Downloaders.Dispatcher.csproj | 1 + Wabbajack.Downloaders.GameFile/GameLocator.cs | 40 ++++---- .../IVerificationCache.cs | 10 ++ .../VerificationCache.cs | 91 +++++++++++++++++++ ...ajack.Downloaders.VerificationCache.csproj | 19 ++++ .../ServiceExtensions.cs | 6 ++ Wabbajack.sln | 7 ++ 11 files changed, 218 insertions(+), 27 deletions(-) create mode 100644 Wabbajack.Downloaders.Dispatcher.Test/VerificationCacheTests.cs create mode 100644 Wabbajack.Downloaders.VerificationCache/IVerificationCache.cs create mode 100644 Wabbajack.Downloaders.VerificationCache/VerificationCache.cs create mode 100644 Wabbajack.Downloaders.VerificationCache/Wabbajack.Downloaders.VerificationCache.csproj diff --git a/Wabbajack.Downloaders.Dispatcher.Test/VerificationCacheTests.cs b/Wabbajack.Downloaders.Dispatcher.Test/VerificationCacheTests.cs new file mode 100644 index 00000000..ac202ca5 --- /dev/null +++ b/Wabbajack.Downloaders.Dispatcher.Test/VerificationCacheTests.cs @@ -0,0 +1,43 @@ +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.Paths.IO; +using Xunit; + +namespace Wabbajack.Downloaders.Dispatcher.Test; + + +public class VerificationCacheTests +{ + private readonly TemporaryFileManager _temp; + private readonly ILogger _logger; + + public VerificationCacheTests(ILogger logger) + { + _logger = logger; + } + + [Fact] + public async Task BasicCacheTests() + { + using var cache = new VerificationCache.VerificationCache(_logger, KnownFolders.EntryPoint.Combine(Guid.NewGuid().ToString()), TimeSpan.FromSeconds(1)); + + var goodState = new DTOs.DownloadStates.Http { Url = new Uri($"https://some.com/{Guid.NewGuid()}/path") }; + var badState = new DTOs.DownloadStates.Http { Url = new Uri($"https://some.com/{Guid.NewGuid()}/path") }; + Assert.True(await cache.Get(goodState) == null); + + await cache.Put(goodState, true); + Assert.True(await cache.Get(goodState)); + + await Task.Delay(TimeSpan.FromSeconds(2)); + + Assert.False(await cache.Get(goodState)); + + await cache.Put(badState, true); + Assert.True(await cache.Get(badState)); + await cache.Put(badState, false); + Assert.Null(await cache.Get(badState)); + + } +} + diff --git a/Wabbajack.Downloaders.Dispatcher.Test/Wabbajack.Downloaders.Dispatcher.Test.csproj b/Wabbajack.Downloaders.Dispatcher.Test/Wabbajack.Downloaders.Dispatcher.Test.csproj index 331179ea..f8211cdd 100644 --- a/Wabbajack.Downloaders.Dispatcher.Test/Wabbajack.Downloaders.Dispatcher.Test.csproj +++ b/Wabbajack.Downloaders.Dispatcher.Test/Wabbajack.Downloaders.Dispatcher.Test.csproj @@ -27,6 +27,7 @@ + diff --git a/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs b/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs index 53014337..72958db2 100644 --- a/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs +++ b/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Wabbajack.Downloaders.Interfaces; +using Wabbajack.Downloaders.VerificationCache; using Wabbajack.DTOs; using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.ServerResponses; @@ -16,6 +17,7 @@ using Wabbajack.Networking.WabbajackClientApi; using Wabbajack.Paths; using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; +using StringExtensions = Wabbajack.Paths.StringExtensions; namespace Wabbajack.Downloaders; @@ -26,15 +28,17 @@ public class DownloadDispatcher private readonly ILogger _logger; private readonly Client _wjClient; private readonly bool _useProxyCache; + private readonly IVerificationCache _verificationCache; public DownloadDispatcher(ILogger logger, IEnumerable downloaders, - IResource limiter, Client wjClient, bool useProxyCache = true) + IResource limiter, Client wjClient, IVerificationCache verificationCache, bool useProxyCache = true) { _downloaders = downloaders.OrderBy(d => d.Priority).ToArray(); _logger = logger; _wjClient = wjClient; _limiter = limiter; _useProxyCache = useProxyCache; + _verificationCache = verificationCache; } public async Task Download(Archive a, AbsolutePath dest, CancellationToken token, bool? proxy = null) @@ -108,13 +112,20 @@ public class DownloadDispatcher { try { + if (await _verificationCache.Get(a.State) == true) + return true; + a = await MaybeProxy(a, token); var downloader = Downloader(a); using var job = await _limiter.Begin($"Verifying {a.State.PrimaryKeyString}", -1, token); - return await downloader.Verify(a, job, token); + var result = await downloader.Verify(a, job, token); + await _verificationCache.Put(a.State, result); + + return result; } catch (HttpException) { + await _verificationCache.Put(a.State, false); return false; } } diff --git a/Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs b/Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs index 90babc45..e392db77 100644 --- a/Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs +++ b/Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs @@ -8,6 +8,7 @@ using Wabbajack.Downloaders.IPS4OAuth2Downloader; using Wabbajack.Downloaders.Manual; using Wabbajack.Downloaders.MediaFire; using Wabbajack.Downloaders.ModDB; +using Wabbajack.Downloaders.VerificationCache; using Wabbajack.DTOs.JsonConverters; using Wabbajack.Networking.WabbajackClientApi; using Wabbajack.RateLimiter; @@ -50,12 +51,13 @@ public static class ServiceExtensions .AddWabbajackClient(); } - services.AddSingleton(s => + services.AddSingleton(s => new DownloadDispatcher(s.GetRequiredService>(), - s.GetRequiredService>(), - s.GetRequiredService>(), - s.GetRequiredService(), - useProxyCache)); + s.GetRequiredService>(), + s.GetRequiredService>(), + s.GetRequiredService(), + s.GetRequiredService(), + useProxyCache)); return services; } diff --git a/Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj b/Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj index dcd9a660..6069eaa0 100644 --- a/Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj +++ b/Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj @@ -19,6 +19,7 @@ + diff --git a/Wabbajack.Downloaders.GameFile/GameLocator.cs b/Wabbajack.Downloaders.GameFile/GameLocator.cs index 07306908..5b6a356b 100644 --- a/Wabbajack.Downloaders.GameFile/GameLocator.cs +++ b/Wabbajack.Downloaders.GameFile/GameLocator.cs @@ -139,34 +139,34 @@ public class GameLocator : IGameLocator { var metaData = game.MetaData(); - int? steamId = metaData.SteamIDs.FirstOrDefault(id => _steamGames.ContainsKey(id)); - if (steamId.HasValue) + foreach (var id in metaData.SteamIDs) { - path = _steamGames[steamId.Value]; - return true; - } - - int? gogId = metaData.GOGIDs.FirstOrDefault(id => _gogGames.ContainsKey(id)); - if (gogId.HasValue) - { - path = _gogGames[gogId.Value]; + if (!_steamGames.TryGetValue(id, out var found)) continue; + path = found; return true; } - var egsId = metaData.EpicGameStoreIDs.FirstOrDefault(id => _egsGames.ContainsKey(id)); - if (egsId is not null) + foreach (var id in metaData.GOGIDs) { - path = _egsGames[egsId]; - return true; - } - - var originId = metaData.OriginIDs.FirstOrDefault(id => _originGames.ContainsKey(id)); - if (originId is not null) - { - path = _originGames[originId]; + if (!_gogGames.TryGetValue(id, out var found)) continue; + path = found; return true; } + foreach (var id in metaData.EpicGameStoreIDs) + { + if (!_egsGames.TryGetValue(id, out var found)) continue; + path = found; + return true; + } + + foreach (var id in metaData.OriginIDs) + { + if (!_originGames.TryGetValue(id, out var found)) continue; + path = found; + return true; + } + path = default; return false; } diff --git a/Wabbajack.Downloaders.VerificationCache/IVerificationCache.cs b/Wabbajack.Downloaders.VerificationCache/IVerificationCache.cs new file mode 100644 index 00000000..be151572 --- /dev/null +++ b/Wabbajack.Downloaders.VerificationCache/IVerificationCache.cs @@ -0,0 +1,10 @@ +using Wabbajack.DTOs; +using Wabbajack.DTOs.DownloadStates; + +namespace Wabbajack.Downloaders.VerificationCache; + +public interface IVerificationCache +{ + Task Get(IDownloadState archive); + Task Put(IDownloadState archive, bool valid); +} \ No newline at end of file diff --git a/Wabbajack.Downloaders.VerificationCache/VerificationCache.cs b/Wabbajack.Downloaders.VerificationCache/VerificationCache.cs new file mode 100644 index 00000000..c6d149d0 --- /dev/null +++ b/Wabbajack.Downloaders.VerificationCache/VerificationCache.cs @@ -0,0 +1,91 @@ +using System.Data.SQLite; +using Microsoft.Extensions.Logging; +using Wabbajack.DTOs; +using Wabbajack.DTOs.DownloadStates; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; + +namespace Wabbajack.Downloaders.VerificationCache; + +public class VerificationCache : IVerificationCache, IDisposable +{ + private readonly AbsolutePath _location; + private readonly string _connectionString; + private readonly SQLiteConnection _conn; + private readonly TimeSpan _expiry; + private readonly ILogger _logger; + + public VerificationCache(ILogger logger, AbsolutePath location, TimeSpan expiry) + { + _logger = logger; + _location = location; + _expiry = expiry; + + if (!_location.Parent.DirectoryExists()) + _location.Parent.CreateDirectory(); + + _connectionString = + string.Intern($"URI=file:{_location};Pooling=True;Max Pool Size=100; Journal Mode=Memory;"); + _conn = new SQLiteConnection(_connectionString); + _conn.Open(); + + + using var cmd = new SQLiteCommand(_conn); + cmd.CommandText = @"CREATE TABLE IF NOT EXISTS VerficationCache ( + PKS TEXT PRIMARY KEY, + LastModified BIGINT) + WITHOUT ROWID"; + cmd.ExecuteNonQuery(); + } + + public async Task Get(IDownloadState archive) + { + var key = archive.PrimaryKeyString; + + await using var cmd = new SQLiteCommand(_conn); + cmd.CommandText = "SELECT LastModified FROM VerficationCache WHERE PKS = @pks"; + cmd.Parameters.AddWithValue("@pks", key); + await cmd.PrepareAsync(); + + await using var reader = await cmd.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + var ts = DateTime.FromFileTimeUtc(reader.GetInt64(0)); + return DateTime.UtcNow - ts <= _expiry; + } + + return null; + } + + public async Task Put(IDownloadState state, bool valid) + { + var key = state.PrimaryKeyString; + if (valid) + { + await using var cmd = new SQLiteCommand(_conn); + cmd.CommandText = @"INSERT INTO VerficationCache (PKS, LastModified) VALUES (@pks, @lastModified) + ON CONFLICT(PKS) DO UPDATE SET LastModified = @lastModified"; + cmd.Parameters.AddWithValue("@pks", key); + cmd.Parameters.AddWithValue("@lastModified", DateTime.UtcNow.ToFileTimeUtc()); + await cmd.PrepareAsync(); + + await cmd.ExecuteNonQueryAsync(); + } + else + { + _logger.LogInformation("Marking {Key} as invalid", key); + await using var cmd = new SQLiteCommand(_conn); + cmd.CommandText = @"DELETE FROM VerficationCache WHERE PKS = @pks"; + cmd.Parameters.AddWithValue("@pks", state.PrimaryKeyString); + await cmd.PrepareAsync(); + await cmd.ExecuteNonQueryAsync(); + } + } + + public void Dispose() + { + _conn.Close(); + _conn.Dispose(); + } +} \ No newline at end of file diff --git a/Wabbajack.Downloaders.VerificationCache/Wabbajack.Downloaders.VerificationCache.csproj b/Wabbajack.Downloaders.VerificationCache/Wabbajack.Downloaders.VerificationCache.csproj new file mode 100644 index 00000000..1e6d05cf --- /dev/null +++ b/Wabbajack.Downloaders.VerificationCache/Wabbajack.Downloaders.VerificationCache.csproj @@ -0,0 +1,19 @@ + + + + net6.0 + enable + enable + + + + + + + + + + + + + diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index 0b2703e4..e406fc76 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -10,6 +10,7 @@ using Microsoft.Extensions.Logging; using Wabbajack.Compiler; using Wabbajack.Downloaders; using Wabbajack.Downloaders.GameFile; +using Wabbajack.Downloaders.VerificationCache; using Wabbajack.DTOs; using Wabbajack.DTOs.Interventions; using Wabbajack.DTOs.Logins; @@ -71,6 +72,11 @@ public static class ServiceExtensions ? new BinaryPatchCache(s.GetRequiredService>(), s.GetService()!.CreateFolder().Path) : new BinaryPatchCache(s.GetRequiredService>(),KnownFolders.WabbajackAppLocal.Combine("PatchCache"))); + + service.AddSingleton(s => options.UseLocalCache + ? new VerificationCache(s.GetRequiredService>(), s.GetService()!.CreateFile().Path, TimeSpan.FromDays(1)) + : new VerificationCache(s.GetRequiredService>(),KnownFolders.WabbajackAppLocal.Combine("VerificationCache"), TimeSpan.FromDays(1))); + service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount}); Func> GetSettings(IServiceProvider provider, string name) diff --git a/Wabbajack.sln b/Wabbajack.sln index 40cfb3a9..533fac81 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -143,6 +143,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VFS.Interfaces", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.CLI.Builder", "Wabbajack.CLI.Builder\Wabbajack.CLI.Builder.csproj", "{99CD51A1-38C9-420E-9497-2C9AEAF0053C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Downloaders.VerificationCache", "Wabbajack.Downloaders.VerificationCache\Wabbajack.Downloaders.VerificationCache.csproj", "{D9560C73-4E58-4463-9DB9-D06491E0E1C8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -393,6 +395,10 @@ Global {99CD51A1-38C9-420E-9497-2C9AEAF0053C}.Debug|Any CPU.Build.0 = Debug|Any CPU {99CD51A1-38C9-420E-9497-2C9AEAF0053C}.Release|Any CPU.ActiveCfg = Release|Any CPU {99CD51A1-38C9-420E-9497-2C9AEAF0053C}.Release|Any CPU.Build.0 = Release|Any CPU + {D9560C73-4E58-4463-9DB9-D06491E0E1C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D9560C73-4E58-4463-9DB9-D06491E0E1C8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D9560C73-4E58-4463-9DB9-D06491E0E1C8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D9560C73-4E58-4463-9DB9-D06491E0E1C8}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -442,6 +448,7 @@ Global {B10BB6D6-B3FC-4A76-8A07-6A0A0ADDE198} = {98B731EE-4FC0-4482-A069-BCBA25497871} {7FC4F129-F0FA-46B7-B7C4-532E371A6326} = {98B731EE-4FC0-4482-A069-BCBA25497871} {E4BDB22D-11A4-452F-8D10-D9CA9777EA22} = {F677890D-5109-43BC-97C7-C4CD47C8EE0C} + {D9560C73-4E58-4463-9DB9-D06491E0E1C8} = {98B731EE-4FC0-4482-A069-BCBA25497871} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {0AA30275-0F38-4A7D-B645-F5505178DDE8}