Implement an archive verification cache

2024-08-30 18:42:17 +00:00 · 2022-10-17 22:48:49 -06:00 · 2022-10-17 22:48:49 -06:00 · 4210234fe5
commit 4210234fe5
parent 320df0d96d
11 changed files with 218 additions and 27 deletions
--- a/Wabbajack.Downloaders.Dispatcher.Test/VerificationCacheTests.cs
+++ b/Wabbajack.Downloaders.Dispatcher.Test/VerificationCacheTests.cs
@ -0,0 +1,43 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.Paths.IO;
+using Xunit;
+
+namespace Wabbajack.Downloaders.Dispatcher.Test;
+
+
+public class VerificationCacheTests
+{
+    private readonly TemporaryFileManager _temp;
+    private readonly ILogger<VerificationCache.VerificationCache> _logger;
+
+    public VerificationCacheTests(ILogger<VerificationCache.VerificationCache> logger)
+    {
+        _logger = logger;
+    }
+
+    [Fact]
+    public async Task BasicCacheTests()
+    {
+        using var cache = new VerificationCache.VerificationCache(_logger,  KnownFolders.EntryPoint.Combine(Guid.NewGuid().ToString()), TimeSpan.FromSeconds(1));
+
+        var goodState = new DTOs.DownloadStates.Http { Url = new Uri($"https://some.com/{Guid.NewGuid()}/path") };
+        var badState = new DTOs.DownloadStates.Http { Url = new Uri($"https://some.com/{Guid.NewGuid()}/path") };
+        Assert.True(await cache.Get(goodState) == null);
+
+        await cache.Put(goodState, true);
+        Assert.True(await cache.Get(goodState));
+
+        await Task.Delay(TimeSpan.FromSeconds(2));
+        
+        Assert.False(await cache.Get(goodState));
+
+        await cache.Put(badState, true);
+        Assert.True(await cache.Get(badState));
+        await cache.Put(badState, false);
+        Assert.Null(await cache.Get(badState));
+
+    }
+}
+
--- a/Wabbajack.Downloaders.Dispatcher.Test/Wabbajack.Downloaders.Dispatcher.Test.csproj
+++ b/Wabbajack.Downloaders.Dispatcher.Test/Wabbajack.Downloaders.Dispatcher.Test.csproj
@ -27,6 +27,7 @@

    <ItemGroup>
        <ProjectReference Include="..\Wabbajack.Downloaders.Dispatcher\Wabbajack.Downloaders.Dispatcher.csproj" />
+        <ProjectReference Include="..\Wabbajack.Downloaders.VerificationCache\Wabbajack.Downloaders.VerificationCache.csproj" />
        <ProjectReference Include="..\Wabbajack.Networking.Http\Wabbajack.Networking.Http.csproj" />
        <ProjectReference Include="..\Wabbajack.Networking.NexusApi.Test\Wabbajack.Networking.NexusApi.Test.csproj" />
        <ProjectReference Include="..\Wabbajack.Networking.NexusApi\Wabbajack.Networking.NexusApi.csproj" />
--- a/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs
+++ b/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs
@ -6,6 +6,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Wabbajack.Downloaders.Interfaces;
+using Wabbajack.Downloaders.VerificationCache;
 using Wabbajack.DTOs;
 using Wabbajack.DTOs.DownloadStates;
 using Wabbajack.DTOs.ServerResponses;
@ -16,6 +17,7 @@ using Wabbajack.Networking.WabbajackClientApi;
 using Wabbajack.Paths;
 using Wabbajack.Paths.IO;
 using Wabbajack.RateLimiter;
+using StringExtensions = Wabbajack.Paths.StringExtensions;

 namespace Wabbajack.Downloaders;

@ -26,15 +28,17 @@ public class DownloadDispatcher
    private readonly ILogger<DownloadDispatcher> _logger;
    private readonly Client _wjClient;
    private readonly bool _useProxyCache;
+    private readonly IVerificationCache _verificationCache;

    public DownloadDispatcher(ILogger<DownloadDispatcher> logger, IEnumerable<IDownloader> downloaders,
-        IResource<DownloadDispatcher> limiter, Client wjClient, bool useProxyCache = true)
+        IResource<DownloadDispatcher> limiter, Client wjClient, IVerificationCache verificationCache, bool useProxyCache = true)
    {
        _downloaders = downloaders.OrderBy(d => d.Priority).ToArray();
        _logger = logger;
        _wjClient = wjClient;
        _limiter = limiter;
        _useProxyCache = useProxyCache;
+        _verificationCache = verificationCache;
    }

    public async Task<Hash> Download(Archive a, AbsolutePath dest, CancellationToken token, bool? proxy = null)
@ -108,13 +112,20 @@ public class DownloadDispatcher
    {
        try
        {
+            if (await _verificationCache.Get(a.State) == true)
+                return true;
+            
            a = await MaybeProxy(a, token);
            var downloader = Downloader(a);
            using var job = await _limiter.Begin($"Verifying {a.State.PrimaryKeyString}", -1, token);
-            return await downloader.Verify(a, job, token);
+            var result = await downloader.Verify(a, job, token);
+            await _verificationCache.Put(a.State, result);
+
+            return result;
        }
        catch (HttpException)
        {
+            await _verificationCache.Put(a.State, false);
            return false;
        }
    }
--- a/Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs
+++ b/Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs
@ -8,6 +8,7 @@ using Wabbajack.Downloaders.IPS4OAuth2Downloader;
 using Wabbajack.Downloaders.Manual;
 using Wabbajack.Downloaders.MediaFire;
 using Wabbajack.Downloaders.ModDB;
+using Wabbajack.Downloaders.VerificationCache;
 using Wabbajack.DTOs.JsonConverters;
 using Wabbajack.Networking.WabbajackClientApi;
 using Wabbajack.RateLimiter;
@ -52,10 +53,11 @@ public static class ServiceExtensions

        services.AddSingleton(s =>
            new DownloadDispatcher(s.GetRequiredService<ILogger<DownloadDispatcher>>(),
-            s.GetRequiredService<IEnumerable<IDownloader>>(),
-            s.GetRequiredService<IResource<DownloadDispatcher>>(),
-            s.GetRequiredService<Client>(),
-            useProxyCache));
+                s.GetRequiredService<IEnumerable<IDownloader>>(),
+                s.GetRequiredService<IResource<DownloadDispatcher>>(),
+                s.GetRequiredService<Client>(),
+                s.GetRequiredService<IVerificationCache>(),
+                useProxyCache));

        return services;
    }
--- a/Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj
+++ b/Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj
@ -19,6 +19,7 @@
        <ProjectReference Include="..\Wabbajack.Downloaders.Mega\Wabbajack.Downloaders.Mega.csproj" />
        <ProjectReference Include="..\Wabbajack.Downloaders.ModDB\Wabbajack.Downloaders.ModDB.csproj" />
        <ProjectReference Include="..\Wabbajack.Downloaders.Nexus\Wabbajack.Downloaders.Nexus.csproj" />
+        <ProjectReference Include="..\Wabbajack.Downloaders.VerificationCache\Wabbajack.Downloaders.VerificationCache.csproj" />
        <ProjectReference Include="..\Wabbajack.Downloaders.WabbajackCDN\Wabbajack.Downloaders.WabbajackCDN.csproj" />
        <ProjectReference Include="..\Wabbajack.Networking.WabbajackClientApi\Wabbajack.Networking.WabbajackClientApi.csproj" />
    </ItemGroup>
--- a/Wabbajack.Downloaders.GameFile/GameLocator.cs
+++ b/Wabbajack.Downloaders.GameFile/GameLocator.cs
@ -139,31 +139,31 @@ public class GameLocator : IGameLocator
    {
        var metaData = game.MetaData();

-        int? steamId = metaData.SteamIDs.FirstOrDefault(id => _steamGames.ContainsKey(id));
-        if (steamId.HasValue)
+        foreach (var id in metaData.SteamIDs)
        {
-            path = _steamGames[steamId.Value];
+            if (!_steamGames.TryGetValue(id, out var found)) continue;
+            path = found;
            return true;
        }
        
-        int? gogId = metaData.GOGIDs.FirstOrDefault(id => _gogGames.ContainsKey(id));
-        if (gogId.HasValue)
+        foreach (var id in metaData.GOGIDs)
        {
-            path = _gogGames[gogId.Value];
+            if (!_gogGames.TryGetValue(id, out var found)) continue;
+            path = found;
            return true;
        }

-        var egsId = metaData.EpicGameStoreIDs.FirstOrDefault(id => _egsGames.ContainsKey(id));
-        if (egsId is not null)
+        foreach (var id in metaData.EpicGameStoreIDs)
        {
-            path = _egsGames[egsId];
+            if (!_egsGames.TryGetValue(id, out var found)) continue;
+            path = found;
            return true;
        }

-        var originId = metaData.OriginIDs.FirstOrDefault(id => _originGames.ContainsKey(id));
-        if (originId is not null)
+        foreach (var id in metaData.OriginIDs)
        {
-            path = _originGames[originId];
+            if (!_originGames.TryGetValue(id, out var found)) continue;
+            path = found;
            return true;
        }
        
--- a/Wabbajack.Downloaders.VerificationCache/IVerificationCache.cs
+++ b/Wabbajack.Downloaders.VerificationCache/IVerificationCache.cs
@ -0,0 +1,10 @@
+using Wabbajack.DTOs;
+using Wabbajack.DTOs.DownloadStates;
+
+namespace Wabbajack.Downloaders.VerificationCache;
+
+public interface IVerificationCache
+{
+    Task<bool?> Get(IDownloadState archive);
+    Task Put(IDownloadState archive, bool valid);
+}
--- a/Wabbajack.Downloaders.VerificationCache/VerificationCache.cs
+++ b/Wabbajack.Downloaders.VerificationCache/VerificationCache.cs
@ -0,0 +1,91 @@
+using System.Data.SQLite;
+using Microsoft.Extensions.Logging;
+using Wabbajack.DTOs;
+using Wabbajack.DTOs.DownloadStates;
+using Wabbajack.Hashing.xxHash64;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+
+namespace Wabbajack.Downloaders.VerificationCache;
+
+public class VerificationCache : IVerificationCache, IDisposable
+{
+    private readonly AbsolutePath _location;
+    private readonly string _connectionString;
+    private readonly SQLiteConnection _conn;
+    private readonly TimeSpan _expiry;
+    private readonly ILogger<VerificationCache> _logger;
+
+    public VerificationCache(ILogger<VerificationCache> logger, AbsolutePath location, TimeSpan expiry)
+    {
+        _logger = logger;
+        _location = location;
+        _expiry = expiry;
+
+        if (!_location.Parent.DirectoryExists())
+            _location.Parent.CreateDirectory();
+
+        _connectionString =
+            string.Intern($"URI=file:{_location};Pooling=True;Max Pool Size=100; Journal Mode=Memory;");
+        _conn = new SQLiteConnection(_connectionString);
+        _conn.Open();
+
+
+        using var cmd = new SQLiteCommand(_conn);
+        cmd.CommandText = @"CREATE TABLE IF NOT EXISTS VerficationCache (
+            PKS TEXT PRIMARY KEY,
+            LastModified BIGINT)
+            WITHOUT ROWID";
+        cmd.ExecuteNonQuery();
+    }
+
+    public async Task<bool?> Get(IDownloadState archive)
+    {
+        var key = archive.PrimaryKeyString;
+
+        await using var cmd = new SQLiteCommand(_conn);
+        cmd.CommandText = "SELECT LastModified FROM VerficationCache WHERE PKS = @pks";
+        cmd.Parameters.AddWithValue("@pks", key);
+        await cmd.PrepareAsync();
+
+        await using var reader = await cmd.ExecuteReaderAsync();
+        while (await reader.ReadAsync())
+        {
+            var ts = DateTime.FromFileTimeUtc(reader.GetInt64(0));
+            return DateTime.UtcNow - ts <= _expiry;
+        }
+
+        return null;
+    }
+
+    public async Task Put(IDownloadState state, bool valid)
+    {
+        var key = state.PrimaryKeyString;
+        if (valid)
+        {
+            await using var cmd = new SQLiteCommand(_conn);
+            cmd.CommandText = @"INSERT INTO VerficationCache (PKS, LastModified) VALUES (@pks, @lastModified)
+            ON CONFLICT(PKS) DO UPDATE SET LastModified = @lastModified";
+            cmd.Parameters.AddWithValue("@pks", key);
+            cmd.Parameters.AddWithValue("@lastModified", DateTime.UtcNow.ToFileTimeUtc());
+            await cmd.PrepareAsync();
+
+            await cmd.ExecuteNonQueryAsync();
+        }
+        else
+        {
+            _logger.LogInformation("Marking {Key} as invalid", key);
+            await using var cmd = new SQLiteCommand(_conn);
+            cmd.CommandText = @"DELETE FROM VerficationCache WHERE PKS = @pks";
+            cmd.Parameters.AddWithValue("@pks", state.PrimaryKeyString);
+            await cmd.PrepareAsync();
+            await cmd.ExecuteNonQueryAsync();
+        }
+    }
+
+    public void Dispose()
+    {
+        _conn.Close();
+        _conn.Dispose();
+    }
+}
--- a/Wabbajack.Downloaders.VerificationCache/Wabbajack.Downloaders.VerificationCache.csproj
+++ b/Wabbajack.Downloaders.VerificationCache/Wabbajack.Downloaders.VerificationCache.csproj
@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net6.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
+      <ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.2-mauipre.1.22102.15" />
+      <PackageReference Include="Stub.System.Data.SQLite.Core.NetStandard" Version="1.0.116" />
+    </ItemGroup>
+
+</Project>
--- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs
+++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs
@ -10,6 +10,7 @@ using Microsoft.Extensions.Logging;
 using Wabbajack.Compiler;
 using Wabbajack.Downloaders;
 using Wabbajack.Downloaders.GameFile;
+using Wabbajack.Downloaders.VerificationCache;
 using Wabbajack.DTOs;
 using Wabbajack.DTOs.Interventions;
 using Wabbajack.DTOs.Logins;
@ -71,6 +72,11 @@ public static class ServiceExtensions
            ? new BinaryPatchCache(s.GetRequiredService<ILogger<BinaryPatchCache>>(), s.GetService<TemporaryFileManager>()!.CreateFolder().Path)
            : new BinaryPatchCache(s.GetRequiredService<ILogger<BinaryPatchCache>>(),KnownFolders.WabbajackAppLocal.Combine("PatchCache")));

+        
+        service.AddSingleton<IVerificationCache>(s => options.UseLocalCache
+            ? new VerificationCache(s.GetRequiredService<ILogger<VerificationCache>>(), s.GetService<TemporaryFileManager>()!.CreateFile().Path, TimeSpan.FromDays(1))
+            : new VerificationCache(s.GetRequiredService<ILogger<VerificationCache>>(),KnownFolders.WabbajackAppLocal.Combine("VerificationCache"), TimeSpan.FromDays(1)));
+
        service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount});

        Func<Task<(int MaxTasks, long MaxThroughput)>> GetSettings(IServiceProvider provider, string name)
--- a/Wabbajack.sln
+++ b/Wabbajack.sln
@ -143,6 +143,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VFS.Interfaces",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.CLI.Builder", "Wabbajack.CLI.Builder\Wabbajack.CLI.Builder.csproj", "{99CD51A1-38C9-420E-9497-2C9AEAF0053C}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Downloaders.VerificationCache", "Wabbajack.Downloaders.VerificationCache\Wabbajack.Downloaders.VerificationCache.csproj", "{D9560C73-4E58-4463-9DB9-D06491E0E1C8}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@ -393,6 +395,10 @@ Global
 		{99CD51A1-38C9-420E-9497-2C9AEAF0053C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{99CD51A1-38C9-420E-9497-2C9AEAF0053C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{99CD51A1-38C9-420E-9497-2C9AEAF0053C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D9560C73-4E58-4463-9DB9-D06491E0E1C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D9560C73-4E58-4463-9DB9-D06491E0E1C8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D9560C73-4E58-4463-9DB9-D06491E0E1C8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D9560C73-4E58-4463-9DB9-D06491E0E1C8}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@ -442,6 +448,7 @@ Global
 		{B10BB6D6-B3FC-4A76-8A07-6A0A0ADDE198} = {98B731EE-4FC0-4482-A069-BCBA25497871}
 		{7FC4F129-F0FA-46B7-B7C4-532E371A6326} = {98B731EE-4FC0-4482-A069-BCBA25497871}
 		{E4BDB22D-11A4-452F-8D10-D9CA9777EA22} = {F677890D-5109-43BC-97C7-C4CD47C8EE0C}
+		{D9560C73-4E58-4463-9DB9-D06491E0E1C8} = {98B731EE-4FC0-4482-A069-BCBA25497871}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {0AA30275-0F38-4A7D-B645-F5505178DDE8}