using System.Data.SQLite; using System.IO; using System.Threading; using System.Threading.Tasks; using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; namespace Wabbajack.VFS; public class FileHashCache { private readonly SQLiteConnection _conn; private readonly string _connectionString; private readonly IResource _limiter; private readonly AbsolutePath _location; public FileHashCache(AbsolutePath location, IResource limiter) { _limiter = limiter; _location = location; if (!_location.Parent.DirectoryExists()) _location.Parent.CreateDirectory(); _connectionString = string.Intern($"URI=file:{_location};Pooling=True;Max Pool Size=100; Journal Mode=Memory;"); _conn = new SQLiteConnection(_connectionString); _conn.Open(); using var cmd = new SQLiteCommand(_conn); cmd.CommandText = @"CREATE TABLE IF NOT EXISTS HashCache ( Path TEXT PRIMARY KEY, LastModified BIGINT, Hash BIGINT) WITHOUT ROWID"; cmd.ExecuteNonQuery(); } private async Task<(AbsolutePath Path, long LastModified, Hash Hash)> Get(AbsolutePath path) { using var cmd = new SQLiteCommand(_conn); cmd.CommandText = "SELECT LastModified, Hash FROM HashCache WHERE Path = @path"; cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant()); await cmd.PrepareAsync(); await using var reader = await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) return (path, reader.GetInt64(0), Hash.FromLong(reader.GetInt64(1))); return default; } public void Purge(AbsolutePath path) { using var cmd = new SQLiteCommand(_conn); cmd.CommandText = "DELETE FROM HashCache WHERE Path = @path"; cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant()); cmd.PrepareAsync(); cmd.ExecuteNonQuery(); } private async Task Upsert(AbsolutePath path, long lastModified, Hash hash) { await using var cmd = new SQLiteCommand(_conn); cmd.CommandText = @"INSERT INTO HashCache (Path, LastModified, Hash) VALUES (@path, @lastModified, @hash) ON CONFLICT(Path) DO UPDATE SET LastModified = @lastModified, Hash = @hash"; cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant()); cmd.Parameters.AddWithValue("@lastModified", lastModified); cmd.Parameters.AddWithValue("@hash", (long) hash); await cmd.PrepareAsync(); await cmd.ExecuteNonQueryAsync(); } public void VacuumDatabase() { using var cmd = new SQLiteCommand(_conn); cmd.CommandText = @"VACUUM"; cmd.PrepareAsync(); cmd.ExecuteNonQuery(); } public async Task TryGetHashCache(AbsolutePath file) { if (!file.FileExists()) return default; var result = await Get(file); if (result == default || result.Hash == default) return default; // Fix for strange issue where dates are messed up on some systems if (file.LastModifiedUtc() < file.CreatedUtc()) file.Touch(); if (result.LastModified == file.LastModifiedUtc().ToFileTimeUtc()) { return result.Hash; } Purge(file); return default; } private async Task WriteHashCache(AbsolutePath file, Hash hash) { if (!file.FileExists()) return; await Upsert(file, file.LastModifiedUtc().ToFileTimeUtc(), hash); } public async Task FileHashWriteCache(AbsolutePath file, Hash hash) { await WriteHashCache(file, hash); } public async Task FileHashCachedAsync(AbsolutePath file, CancellationToken token) { var hash = await TryGetHashCache(file); if (hash != default) return hash; using var job = await _limiter.Begin($"Hashing {file.FileName}", file.Size(), token); await using var fs = file.Open(FileMode.Open, FileAccess.Read, FileShare.Read); hash = await fs.HashingCopy(Stream.Null, token, job); if (hash != default) await WriteHashCache(file, hash); return hash; } }