From a4e5d416030f119b90c1d39421fcaef51f3406e5 Mon Sep 17 00:00:00 2001 From: Halgari Date: Fri, 30 Sep 2022 23:21:58 -0600 Subject: [PATCH] 3.0.1.8 fixes --- CHANGELOG.md | 8 ++++ Wabbajack.CLI/Verbs/DownloadAll.cs | 2 +- Wabbajack.Common/AsyncParallelExtensions.cs | 15 +++++++ Wabbajack.Installer/AInstaller.cs | 34 +++++++------- Wabbajack.Installer/StandardInstaller.cs | 39 +++++++++------- .../Services/ModListDownloadMaintainer.cs | 4 +- Wabbajack.VFS.Test/HashCacheTest.cs | 8 ++-- Wabbajack.VFS/FileHashCache.cs | 45 +++++++++---------- 8 files changed, 92 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1329a599..4cabe541 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ### Changelog +#### Version - 3.0.1.8 - 10/??/2022 +* Fix broken ZEditMerge code (this stream is not readable) +* Update out-of-date dependencies +* Update CLI to perform lazy initialization of command components (faster startup) +* Fix some status messages during installation +* Optimize the modlist optimizer so runs a bit faster +* Rework the file hash cache so it doesn't block the UI thread + #### Version - 3.0.1.7 - 9/27/2022 * HOTFIX: fix "Could not find part of path" bug related to the profiles folder diff --git a/Wabbajack.CLI/Verbs/DownloadAll.cs b/Wabbajack.CLI/Verbs/DownloadAll.cs index 04a1c29d..3dacba1f 100644 --- a/Wabbajack.CLI/Verbs/DownloadAll.cs +++ b/Wabbajack.CLI/Verbs/DownloadAll.cs @@ -111,7 +111,7 @@ public class DownloadAll : IVerb return; } - _cache.FileHashWriteCache(output, result.Item2); + await _cache.FileHashWriteCache(output, result.Item2); var metaFile = outputFile.WithExtension(Ext.Meta); await metaFile.WriteAllTextAsync(_dispatcher.MetaIniSection(file), token: token); diff --git a/Wabbajack.Common/AsyncParallelExtensions.cs b/Wabbajack.Common/AsyncParallelExtensions.cs index 71ae5032..bab43637 100644 --- a/Wabbajack.Common/AsyncParallelExtensions.cs +++ b/Wabbajack.Common/AsyncParallelExtensions.cs @@ -125,6 +125,21 @@ public static class AsyncParallelExtensions await foreach (var itm in coll) lst.Add(itm); return lst; } + + /// + /// Consumes a IAsyncEnumerable without doing anything with it + /// + /// + /// + public static async Task Sink(this IAsyncEnumerable coll) + { + long count = 0; + await foreach (var itm in coll) + { + count++; + + } + } public static async Task ToArray(this IAsyncEnumerable coll) { diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index cc25929c..ae051cf1 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -232,7 +232,7 @@ public abstract class AInstaller { var file = directive.Directive; UpdateProgress(file.Size); - + var destPath = file.To.RelativeTo(_configuration.Install); switch (file) { case PatchedFromArchive pfa: @@ -240,9 +240,8 @@ public abstract class AInstaller await using var s = await sf.GetStream(); s.Position = 0; await using var patchDataStream = await InlinedFileStream(pfa.PatchID); - var toFile = file.To.RelativeTo(_configuration.Install); { - await using var os = toFile.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); + await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); await BinaryPatching.ApplyPatch(s, patchDataStream, os); } } @@ -252,8 +251,7 @@ public abstract class AInstaller case TransformedTexture tt: { await using var s = await sf.GetStream(); - await using var of = directive.Directive.To.RelativeTo(_configuration.Install) - .Open(FileMode.Create, FileAccess.Write); + await using var of = destPath.Open(FileMode.Create, FileAccess.Write); _logger.LogInformation("Recompressing {Filename}", tt.To.FileName); await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.ImageState.Format, of, token); @@ -264,19 +262,19 @@ public abstract class AInstaller case FromArchive _: if (grouped[vf].Count() == 1) { - await sf.Move(directive.Directive.To.RelativeTo(_configuration.Install), token); + await sf.Move(destPath, token); } else { await using var s = await sf.GetStream(); - await directive.Directive.To.RelativeTo(_configuration.Install) - .WriteAllAsync(s, token, false); + await destPath.WriteAllAsync(s, token, false); } break; default: throw new Exception($"No handler for {directive}"); } + await FileHashCache.FileHashWriteCache(destPath, file.Hash); await job.Report((int) directive.VF.Size, token); } @@ -383,7 +381,7 @@ public abstract class AInstaller } if (hash != default) - FileHashCache.FileHashWriteCache(destination.Value, hash); + await FileHashCache.FileHashWriteCache(destination.Value, hash); if (result == DownloadResult.Update) await destination.Value.MoveToAsync(destination.Value.Parent.Combine(archive.Hash.ToHex()), true, @@ -487,25 +485,26 @@ public abstract class AInstaller NextStep(Consts.StepPreparing, "Looking for files to delete", 0); await _configuration.Install.EnumerateFiles() - .PDoAll(_limiter, async f => + .PMapAllBatched(_limiter, async f => { var relativeTo = f.RelativeTo(_configuration.Install); if (indexed.ContainsKey(relativeTo) || f.InFolder(_configuration.Downloads)) - return ; + return f; - if (f.InFolder(profileFolder) && f.Parent.FileName == savePath) return; + if (f.InFolder(profileFolder) && f.Parent.FileName == savePath) return f; if (NoDeleteRegex.IsMatch(f.ToString())) - return ; + return f; if (bsaPathsToNotBuild.Contains(f)) - return ; + return f; _logger.LogInformation("Deleting {RelativePath} it's not part of this ModList", relativeTo); f.Delete(); - }); + return f; + }).Sink(); - _logger.LogInformation("Cleaning empty folders"); + NextStep(Consts.StepPreparing, "Cleaning empty folders", 0); var expectedFolders = indexed.Keys .Select(f => f.RelativeTo(_configuration.Install)) // We ignore the last part of the path, so we need a dummy file name @@ -542,12 +541,11 @@ public abstract class AInstaller var existingfiles = _configuration.Install.EnumerateFiles().ToHashSet(); NextStep(Consts.StepPreparing, "Looking for unmodified files", 0); - await indexed.Values.PMapAll(async d => + await indexed.Values.PMapAllBatched(_limiter, async d => { // Bit backwards, but we want to return null for // all files we *want* installed. We return the files // to remove from the install list. - using var job = await _limiter.Begin($"Hashing File {d.To}", 0, token); var path = _configuration.Install.Combine(d.To); if (!existingfiles.Contains(path)) return null; diff --git a/Wabbajack.Installer/StandardInstaller.cs b/Wabbajack.Installer/StandardInstaller.cs index 01627dd7..3d14123a 100644 --- a/Wabbajack.Installer/StandardInstaller.cs +++ b/Wabbajack.Installer/StandardInstaller.cs @@ -271,9 +271,11 @@ public class StandardInstaller : AInstaller { var bsas = ModList.Directives.OfType().ToList(); _logger.LogInformation("Building {bsasCount} bsa files", bsas.Count); + NextStep("Installing", "Building BSAs", bsas.Count); foreach (var bsa in bsas) { + UpdateProgress(1); _logger.LogInformation("Building {bsaTo}", bsa.To.FileName); var sourceDir = _configuration.Install.Combine(BSACreationDir, bsa.TempID); @@ -295,9 +297,7 @@ public class StandardInstaller : AInstaller await a.Build(outStream, token); streams.Do(s => s.Dispose()); - FileHashCache.FileHashWriteCache(outPath, bsa.Hash); - - + await FileHashCache.FileHashWriteCache(outPath, bsa.Hash); sourceDir.DeleteDirectory(); } @@ -325,9 +325,11 @@ public class StandardInstaller : AInstaller { case RemappedInlineFile file: await WriteRemappedFile(file); + await FileHashCache.FileHashCachedAsync(outPath, token); break; default: await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID), token); + await FileHashCache.FileHashWriteCache(outPath, directive.Hash); break; } }); @@ -453,24 +455,29 @@ public class StandardInstaller : AInstaller public async Task GenerateZEditMerges(CancellationToken token) { - await _configuration.ModList + var patches = _configuration.ModList .Directives .OfType() - .PDoAll(async m => - { - _logger.LogInformation("Generating zEdit merge: {to}", m.To); + .ToList(); + NextStep("Installing", "Generating ZEdit Merges", patches.Count); - var srcData = (await m.Sources.SelectAsync(async s => - await _configuration.Install.Combine(s.RelativePath).ReadAllBytesAsync(token)) - .ToReadOnlyCollection()) - .ConcatArrays(); + await patches.PMapAllBatched(_limiter, async m => + { + UpdateProgress(1); + _logger.LogInformation("Generating zEdit merge: {to}", m.To); - var patchData = await LoadBytesFromPath(m.PatchID); + var srcData = (await m.Sources.SelectAsync(async s => + await _configuration.Install.Combine(s.RelativePath).ReadAllBytesAsync(token)) + .ToReadOnlyCollection()) + .ConcatArrays(); - await using var fs = _configuration.Install.Combine(m.To) - .Open(FileMode.Create, FileAccess.Write, FileShare.None); - await BinaryPatching.ApplyPatch(new MemoryStream(srcData), new MemoryStream(patchData), fs); - }); + var patchData = await LoadBytesFromPath(m.PatchID); + + await using var fs = _configuration.Install.Combine(m.To) + .Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); + await BinaryPatching.ApplyPatch(new MemoryStream(srcData), new MemoryStream(patchData), fs); + return m; + }).ToList(); } public static async Task Load(DTOSerializer dtos, DownloadDispatcher dispatcher, ModlistMetadata metadata, CancellationToken token) diff --git a/Wabbajack.Services.OSIntegrated/Services/ModListDownloadMaintainer.cs b/Wabbajack.Services.OSIntegrated/Services/ModListDownloadMaintainer.cs index 323a96be..31e8a900 100644 --- a/Wabbajack.Services.OSIntegrated/Services/ModListDownloadMaintainer.cs +++ b/Wabbajack.Services.OSIntegrated/Services/ModListDownloadMaintainer.cs @@ -48,7 +48,7 @@ public class ModListDownloadMaintainer var path = ModListPath(metadata); if (!path.FileExists()) return false; - if (_hashCache.TryGetHashCache(path, out var hash) && hash == metadata.DownloadMetadata!.Hash) return true; + if (await _hashCache.TryGetHashCache(path) == metadata.DownloadMetadata!.Hash) return true; if (_downloadingCount > 0) return false; return await _hashCache.FileHashCachedAsync(path, token.Value) == metadata.DownloadMetadata!.Hash; @@ -80,7 +80,7 @@ public class ModListDownloadMaintainer Hash = metadata.DownloadMetadata.Hash }, path, job, token.Value); - _hashCache.FileHashWriteCache(path, hash); + await _hashCache.FileHashWriteCache(path, hash); await path.WithExtension(Ext.MetaData).WriteAllTextAsync(JsonSerializer.Serialize(metadata)); } finally diff --git a/Wabbajack.VFS.Test/HashCacheTest.cs b/Wabbajack.VFS.Test/HashCacheTest.cs index e1a440d0..b703ad62 100644 --- a/Wabbajack.VFS.Test/HashCacheTest.cs +++ b/Wabbajack.VFS.Test/HashCacheTest.cs @@ -26,13 +26,15 @@ public class HashCacheTest Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await _cache.FileHashCachedAsync(testFile.Path, CancellationToken.None)); - Assert.True(_cache.TryGetHashCache(testFile.Path, out var hash)); + Assert.True(await _cache.TryGetHashCache(testFile.Path) != default); _cache.Purge(testFile.Path); - Assert.False(_cache.TryGetHashCache(testFile.Path, out _)); + var hash = await testFile.Path.Hash(CancellationToken.None); + Assert.NotEqual(hash, default); + Assert.NotEqual(hash, await _cache.TryGetHashCache(testFile.Path)); Assert.Equal(hash, await _cache.FileHashCachedAsync(testFile.Path, CancellationToken.None)); - Assert.True(_cache.TryGetHashCache(testFile.Path, out _)); + Assert.Equal(hash, await _cache.TryGetHashCache(testFile.Path)); _cache.VacuumDatabase(); } diff --git a/Wabbajack.VFS/FileHashCache.cs b/Wabbajack.VFS/FileHashCache.cs index ef141398..64dfd90c 100644 --- a/Wabbajack.VFS/FileHashCache.cs +++ b/Wabbajack.VFS/FileHashCache.cs @@ -39,15 +39,15 @@ public class FileHashCache cmd.ExecuteNonQuery(); } - private (AbsolutePath Path, long LastModified, Hash Hash) Get(AbsolutePath path) + private async Task<(AbsolutePath Path, long LastModified, Hash Hash)> Get(AbsolutePath path) { using var cmd = new SQLiteCommand(_conn); cmd.CommandText = "SELECT LastModified, Hash FROM HashCache WHERE Path = @path"; cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant()); - cmd.PrepareAsync(); + await cmd.PrepareAsync(); - using var reader = cmd.ExecuteReader(); - while (reader.Read()) return (path, reader.GetInt64(0), Hash.FromLong(reader.GetInt64(1))); + await using var reader = await cmd.ExecuteReaderAsync(); + while (await reader.ReadAsync()) return (path, reader.GetInt64(0), Hash.FromLong(reader.GetInt64(1))); return default; } @@ -62,17 +62,17 @@ public class FileHashCache cmd.ExecuteNonQuery(); } - private void Upsert(AbsolutePath path, long lastModified, Hash hash) + private async Task Upsert(AbsolutePath path, long lastModified, Hash hash) { - using var cmd = new SQLiteCommand(_conn); + await using var cmd = new SQLiteCommand(_conn); cmd.CommandText = @"INSERT INTO HashCache (Path, LastModified, Hash) VALUES (@path, @lastModified, @hash) ON CONFLICT(Path) DO UPDATE SET LastModified = @lastModified, Hash = @hash"; cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant()); cmd.Parameters.AddWithValue("@lastModified", lastModified); cmd.Parameters.AddWithValue("@hash", (long) hash); - cmd.PrepareAsync(); + await cmd.PrepareAsync(); - cmd.ExecuteNonQuery(); + await cmd.ExecuteNonQueryAsync(); } public void VacuumDatabase() @@ -84,46 +84,45 @@ public class FileHashCache cmd.ExecuteNonQuery(); } - public bool TryGetHashCache(AbsolutePath file, out Hash hash) + public async Task TryGetHashCache(AbsolutePath file) { - hash = default; - if (!file.FileExists()) return false; + if (!file.FileExists()) return default; - var result = Get(file); + var result = await Get(file); if (result == default || result.Hash == default) - return false; + return default; if (result.LastModified == file.LastModifiedUtc().ToFileTimeUtc()) { - hash = result.Hash; - return true; + return result.Hash; } Purge(file); - return false; + return default; } - private void WriteHashCache(AbsolutePath file, Hash hash) + private async Task WriteHashCache(AbsolutePath file, Hash hash) { if (!file.FileExists()) return; - Upsert(file, file.LastModifiedUtc().ToFileTimeUtc(), hash); + await Upsert(file, file.LastModifiedUtc().ToFileTimeUtc(), hash); } - public void FileHashWriteCache(AbsolutePath file, Hash hash) + public async Task FileHashWriteCache(AbsolutePath file, Hash hash) { - WriteHashCache(file, hash); + await WriteHashCache(file, hash); } public async Task FileHashCachedAsync(AbsolutePath file, CancellationToken token) { - if (TryGetHashCache(file, out var foundHash)) return foundHash; + var hash = await TryGetHashCache(file); + if (hash != default) return hash; using var job = await _limiter.Begin($"Hashing {file.FileName}", file.Size(), token); await using var fs = file.Open(FileMode.Open, FileAccess.Read, FileShare.Read); - var hash = await fs.HashingCopy(Stream.Null, token, job); + hash = await fs.HashingCopy(Stream.Null, token, job); if (hash != default) - WriteHashCache(file, hash); + await WriteHashCache(file, hash); return hash; } } \ No newline at end of file