diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1329a599..4cabe541 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,13 @@
### Changelog
+#### Version - 3.0.1.8 - 10/??/2022
+* Fix broken ZEditMerge code (this stream is not readable)
+* Update out-of-date dependencies
+* Update CLI to perform lazy initialization of command components (faster startup)
+* Fix some status messages during installation
+* Optimize the modlist optimizer so runs a bit faster
+* Rework the file hash cache so it doesn't block the UI thread
+
#### Version - 3.0.1.7 - 9/27/2022
* HOTFIX: fix "Could not find part of path" bug related to the profiles folder
diff --git a/Wabbajack.CLI/Verbs/DownloadAll.cs b/Wabbajack.CLI/Verbs/DownloadAll.cs
index 04a1c29d..3dacba1f 100644
--- a/Wabbajack.CLI/Verbs/DownloadAll.cs
+++ b/Wabbajack.CLI/Verbs/DownloadAll.cs
@@ -111,7 +111,7 @@ public class DownloadAll : IVerb
return;
}
- _cache.FileHashWriteCache(output, result.Item2);
+ await _cache.FileHashWriteCache(output, result.Item2);
var metaFile = outputFile.WithExtension(Ext.Meta);
await metaFile.WriteAllTextAsync(_dispatcher.MetaIniSection(file), token: token);
diff --git a/Wabbajack.Common/AsyncParallelExtensions.cs b/Wabbajack.Common/AsyncParallelExtensions.cs
index 71ae5032..bab43637 100644
--- a/Wabbajack.Common/AsyncParallelExtensions.cs
+++ b/Wabbajack.Common/AsyncParallelExtensions.cs
@@ -125,6 +125,21 @@ public static class AsyncParallelExtensions
await foreach (var itm in coll) lst.Add(itm);
return lst;
}
+
+ ///
+ /// Consumes a IAsyncEnumerable without doing anything with it
+ ///
+ ///
+ ///
+ public static async Task Sink(this IAsyncEnumerable coll)
+ {
+ long count = 0;
+ await foreach (var itm in coll)
+ {
+ count++;
+
+ }
+ }
public static async Task ToArray(this IAsyncEnumerable coll)
{
diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs
index cc25929c..ae051cf1 100644
--- a/Wabbajack.Installer/AInstaller.cs
+++ b/Wabbajack.Installer/AInstaller.cs
@@ -232,7 +232,7 @@ public abstract class AInstaller
{
var file = directive.Directive;
UpdateProgress(file.Size);
-
+ var destPath = file.To.RelativeTo(_configuration.Install);
switch (file)
{
case PatchedFromArchive pfa:
@@ -240,9 +240,8 @@ public abstract class AInstaller
await using var s = await sf.GetStream();
s.Position = 0;
await using var patchDataStream = await InlinedFileStream(pfa.PatchID);
- var toFile = file.To.RelativeTo(_configuration.Install);
{
- await using var os = toFile.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None);
+ await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None);
await BinaryPatching.ApplyPatch(s, patchDataStream, os);
}
}
@@ -252,8 +251,7 @@ public abstract class AInstaller
case TransformedTexture tt:
{
await using var s = await sf.GetStream();
- await using var of = directive.Directive.To.RelativeTo(_configuration.Install)
- .Open(FileMode.Create, FileAccess.Write);
+ await using var of = destPath.Open(FileMode.Create, FileAccess.Write);
_logger.LogInformation("Recompressing {Filename}", tt.To.FileName);
await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.ImageState.Format,
of, token);
@@ -264,19 +262,19 @@ public abstract class AInstaller
case FromArchive _:
if (grouped[vf].Count() == 1)
{
- await sf.Move(directive.Directive.To.RelativeTo(_configuration.Install), token);
+ await sf.Move(destPath, token);
}
else
{
await using var s = await sf.GetStream();
- await directive.Directive.To.RelativeTo(_configuration.Install)
- .WriteAllAsync(s, token, false);
+ await destPath.WriteAllAsync(s, token, false);
}
break;
default:
throw new Exception($"No handler for {directive}");
}
+ await FileHashCache.FileHashWriteCache(destPath, file.Hash);
await job.Report((int) directive.VF.Size, token);
}
@@ -383,7 +381,7 @@ public abstract class AInstaller
}
if (hash != default)
- FileHashCache.FileHashWriteCache(destination.Value, hash);
+ await FileHashCache.FileHashWriteCache(destination.Value, hash);
if (result == DownloadResult.Update)
await destination.Value.MoveToAsync(destination.Value.Parent.Combine(archive.Hash.ToHex()), true,
@@ -487,25 +485,26 @@ public abstract class AInstaller
NextStep(Consts.StepPreparing, "Looking for files to delete", 0);
await _configuration.Install.EnumerateFiles()
- .PDoAll(_limiter, async f =>
+ .PMapAllBatched(_limiter, async f =>
{
var relativeTo = f.RelativeTo(_configuration.Install);
if (indexed.ContainsKey(relativeTo) || f.InFolder(_configuration.Downloads))
- return ;
+ return f;
- if (f.InFolder(profileFolder) && f.Parent.FileName == savePath) return;
+ if (f.InFolder(profileFolder) && f.Parent.FileName == savePath) return f;
if (NoDeleteRegex.IsMatch(f.ToString()))
- return ;
+ return f;
if (bsaPathsToNotBuild.Contains(f))
- return ;
+ return f;
_logger.LogInformation("Deleting {RelativePath} it's not part of this ModList", relativeTo);
f.Delete();
- });
+ return f;
+ }).Sink();
- _logger.LogInformation("Cleaning empty folders");
+ NextStep(Consts.StepPreparing, "Cleaning empty folders", 0);
var expectedFolders = indexed.Keys
.Select(f => f.RelativeTo(_configuration.Install))
// We ignore the last part of the path, so we need a dummy file name
@@ -542,12 +541,11 @@ public abstract class AInstaller
var existingfiles = _configuration.Install.EnumerateFiles().ToHashSet();
NextStep(Consts.StepPreparing, "Looking for unmodified files", 0);
- await indexed.Values.PMapAll(async d =>
+ await indexed.Values.PMapAllBatched(_limiter, async d =>
{
// Bit backwards, but we want to return null for
// all files we *want* installed. We return the files
// to remove from the install list.
- using var job = await _limiter.Begin($"Hashing File {d.To}", 0, token);
var path = _configuration.Install.Combine(d.To);
if (!existingfiles.Contains(path)) return null;
diff --git a/Wabbajack.Installer/StandardInstaller.cs b/Wabbajack.Installer/StandardInstaller.cs
index 01627dd7..3d14123a 100644
--- a/Wabbajack.Installer/StandardInstaller.cs
+++ b/Wabbajack.Installer/StandardInstaller.cs
@@ -271,9 +271,11 @@ public class StandardInstaller : AInstaller
{
var bsas = ModList.Directives.OfType().ToList();
_logger.LogInformation("Building {bsasCount} bsa files", bsas.Count);
+ NextStep("Installing", "Building BSAs", bsas.Count);
foreach (var bsa in bsas)
{
+ UpdateProgress(1);
_logger.LogInformation("Building {bsaTo}", bsa.To.FileName);
var sourceDir = _configuration.Install.Combine(BSACreationDir, bsa.TempID);
@@ -295,9 +297,7 @@ public class StandardInstaller : AInstaller
await a.Build(outStream, token);
streams.Do(s => s.Dispose());
- FileHashCache.FileHashWriteCache(outPath, bsa.Hash);
-
-
+ await FileHashCache.FileHashWriteCache(outPath, bsa.Hash);
sourceDir.DeleteDirectory();
}
@@ -325,9 +325,11 @@ public class StandardInstaller : AInstaller
{
case RemappedInlineFile file:
await WriteRemappedFile(file);
+ await FileHashCache.FileHashCachedAsync(outPath, token);
break;
default:
await outPath.WriteAllBytesAsync(await LoadBytesFromPath(directive.SourceDataID), token);
+ await FileHashCache.FileHashWriteCache(outPath, directive.Hash);
break;
}
});
@@ -453,24 +455,29 @@ public class StandardInstaller : AInstaller
public async Task GenerateZEditMerges(CancellationToken token)
{
- await _configuration.ModList
+ var patches = _configuration.ModList
.Directives
.OfType()
- .PDoAll(async m =>
- {
- _logger.LogInformation("Generating zEdit merge: {to}", m.To);
+ .ToList();
+ NextStep("Installing", "Generating ZEdit Merges", patches.Count);
- var srcData = (await m.Sources.SelectAsync(async s =>
- await _configuration.Install.Combine(s.RelativePath).ReadAllBytesAsync(token))
- .ToReadOnlyCollection())
- .ConcatArrays();
+ await patches.PMapAllBatched(_limiter, async m =>
+ {
+ UpdateProgress(1);
+ _logger.LogInformation("Generating zEdit merge: {to}", m.To);
- var patchData = await LoadBytesFromPath(m.PatchID);
+ var srcData = (await m.Sources.SelectAsync(async s =>
+ await _configuration.Install.Combine(s.RelativePath).ReadAllBytesAsync(token))
+ .ToReadOnlyCollection())
+ .ConcatArrays();
- await using var fs = _configuration.Install.Combine(m.To)
- .Open(FileMode.Create, FileAccess.Write, FileShare.None);
- await BinaryPatching.ApplyPatch(new MemoryStream(srcData), new MemoryStream(patchData), fs);
- });
+ var patchData = await LoadBytesFromPath(m.PatchID);
+
+ await using var fs = _configuration.Install.Combine(m.To)
+ .Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None);
+ await BinaryPatching.ApplyPatch(new MemoryStream(srcData), new MemoryStream(patchData), fs);
+ return m;
+ }).ToList();
}
public static async Task Load(DTOSerializer dtos, DownloadDispatcher dispatcher, ModlistMetadata metadata, CancellationToken token)
diff --git a/Wabbajack.Services.OSIntegrated/Services/ModListDownloadMaintainer.cs b/Wabbajack.Services.OSIntegrated/Services/ModListDownloadMaintainer.cs
index 323a96be..31e8a900 100644
--- a/Wabbajack.Services.OSIntegrated/Services/ModListDownloadMaintainer.cs
+++ b/Wabbajack.Services.OSIntegrated/Services/ModListDownloadMaintainer.cs
@@ -48,7 +48,7 @@ public class ModListDownloadMaintainer
var path = ModListPath(metadata);
if (!path.FileExists()) return false;
- if (_hashCache.TryGetHashCache(path, out var hash) && hash == metadata.DownloadMetadata!.Hash) return true;
+ if (await _hashCache.TryGetHashCache(path) == metadata.DownloadMetadata!.Hash) return true;
if (_downloadingCount > 0) return false;
return await _hashCache.FileHashCachedAsync(path, token.Value) == metadata.DownloadMetadata!.Hash;
@@ -80,7 +80,7 @@ public class ModListDownloadMaintainer
Hash = metadata.DownloadMetadata.Hash
}, path, job, token.Value);
- _hashCache.FileHashWriteCache(path, hash);
+ await _hashCache.FileHashWriteCache(path, hash);
await path.WithExtension(Ext.MetaData).WriteAllTextAsync(JsonSerializer.Serialize(metadata));
}
finally
diff --git a/Wabbajack.VFS.Test/HashCacheTest.cs b/Wabbajack.VFS.Test/HashCacheTest.cs
index e1a440d0..b703ad62 100644
--- a/Wabbajack.VFS.Test/HashCacheTest.cs
+++ b/Wabbajack.VFS.Test/HashCacheTest.cs
@@ -26,13 +26,15 @@ public class HashCacheTest
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="),
await _cache.FileHashCachedAsync(testFile.Path, CancellationToken.None));
- Assert.True(_cache.TryGetHashCache(testFile.Path, out var hash));
+ Assert.True(await _cache.TryGetHashCache(testFile.Path) != default);
_cache.Purge(testFile.Path);
- Assert.False(_cache.TryGetHashCache(testFile.Path, out _));
+ var hash = await testFile.Path.Hash(CancellationToken.None);
+ Assert.NotEqual(hash, default);
+ Assert.NotEqual(hash, await _cache.TryGetHashCache(testFile.Path));
Assert.Equal(hash, await _cache.FileHashCachedAsync(testFile.Path, CancellationToken.None));
- Assert.True(_cache.TryGetHashCache(testFile.Path, out _));
+ Assert.Equal(hash, await _cache.TryGetHashCache(testFile.Path));
_cache.VacuumDatabase();
}
diff --git a/Wabbajack.VFS/FileHashCache.cs b/Wabbajack.VFS/FileHashCache.cs
index ef141398..64dfd90c 100644
--- a/Wabbajack.VFS/FileHashCache.cs
+++ b/Wabbajack.VFS/FileHashCache.cs
@@ -39,15 +39,15 @@ public class FileHashCache
cmd.ExecuteNonQuery();
}
- private (AbsolutePath Path, long LastModified, Hash Hash) Get(AbsolutePath path)
+ private async Task<(AbsolutePath Path, long LastModified, Hash Hash)> Get(AbsolutePath path)
{
using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = "SELECT LastModified, Hash FROM HashCache WHERE Path = @path";
cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant());
- cmd.PrepareAsync();
+ await cmd.PrepareAsync();
- using var reader = cmd.ExecuteReader();
- while (reader.Read()) return (path, reader.GetInt64(0), Hash.FromLong(reader.GetInt64(1)));
+ await using var reader = await cmd.ExecuteReaderAsync();
+ while (await reader.ReadAsync()) return (path, reader.GetInt64(0), Hash.FromLong(reader.GetInt64(1)));
return default;
}
@@ -62,17 +62,17 @@ public class FileHashCache
cmd.ExecuteNonQuery();
}
- private void Upsert(AbsolutePath path, long lastModified, Hash hash)
+ private async Task Upsert(AbsolutePath path, long lastModified, Hash hash)
{
- using var cmd = new SQLiteCommand(_conn);
+ await using var cmd = new SQLiteCommand(_conn);
cmd.CommandText = @"INSERT INTO HashCache (Path, LastModified, Hash) VALUES (@path, @lastModified, @hash)
ON CONFLICT(Path) DO UPDATE SET LastModified = @lastModified, Hash = @hash";
cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant());
cmd.Parameters.AddWithValue("@lastModified", lastModified);
cmd.Parameters.AddWithValue("@hash", (long) hash);
- cmd.PrepareAsync();
+ await cmd.PrepareAsync();
- cmd.ExecuteNonQuery();
+ await cmd.ExecuteNonQueryAsync();
}
public void VacuumDatabase()
@@ -84,46 +84,45 @@ public class FileHashCache
cmd.ExecuteNonQuery();
}
- public bool TryGetHashCache(AbsolutePath file, out Hash hash)
+ public async Task TryGetHashCache(AbsolutePath file)
{
- hash = default;
- if (!file.FileExists()) return false;
+ if (!file.FileExists()) return default;
- var result = Get(file);
+ var result = await Get(file);
if (result == default || result.Hash == default)
- return false;
+ return default;
if (result.LastModified == file.LastModifiedUtc().ToFileTimeUtc())
{
- hash = result.Hash;
- return true;
+ return result.Hash;
}
Purge(file);
- return false;
+ return default;
}
- private void WriteHashCache(AbsolutePath file, Hash hash)
+ private async Task WriteHashCache(AbsolutePath file, Hash hash)
{
if (!file.FileExists()) return;
- Upsert(file, file.LastModifiedUtc().ToFileTimeUtc(), hash);
+ await Upsert(file, file.LastModifiedUtc().ToFileTimeUtc(), hash);
}
- public void FileHashWriteCache(AbsolutePath file, Hash hash)
+ public async Task FileHashWriteCache(AbsolutePath file, Hash hash)
{
- WriteHashCache(file, hash);
+ await WriteHashCache(file, hash);
}
public async Task FileHashCachedAsync(AbsolutePath file, CancellationToken token)
{
- if (TryGetHashCache(file, out var foundHash)) return foundHash;
+ var hash = await TryGetHashCache(file);
+ if (hash != default) return hash;
using var job = await _limiter.Begin($"Hashing {file.FileName}", file.Size(), token);
await using var fs = file.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
- var hash = await fs.HashingCopy(Stream.Null, token, job);
+ hash = await fs.HashingCopy(Stream.Null, token, job);
if (hash != default)
- WriteHashCache(file, hash);
+ await WriteHashCache(file, hash);
return hash;
}
}
\ No newline at end of file