From 7f7d7c0703a1b961980153e7e19480db6267d53a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 22 Aug 2022 09:34:19 -0600 Subject: [PATCH] Several more optimizations --- Wabbajack.Common/AsyncParallelExtensions.cs | 5 ++- Wabbajack.Compiler.Test/Startup.cs | 26 +++++++++++++ .../CompilationSteps/IgnoreEndsWith.cs | 6 +-- Wabbajack.Paths/AbsolutePath.cs | 6 ++- Wabbajack.Paths/RelativePath.cs | 2 +- .../ServiceExtensions.cs | 2 +- Wabbajack.VFS/Context.cs | 39 ++----------------- Wabbajack.VFS/FileHashCache.cs | 6 +-- Wabbajack.VFS/VFSCache.cs | 4 +- Wabbajack.VFS/Wabbajack.VFS.csproj | 1 + 10 files changed, 48 insertions(+), 49 deletions(-) diff --git a/Wabbajack.Common/AsyncParallelExtensions.cs b/Wabbajack.Common/AsyncParallelExtensions.cs index 418f60a7..260b366b 100644 --- a/Wabbajack.Common/AsyncParallelExtensions.cs +++ b/Wabbajack.Common/AsyncParallelExtensions.cs @@ -29,7 +29,10 @@ public static class AsyncParallelExtensions public static async IAsyncEnumerable PMapAll(this IEnumerable coll, Func> mapFn) { - var tasks = coll.Select(mapFn).ToList(); + var tasks = coll.Select(async x => + { + return await Task.Run(() => mapFn(x)); + }).ToList(); foreach (var itm in tasks) yield return await itm; } diff --git a/Wabbajack.Compiler.Test/Startup.cs b/Wabbajack.Compiler.Test/Startup.cs index b4b0edfb..7b001891 100644 --- a/Wabbajack.Compiler.Test/Startup.cs +++ b/Wabbajack.Compiler.Test/Startup.cs @@ -1,5 +1,8 @@ +using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Wabbajack.DTOs.Interventions; +using Wabbajack.Networking.Steam.UserInterventions; using Wabbajack.Networking.WabbajackClientApi; using Wabbajack.Services.OSIntegrated; using Xunit.DependencyInjection; @@ -19,10 +22,33 @@ public class Startup }); service.AddScoped(); + service.AddSingleton(); } public void Configure(ILoggerFactory loggerFactory, ITestOutputHelperAccessor accessor) { loggerFactory.AddProvider(new XunitTestOutputLoggerProvider(accessor, delegate { return true; })); } + + public class UserInterventionHandler : IUserInterventionHandler + { + public void Raise(IUserIntervention intervention) + { + if (intervention is GetAuthCode gac) + { + switch (gac.Type) + { + case GetAuthCode.AuthType.EmailCode: + Console.WriteLine("Please enter the Steam code that was just emailed to you"); + break; + case GetAuthCode.AuthType.TwoFactorAuth: + Console.WriteLine("Please enter your 2FA code for Steam"); + break; + default: + throw new ArgumentOutOfRangeException(); + } + gac.Finish(Console.ReadLine()!.Trim()); + } + } + } } \ No newline at end of file diff --git a/Wabbajack.Compiler/CompilationSteps/IgnoreEndsWith.cs b/Wabbajack.Compiler/CompilationSteps/IgnoreEndsWith.cs index 0d728922..9f8fbdf3 100644 --- a/Wabbajack.Compiler/CompilationSteps/IgnoreEndsWith.cs +++ b/Wabbajack.Compiler/CompilationSteps/IgnoreEndsWith.cs @@ -7,18 +7,18 @@ namespace Wabbajack.Compiler.CompilationSteps; public class IgnoreFilename : ACompilationStep { - private readonly RelativePath _postfix; + private readonly string _postfix; private readonly string _reason; public IgnoreFilename(ACompiler compiler, RelativePath postfix) : base(compiler) { - _postfix = postfix; + _postfix = postfix.FileName.ToString(); _reason = $"Ignored because path ends with {postfix}"; } public override async ValueTask Run(RawSourceFile source) { - if (source.Path.FileName != _postfix.FileName) return null; + if (!source.Path.EndsWith(_postfix)) return null; var result = source.EvolveTo(); result.Reason = _reason; return result; diff --git a/Wabbajack.Paths/AbsolutePath.cs b/Wabbajack.Paths/AbsolutePath.cs index e3905cc0..04369eba 100644 --- a/Wabbajack.Paths/AbsolutePath.cs +++ b/Wabbajack.Paths/AbsolutePath.cs @@ -95,8 +95,10 @@ public struct AbsolutePath : IPath, IComparable, IEquatable current ^ part.GetHashCode(StringComparison.CurrentCultureIgnoreCase)); + var result = 0; + foreach (var part in Parts) + result = result ^ part.GetHashCode(StringComparison.CurrentCultureIgnoreCase); + _hashCode = result; return _hashCode; } diff --git a/Wabbajack.Paths/RelativePath.cs b/Wabbajack.Paths/RelativePath.cs index e9cabbe5..31452a0b 100644 --- a/Wabbajack.Paths/RelativePath.cs +++ b/Wabbajack.Paths/RelativePath.cs @@ -175,7 +175,7 @@ public struct RelativePath : IPath, IEquatable, IComparable Parts[^1][..Extension.ToString().Length].ToRelativePath(); public int Level => Parts.Length; - public bool EndsWith(string postfix) + public readonly bool EndsWith(string postfix) { return Parts[^1].EndsWith(postfix); } diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index b29c6b3a..dc3f28f9 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -70,7 +70,7 @@ public static class ServiceExtensions service.AddSingleton(s => options.UseLocalCache ? new BinaryPatchCache(s.GetService()!.CreateFile().Path) - : new BinaryPatchCache(KnownFolders.EntryPoint.Combine("patchCache.sqlite"))); + : new BinaryPatchCache(KnownFolders.WabbajackAppLocal.Combine("patchCache.sqlite"))); service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount}); diff --git a/Wabbajack.VFS/Context.cs b/Wabbajack.VFS/Context.cs index c2762d04..cbea822d 100644 --- a/Wabbajack.VFS/Context.cs +++ b/Wabbajack.VFS/Context.cs @@ -55,43 +55,10 @@ public class Context public async Task AddRoot(AbsolutePath root, CancellationToken token) { - var filtered = Index.AllFiles.Where(file => file.IsNative && ((AbsolutePath) file.Name).FileExists()) - .ToList(); - - var byPath = filtered.ToDictionary(f => f.Name); - - var filesToIndex = root.EnumerateFiles().Distinct().ToList(); - - var allFiles = await filesToIndex - .PMapAll(async f => - { - using var job = await Limiter.Begin($"Analyzing {f}", 0, token); - if (byPath.TryGetValue(f, out var found)) - if (found.LastModified == f.LastModifiedUtc().AsUnixTime() && found.Size == f.Size()) - return found; - - try - { - return await VirtualFile.Analyze(this, null, new NativeFileStreamFactory(f), f, token, job: job); - } - catch (Exception ex) - { - Logger.LogError(ex, "While analyzing {File}", f); - throw; - } - }).ToList(); - - var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList()); - - lock (this) - { - Index = newIndex; - } - - return newIndex; + return await AddRoots(new[] {root}, token); } - public async Task AddRoots(List roots, CancellationToken token, Func? updateFunction = null) + public async Task AddRoots(IEnumerable roots, CancellationToken token, Func? updateFunction = null) { var native = Index.AllFiles.Where(file => file.IsNative).ToDictionary(file => file.FullPath.Base); @@ -120,7 +87,7 @@ public class Context Index = newIndex; } - VfsCache.Clean(); + await VfsCache.Clean(); return newIndex; } diff --git a/Wabbajack.VFS/FileHashCache.cs b/Wabbajack.VFS/FileHashCache.cs index d62038ee..ef141398 100644 --- a/Wabbajack.VFS/FileHashCache.cs +++ b/Wabbajack.VFS/FileHashCache.cs @@ -43,7 +43,7 @@ public class FileHashCache { using var cmd = new SQLiteCommand(_conn); cmd.CommandText = "SELECT LastModified, Hash FROM HashCache WHERE Path = @path"; - cmd.Parameters.AddWithValue("@path", path.ToString()); + cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant()); cmd.PrepareAsync(); using var reader = cmd.ExecuteReader(); @@ -56,7 +56,7 @@ public class FileHashCache { using var cmd = new SQLiteCommand(_conn); cmd.CommandText = "DELETE FROM HashCache WHERE Path = @path"; - cmd.Parameters.AddWithValue("@path", path.ToString()); + cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant()); cmd.PrepareAsync(); cmd.ExecuteNonQuery(); @@ -67,7 +67,7 @@ public class FileHashCache using var cmd = new SQLiteCommand(_conn); cmd.CommandText = @"INSERT INTO HashCache (Path, LastModified, Hash) VALUES (@path, @lastModified, @hash) ON CONFLICT(Path) DO UPDATE SET LastModified = @lastModified, Hash = @hash"; - cmd.Parameters.AddWithValue("@path", path.ToString()); + cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant()); cmd.Parameters.AddWithValue("@lastModified", lastModified); cmd.Parameters.AddWithValue("@hash", (long) hash); cmd.PrepareAsync(); diff --git a/Wabbajack.VFS/VFSCache.cs b/Wabbajack.VFS/VFSCache.cs index 557cc1fe..421a1ffd 100644 --- a/Wabbajack.VFS/VFSCache.cs +++ b/Wabbajack.VFS/VFSCache.cs @@ -50,8 +50,8 @@ public class VFSDiskCache : IVfsCache cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash"; cmd.Parameters.AddWithValue("@hash", (long) hash); - await using var rdr = cmd.ExecuteReader(); - while (rdr.Read()) + await using var rdr = await cmd.ExecuteReaderAsync(token); + while (await rdr.ReadAsync(token)) { var data = IndexedVirtualFileExtensions.Read(rdr.GetStream(0)); return data; diff --git a/Wabbajack.VFS/Wabbajack.VFS.csproj b/Wabbajack.VFS/Wabbajack.VFS.csproj index 0c506591..4107738d 100644 --- a/Wabbajack.VFS/Wabbajack.VFS.csproj +++ b/Wabbajack.VFS/Wabbajack.VFS.csproj @@ -9,6 +9,7 @@ +