Several more optimizations

Wabbajack.Common/AsyncParallelExtensions.cs

@@ -29,7 +29,10 @@ public static class AsyncParallelExtensions
     public static async IAsyncEnumerable<TOut> PMapAll<TIn, TOut>(this IEnumerable<TIn> coll,
         Func<TIn, Task<TOut>> mapFn)
     {
-        var tasks = coll.Select(mapFn).ToList();
+        var tasks = coll.Select(async x =>
+        {
+            return await Task.Run(() => mapFn(x));
+        }).ToList();
         foreach (var itm in tasks) yield return await itm;
     }
     

Wabbajack.Compiler.Test/Startup.cs

@@ -1,5 +1,8 @@
+using System;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Wabbajack.DTOs.Interventions;
+using Wabbajack.Networking.Steam.UserInterventions;
 using Wabbajack.Networking.WabbajackClientApi;
 using Wabbajack.Services.OSIntegrated;
 using Xunit.DependencyInjection;
@@ -19,10 +22,33 @@ public class Startup
         });
 
         service.AddScoped<ModListHarness>();
+        service.AddSingleton<IUserInterventionHandler, UserInterventionHandler>();
     }
 
     public void Configure(ILoggerFactory loggerFactory, ITestOutputHelperAccessor accessor)
     {
         loggerFactory.AddProvider(new XunitTestOutputLoggerProvider(accessor, delegate { return true; }));
     }
+    
+    public class UserInterventionHandler : IUserInterventionHandler
+    {
+        public void Raise(IUserIntervention intervention)
+        {
+            if (intervention is GetAuthCode gac)
+            {
+                switch (gac.Type)
+                {
+                    case GetAuthCode.AuthType.EmailCode:
+                        Console.WriteLine("Please enter the Steam code that was just emailed to you");
+                        break;
+                    case GetAuthCode.AuthType.TwoFactorAuth:
+                        Console.WriteLine("Please enter your 2FA code for Steam");
+                        break;
+                    default:
+                        throw new ArgumentOutOfRangeException();
+                }
+                gac.Finish(Console.ReadLine()!.Trim());
+            }
+        }
+    }
 }

Wabbajack.Compiler/CompilationSteps/IgnoreEndsWith.cs

@@ -7,18 +7,18 @@ namespace Wabbajack.Compiler.CompilationSteps;
 
 public class IgnoreFilename : ACompilationStep
 {
-    private readonly RelativePath _postfix;
+    private readonly string _postfix;
     private readonly string _reason;
 
     public IgnoreFilename(ACompiler compiler, RelativePath postfix) : base(compiler)
     {
-        _postfix = postfix;
+        _postfix = postfix.FileName.ToString();
         _reason = $"Ignored because path ends with {postfix}";
     }
 
     public override async ValueTask<Directive?> Run(RawSourceFile source)
     {
-        if (source.Path.FileName != _postfix.FileName) return null;
+        if (!source.Path.EndsWith(_postfix)) return null;
         var result = source.EvolveTo<IgnoredDirectly>();
         result.Reason = _reason;
         return result;

Wabbajack.Paths/AbsolutePath.cs

@@ -95,8 +95,10 @@ public struct AbsolutePath : IPath, IComparable<AbsolutePath>, IEquatable<Absolu
     public override int GetHashCode()
     {
         if (_hashCode != 0) return _hashCode;
-        _hashCode = Parts.Aggregate(0,
+        var result = 0;
-            (current, part) => current ^ part.GetHashCode(StringComparison.CurrentCultureIgnoreCase));
+        foreach (var part in Parts) 
+            result = result ^ part.GetHashCode(StringComparison.CurrentCultureIgnoreCase);
+        _hashCode = result;
         return _hashCode;
     }
 

Wabbajack.Paths/RelativePath.cs

@@ -175,7 +175,7 @@ public struct RelativePath : IPath, IEquatable<RelativePath>, IComparable<Relati
     public RelativePath FileNameWithoutExtension => Parts[^1][..Extension.ToString().Length].ToRelativePath();
     public int Level => Parts.Length;
 
-    public bool EndsWith(string postfix)
+    public readonly bool EndsWith(string postfix)
     {
         return Parts[^1].EndsWith(postfix);
     }

Wabbajack.Services.OSIntegrated/ServiceExtensions.cs

@@ -70,7 +70,7 @@ public static class ServiceExtensions
 
         service.AddSingleton<IBinaryPatchCache>(s => options.UseLocalCache
             ? new BinaryPatchCache(s.GetService<TemporaryFileManager>()!.CreateFile().Path)
-            : new BinaryPatchCache(KnownFolders.EntryPoint.Combine("patchCache.sqlite")));
+            : new BinaryPatchCache(KnownFolders.WabbajackAppLocal.Combine("patchCache.sqlite")));
 
         service.AddSingleton(new ParallelOptions {MaxDegreeOfParallelism = Environment.ProcessorCount});
 

Wabbajack.VFS/Context.cs

@@ -55,43 +55,10 @@ public class Context
 
     public async Task<IndexRoot> AddRoot(AbsolutePath root, CancellationToken token)
     {
-        var filtered = Index.AllFiles.Where(file => file.IsNative && ((AbsolutePath) file.Name).FileExists())
+        return await AddRoots(new[] {root}, token);
-            .ToList();
-
-        var byPath = filtered.ToDictionary(f => f.Name);
-
-        var filesToIndex = root.EnumerateFiles().Distinct().ToList();
-
-        var allFiles = await filesToIndex
-            .PMapAll(async f =>
-            {
-                using var job = await Limiter.Begin($"Analyzing {f}", 0, token);
-                if (byPath.TryGetValue(f, out var found))
-                    if (found.LastModified == f.LastModifiedUtc().AsUnixTime() && found.Size == f.Size())
-                        return found;
-
-                try
-                {
-                    return await VirtualFile.Analyze(this, null, new NativeFileStreamFactory(f), f, token, job: job);
-                }
-                catch (Exception ex)
-                {
-                    Logger.LogError(ex, "While analyzing {File}", f);
-                    throw;
-                }
-            }).ToList();
-
-        var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
-
-        lock (this)
-        {
-            Index = newIndex;
-        }
-
-        return newIndex;
     }
 
-    public async Task<IndexRoot> AddRoots(List<AbsolutePath> roots, CancellationToken token, Func<long, long, Task>? updateFunction = null)
+    public async Task<IndexRoot> AddRoots(IEnumerable<AbsolutePath> roots, CancellationToken token, Func<long, long, Task>? updateFunction = null)
     {
         var native = Index.AllFiles.Where(file => file.IsNative).ToDictionary(file => file.FullPath.Base);
 
@@ -120,7 +87,7 @@ public class Context
             Index = newIndex;
         }
 
-        VfsCache.Clean();
+        await VfsCache.Clean();
 
         return newIndex;
     }

Wabbajack.VFS/FileHashCache.cs

@@ -43,7 +43,7 @@ public class FileHashCache
     {
         using var cmd = new SQLiteCommand(_conn);
         cmd.CommandText = "SELECT LastModified, Hash FROM HashCache WHERE Path = @path";
-        cmd.Parameters.AddWithValue("@path", path.ToString());
+        cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant());
         cmd.PrepareAsync();
 
         using var reader = cmd.ExecuteReader();
@@ -56,7 +56,7 @@ public class FileHashCache
     {
         using var cmd = new SQLiteCommand(_conn);
         cmd.CommandText = "DELETE FROM HashCache WHERE Path = @path";
-        cmd.Parameters.AddWithValue("@path", path.ToString());
+        cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant());
         cmd.PrepareAsync();
 
         cmd.ExecuteNonQuery();
@@ -67,7 +67,7 @@ public class FileHashCache
         using var cmd = new SQLiteCommand(_conn);
         cmd.CommandText = @"INSERT INTO HashCache (Path, LastModified, Hash) VALUES (@path, @lastModified, @hash)
             ON CONFLICT(Path) DO UPDATE SET LastModified = @lastModified, Hash = @hash";
-        cmd.Parameters.AddWithValue("@path", path.ToString());
+        cmd.Parameters.AddWithValue("@path", path.ToString().ToLowerInvariant());
         cmd.Parameters.AddWithValue("@lastModified", lastModified);
         cmd.Parameters.AddWithValue("@hash", (long) hash);
         cmd.PrepareAsync();

Wabbajack.VFS/VFSCache.cs

@@ -50,8 +50,8 @@ public class VFSDiskCache : IVfsCache
         cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash";
         cmd.Parameters.AddWithValue("@hash", (long) hash);
 
-        await using var rdr = cmd.ExecuteReader();
+        await using var rdr = await cmd.ExecuteReaderAsync(token);
-        while (rdr.Read())
+        while (await rdr.ReadAsync(token))
         {
             var data = IndexedVirtualFileExtensions.Read(rdr.GetStream(0));
             return data;

Wabbajack.VFS/Wabbajack.VFS.csproj

@@ -9,6 +9,7 @@
 
     <ItemGroup>
         <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.2-mauipre.1.22054.8" />
+        <PackageReference Include="Microsoft.FASTER.Core" Version="2.0.16" />
         <PackageReference Include="System.Data.SQLite.Core" Version="1.0.116" />
     </ItemGroup>