Tons of compiler optimziations

Wabbajack.App.Wpf/Models/LogStream.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.ObjectModel;
 using System.Reactive.Disposables;
+using System.Reactive.Linq;
 using System.Reactive.Subjects;
 using System.Text;
 using System.Windows.Data;
@@ -31,10 +32,11 @@ public class LogStream : TargetWithLayout
     {
         _disposables = new CompositeDisposable();
         _messageLog.Connect()
+            .LimitSizeTo(200)
             .Bind(out _messagesFiltered)
             .Subscribe()
             .DisposeWith(_disposables);
-        
+
         Messages
             .Subscribe(m =>
             {

Wabbajack.App.Wpf/Settings.cs

@@ -21,12 +21,12 @@ namespace Wabbajack
         public double PosY { get; set; }
         public double Height { get; set; }
         public double Width { get; set; }
-        public InstallerSettings Installer { get; set; } = new InstallerSettings();
-        public FiltersSettings Filters { get; set; } = new FiltersSettings();
-        public CompilerSettings Compiler { get; set; } = new CompilerSettings();
-        public PerformanceSettings Performance { get; set; } = new PerformanceSettings();
+        public InstallerSettings Installer { get; set; } = new();
+        public FiltersSettings Filters { get; set; } = new();
+        public CompilerSettings Compiler { get; set; } = new();
+        public PerformanceSettings Performance { get; set; } = new();
 
-        private Subject<Unit> _saveSignal = new Subject<Unit>();
+        private Subject<Unit> _saveSignal = new();
         [JsonIgnore]
         public IObservable<Unit> SaveSignal => _saveSignal;
 

Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs

@@ -13,7 +13,6 @@ using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Windows.Media;
-using System.Windows.Threading;
 using DynamicData;
 using Microsoft.WindowsAPICodePack.Dialogs;
 using ReactiveUI.Fody.Helpers;

Wabbajack.App.Wpf/Views/Common/LogView.xaml.cs

@@ -1,17 +1,5 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using System.Windows;
+using System.Windows;
 using System.Windows.Controls;
-using System.Windows.Data;
-using System.Windows.Documents;
-using System.Windows.Input;
-using System.Windows.Media;
-using System.Windows.Media.Imaging;
-using System.Windows.Navigation;
-using System.Windows.Shapes;
 
 namespace Wabbajack
 {

Wabbajack.Common/AsyncParallelExtensions.cs

@@ -103,6 +103,38 @@ public static class AsyncParallelExtensions
         }
     }
     
+    /// <summary>
+    /// Faster version of PMapAll for when the function invocation will take a very small amount of time
+    /// batches all the inputs into N groups and executes them all on one task, where N is the number of
+    /// threads supported by the limiter
+    /// </summary>
+    /// <param name="coll"></param>
+    /// <param name="limiter"></param>
+    /// <param name="mapFn"></param>
+    /// <typeparam name="TIn"></typeparam>
+    /// <typeparam name="TJob"></typeparam>
+    /// <typeparam name="TOut"></typeparam>
+    /// <returns></returns>
+    public static async Task PDoAllBatched<TIn, TJob, TOut>(this IEnumerable<TIn> coll,
+        IResource<TJob> limiter, Func<TIn, Task> mapFn)
+    {
+        var asList = coll.ToList();
+        
+        var tasks = new List<Task>();
+
+        tasks.AddRange(Enumerable.Range(0, limiter.MaxTasks).Select(i => Task.Run(async () =>
+        {
+            using var job = await limiter.Begin(limiter.Name, asList.Count / limiter.MaxTasks, CancellationToken.None);
+            for (var idx = i; idx < asList.Count; idx += limiter.MaxTasks)
+            {
+                job.ReportNoWait(1);
+                await mapFn(asList[idx]);
+            }
+        })));
+
+        await Task.WhenAll(tasks);
+    }
+    
     public static async IAsyncEnumerable<TOut> PKeepAll<TIn, TJob, TOut>(this IEnumerable<TIn> coll,
         IResource<TJob> limiter, Func<TIn, Task<TOut>> mapFn)
     where TOut : class

Wabbajack.Compiler/ACompiler.cs

@@ -476,7 +476,7 @@ public abstract class ACompiler
 
             NextStep("Compiling", "Generating Patches", toBuild.Length);
 
-            var allFiles = toBuild.SelectMany(f =>
+            var allFiles = (await toBuild.PMapAllBatched(CompilerLimiter, async f =>
                 {
                     UpdateProgress(1);
                     return new[]
@@ -484,7 +484,8 @@ public abstract class ACompiler
                         _vfs.Index.FileForArchiveHashPath(f.ArchiveHashPath),
                         FindDestFile(f.To)
                     };
-                })
+                }).ToList())
+                .SelectMany(x => x)
                 .DistinctBy(f => f.Hash)
                 .ToHashSet();
             

Wabbajack.Compiler/CompilationSteps/DeconstructBSAs.cs

@@ -20,6 +20,11 @@ public class DeconstructBSAs : ACompilationStep
     private readonly Func<VirtualFile, List<ICompilationStep>> _microstack;
     private readonly Func<VirtualFile, List<ICompilationStep>> _microstackWithInclude;
     private readonly MO2Compiler _mo2Compiler;
+    private readonly DirectMatch _directMatch;
+    private readonly MatchSimilarTextures _matchSimilar;
+    private readonly IncludePatches _includePatches;
+    private readonly DropAll _dropAll;
+    private readonly IncludeAll _includeAll;
 
     public DeconstructBSAs(ACompiler compiler) : base(compiler)
     {
@@ -38,20 +43,27 @@ public class DeconstructBSAs : ACompilationStep
             .Select(kv => kv.Key.RelativeTo(_mo2Compiler._settings.Source))
             .ToList();
 
+        // Cache these so their internal caches aren't recreated on every use
+        _directMatch = new DirectMatch(_mo2Compiler);
+        _matchSimilar = new MatchSimilarTextures(_mo2Compiler);
+        _includePatches = new IncludePatches(_mo2Compiler);
+        _dropAll = new DropAll(_mo2Compiler);
+        _includeAll = new IncludeAll(_mo2Compiler);
+
         _microstack = bsa => new List<ICompilationStep>
         {
-            new DirectMatch(_mo2Compiler),
-            new MatchSimilarTextures(_mo2Compiler),
-            new IncludePatches(_mo2Compiler, bsa),
-            new DropAll(_mo2Compiler)
+            _directMatch,
+            _matchSimilar,
+            _includePatches.WithBSA(bsa),
+            _dropAll
         };
 
         _microstackWithInclude = bsa => new List<ICompilationStep>
         {
-            new DirectMatch(_mo2Compiler),
-            new MatchSimilarTextures(_mo2Compiler),
-            new IncludePatches(_mo2Compiler, bsa),
-            new IncludeAll(_mo2Compiler)
+            _directMatch,
+            _matchSimilar,
+            _includePatches.WithBSA(bsa),
+            _includeAll
         };
     }
 

Wabbajack.Compiler/CompilationSteps/IncludePatches.cs

@@ -37,6 +37,21 @@ public class IncludePatches : ACompilationStep
         _isGenericGame = _compiler._settings.Game.MetaData().IsGenericMO2Plugin;
     }
 
+    private IncludePatches(ACompiler compiler, 
+        VirtualFile bsa,
+        Dictionary<RelativePath, IEnumerable<VirtualFile>> indexedByName,
+        Dictionary<RelativePath, IGrouping<RelativePath, VirtualFile>> indexed) : base(compiler)
+    {
+        _bsa = bsa;
+        _indexedByName = indexedByName;
+        _indexed = indexed;
+    }
+
+    public IncludePatches WithBSA(VirtualFile constructingFromBSA)
+    {
+        return new IncludePatches(_compiler, constructingFromBSA, _indexedByName, _indexed);
+    }
+
     public override async ValueTask<Directive?> Run(RawSourceFile source)
     {
         if (_isGenericGame)

Wabbajack.Compiler/MO2Compiler.cs

@@ -151,6 +151,7 @@ public class MO2Compiler : ACompiler
         // Add the extra files that were generated by the stack
         results = results.Concat(ExtraFiles).ToList();
 
+        NextStep("Compiling", "Finding Errors");
         var noMatch = results.OfType<NoMatch>().ToArray();
         PrintNoMatches(noMatch);
         if (CheckForNoMatchExit(noMatch)) return false;

Wabbajack.Compiler/PatchCache/BinaryPatchCache.cs

@@ -42,18 +42,17 @@ public class BinaryPatchCache : IBinaryPatchCache
         
         await using var sigStream = new MemoryStream();
         var tempName = _location.Combine(Guid.NewGuid().ToString()).WithExtension(Ext.Temp);
-        await using var patchStream = tempName.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None);
         try
         {
 
-            OctoDiff.Create(srcStream, destStream, sigStream, patchStream, job);
-
-            patchStream.Close();
+            {
+                await using var patchStream = tempName.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None);
+                OctoDiff.Create(srcStream, destStream, sigStream, patchStream, job);
+            }
             await tempName.MoveToAsync(location, true, CancellationToken.None);
         }
         finally
         {
-            await patchStream.DisposeAsync();
             if (tempName.FileExists())
                 tempName.Delete();
         }

Wabbajack.Paths.IO/AbsolutePathExtensions.cs

@@ -183,16 +183,22 @@ public static class AbsolutePathExtensions
         {
             fid.IsReadOnly = false;
         }
-        
-        
-        
-        try
+
+        var retries = 0;
+        while (true)
         {
-            File.Move(srcStr, destStr, overwrite);
-        }
-        catch (Exception)
-        {
-            
+            try
+            {
+                File.Move(srcStr, destStr, overwrite);
+                return;
+            }
+            catch (Exception ex)
+            {
+                if (retries > 10)
+                    throw;
+                retries++;
+                await Task.Delay(TimeSpan.FromSeconds(1), token);
+            }
         }
     }