diff --git a/Wabbajack.App.Wpf/Models/LogStream.cs b/Wabbajack.App.Wpf/Models/LogStream.cs index 52a6ec10..5a997c01 100644 --- a/Wabbajack.App.Wpf/Models/LogStream.cs +++ b/Wabbajack.App.Wpf/Models/LogStream.cs @@ -1,6 +1,7 @@ using System; using System.Collections.ObjectModel; using System.Reactive.Disposables; +using System.Reactive.Linq; using System.Reactive.Subjects; using System.Text; using System.Windows.Data; @@ -31,10 +32,11 @@ public class LogStream : TargetWithLayout { _disposables = new CompositeDisposable(); _messageLog.Connect() + .LimitSizeTo(200) .Bind(out _messagesFiltered) .Subscribe() .DisposeWith(_disposables); - + Messages .Subscribe(m => { diff --git a/Wabbajack.App.Wpf/Settings.cs b/Wabbajack.App.Wpf/Settings.cs index 68a9d89a..4a65f6d7 100644 --- a/Wabbajack.App.Wpf/Settings.cs +++ b/Wabbajack.App.Wpf/Settings.cs @@ -21,12 +21,12 @@ namespace Wabbajack public double PosY { get; set; } public double Height { get; set; } public double Width { get; set; } - public InstallerSettings Installer { get; set; } = new InstallerSettings(); - public FiltersSettings Filters { get; set; } = new FiltersSettings(); - public CompilerSettings Compiler { get; set; } = new CompilerSettings(); - public PerformanceSettings Performance { get; set; } = new PerformanceSettings(); + public InstallerSettings Installer { get; set; } = new(); + public FiltersSettings Filters { get; set; } = new(); + public CompilerSettings Compiler { get; set; } = new(); + public PerformanceSettings Performance { get; set; } = new(); - private Subject _saveSignal = new Subject(); + private Subject _saveSignal = new(); [JsonIgnore] public IObservable SaveSignal => _saveSignal; diff --git a/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs b/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs index 6d1d60d3..d0575cb0 100644 --- a/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs +++ b/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs @@ -13,7 +13,6 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; using System.Windows.Media; -using System.Windows.Threading; using DynamicData; using Microsoft.WindowsAPICodePack.Dialogs; using ReactiveUI.Fody.Helpers; diff --git a/Wabbajack.App.Wpf/Views/Common/LogView.xaml.cs b/Wabbajack.App.Wpf/Views/Common/LogView.xaml.cs index fcb2f5c2..ff853410 100644 --- a/Wabbajack.App.Wpf/Views/Common/LogView.xaml.cs +++ b/Wabbajack.App.Wpf/Views/Common/LogView.xaml.cs @@ -1,17 +1,5 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Windows; +using System.Windows; using System.Windows.Controls; -using System.Windows.Data; -using System.Windows.Documents; -using System.Windows.Input; -using System.Windows.Media; -using System.Windows.Media.Imaging; -using System.Windows.Navigation; -using System.Windows.Shapes; namespace Wabbajack { diff --git a/Wabbajack.Common/AsyncParallelExtensions.cs b/Wabbajack.Common/AsyncParallelExtensions.cs index bab43637..5e800436 100644 --- a/Wabbajack.Common/AsyncParallelExtensions.cs +++ b/Wabbajack.Common/AsyncParallelExtensions.cs @@ -103,6 +103,38 @@ public static class AsyncParallelExtensions } } + /// + /// Faster version of PMapAll for when the function invocation will take a very small amount of time + /// batches all the inputs into N groups and executes them all on one task, where N is the number of + /// threads supported by the limiter + /// + /// + /// + /// + /// + /// + /// + /// + public static async Task PDoAllBatched(this IEnumerable coll, + IResource limiter, Func mapFn) + { + var asList = coll.ToList(); + + var tasks = new List(); + + tasks.AddRange(Enumerable.Range(0, limiter.MaxTasks).Select(i => Task.Run(async () => + { + using var job = await limiter.Begin(limiter.Name, asList.Count / limiter.MaxTasks, CancellationToken.None); + for (var idx = i; idx < asList.Count; idx += limiter.MaxTasks) + { + job.ReportNoWait(1); + await mapFn(asList[idx]); + } + }))); + + await Task.WhenAll(tasks); + } + public static async IAsyncEnumerable PKeepAll(this IEnumerable coll, IResource limiter, Func> mapFn) where TOut : class diff --git a/Wabbajack.Compiler/ACompiler.cs b/Wabbajack.Compiler/ACompiler.cs index dd614b36..c7e140e6 100644 --- a/Wabbajack.Compiler/ACompiler.cs +++ b/Wabbajack.Compiler/ACompiler.cs @@ -476,7 +476,7 @@ public abstract class ACompiler NextStep("Compiling", "Generating Patches", toBuild.Length); - var allFiles = toBuild.SelectMany(f => + var allFiles = (await toBuild.PMapAllBatched(CompilerLimiter, async f => { UpdateProgress(1); return new[] @@ -484,7 +484,8 @@ public abstract class ACompiler _vfs.Index.FileForArchiveHashPath(f.ArchiveHashPath), FindDestFile(f.To) }; - }) + }).ToList()) + .SelectMany(x => x) .DistinctBy(f => f.Hash) .ToHashSet(); diff --git a/Wabbajack.Compiler/CompilationSteps/DeconstructBSAs.cs b/Wabbajack.Compiler/CompilationSteps/DeconstructBSAs.cs index 4aac784e..6c8da798 100644 --- a/Wabbajack.Compiler/CompilationSteps/DeconstructBSAs.cs +++ b/Wabbajack.Compiler/CompilationSteps/DeconstructBSAs.cs @@ -20,6 +20,11 @@ public class DeconstructBSAs : ACompilationStep private readonly Func> _microstack; private readonly Func> _microstackWithInclude; private readonly MO2Compiler _mo2Compiler; + private readonly DirectMatch _directMatch; + private readonly MatchSimilarTextures _matchSimilar; + private readonly IncludePatches _includePatches; + private readonly DropAll _dropAll; + private readonly IncludeAll _includeAll; public DeconstructBSAs(ACompiler compiler) : base(compiler) { @@ -38,20 +43,27 @@ public class DeconstructBSAs : ACompilationStep .Select(kv => kv.Key.RelativeTo(_mo2Compiler._settings.Source)) .ToList(); + // Cache these so their internal caches aren't recreated on every use + _directMatch = new DirectMatch(_mo2Compiler); + _matchSimilar = new MatchSimilarTextures(_mo2Compiler); + _includePatches = new IncludePatches(_mo2Compiler); + _dropAll = new DropAll(_mo2Compiler); + _includeAll = new IncludeAll(_mo2Compiler); + _microstack = bsa => new List { - new DirectMatch(_mo2Compiler), - new MatchSimilarTextures(_mo2Compiler), - new IncludePatches(_mo2Compiler, bsa), - new DropAll(_mo2Compiler) + _directMatch, + _matchSimilar, + _includePatches.WithBSA(bsa), + _dropAll }; _microstackWithInclude = bsa => new List { - new DirectMatch(_mo2Compiler), - new MatchSimilarTextures(_mo2Compiler), - new IncludePatches(_mo2Compiler, bsa), - new IncludeAll(_mo2Compiler) + _directMatch, + _matchSimilar, + _includePatches.WithBSA(bsa), + _includeAll }; } diff --git a/Wabbajack.Compiler/CompilationSteps/IncludePatches.cs b/Wabbajack.Compiler/CompilationSteps/IncludePatches.cs index 6a10d288..f6363c8b 100644 --- a/Wabbajack.Compiler/CompilationSteps/IncludePatches.cs +++ b/Wabbajack.Compiler/CompilationSteps/IncludePatches.cs @@ -37,6 +37,21 @@ public class IncludePatches : ACompilationStep _isGenericGame = _compiler._settings.Game.MetaData().IsGenericMO2Plugin; } + private IncludePatches(ACompiler compiler, + VirtualFile bsa, + Dictionary> indexedByName, + Dictionary> indexed) : base(compiler) + { + _bsa = bsa; + _indexedByName = indexedByName; + _indexed = indexed; + } + + public IncludePatches WithBSA(VirtualFile constructingFromBSA) + { + return new IncludePatches(_compiler, constructingFromBSA, _indexedByName, _indexed); + } + public override async ValueTask Run(RawSourceFile source) { if (_isGenericGame) diff --git a/Wabbajack.Compiler/MO2Compiler.cs b/Wabbajack.Compiler/MO2Compiler.cs index fa9e63a3..32931b27 100644 --- a/Wabbajack.Compiler/MO2Compiler.cs +++ b/Wabbajack.Compiler/MO2Compiler.cs @@ -151,6 +151,7 @@ public class MO2Compiler : ACompiler // Add the extra files that were generated by the stack results = results.Concat(ExtraFiles).ToList(); + NextStep("Compiling", "Finding Errors"); var noMatch = results.OfType().ToArray(); PrintNoMatches(noMatch); if (CheckForNoMatchExit(noMatch)) return false; diff --git a/Wabbajack.Compiler/PatchCache/BinaryPatchCache.cs b/Wabbajack.Compiler/PatchCache/BinaryPatchCache.cs index 76e47e89..bff35753 100644 --- a/Wabbajack.Compiler/PatchCache/BinaryPatchCache.cs +++ b/Wabbajack.Compiler/PatchCache/BinaryPatchCache.cs @@ -42,18 +42,17 @@ public class BinaryPatchCache : IBinaryPatchCache await using var sigStream = new MemoryStream(); var tempName = _location.Combine(Guid.NewGuid().ToString()).WithExtension(Ext.Temp); - await using var patchStream = tempName.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); try { - OctoDiff.Create(srcStream, destStream, sigStream, patchStream, job); - - patchStream.Close(); + { + await using var patchStream = tempName.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); + OctoDiff.Create(srcStream, destStream, sigStream, patchStream, job); + } await tempName.MoveToAsync(location, true, CancellationToken.None); } finally { - await patchStream.DisposeAsync(); if (tempName.FileExists()) tempName.Delete(); } diff --git a/Wabbajack.Paths.IO/AbsolutePathExtensions.cs b/Wabbajack.Paths.IO/AbsolutePathExtensions.cs index 25b06a87..6ef6ac4b 100644 --- a/Wabbajack.Paths.IO/AbsolutePathExtensions.cs +++ b/Wabbajack.Paths.IO/AbsolutePathExtensions.cs @@ -183,16 +183,22 @@ public static class AbsolutePathExtensions { fid.IsReadOnly = false; } - - - - try + + var retries = 0; + while (true) { - File.Move(srcStr, destStr, overwrite); - } - catch (Exception) - { - + try + { + File.Move(srcStr, destStr, overwrite); + return; + } + catch (Exception ex) + { + if (retries > 10) + throw; + retries++; + await Task.Delay(TimeSpan.FromSeconds(1), token); + } } }