Merge pull request #1085 from wabbajack-tools/reduce-extraction-memory-usage

Reduce extraction memory usage

CHANGELOG.md

@@ -5,6 +5,11 @@
 * Reworked IPS4 integration to reduce download failures
 * Profiles can now contain an (optional) file `compiler_settings.json` that includes options for other games to be used during install.
 This is now the only way to include extra games in the install process, implicit inclusion has been removed. 
+* Number of download/install threads is now manually set (defaults to CPU cores) and is independently configurable
+* Includes a "Favor performance over RAM" optional mode (defaults to off) that will use excessive amounts of RAM in exchange
+for almost 1GB/sec install speed on the correct hardware. Don't enable this unless you have a fast SSD and at least 2.5GB of RAM for every
+install thread.
+
 
 #### Version - 2.2.2.0 - 8/31/2020
 * Route CDN requests through a reverse proxy to improve reliability

Wabbajack.Lib/ABatchProcessor.cs

@@ -15,6 +15,10 @@ namespace Wabbajack.Lib
     {
         public WorkQueue Queue { get; } = new WorkQueue();
 
+        public int DownloadThreads { get; set; } = Environment.ProcessorCount;
+        public int DiskThreads { get; set; } = Environment.ProcessorCount;
+        public bool FavorPerfOverRam { get; set; } = false;
+        
         public Context VFS { get; }
 
         protected StatusUpdateTracker UpdateTracker { get; }
@@ -51,6 +55,7 @@ namespace Wabbajack.Lib
         public BehaviorSubject<bool> ManualCoreLimit = new BehaviorSubject<bool>(true);
         public BehaviorSubject<byte> MaxCores = new BehaviorSubject<byte>(byte.MaxValue);
         public BehaviorSubject<Percent> TargetUsagePercent = new BehaviorSubject<Percent>(Percent.One);
+        public Subject<int> DesiredThreads { get; set; }
 
         public ABatchProcessor(int steps)
         {
@@ -62,8 +67,11 @@ namespace Wabbajack.Lib
                 .DisposeWith(_subs);
             UpdateTracker.Progress.Subscribe(_percentCompleted);
             UpdateTracker.StepName.Subscribe(_textStatus);
+            DesiredThreads = new Subject<int>();
+            Queue.SetActiveThreadsObservable(DesiredThreads);
         }
 
+        
         /// <summary>
         /// Gets the recommended maximum number of threads that should be used for the current machine.
         /// This will either run a heavy processing job to do the measurement in the current folder, or refer to caches.

Wabbajack.Lib/AInstaller.cs

@@ -216,6 +216,7 @@ namespace Wabbajack.Lib
                 }
             }
 
+            DesiredThreads.OnNext(DownloadThreads);
             await missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State))
                 .PMap(Queue, async archive =>
                 {
@@ -236,6 +237,9 @@ namespace Wabbajack.Lib
 
                     return await DownloadArchive(archive, download, outputPath);
                 });
+            
+            DesiredThreads.OnNext(DiskThreads);
+
         }
 
         public async Task<bool> DownloadArchive(Archive archive, bool download, AbsolutePath? destination = null)

Wabbajack.Lib/MO2Compiler.cs

@@ -84,7 +84,10 @@ namespace Wabbajack.Lib
         {
             await Metrics.Send("begin_compiling", MO2Profile ?? "unknown");
             if (cancel.IsCancellationRequested) return false;
-            Queue.SetActiveThreadsObservable(ConstructDynamicNumThreads(await RecommendQueueSize()));
+
+            DesiredThreads.OnNext(DiskThreads);
+            FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;
+
             UpdateTracker.Reset();
             UpdateTracker.NextStep("Gathering information");
 

Wabbajack.Lib/MO2Installer.cs

@@ -22,6 +22,7 @@ using File = Alphaleonis.Win32.Filesystem.File;
 using Path = Alphaleonis.Win32.Filesystem.Path;
 using SectionData = Wabbajack.Common.SectionData;
 using System.Collections.Generic;
+using Wabbajack.VirtualFileSystem;
 
 namespace Wabbajack.Lib
 {
@@ -55,7 +56,8 @@ namespace Wabbajack.Lib
             await Metrics.Send(Metrics.BeginInstall, ModList.Name);
             Utils.Log("Configuring Processor");
 
-            Queue.SetActiveThreadsObservable(ConstructDynamicNumThreads(await RecommendQueueSize()));
+            DesiredThreads.OnNext(DiskThreads);
+            FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam;
 
             if (GameFolder == null)
                 GameFolder = Game.TryGetGameLocation();

Wabbajack.Server/Services/MirrorUploader.cs

@@ -1,10 +1,13 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.IO.Compression;
+using System.Linq;
 using System.Net;
 using System.Threading.Tasks;
 using FluentFTP;
 using Microsoft.Extensions.Logging;
+using Org.BouncyCastle.Utilities.Collections;
 using Wabbajack.BuildServer;
 using Wabbajack.BuildServer.Controllers;
 using Wabbajack.Common;
@@ -38,14 +41,28 @@ namespace Wabbajack.Server.Services
 
             try
             {
+                var creds = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
+
                 using var queue = new WorkQueue();
                 if (_archives.TryGetPath(toUpload.Hash, out var path))
                 {
                     _logger.LogInformation($"Uploading mirror file {toUpload.Hash} {path.Size.FileSizeToString()}");
 
+                    bool exists = false;
+                    using (var client = await GetClient(creds))
+                    {
+                        exists = await client.FileExistsAsync($"{toUpload.Hash.ToHex()}/definition.json.gz");
+                    }
+                    
+                    if (exists)
+                    {
+                        _logger.LogInformation($"Skipping {toUpload.Hash} it's already on the server");
+                        await toUpload.Finish(_sql);
+                        goto TOP;
+                    }
+
                     var definition = await Client.GenerateFileDefinition(queue, path, (s, percent) => { });
 
-                    var creds = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors);
                     using (var client = await GetClient(creds))
                     {
                         await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}");

Wabbajack.Test/DownloaderTests.cs

@@ -569,7 +569,7 @@ namespace Wabbajack.Test
             public TestInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters parameters)
                 : base(archive, modList, outputFolder, downloadFolder, parameters, steps: 1, modList.GameType)
             {
-                Queue.SetActiveThreadsObservable(Observable.Return(1));
+                DesiredThreads.OnNext(1);
             }
 
             protected override Task<bool> _Begin(CancellationToken cancel)

Wabbajack.VirtualFileSystem/Context.cs

@@ -43,6 +43,8 @@ namespace Wabbajack.VirtualFileSystem
         public WorkQueue  Queue { get; }
         public bool UseExtendedHashes { get; set; }
         
+        public bool FavorPerfOverRAM { get; set; }
+
         public Context(WorkQueue queue, bool extendedHashes = false)
         {
             Queue = queue;
@@ -234,6 +236,7 @@ namespace Wabbajack.VirtualFileSystem
 
         private List<HashRelativePath> _knownFiles = new List<HashRelativePath>();
         private Dictionary<Hash, AbsolutePath> _knownArchives = new Dictionary<Hash, AbsolutePath>();
+
         public void AddKnown(IEnumerable<HashRelativePath> known, Dictionary<Hash, AbsolutePath> archives)
         {
             _knownFiles.AddRange(known);

Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs

@@ -27,6 +27,12 @@ namespace Wabbajack.VirtualFileSystem
         
         private static Extension OMODExtension = new Extension(".omod");
         
+        /// <summary>
+        /// When true, will allow 7z to use multiple threads and cache more data in memory, potentially
+        /// using many GB of RAM during extraction but vastly reducing extraction times in the process.
+        /// </summary>
+        public static bool FavorPerfOverRAM { get; set; }
+
 
         public static async Task<Dictionary<RelativePath, T>> GatheringExtract<T>(IStreamFactory sFn,
             Predicate<RelativePath> shouldExtract, Func<RelativePath, IStreamFactory, ValueTask<T>> mapfn)

Wabbajack.VirtualFileSystem/FileExtractor2/GatheringExtractor.cs

@@ -18,10 +18,10 @@ namespace Wabbajack.VirtualFileSystem
         private Predicate<RelativePath> _shouldExtract;
         private Func<RelativePath, IStreamFactory, ValueTask<T>> _mapFn;
         private Dictionary<RelativePath, T> _results;
-        private Dictionary<uint, (RelativePath, ulong)> _indexes;
         private Stream _stream;
         private Definitions.FileType _sig;
         private Exception _killException;
+        private uint _itemsCount;
 
         public GatheringExtractor(Stream stream, Definitions.FileType sig, Predicate<RelativePath> shouldExtract, Func<RelativePath,IStreamFactory, ValueTask<T>> mapfn)
         {
@@ -41,14 +41,9 @@ namespace Wabbajack.VirtualFileSystem
                 try
                 {
                     _archive = ArchiveFile.Open(_stream, _sig).Result;
-                    _indexes = _archive.Entries
+                    ulong checkPos = 1024 * 32;
-                        .Select((entry, idx) => (entry, (uint)idx))
+                    _archive._archive.Open(_archive._archiveStream, ref checkPos, null);
-                        .Where(f => !f.entry.IsFolder)
+                    _itemsCount = _archive._archive.GetNumberOfItems();
-                        .Select(t => ((RelativePath)t.entry.FileName, t.Item2, t.entry.Size))
-                        .Where(t => _shouldExtract(t.Item1))
-                        .ToDictionary(t => t.Item2, t => (t.Item1, t.Size));
-
-
                     _archive._archive.Extract(null, 0xFFFFFFFF, 0, this);
                     _archive.Dispose();
                     if (_killException != null)
@@ -86,21 +81,22 @@ namespace Wabbajack.VirtualFileSystem
 
         public int GetStream(uint index, out ISequentialOutStream outStream, AskMode askExtractMode)
         {
-            if (_indexes.ContainsKey(index))
+            var entry = _archive.GetEntry(index);
+            var path = (RelativePath)entry.FileName;
+            if (entry.IsFolder || !_shouldExtract(path))
             {
-                var path = _indexes[index].Item1;
+                outStream = null;
-                Utils.Status($"Extracting {path}", Percent.FactoryPutInRange(_results.Count, _indexes.Count));
+                return 0;
+            }
+
+            Utils.Status($"Extracting {path}", Percent.FactoryPutInRange(_results.Count, _itemsCount));
             // Empty files are never extracted via a write call, so we have to fake that now
-                if (_indexes[index].Item2 == 0)
+            if (entry.Size == 0)
             {
                 var result = _mapFn(path, new MemoryStreamFactory(new MemoryStream(), path)).Result;
                 _results.Add(path, result);
             }
-                outStream =  new GatheringExtractorStream<T>(this, index);
+            outStream =  new GatheringExtractorStream<T>(this, entry, path);
-                return 0;
-            }
-
-            outStream = null;
             return 0;
         }
 
@@ -117,25 +113,23 @@ namespace Wabbajack.VirtualFileSystem
         private class GatheringExtractorStream<T> : ISequentialOutStream, IOutStream
         {
             private GatheringExtractor<T> _extractor;
-            private uint _index;
-            private bool _written;
             private ulong _totalSize;
             private Stream _tmpStream;
             private TempFile _tmpFile;
-            private IStreamFactory _factory;
             private bool _diskCached;
+            private RelativePath _path;
 
-            public GatheringExtractorStream(GatheringExtractor<T> extractor, uint index)
+            public GatheringExtractorStream(GatheringExtractor<T> extractor, Entry entry, RelativePath path)
             {
+                _path = path;
                 _extractor = extractor;
-                _index = index;
+                _totalSize = entry.Size;
-                _totalSize = extractor._indexes[index].Item2;
+                _diskCached = _totalSize >= int.MaxValue - 1024;
-                _diskCached = _totalSize >= 500_000_000;
             }
 
             private IPath GetPath()
             {
-                return _extractor._indexes[_index].Item1;
+                return _path;
             }
 
             public int Write(byte[] data, uint size, IntPtr processedSize)
@@ -167,7 +161,7 @@ namespace Wabbajack.VirtualFileSystem
 
             private void WriteSingleCall(byte[] data, in uint size)
             {
-                var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryBufferFactory(data, (int)size, GetPath())).Result;
+                var result = _extractor._mapFn(_path, new MemoryBufferFactory(data, (int)size, GetPath())).Result;
                 AddResult(result);
                 Cleanup();
             }
@@ -180,7 +174,7 @@ namespace Wabbajack.VirtualFileSystem
 
             private void AddResult(T result)
             {
-                _extractor._results.Add(_extractor._indexes[_index].Item1, result);
+                _extractor._results.Add(_path, result);
             }
 
             private void WriteMemoryCached(byte[] data, in uint size)
@@ -193,7 +187,7 @@ namespace Wabbajack.VirtualFileSystem
 
                 _tmpStream.Flush();
                 _tmpStream.Position = 0;
-                var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new MemoryStreamFactory((MemoryStream)_tmpStream, GetPath())).Result;
+                var result = _extractor._mapFn(_path, new MemoryStreamFactory((MemoryStream)_tmpStream, GetPath())).Result;
                 AddResult(result);
                 Cleanup();
             }
@@ -213,7 +207,7 @@ namespace Wabbajack.VirtualFileSystem
                 _tmpStream.Flush();
                 _tmpStream.Close();
                 
-                var result = _extractor._mapFn(_extractor._indexes[_index].Item1, new NativeFileStreamFactory(_tmpFile.Path, GetPath())).Result;
+                var result = _extractor._mapFn(_path, new NativeFileStreamFactory(_tmpFile.Path, GetPath())).Result;
                 AddResult(result);
                 Cleanup();
             }

Wabbajack.VirtualFileSystem/SevenZipExtractor/ArchiveFile.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
@@ -13,7 +14,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
     {
         private SevenZipHandle _sevenZipHandle;
         internal IInArchive _archive;
-        private InStreamWrapper _archiveStream;
+        public InStreamWrapper _archiveStream;
         private IList<Entry> _entries;
 
         private static readonly AbsolutePath LibraryFilePath = @"Extractors\7z.dll".RelativeTo(AbsolutePath.EntryPoint);
@@ -24,10 +25,112 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
             var self = new ArchiveFile();
             self.InitializeAndValidateLibrary();
             self._archive = self._sevenZipHandle.CreateInArchive(Formats.FileTypeGuidMapping[format]);
+
+            if (!FileExtractor2.FavorPerfOverRAM)
+            {
+                self.SetCompressionProperties(new Dictionary<string, string>() {{"mt", "off"}});
+            }
+
+
             self._archiveStream = new InStreamWrapper(archiveStream);
             return self;
         }
 
+        /// <summary>
+        /// Sets the compression properties
+        /// </summary>
+        private void SetCompressionProperties(Dictionary<string, string> CustomParameters)
+        {
+
+
+            {
+                ISetProperties setter;
+                try
+                {
+                    setter = (ISetProperties)_archive;
+                }
+                catch (InvalidCastException _)
+                {
+                    return;
+                }
+
+                var names = new List<IntPtr>(1 + CustomParameters.Count);
+                var values = new List<PropVariant>(1 + CustomParameters.Count);
+                //var sp = new SecurityPermission(SecurityPermissionFlag.UnmanagedCode);
+                //sp.Demand();
+
+                #region Initialize compression properties
+
+                names.Add(Marshal.StringToBSTR("x"));
+                values.Add(new PropVariant());
+
+                foreach (var pair in CustomParameters)
+                {
+
+                    names.Add(Marshal.StringToBSTR(pair.Key));
+                    var pv = new PropVariant();
+
+                    #region List of parameters to cast as integers
+
+                    var integerParameters = new HashSet<string>
+                    {
+                        "fb",
+                        "pass",
+                        "o",
+                        "yx",
+                        "a",
+                        "mc",
+                        "lc",
+                        "lp",
+                        "pb",
+                        "cp"
+                    };
+
+                    #endregion
+
+                    if (integerParameters.Contains(pair.Key))
+                    {
+                        pv.VarType = VarEnum.VT_UI4;
+                        pv.UInt32Value = Convert.ToUInt32(pair.Value, CultureInfo.InvariantCulture);
+                    }
+                    else
+                    {
+                        pv.VarType = VarEnum.VT_BSTR;
+                        pv.pointerValue = Marshal.StringToBSTR(pair.Value);
+                    }
+
+                    values.Add(pv);
+                }
+
+                #endregion
+
+                #region Set compression level
+
+                var clpv = values[0];
+                clpv.VarType = VarEnum.VT_UI4;
+                clpv.UInt32Value = 0;
+
+                values[0] = clpv;
+
+                #endregion
+
+                var namesHandle = GCHandle.Alloc(names.ToArray(), GCHandleType.Pinned);
+                var valuesHandle = GCHandle.Alloc(values.ToArray(), GCHandleType.Pinned);
+
+                try
+                {
+                    setter?.SetProperties(namesHandle.AddrOfPinnedObject(), valuesHandle.AddrOfPinnedObject(),
+                        names.Count);
+                }
+                finally
+                {
+                    namesHandle.Free();
+                    valuesHandle.Free();
+                }
+
+            }
+        }
+        
         public IList<Entry> Entries
         {
             get
@@ -50,6 +153,16 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
                 this._entries = new List<Entry>();
 
                 for (uint fileIndex = 0; fileIndex < itemsCount; fileIndex++)
+                {
+                    var entry = GetEntry(fileIndex);
+                    this._entries.Add(entry);
+                }
+
+                return this._entries;
+            }
+        }
+
+        internal Entry GetEntry(uint fileIndex)
         {
             string fileName = this.GetProperty<string>(fileIndex, ItemPropId.kpidPath);
             bool isFolder = this.GetProperty<bool>(fileIndex, ItemPropId.kpidIsFolder);
@@ -68,7 +181,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
             bool isSplitBefore = this.GetPropertySafe<bool>(fileIndex, ItemPropId.kpidSplitBefore);
             bool isSplitAfter = this.GetPropertySafe<bool>(fileIndex, ItemPropId.kpidSplitAfter);
 
-                    this._entries.Add(new Entry(this._archive, fileIndex)
+            var entry = new Entry(this._archive, fileIndex)
             {
                 FileName = fileName,
                 IsFolder = isFolder,
@@ -85,11 +198,8 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
                 Method = method,
                 IsSplitBefore = isSplitBefore,
                 IsSplitAfter = isSplitAfter
-                    });
+            };
-                }
+            return entry;
-
-                return this._entries;
-            }
         }
 
         private T GetPropertySafe<T>(uint fileIndex, ItemPropId name)

Wabbajack.VirtualFileSystem/SevenZipExtractor/SevenZipInterface.cs

@@ -26,6 +26,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
         [FieldOffset(8)] public IntPtr pointerValue;
         [FieldOffset(8)] public byte byteValue;
         [FieldOffset(8)] public long longValue;
+        [FieldOffset(8)] public UInt32 UInt32Value;
         [FieldOffset(8)] public System.Runtime.InteropServices.ComTypes.FILETIME filetime;
         [FieldOffset(8)] public PropArray propArray;
 
@@ -35,6 +36,10 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
             {
                 return (VarEnum) this.vt;
             }
+            set
+            {
+                vt = (ushort)value;
+            }
         }
 
         public void Clear()
@@ -305,6 +310,24 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
         kpidUserDefined = 0x10000
     }
     
+    /// <summary>
+    /// 7-zip ISetProperties interface for setting various archive properties
+    /// </summary>
+    [ComImport]
+    [Guid("23170F69-40C1-278A-0000-000600030000")]
+    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
+    internal interface ISetProperties
+    {
+        /// <summary>
+        /// Sets the archive properties
+        /// </summary>
+        /// <param name="names">The names of the properties</param>
+        /// <param name="values">The values of the properties</param>
+        /// <param name="numProperties">The properties count</param>
+        /// <returns></returns>        
+        int SetProperties(IntPtr names, IntPtr values, int numProperties);
+    }
+
 
     [ComImport]
     [Guid("23170F69-40C1-278A-0000-000600600000")]
@@ -395,7 +418,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
         ArchivePropId propID,
         ref PropVariant value); // PROPVARIANT
 
-    internal class StreamWrapper : IDisposable
+    public class StreamWrapper : IDisposable
     {
         protected Stream BaseStream;
 
@@ -420,7 +443,7 @@ namespace Wabbajack.VirtualFileSystem.SevenZipExtractor
         }
     }
 
-    internal class InStreamWrapper : StreamWrapper, ISequentialInStream, IInStream
+    public class InStreamWrapper : StreamWrapper, ISequentialInStream, IInStream
     {
         public InStreamWrapper(Stream baseStream) : base(baseStream)
         {

Wabbajack/Settings.cs

@@ -113,26 +113,28 @@ namespace Wabbajack
     [JsonObject(MemberSerialization.OptOut)]
     public class PerformanceSettings : ViewModel
     {
-        private bool _manual;
+        public PerformanceSettings()
-        public bool Manual { get => _manual; set => RaiseAndSetIfChanged(ref _manual, value); }
-
-        private byte _maxCores = byte.MaxValue;
-        public byte MaxCores { get => _maxCores; set => RaiseAndSetIfChanged(ref _maxCores, value); }
-
-        private Percent _targetUsage = Percent.One;
-        public Percent TargetUsage { get => _targetUsage; set => RaiseAndSetIfChanged(ref _targetUsage, value); }
-
-        public void AttachToBatchProcessor(ABatchProcessor processor)
         {
-            processor.Add(
+            _favorPerfOverRam = false;
-                this.WhenAny(x => x.Manual)
+            _diskThreads = Environment.ProcessorCount;
-                    .Subscribe(processor.ManualCoreLimit));
+            _downloadThreads = Environment.ProcessorCount;
-            processor.Add(
+        }
-                this.WhenAny(x => x.MaxCores)
+
-                    .Subscribe(processor.MaxCores));
+        private int _downloadThreads;
-            processor.Add(
+        public int DownloadThreads { get => _downloadThreads; set => RaiseAndSetIfChanged(ref _downloadThreads, value); }
-                this.WhenAny(x => x.TargetUsage)
+        
-                    .Subscribe(processor.TargetUsagePercent));
+        private int _diskThreads;
+        public int DiskThreads { get => _diskThreads; set => RaiseAndSetIfChanged(ref _diskThreads, value); }
+        
+        private bool _favorPerfOverRam;
+        public bool FavorPerfOverRam { get => _favorPerfOverRam; set => RaiseAndSetIfChanged(ref _favorPerfOverRam, value); }
+
+
+        public void SetProcessorSettings(ABatchProcessor processor)
+        {
+            processor.DownloadThreads = DownloadThreads;
+            processor.DiskThreads = DiskThreads;
+            processor.FavorPerfOverRam = FavorPerfOverRam;
         }
     }
 

Wabbajack/View Models/Compilers/MO2CompilerVM.cs

@@ -195,7 +195,7 @@ namespace Wabbajack
                     ModlistIsNSFW = ModlistSettings.IsNSFW
                 })
                 {
-                    Parent.MWVM.Settings.Performance.AttachToBatchProcessor(ActiveCompilation);
+                    Parent.MWVM.Settings.Performance.SetProcessorSettings(ActiveCompilation);
 
                     var success = await ActiveCompilation.Begin();
                     return GetResponse<ModList>.Create(success, ActiveCompilation.ModList);

Wabbajack/View Models/Installers/MO2InstallerVM.cs

@@ -157,7 +157,7 @@ namespace Wabbajack
                 parameters: SystemParametersConstructor.Create()))
             {
                 installer.UseCompression = Parent.MWVM.Settings.Filters.UseCompression;
-                Parent.MWVM.Settings.Performance.AttachToBatchProcessor(installer);
+                Parent.MWVM.Settings.Performance.SetProcessorSettings(installer);
 
                 return await Task.Run(async () =>
                 {

Wabbajack/Views/Common/CpuView.xaml.cs

@@ -51,32 +51,11 @@ namespace Wabbajack
             InitializeComponent();
             this.WhenActivated(disposable =>
             {
-                Observable.CombineLatest(
-                        this.WhenAny(x => x.ControlGrid.IsMouseOver),
-                        this.WhenAny(x => x.SettingsHook.Performance.Manual)
-                            .StartWith(true),
-                        resultSelector: (over, manual) => over && !manual)
-                    .Select(showing => showing ? Visibility.Visible : Visibility.Collapsed)
-                    .BindToStrict(this, x => x.SettingsBar.Visibility)
-                    .DisposeWith(disposable);
                
                 this.WhenAny(x => x.ViewModel.StatusList)
                     .BindToStrict(this, x => x.CpuListControl.ItemsSource)
                     .DisposeWith(disposable);
 
-                this.BindStrict(
-                        this.ViewModel,
-                        x => x.MWVM.Settings.Performance.TargetUsage,
-                        x => x.TargetPercentageSlider.Value,
-                        vmToViewConverter: p => p.Value,
-                        viewToVmConverter: d => new Percent(d))
-                    .DisposeWith(disposable);
-
-                this.WhenAny(x => x.ViewModel.MWVM.Settings.Performance.TargetUsage)
-                    .Select(p => p.ToString(0))
-                    .BindToStrict(this, x => x.PercentageText.Text)
-                    .DisposeWith(disposable);
-
                 this.WhenAny(x => x.ViewModel.CurrentCpuCount)
                     .DistinctUntilChanged()
                     .Select(x => $"{x.CurrentCPUs} / {x.DesiredCPUs}")

Wabbajack/Views/Settings/PerformanceSettingsView.xaml

@@ -25,6 +25,7 @@
                 <RowDefinition Height="Auto" />
                 <RowDefinition Height="25" />
                 <RowDefinition Height="25" />
+                <RowDefinition Height="25" />
                 <RowDefinition Height="*" />
             </Grid.RowDefinitions>
             <Grid.ColumnDefinitions>
@@ -37,67 +38,44 @@
                 FontSize="20"
                 FontWeight="Bold"
                 Text="Performance" />
-            <Grid Grid.Row="2" Grid.Column="0" Grid.ColumnSpan="3"
-                Height="25"
-                Margin="0,0,0,10">
-                <Grid.Resources>
-                    <Style BasedOn="{StaticResource MainButtonStyle}" TargetType="Button">
-                        <Style.Triggers>
-                            <Trigger Property="IsEnabled" Value="false">
-                                <Setter Property="Foreground" Value="{StaticResource ForegroundBrush}" />
-                                <Setter Property="Background" Value="{StaticResource SecondaryBackgroundBrush}" />
-                                <Setter Property="BorderBrush" Value="{StaticResource DarkSecondaryBrush}" />
-                            </Trigger>
-                        </Style.Triggers>
-                    </Style>
-                </Grid.Resources>
-                <Grid.ColumnDefinitions>
-                    <ColumnDefinition Width="*" />
-                    <ColumnDefinition Width="5" />
-                    <ColumnDefinition Width="*" />
-                </Grid.ColumnDefinitions>
-                <Button Grid.Column="0"
-                    x:Name="ManualButton"
-                    Content="Manual"
-                    ToolTip="Control the number of cores by setting the max limit manually" />
-                <Button Grid.Column="2"
-                    x:Name="AutoButton"
-                    Content="Auto"
-                    ToolTip="Control the number of cores by scaling it to a percentage of what WJ would use at full speed" />
-            </Grid>
             <TextBlock Grid.Row="3" Grid.Column="0"
-                x:Name="MaxCoresLabel"
+                       x:Name="DownloadThreadsLabel"
                        VerticalAlignment="Center"
-                Text="Max Cores" />
+                       Text="Download Threads" />
-            <xwpf:IntegerUpDown Grid.Row="3" Grid.Column="2"
-                x:Name="MaxCoresSpinner"
-                MinWidth="75"
-                HorizontalAlignment="Right"
-                VerticalAlignment="Center"
-                Foreground="White"
-                Maximum="255"
-                Minimum="1" />
-            <TextBlock Grid.Row="3" Grid.Column="0"
-                x:Name="TargetUsageLabel"
-                VerticalAlignment="Center"
-                Text="Target Percent Usage" />
             <xwpf:DoubleUpDown Grid.Row="3" Grid.Column="2"
-                x:Name="TargetUsageSpinner"
+                               x:Name="DownloadThreads"
                                MinWidth="75"
                                HorizontalAlignment="Right"
                                VerticalAlignment="Center"
                                Foreground="White"
-                FormatString="F2"
+                               FormatString="F0"
-                Increment="0.1"
+                               Increment="2"
-                Maximum="1"
+                               Maximum="128"
-                Minimum="0.1" />
+                               Minimum="2" />
-            <Slider Grid.Row="4" Grid.Column="0" Grid.ColumnSpan="3"
+            <TextBlock Grid.Row="4" Grid.Column="0"
-                x:Name="TargetUsageSlider"
+                       x:Name="DiskThreadsLabel"
-                IsSnapToTickEnabled="True"
+                       VerticalAlignment="Center"
-                LargeChange="0.1"
+                       Text="Disk Threads" />
-                Maximum="1"
+            <xwpf:DoubleUpDown Grid.Row="4" Grid.Column="2"
-                Minimum="0.1"
+                               x:Name="DiskThreads"
-                TickFrequency="0.05" />
+                               MinWidth="75"
+                               HorizontalAlignment="Right"
+                               VerticalAlignment="Center"
+                               Foreground="White"
+                               FormatString="F0"
+                               Increment="2"
+                               Maximum="128"
+                               Minimum="2" />
+            <TextBlock Grid.Row="5" Grid.Column="0"
+                       x:Name="FavorPerfOverRamLabel"
+                       VerticalAlignment="Center"
+                       Text="Favor performance over RAM Usage" />
+            <CheckBox Grid.Row="5" Grid.Column="2"
+                      x:Name="FavorPerfOverRam"
+                      HorizontalAlignment="Right"
+                      VerticalAlignment="Center"
+                      Foreground="White"
+                ></CheckBox>
         </Grid>
     </Border>
 </rxui:ReactiveUserControl>

Wabbajack/Views/Settings/PerformanceSettingsView.xaml.cs

@@ -28,45 +28,18 @@ namespace Wabbajack
         {
             InitializeComponent();
 
-            this.AutoButton.Command = ReactiveCommand.Create(
-                execute: () => this.ViewModel.Manual = false,
-                canExecute: this.WhenAny(x => x.ViewModel.Manual)
-                    .ObserveOnGuiThread());
-            this.ManualButton.Command = ReactiveCommand.Create(
-                execute: () => this.ViewModel.Manual = true,
-                canExecute: this.WhenAny(x => x.ViewModel.Manual)
-                    .Select(x => !x)
-                    .ObserveOnGuiThread());
-
             this.WhenActivated(disposable =>
             {
-                // Bind mode buttons
-
-                // Modify visibility of controls based on if auto
-                this.OneWayBindStrict(this.ViewModel, x => x.Manual, x => x.MaxCoresLabel.Visibility,
-                        b => b ? Visibility.Visible : Visibility.Collapsed)
-                    .DisposeWith(disposable);
-                this.OneWayBindStrict(this.ViewModel, x => x.Manual, x => x.MaxCoresSpinner.Visibility,
-                        b => b ? Visibility.Visible : Visibility.Collapsed)
-                    .DisposeWith(disposable);
-                this.OneWayBindStrict(this.ViewModel, x => x.Manual, x => x.TargetUsageLabel.Visibility,
-                        b => b ? Visibility.Collapsed : Visibility.Visible)
-                    .DisposeWith(disposable);
-                this.OneWayBindStrict(this.ViewModel, x => x.Manual, x => x.TargetUsageSpinner.Visibility,
-                        b => b ? Visibility.Collapsed : Visibility.Visible)
-                    .DisposeWith(disposable);
-                this.OneWayBindStrict(this.ViewModel, x => x.Manual, x => x.TargetUsageSlider.Visibility,
-                        b => b ? Visibility.Collapsed : Visibility.Visible)
-                    .DisposeWith(disposable);
-
                 // Bind Values
-                this.BindStrict(this.ViewModel, x => x.MaxCores, x => x.MaxCoresSpinner.Value,
+                this.BindStrict(this.ViewModel, x => x.DiskThreads, x => x.DiskThreads.Value,
                         vmToViewConverter: x => x,
-                        viewToVmConverter: x => (byte)(x ?? 0))
+                        viewToVmConverter: x => (int)(x ?? 0))
                     .DisposeWith(disposable);
-                this.Bind(this.ViewModel, x => x.TargetUsage, x => x.TargetUsageSpinner.Value)
+                this.BindStrict(this.ViewModel, x => x.DownloadThreads, x => x.DownloadThreads.Value,
+                        vmToViewConverter: x => x,
+                        viewToVmConverter: x => (int)(x ?? 0))
                     .DisposeWith(disposable);
-                this.Bind(this.ViewModel, x => x.TargetUsage, x => x.TargetUsageSlider.Value)
+                this.BindStrict(this.ViewModel, x => x.FavorPerfOverRam, x => x.FavorPerfOverRam.IsChecked)
                     .DisposeWith(disposable);
             });
         }