Set the queue size during installation based on the disk performance. Abort installation if there isn't enough disk space to perform the installation.

Wabbajack.Common/Utils.cs

@@ -697,6 +697,34 @@ namespace Wabbajack.Common
             Log(s);
         }
 
+        public static long TestDiskSpeed(WorkQueue queue, string path)
+        {
+            var start_time = DateTime.Now;
+            var seconds = 2;
+            return Enumerable.Range(0, queue.ThreadCount)
+                .PMap(queue, idx =>
+                {
+                    var random = new Random();
+
+                    var file = Path.Combine(path, $"size_test{idx}.bin");
+                    long size = 0;
+                    byte[] buffer = new byte[1024 * 8];
+                    random.NextBytes(buffer);
+                    using (var fs = File.OpenWrite(file))
+                    {
+                        while (DateTime.Now < start_time + new TimeSpan(0, 0, seconds))
+                        {
+                            fs.Write(buffer, 0, buffer.Length);
+                            // Flush to make sure large buffers don't cause the rate to be higher than it should
+                            fs.Flush();
+                            size += buffer.Length;
+                        }
+                    }
+                    File.Delete(file);
+                    return size;
+                }).Sum() / seconds;
+        }
+
         /// https://stackoverflow.com/questions/422090/in-c-sharp-check-that-filename-is-possibly-valid-not-that-it-exists
         public static IErrorResponse IsFilePathValid(string path)
         {

Wabbajack.Common/WorkQueue.cs

@@ -10,7 +10,7 @@ using System.Threading;
 
 namespace Wabbajack.Common
 {
-    public class WorkQueue
+    public class WorkQueue : IDisposable
     {
         internal BlockingCollection<Action>
             Queue = new BlockingCollection<Action>(new ConcurrentStack<Action>());
@@ -32,6 +32,7 @@ namespace Wabbajack.Common
 
         private void StartThreads(int threadCount)
         {
+            ThreadCount = threadCount;
             Threads = Enumerable.Range(0, threadCount)
                 .Select(idx =>
                 {
@@ -44,6 +45,8 @@ namespace Wabbajack.Common
                 }).ToList();
         }
 
+        public int ThreadCount { get; private set; }
+
         private void ThreadBody(int idx)
         {
             CpuId = idx;
@@ -77,6 +80,12 @@ namespace Wabbajack.Common
         {
             Threads.Do(th => th.Abort());
         }
+
+        public void Dispose()
+        {
+            Shutdown();
+            Queue?.Dispose();
+        }
     }
 
     public class CPUStatus
@@ -85,4 +94,4 @@ namespace Wabbajack.Common
         public string Msg { get; internal set; }
         public int ID { get; internal set; }
     }
-}
+}

Wabbajack.Lib/ABatchProcessor.cs

@@ -61,6 +61,23 @@ namespace Wabbajack.Lib
             _configured = true;
         }
 
+        public static int RecommendQueueSize(string folder)
+        {
+            if (!Directory.Exists(folder))
+                Directory.CreateDirectory(folder);
+
+            using (var queue = new WorkQueue())
+            {
+                Utils.Log($"Benchmarking {folder}");
+                var raw_speed = Utils.TestDiskSpeed(queue, folder);
+                Utils.Log($"{raw_speed.ToFileSizeString()}/sec for {folder}");
+                int speed = (int)(raw_speed / 1024 / 1024);
+
+                // Less than 100MB/sec, stick with two threads.
+                return speed < 100 ? 2 : Math.Min(Environment.ProcessorCount, speed / 100 * 2);
+            }
+        }
+
         protected abstract bool _Begin();
         public Task<bool> Begin()
         {

Wabbajack.Lib/AInstaller.cs

@@ -11,6 +11,7 @@ using Wabbajack.Lib.Downloaders;
 using Wabbajack.VirtualFileSystem;
 using Context = Wabbajack.VirtualFileSystem.Context;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
+using DriveInfo = Alphaleonis.Win32.Filesystem.DriveInfo;
 using File = System.IO.File;
 using FileInfo = System.IO.FileInfo;
 using Path = Alphaleonis.Win32.Filesystem.Path;
@@ -286,6 +287,39 @@ namespace Wabbajack.Lib
                 .ToDictionary(e => e.Item1, e => e.Item2);
         }
 
+        public void ValidateFreeSpace()
+        {
+            DiskSpaceInfo DriveInfo(string path)
+            {
+                return Volume.GetDiskFreeSpace(Volume.GetUniqueVolumeNameForPath(path));
+            }
+
+            var paths = new[] {(OutputFolder, ModList.InstallSize),
+                               (DownloadFolder, ModList.DownloadSize),
+                               (Directory.GetCurrentDirectory(), ModList.ScratchSpaceSize)};
+            paths.GroupBy(f => DriveInfo(f.Item1).DriveName)
+                .Do(g =>
+                {
+                    var required = g.Sum(i => i.Item2);
+                    var available = DriveInfo(g.Key).FreeBytesAvailable;
+                    if (required > available)
+                        throw new NotEnoughDiskSpaceException(
+                            $"This modlist requires {required.ToFileSizeString()} on {g.Key} but only {available.ToFileSizeString()} is available.");
+                });
+
+        }
+
+        public int RecommendQueueSize()
+        {
+            var output_size = RecommendQueueSize(OutputFolder);
+            var download_size = RecommendQueueSize(DownloadFolder);
+            var scratch_size = RecommendQueueSize(Directory.GetCurrentDirectory());
+            var result =  Math.Min(output_size, Math.Min(download_size, scratch_size));
+            Utils.Log($"Recommending a queue size of {result} based on disk performance and number of cores");
+            return result;
+        }
+
+
         /// <summary>
         /// The user may already have some files in the OutputFolder. If so we can go through these and
         /// figure out which need to be updated, deleted, or left alone
@@ -336,4 +370,11 @@ namespace Wabbajack.Lib
 
         }
     }
+
+    public class NotEnoughDiskSpaceException : Exception
+    {
+        public NotEnoughDiskSpaceException(string s) : base(s)
+        {
+        }
+    }
 }

Wabbajack.Lib/Data.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using Ceras;
 using Compression.BSA;
 using Wabbajack.Common;
@@ -94,6 +95,24 @@ namespace Wabbajack.Lib
         ///     Content Report in HTML form
         /// </summary>
         public string ReportHTML;
+
+        /// <summary>
+        /// The size of all the archives once they're downloaded
+        /// </summary>
+        public long DownloadSize => Archives.Sum(a => a.Size);
+
+        /// <summary>
+        /// The size of all the files once they are installed (excluding downloaded archives)
+        /// </summary>
+        public long InstallSize => Directives.Sum(s => s.Size);
+
+        /// <summary>
+        /// Estimate of the amount of space required in the VFS staging folders during installation
+        /// </summary>
+        public long ScratchSpaceSize => Archives.OrderByDescending(a => a.Size)
+                                            .Take(Environment.ProcessorCount)
+                                            .Sum(a => a.Size) * 2;
+
     }
 
     public class Directive
@@ -267,4 +286,4 @@ namespace Wabbajack.Lib
         /// </summary>
         public string BSAHash;
     }
-}
+}

Wabbajack.Lib/MO2Compiler.cs

@@ -443,4 +443,4 @@ namespace Wabbajack.Lib
             public DateTime LastModified;
         }
     }
-}
+}

Wabbajack.Lib/MO2Installer.cs

@@ -33,7 +33,8 @@ namespace Wabbajack.Lib
 
         protected override bool _Begin()
         {
-            ConfigureProcessor(10);
+            ConfigureProcessor(RecommendQueueSize());
+            ValidateFreeSpace();
             var game = GameRegistry.Games[ModList.GameType];
 
             if (GameFolder == null)

Wabbajack.Lib/VortexInstaller.cs

@@ -25,7 +25,7 @@ namespace Wabbajack.Lib
 
         protected override bool _Begin()
         {
-            ConfigureProcessor(10);
+            ConfigureProcessor(10, RecommendQueueSize());
             Directory.CreateDirectory(DownloadFolder);
 
             HashArchives();

Wabbajack.Test/MiscTests.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using MahApps.Metro.Controls;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Utils = Wabbajack.Common.Utils;
+
+namespace Wabbajack.Test
+{
+    [TestClass]
+    public class MiscTests
+    {
+        [TestMethod]
+        public void TestDiskSpeed()
+        {
+            using (var queue = new WorkQueue())
+            {
+                var speed = Utils.TestDiskSpeed(queue, @".\");
+            }
+        } 
+    }
+}

Wabbajack.Test/Wabbajack.Test.csproj

@@ -100,6 +100,7 @@
     <Compile Include="DownloaderTests.cs" />
     <Compile Include="EndToEndTests.cs" />
     <Compile Include="Extensions.cs" />
+    <Compile Include="MiscTests.cs" />
     <Compile Include="ModlistMetadataTests.cs" />
     <Compile Include="TestUtils.cs" />
     <Compile Include="SanityTests.cs" />