From 27964f03486aee60b733a56e50ff24b4a0a2b459 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 20 Nov 2019 16:39:03 -0700 Subject: [PATCH] Set the queue size during installation based on the disk performance. Abort installation if there isn't enough disk space to perform the installation. --- Wabbajack.Common/Utils.cs | 28 +++++++++++++++++++ Wabbajack.Common/WorkQueue.cs | 13 +++++++-- Wabbajack.Lib/ABatchProcessor.cs | 17 ++++++++++++ Wabbajack.Lib/AInstaller.cs | 41 ++++++++++++++++++++++++++++ Wabbajack.Lib/Data.cs | 21 +++++++++++++- Wabbajack.Lib/MO2Compiler.cs | 2 +- Wabbajack.Lib/MO2Installer.cs | 3 +- Wabbajack.Lib/VortexInstaller.cs | 2 +- Wabbajack.Test/MiscTests.cs | 25 +++++++++++++++++ Wabbajack.Test/Wabbajack.Test.csproj | 1 + 10 files changed, 147 insertions(+), 6 deletions(-) create mode 100644 Wabbajack.Test/MiscTests.cs diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index c2d839c4..b026426e 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -697,6 +697,34 @@ namespace Wabbajack.Common Log(s); } + public static long TestDiskSpeed(WorkQueue queue, string path) + { + var start_time = DateTime.Now; + var seconds = 2; + return Enumerable.Range(0, queue.ThreadCount) + .PMap(queue, idx => + { + var random = new Random(); + + var file = Path.Combine(path, $"size_test{idx}.bin"); + long size = 0; + byte[] buffer = new byte[1024 * 8]; + random.NextBytes(buffer); + using (var fs = File.OpenWrite(file)) + { + while (DateTime.Now < start_time + new TimeSpan(0, 0, seconds)) + { + fs.Write(buffer, 0, buffer.Length); + // Flush to make sure large buffers don't cause the rate to be higher than it should + fs.Flush(); + size += buffer.Length; + } + } + File.Delete(file); + return size; + }).Sum() / seconds; + } + /// https://stackoverflow.com/questions/422090/in-c-sharp-check-that-filename-is-possibly-valid-not-that-it-exists public static IErrorResponse IsFilePathValid(string path) { diff --git a/Wabbajack.Common/WorkQueue.cs b/Wabbajack.Common/WorkQueue.cs index 37a74cee..33095b8f 100644 --- a/Wabbajack.Common/WorkQueue.cs +++ b/Wabbajack.Common/WorkQueue.cs @@ -10,7 +10,7 @@ using System.Threading; namespace Wabbajack.Common { - public class WorkQueue + public class WorkQueue : IDisposable { internal BlockingCollection Queue = new BlockingCollection(new ConcurrentStack()); @@ -32,6 +32,7 @@ namespace Wabbajack.Common private void StartThreads(int threadCount) { + ThreadCount = threadCount; Threads = Enumerable.Range(0, threadCount) .Select(idx => { @@ -44,6 +45,8 @@ namespace Wabbajack.Common }).ToList(); } + public int ThreadCount { get; private set; } + private void ThreadBody(int idx) { CpuId = idx; @@ -77,6 +80,12 @@ namespace Wabbajack.Common { Threads.Do(th => th.Abort()); } + + public void Dispose() + { + Shutdown(); + Queue?.Dispose(); + } } public class CPUStatus @@ -85,4 +94,4 @@ namespace Wabbajack.Common public string Msg { get; internal set; } public int ID { get; internal set; } } -} \ No newline at end of file +} diff --git a/Wabbajack.Lib/ABatchProcessor.cs b/Wabbajack.Lib/ABatchProcessor.cs index 212d8826..e211b6d8 100644 --- a/Wabbajack.Lib/ABatchProcessor.cs +++ b/Wabbajack.Lib/ABatchProcessor.cs @@ -61,6 +61,23 @@ namespace Wabbajack.Lib _configured = true; } + public static int RecommendQueueSize(string folder) + { + if (!Directory.Exists(folder)) + Directory.CreateDirectory(folder); + + using (var queue = new WorkQueue()) + { + Utils.Log($"Benchmarking {folder}"); + var raw_speed = Utils.TestDiskSpeed(queue, folder); + Utils.Log($"{raw_speed.ToFileSizeString()}/sec for {folder}"); + int speed = (int)(raw_speed / 1024 / 1024); + + // Less than 100MB/sec, stick with two threads. + return speed < 100 ? 2 : Math.Min(Environment.ProcessorCount, speed / 100 * 2); + } + } + protected abstract bool _Begin(); public Task Begin() { diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index 2c775784..a76e9a47 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -11,6 +11,7 @@ using Wabbajack.Lib.Downloaders; using Wabbajack.VirtualFileSystem; using Context = Wabbajack.VirtualFileSystem.Context; using Directory = Alphaleonis.Win32.Filesystem.Directory; +using DriveInfo = Alphaleonis.Win32.Filesystem.DriveInfo; using File = System.IO.File; using FileInfo = System.IO.FileInfo; using Path = Alphaleonis.Win32.Filesystem.Path; @@ -286,6 +287,39 @@ namespace Wabbajack.Lib .ToDictionary(e => e.Item1, e => e.Item2); } + public void ValidateFreeSpace() + { + DiskSpaceInfo DriveInfo(string path) + { + return Volume.GetDiskFreeSpace(Volume.GetUniqueVolumeNameForPath(path)); + } + + var paths = new[] {(OutputFolder, ModList.InstallSize), + (DownloadFolder, ModList.DownloadSize), + (Directory.GetCurrentDirectory(), ModList.ScratchSpaceSize)}; + paths.GroupBy(f => DriveInfo(f.Item1).DriveName) + .Do(g => + { + var required = g.Sum(i => i.Item2); + var available = DriveInfo(g.Key).FreeBytesAvailable; + if (required > available) + throw new NotEnoughDiskSpaceException( + $"This modlist requires {required.ToFileSizeString()} on {g.Key} but only {available.ToFileSizeString()} is available."); + }); + + } + + public int RecommendQueueSize() + { + var output_size = RecommendQueueSize(OutputFolder); + var download_size = RecommendQueueSize(DownloadFolder); + var scratch_size = RecommendQueueSize(Directory.GetCurrentDirectory()); + var result = Math.Min(output_size, Math.Min(download_size, scratch_size)); + Utils.Log($"Recommending a queue size of {result} based on disk performance and number of cores"); + return result; + } + + /// /// The user may already have some files in the OutputFolder. If so we can go through these and /// figure out which need to be updated, deleted, or left alone @@ -336,4 +370,11 @@ namespace Wabbajack.Lib } } + + public class NotEnoughDiskSpaceException : Exception + { + public NotEnoughDiskSpaceException(string s) : base(s) + { + } + } } diff --git a/Wabbajack.Lib/Data.cs b/Wabbajack.Lib/Data.cs index 4c3bf9bd..1a312bde 100644 --- a/Wabbajack.Lib/Data.cs +++ b/Wabbajack.Lib/Data.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using Ceras; using Compression.BSA; using Wabbajack.Common; @@ -94,6 +95,24 @@ namespace Wabbajack.Lib /// Content Report in HTML form /// public string ReportHTML; + + /// + /// The size of all the archives once they're downloaded + /// + public long DownloadSize => Archives.Sum(a => a.Size); + + /// + /// The size of all the files once they are installed (excluding downloaded archives) + /// + public long InstallSize => Directives.Sum(s => s.Size); + + /// + /// Estimate of the amount of space required in the VFS staging folders during installation + /// + public long ScratchSpaceSize => Archives.OrderByDescending(a => a.Size) + .Take(Environment.ProcessorCount) + .Sum(a => a.Size) * 2; + } public class Directive @@ -267,4 +286,4 @@ namespace Wabbajack.Lib /// public string BSAHash; } -} \ No newline at end of file +} diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index b13e67da..2590d445 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -443,4 +443,4 @@ namespace Wabbajack.Lib public DateTime LastModified; } } -} \ No newline at end of file +} diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index 3658ba1a..5d0f992d 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -33,7 +33,8 @@ namespace Wabbajack.Lib protected override bool _Begin() { - ConfigureProcessor(10); + ConfigureProcessor(RecommendQueueSize()); + ValidateFreeSpace(); var game = GameRegistry.Games[ModList.GameType]; if (GameFolder == null) diff --git a/Wabbajack.Lib/VortexInstaller.cs b/Wabbajack.Lib/VortexInstaller.cs index 63e5dc39..30e278a7 100644 --- a/Wabbajack.Lib/VortexInstaller.cs +++ b/Wabbajack.Lib/VortexInstaller.cs @@ -25,7 +25,7 @@ namespace Wabbajack.Lib protected override bool _Begin() { - ConfigureProcessor(10); + ConfigureProcessor(10, RecommendQueueSize()); Directory.CreateDirectory(DownloadFolder); HashArchives(); diff --git a/Wabbajack.Test/MiscTests.cs b/Wabbajack.Test/MiscTests.cs new file mode 100644 index 00000000..9de3aa50 --- /dev/null +++ b/Wabbajack.Test/MiscTests.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common; +using MahApps.Metro.Controls; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Utils = Wabbajack.Common.Utils; + +namespace Wabbajack.Test +{ + [TestClass] + public class MiscTests + { + [TestMethod] + public void TestDiskSpeed() + { + using (var queue = new WorkQueue()) + { + var speed = Utils.TestDiskSpeed(queue, @".\"); + } + } + } +} diff --git a/Wabbajack.Test/Wabbajack.Test.csproj b/Wabbajack.Test/Wabbajack.Test.csproj index b7d54c82..116e44f2 100644 --- a/Wabbajack.Test/Wabbajack.Test.csproj +++ b/Wabbajack.Test/Wabbajack.Test.csproj @@ -100,6 +100,7 @@ +