diff --git a/CHANGELOG.md b/CHANGELOG.md index faa51493..70aed2fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ### Changelog +* Run disk benchmarks for 10 seconds and use a tiered approach to disk queue size calculation of disks + #### Version - 2.0.7.0 - 5/28/2020 * Code is now robust when dealing with invasive Anti-virus software. We'll retry deletes/opens if the file is in use * Rework HTTP retries for all sites to reduce the amount of 503 errors we get from LL diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index a673ea45..5eb8d400 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -908,11 +908,13 @@ namespace Wabbajack.Common private static async Task TestDiskSpeedInner(WorkQueue queue, AbsolutePath path) { - var startTime = DateTime.Now; - var seconds = 2; - var results = await Enumerable.Range(0, queue.DesiredNumWorkers) + var seconds = 10; + var runTime = new TimeSpan(0, 0, seconds); + Log($"Running disk benchmark, this will take {seconds} seconds"); + var results = Enumerable.Range(0, queue.DesiredNumWorkers) .PMap(queue, async idx => { + var startTime = DateTime.Now; var random = new Random(); var file = path.Combine($"size_test{idx}.bin"); @@ -921,7 +923,7 @@ namespace Wabbajack.Common random.NextBytes(buffer); await using (var fs = await file.Create()) { - while (DateTime.Now < startTime + new TimeSpan(0, 0, seconds)) + while (DateTime.Now - startTime < runTime) { fs.Write(buffer, 0, buffer.Length); // Flush to make sure large buffers don't cause the rate to be higher than it should @@ -932,7 +934,14 @@ namespace Wabbajack.Common await file.DeleteAsync(); return size; }); - return results.Sum() / seconds; + + for (int x = 0; x < seconds; x++) + { + Log($"Running Disk benchmark {Percent.FactoryPutInRange(x, seconds)} complete"); + await Task.Delay(TimeSpan.FromSeconds(1)); + } + + return (await results).Sum() / seconds; } public static async Task TestDiskSpeed(WorkQueue queue, AbsolutePath path) diff --git a/Wabbajack.Lib/ABatchProcessor.cs b/Wabbajack.Lib/ABatchProcessor.cs index b2734334..28a14bdc 100644 --- a/Wabbajack.Lib/ABatchProcessor.cs +++ b/Wabbajack.Lib/ABatchProcessor.cs @@ -75,9 +75,9 @@ namespace Wabbajack.Lib // Most of the heavy lifting is done on the scratch disk, so we'll use the value from that disk var memory = Utils.GetMemoryStatus(); // Assume roughly 2GB of ram needed to extract each 7zip archive, and then leave 2GB for the OS. If calculation is lower or equal to 1 GB, use 1GB - var based_on_memory = Math.Max((memory.ullTotalPhys - (2 * GB)) / (2 * GB), 1); - var scratch_size = await RecommendQueueSize(AbsolutePath.EntryPoint); - var result = Math.Min((int)based_on_memory, (int)scratch_size); + var basedOnMemory = Math.Max((memory.ullTotalPhys - (2 * GB)) / (2 * GB), 1); + var scratchSize = await RecommendQueueSize(AbsolutePath.EntryPoint); + var result = Math.Min((int)basedOnMemory, (int)scratchSize); Utils.Log($"Recommending a queue size of {result} based on disk performance, number of cores, and {((long)memory.ullTotalPhys).ToFileSizeString()} of system RAM"); return result; } @@ -99,8 +99,12 @@ namespace Wabbajack.Lib Utils.Log($"{raw_speed.ToFileSizeString()}/sec for {folder}"); int speed = (int)(raw_speed / 1024 / 1024); - // Less than 100MB/sec, stick with two threads. - return speed < 100 ? 2 : Math.Min(Environment.ProcessorCount, speed / 100 * 2); + // Less than 200, it's probably a HDD, so we can't go higher than 2 + if (speed < 200) return 2; + // SATA SSD, so stick with 8 thread maximum + if (speed < 600) return Math.Min(Environment.ProcessorCount, 8); + // Anything higher is probably a NVME or a really good SSD, so take off the reins + return Environment.ProcessorCount; } ///