Few small fixes and optimizations

Wabbajack.CLI/OptionsDefinition.cs

@@ -39,7 +39,8 @@ namespace Wabbajack.CLI
             typeof(AllKnownDownloadStates),
             typeof(VerifyAllDownloads),
             typeof(HashBenchmark),
-            typeof(StressTestURL)
+            typeof(StressTestURL),
+            typeof(MirrorFolder)
         };
     }
 }

Wabbajack.CLI/Verbs/MirrorFolder.cs

@@ -0,0 +1,46 @@
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using CommandLine;
+using Wabbajack.Common;
+
+namespace Wabbajack.CLI.Verbs
+{
+    [Verb("mirror-folder", HelpText = "Copies the files from one folder to the other, skipping file of the same size and copies in parallel")]
+    public class MirrorFolder : AVerb
+    {
+        [IsDirectory(CustomMessage = "Downloads folder at %1 does not exist!")]
+        [Option('f', "from", HelpText = "From folder", Required = true)]
+        public string _FromFolder { get; set; } = "";
+
+        public AbsolutePath FromFolder => (AbsolutePath)_FromFolder;
+        
+        
+        [IsDirectory(CustomMessage = "Downloads folder at %1 does not exist!")]
+        [Option('t', "to", HelpText = "To folder", Required = true)]
+        public string _ToFolder { get; set; } = "";
+
+        public AbsolutePath ToFolder => (AbsolutePath)_ToFolder;
+        protected override async Task<ExitCode> Run()
+        {
+            var queue = new WorkQueue();
+
+            var src = FromFolder.EnumerateFiles().Where(f => f.IsFile).ToList();
+            Console.WriteLine($"Found {src.Count} files");
+            int idx = 0;
+
+            await src.PMap(queue, async f =>
+            {
+                var thisidx = Interlocked.Increment(ref idx);
+                var dest = f.RelativeTo(FromFolder).RelativeTo(ToFolder);
+
+                if (dest.IsFile && f.Size == dest.Size) return;
+
+                Console.WriteLine($"({thisidx}/{src.Count}) Copying {f.RelativeTo(FromFolder)} - {f.Size.ToFileSizeString()}");
+                await f.CopyToAsync(dest);
+            });
+            return ExitCode.Ok;
+        }
+    }
+}

Wabbajack.Common/Hash.cs

@@ -289,6 +289,7 @@ namespace Wabbajack.Common
             try
             {
                 await using var fs = await file.OpenRead();
+                await using var bs = new BufferedStream(fs, 1024 * 1024 * 10);
                 var config = new xxHashConfig { HashSizeInBits = 64 };
                 await using var hs = new StatusFileStream(fs, $"Hashing {file}");
                 var value = await xxHashFactory.Instance.Create(config).ComputeHashAsync(hs);

Wabbajack.Server/Services/NonNexusDownloadValidator.cs

@@ -27,13 +27,16 @@ namespace Wabbajack.Server.Services
         {
             var archives = await _sql.GetNonNexusModlistArchives();
             _logger.Log(LogLevel.Information, $"Validating {archives.Count} non-Nexus archives");
-            using var queue = new WorkQueue();
+            using var queue = new WorkQueue(10);
             await DownloadDispatcher.PrepareAll(archives.Select(a => a.State));
 
+            var random = new Random();
             var results = await archives.PMap(queue, async archive =>
             {
                 try
                 {
+                    await Task.Delay(random.Next(1000, 5000));
+                    
                     var token = new CancellationTokenSource();
                     token.CancelAfter(TimeSpan.FromMinutes(10));
                     
@@ -43,6 +46,9 @@ namespace Wabbajack.Server.Services
                     {
                         //case WabbajackCDNDownloader.State _: 
                         //case GoogleDriveDownloader.State _: // Let's try validating Google again 2/10/2021
+                        case GameFileSourceDownloader.State _:
+                            isValid = true;
+                            break;
                         case ManualDownloader.State _:
                         case ModDBDownloader.State _:
                         case HTTPDownloader.State h when h.Url.StartsWith("https://wabbajack"):