From e7db21d0fe01017e176e750fc280a7a323f283b8 Mon Sep 17 00:00:00 2001
From: Timothy Baldridge <tbaldridge@gmail.com>
Date: Sun, 10 Jan 2021 19:08:14 -0700
Subject: [PATCH] Pass the csrf token into the IPS4 downloader page

---
 Wabbajack.CLI/OptionsDefinition.cs            |  3 +-
 Wabbajack.CLI/Verbs/HashBenchmark.cs          | 46 +++++++++++++++++++
 .../Downloaders/AbstractIPS4Downloader.cs     | 33 +++++++++----
 3 files changed, 71 insertions(+), 11 deletions(-)
 create mode 100644 Wabbajack.CLI/Verbs/HashBenchmark.cs

diff --git a/Wabbajack.CLI/OptionsDefinition.cs b/Wabbajack.CLI/OptionsDefinition.cs
index 64031c98..c2d93bec 100644
--- a/Wabbajack.CLI/OptionsDefinition.cs
+++ b/Wabbajack.CLI/OptionsDefinition.cs
@@ -37,7 +37,8 @@ namespace Wabbajack.CLI
             typeof(Restore),
             typeof(PurgeArchive),
             typeof(AllKnownDownloadStates),
-            typeof(VerifyAllDownloads)
+            typeof(VerifyAllDownloads),
+            typeof(HashBenchmark)
         };
     }
 }
diff --git a/Wabbajack.CLI/Verbs/HashBenchmark.cs b/Wabbajack.CLI/Verbs/HashBenchmark.cs
new file mode 100644
index 00000000..019ed489
--- /dev/null
+++ b/Wabbajack.CLI/Verbs/HashBenchmark.cs
@@ -0,0 +1,46 @@
+using System;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading.Tasks;
+using CommandLine;
+using Wabbajack.Common;
+
+namespace Wabbajack.CLI.Verbs
+{
+    [Verb("hash-benchmark", HelpText = "Perform a benchmark of the hash routines by benchmarking all files in a folder")]
+    public class HashBenchmark : AVerb
+    {
+        [Option('i', "input", Required = true, HelpText = "Input Folder")]
+        public string _input { get; set; } = "";
+
+        public AbsolutePath Input => (AbsolutePath)_input;
+
+        [Option('t', "threads", Required = false, HelpText = "Thread Count (defaults to number of logical cores)")]
+        public int ThreadCount { get; set; } = Environment.ProcessorCount;
+
+
+        protected override async Task<ExitCode> Run()
+        {
+            using var queue = new WorkQueue(ThreadCount);
+
+            var files = Input.EnumerateFiles().Select(f => (f, f.Size)).Take(1000).ToArray();
+            var totalSize = files.Sum(f => f.Size);
+            Console.WriteLine($"Found {files.Length} files and {totalSize.ToFileSizeString()} to hash");
+
+            var stopwatch = new Stopwatch();
+            stopwatch.Start();
+
+            var su = new StatusUpdateTracker(1);
+            
+            await files.PMap(queue, su, async f =>
+            {
+                await f.f.FileHashAsync();
+            });
+            
+            stopwatch.Stop();
+            Console.WriteLine($"Hashed {totalSize.ToFileSizeString()} in {stopwatch.Elapsed.TotalSeconds} or {((long)(totalSize/stopwatch.Elapsed.TotalSeconds)).ToFileSizeString()}/sec");
+
+            return ExitCode.Ok;
+        }
+    }
+}
diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
index a8c3117f..f46e5a5d 100644
--- a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
+++ b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
@@ -198,15 +198,7 @@ namespace Wabbajack.Lib.Downloaders
                 }
                 else
                 {
-                    var csrfURL = string.IsNullOrWhiteSpace(FileID)
-                        ? $"{Site}/files/file/{FileName}/?do=download"
-                        : $"{Site}/files/file/{FileName}/?do=download&r={FileID}";
-                    var html = await GetStringAsync(new Uri(csrfURL), token);
-
-                    var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])");
-                    var matches = pattern.Matches(html).Cast<Match>();
-                    
-                    var csrfKey = matches.Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault();
+                    var csrfKey = await GetCsrfKey(token);
 
                     if (!Downloader.IsCloudFlareProtected && csrfKey == null)
                     {
@@ -291,6 +283,20 @@ namespace Wabbajack.Lib.Downloaders
                 goto TOP;
             }
 
+            private async Task<string?> GetCsrfKey(CancellationToken? token)
+            {
+                var csrfURL = string.IsNullOrWhiteSpace(FileID)
+                    ? $"{Site}/files/file/{FileName}/?do=download"
+                    : $"{Site}/files/file/{FileName}/?do=download&r={FileID}";
+                var html = await GetStringAsync(new Uri(csrfURL), token);
+
+                var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])");
+                var matches = pattern.Matches(html).Cast<Match>();
+
+                var csrfKey = matches.Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault();
+                return csrfKey;
+            }
+
             private async Task DeleteOldDownloadCookies(Driver driver)
             {
                 await driver.DeleteCookiesWhere(c => c.Name.StartsWith("ips4_downloads_delay_") && c.Value == "-1");
@@ -354,8 +360,15 @@ namespace Wabbajack.Lib.Downloaders
             {
                 var token = new CancellationTokenSource();
                 token.CancelAfter(TimeSpan.FromMinutes(10));
+
+                var csrfKey = await GetCsrfKey(token.Token);
+                if (csrfKey == null)
+                {
+                    Utils.Log($"Could not load CRSF key");
+                    return new List<Archive>();
+                }
                 
-                var others = await GetHtmlAsync(new Uri($"{Site}/files/file/{FileName}?do=download"), token.Token);
+                var others = await GetHtmlAsync(new Uri($"{Site}/files/file/{FileName}?do=download&csrfKey={csrfKey}"), token.Token);
 
                 var pairs = others.DocumentNode.SelectNodes("//a[@data-action='download']")
                     .Select(item => (item.GetAttributeValue("href", ""),