Merge pull request #241 from Noggog/async-exploration

Async exploration

Compression.BSA.Test/BSATests.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Newtonsoft.Json;
@@ -28,7 +29,7 @@ namespace Compression.BSA.Test
         private static WorkQueue Queue { get; set; }
 
         [ClassInitialize]
-        public static void Setup(TestContext testContext)
+        public static async Task Setup(TestContext testContext)
         {
             Queue = new WorkQueue();
             Utils.LogMessages.Subscribe(f => testContext.WriteLine(f.ShortDescription));
@@ -46,24 +47,21 @@ namespace Compression.BSA.Test
                 (Game.Fallout4, 22223) // 10mm SMG
             };
 
-            foreach (var info in modIDs)
+            await Task.WhenAll(modIDs.Select(async (info) =>
             {
-                var filename = DownloadMod(info);
+                var filename = await DownloadMod(info);
                 var folder = Path.Combine(_bsaFolder, info.Item1.ToString(), info.Item2.ToString());
                 if (!Directory.Exists(folder))
                     Directory.CreateDirectory(folder);
-                FileExtractor.ExtractAll(Queue, filename, folder);
-            }
+                await FileExtractor.ExtractAll(Queue, filename, folder);
+            }));
         }
 
-
-
-
-        private static string DownloadMod((Game, int) info)
+        private static async Task<string> DownloadMod((Game, int) info)
         {
-            using (var client = new NexusApiClient())
+            using (var client = await NexusApiClient.Get())
             {
-                var results = client.GetModFiles(info.Item1, info.Item2);
+                var results = await client.GetModFiles(info.Item1, info.Item2);
                 var file = results.files.FirstOrDefault(f => f.is_primary) ??
                            results.files.OrderByDescending(f => f.uploaded_timestamp).First();
                 var src = Path.Combine(_stagingFolder, file.file_name);
@@ -76,7 +74,7 @@ namespace Compression.BSA.Test
                     GameName = info.Item1.MetaData().NexusName,
                     FileID = file.file_id.ToString()
                 };
-                state.Download(src);
+                await state.Download(src);
                 return src;
             }
         }
@@ -91,7 +89,7 @@ namespace Compression.BSA.Test
         [TestMethod]
         [DataTestMethod]
         [DynamicData(nameof(BSAs), DynamicDataSourceType.Method)]
-        public void BSACompressionRecompression(string bsa)
+        public async Task BSACompressionRecompression(string bsa)
         {
             TestContext.WriteLine($"From {bsa}");
             TestContext.WriteLine("Cleaning Output Dir");
@@ -102,7 +100,7 @@ namespace Compression.BSA.Test
             string tempFile = Path.Combine("tmp.bsa");
             using (var a = BSADispatch.OpenRead(bsa))
             {
-                a.Files.PMap(Queue, file =>
+                await a.Files.PMap(Queue, file =>
                 {
                     var absName = Path.Combine(_tempDir, file.Path);
                     ViaJson(file.State);
@@ -123,7 +121,7 @@ namespace Compression.BSA.Test
 
                 using (var w = ViaJson(a.State).MakeBuilder())
                 {
-                    a.Files.PMap(Queue, file =>
+                    await a.Files.PMap(Queue, file =>
                     {
                         var absPath = Path.Combine(_tempDir, file.Path);
                         using (var str = File.OpenRead(absPath))
@@ -145,7 +143,7 @@ namespace Compression.BSA.Test
                     Assert.AreEqual(a.Files.Count(), b.Files.Count());
                     var idx = 0;
 
-                    a.Files.Zip(b.Files, (ai, bi) => (ai, bi))
+                    await a.Files.Zip(b.Files, (ai, bi) => (ai, bi))
                                 .PMap(Queue, pair =>
                                 {
                                     idx++;

Wabbajack.CacheServer/NexusCacheModule.cs

@@ -18,6 +18,9 @@ namespace Wabbajack.CacheServer
 
         public NexusCacheModule() : base("/")
         {
+            // ToDo
+            // Handle what to do with the fact that lots of these are now a tasks
+            throw new NotImplementedException("Unsure if following functions still work when taking in a Task");
             Get("/v1/games/{GameName}/mods/{ModID}/files/{FileID}.json", HandleFileID);
             Get("/v1/games/{GameName}/mods/{ModID}/files.json", HandleGetFiles);
             Get("/v1/games/{GameName}/mods/{ModID}.json", HandleModInfo);
@@ -26,10 +29,10 @@ namespace Wabbajack.CacheServer
             Get("/nexus_api_cache/update", UpdateCache);
         }
 
-        private object UpdateCache(object arg)
+        private async Task<object> UpdateCache(object arg)
         {
-            var api = new NexusApiClient(Request.Headers["apikey"].FirstOrDefault());
-            api.ClearUpdatedModsInCache();
+            var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
+            await api.ClearUpdatedModsInCache();
             return "Done";
         }
 
@@ -47,17 +50,17 @@ namespace Wabbajack.CacheServer
                 }));
         }
 
-        private object HandleModInfo(dynamic arg)
+        private async Task<object> HandleModInfo(dynamic arg)
         {
             Utils.Log($"{DateTime.Now} - Mod Info - {arg.GameName}/{arg.ModID}/");
-            var api = new NexusApiClient(Request.Headers["apikey"].FirstOrDefault());
+            var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
             return api.GetModInfo(GameRegistry.GetByNexusName((string)arg.GameName).Game, (string)arg.ModID).ToJSON();
         }
 
-        private object HandleFileID(dynamic arg)
+        private async Task<object> HandleFileID(dynamic arg)
         {
             Utils.Log($"{DateTime.Now} - File Info - {arg.GameName}/{arg.ModID}/{arg.FileID}");
-            var api = new NexusApiClient(Request.Headers["apikey"].FirstOrDefault());
+            var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
             return api.GetFileInfo(new NexusDownloader.State
             {
                 GameName = arg.GameName,
@@ -66,14 +69,14 @@ namespace Wabbajack.CacheServer
             }).ToJSON();
         }
 
-        private object HandleGetFiles(dynamic arg)
+        private async Task<object> HandleGetFiles(dynamic arg)
         {
             Utils.Log($"{DateTime.Now} - Mod Files - {arg.GameName} {arg.ModID}");
-            var api = new NexusApiClient(Request.Headers["apikey"].FirstOrDefault());
+            var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
             return api.GetModFiles(GameRegistry.GetByNexusName((string)arg.GameName).Game, (int)arg.ModID).ToJSON();
         }
 
-        private string HandleCacheCall(dynamic arg)
+        private async Task<string> HandleCacheCall(dynamic arg)
         {
             try
             {
@@ -87,7 +90,7 @@ namespace Wabbajack.CacheServer
                     var client = new HttpClient();
                     var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 8080, Scheme = "http"};
                     client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]);
-                    client.GetStringSync(builder.Uri.ToString());
+                    await client.GetStringAsync(builder.Uri.ToString());
                     if (!File.Exists(path))
                     {
                         Utils.Log($"Still not cached : {path}");

Wabbajack.CacheServer/Wabbajack.CacheServer.csproj

@@ -60,10 +60,6 @@
     <Prefer32Bit>true</Prefer32Bit>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="ReactiveUI, Version=10.5.0.0, Culture=neutral, processorArchitecture=MSIL">
-      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\Users\tbald\.nuget\packages\reactiveui\10.5.7\lib\net461\ReactiveUI.dll</HintPath>
-    </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />
     <Reference Include="System.Windows" />
@@ -103,6 +99,9 @@
     <PackageReference Include="Newtonsoft.Json">
       <Version>12.0.3</Version>
     </PackageReference>
+    <PackageReference Include="ReactiveUI">
+      <Version>11.0.1</Version>
+    </PackageReference>
     <PackageReference Include="System.Reactive">
       <Version>4.3.1</Version>
     </PackageReference>

Wabbajack.Common/FileExtractor.cs

@@ -5,6 +5,7 @@ using System.Linq;
 using System.Net.Http;
 using System.Reflection;
 using System.Security.Cryptography;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Compression.BSA;
 using ICSharpCode.SharpZipLib.GZip;
@@ -37,12 +38,12 @@ namespace Wabbajack.Common
         }
 
 
-        public static void ExtractAll(WorkQueue queue, string source, string dest)
+        public static async Task ExtractAll(WorkQueue queue, string source, string dest)
         {
             try
             {
                 if (Consts.SupportedBSAs.Any(b => source.ToLower().EndsWith(b)))
-                    ExtractAllWithBSA(queue, source, dest);
+                    await ExtractAllWithBSA(queue, source, dest);
                 else if (source.EndsWith(".omod"))
                     ExtractAllWithOMOD(source, dest);
                 else if (source.EndsWith(".exe"))
@@ -125,13 +126,13 @@ namespace Wabbajack.Common
             return dest;
         }
 
-        private static void ExtractAllWithBSA(WorkQueue queue, string source, string dest)
+        private static async Task ExtractAllWithBSA(WorkQueue queue, string source, string dest)
         {
             try
             {
                 using (var arch = BSADispatch.OpenRead(source))
                 {
-                    arch.Files
+                    await arch.Files
                         .PMap(queue, f =>
                         {
                             var path = f.Path;

Wabbajack.Common/StatusFeed/Errors/GenericException.cs

@@ -12,9 +12,9 @@ namespace Wabbajack.Common.StatusFeed.Errors
 
         public DateTime Timestamp { get; } = DateTime.Now;
 
-        public string ShortDescription => ExtraMessage + " - " + Exception?.Message;
+        public string ShortDescription => ExtraMessage == null ? Exception?.Message : $"{ExtraMessage} - {Exception?.Message}";
 
-        public string ExtendedDescription => $"{ExtraMessage}: {Exception?.ToString()}";
+        public string ExtendedDescription => ExtraMessage == null ? Exception?.ToString() : $"{ExtraMessage} - {Exception?.ToString()}";
 
         public Exception Exception { get; }
 

Wabbajack.Common/StatusFeed/GenericInfo.cs

@@ -9,12 +9,13 @@ namespace Wabbajack.Common.StatusFeed
     public class GenericInfo : AStatusMessage, IInfo
     {
         public override string ShortDescription { get; }
-        public override string ExtendedDescription { get;}
+        private readonly string _extendedDescription;
+        public override string ExtendedDescription => _extendedDescription ?? ShortDescription;
 
-        public GenericInfo(string short_description, string long_description = "")
+        public GenericInfo(string short_description, string long_description = null)
         {
             ShortDescription = short_description;
-            ExtendedDescription = long_description;
+            _extendedDescription = long_description;
         }
 
         public override string ToString()

Wabbajack.Common/StatusFeed/Interventions/AUserIntervention.cs

@@ -17,7 +17,7 @@ namespace Wabbajack.Common
         private bool _handled;
         public bool Handled { get => _handled; set => this.RaiseAndSetIfChanged(ref _handled, value); }
 
-        public int CpuID { get; } = WorkQueue.CpuId;
+        public int CpuID { get; } = WorkQueue.AsyncLocalCurrentQueue.Value?.CpuId ?? WorkQueue.UnassignedCpuId;
 
         public abstract void Cancel();
         public ICommand CancelCommand { get; }

Wabbajack.Common/Utils.cs

@@ -65,7 +65,7 @@ namespace Wabbajack.Common
 
         public static T Log<T>(T msg) where T : IStatusMessage
         {
-            LogToFile(msg.ShortDescription);
+            LogToFile(msg.ExtendedDescription);
             LoggerSubj.OnNext(msg);
             return msg;
         }
@@ -103,7 +103,7 @@ namespace Wabbajack.Common
 
         public static void Status(string msg, int progress = 0)
         {
-            WorkQueue.CurrentQueue?.Report(msg, progress);
+            WorkQueue.AsyncLocalCurrentQueue.Value?.Report(msg, progress);
         }
 
         /// <summary>
@@ -483,24 +483,36 @@ namespace Wabbajack.Common
             }
         }
 
-        public static TR[] PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue, StatusUpdateTracker updateTracker,
+        public static async Task<TR[]> PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue, StatusUpdateTracker updateTracker,
             Func<TI, TR> f)
         {
             var cnt = 0;
             var collist = coll.ToList();
-            return collist.PMap(queue, itm =>
+            return await collist.PMap(queue, itm =>
             {
                 updateTracker.MakeUpdate(collist.Count, Interlocked.Increment(ref cnt));
                 return f(itm);
             });
         }
 
-        public static void PMap<TI>(this IEnumerable<TI> coll, WorkQueue queue, StatusUpdateTracker updateTracker,
+        public static async Task<TR[]> PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue, StatusUpdateTracker updateTracker,
+            Func<TI, Task<TR>> f)
+        {
+            var cnt = 0;
+            var collist = coll.ToList();
+            return await collist.PMap(queue, itm =>
+            {
+                updateTracker.MakeUpdate(collist.Count, Interlocked.Increment(ref cnt));
+                return f(itm);
+            });
+        }
+
+        public static async Task PMap<TI>(this IEnumerable<TI> coll, WorkQueue queue, StatusUpdateTracker updateTracker,
             Action<TI> f)
         {
             var cnt = 0;
             var collist = coll.ToList();
-            collist.PMap(queue, itm =>
+            await collist.PMap(queue, itm =>
             {
                 updateTracker.MakeUpdate(collist.Count, Interlocked.Increment(ref cnt));
                 f(itm);
@@ -509,17 +521,17 @@ namespace Wabbajack.Common
         }
 
 
-        public static TR[] PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue,
+        public static async Task<TR[]> PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue,
             Func<TI, TR> f)
         {
             var colllst = coll.ToList();
 
             var remainingTasks = colllst.Count;
 
-            var tasks = coll.Select(i =>
+            var tasks = colllst.Select(i =>
             {
                 var tc = new TaskCompletionSource<TR>();
-                queue.QueueTask(() =>
+                queue.QueueTask(async () =>
                 {
                     try
                     {
@@ -538,20 +550,55 @@ namespace Wabbajack.Common
             if (WorkQueue.WorkerThread)
                 while (remainingTasks > 0)
                     if (queue.Queue.TryTake(out var a, 500))
-                        a();
+                    {
+                        WorkQueue.AsyncLocalCurrentQueue.Value = WorkQueue.ThreadLocalCurrentQueue.Value;
+                        await a();
+                    }
 
-            return tasks.Select(t =>
-            {
-                t.Wait();
-                if (t.IsFaulted)
-                    throw t.Exception;
-                return t.Result;
-            }).ToArray();
+            return await Task.WhenAll(tasks);
         }
 
-        public static void PMap<TI>(this IEnumerable<TI> coll, WorkQueue queue, Action<TI> f)
+
+        public static async Task<TR[]> PMap<TI, TR>(this IEnumerable<TI> coll, WorkQueue queue,
+            Func<TI, Task<TR>> f)
         {
-            coll.PMap(queue, i =>
+            var colllst = coll.ToList();
+
+            var remainingTasks = colllst.Count;
+
+            var tasks = colllst.Select(i =>
+            {
+                var tc = new TaskCompletionSource<TR>();
+                queue.QueueTask(async () =>
+                {
+                    try
+                    {
+                        tc.SetResult(await f(i));
+                    }
+                    catch (Exception ex)
+                    {
+                        tc.SetException(ex);
+                    }
+                    Interlocked.Decrement(ref remainingTasks);
+                });
+                return tc.Task;
+            }).ToList();
+
+            // To avoid thread starvation, we'll start to help out in the work queue
+            if (WorkQueue.WorkerThread)
+                while (remainingTasks > 0)
+                    if (queue.Queue.TryTake(out var a, 500))
+                    {
+                        WorkQueue.AsyncLocalCurrentQueue.Value = WorkQueue.ThreadLocalCurrentQueue.Value;
+                        await a();
+                    }
+
+            return await Task.WhenAll(tasks);
+        }
+
+        public static async Task PMap<TI>(this IEnumerable<TI> coll, WorkQueue queue, Action<TI> f)
+        {
+            await coll.PMap(queue, i =>
             {
                 f(i);
                 return false;
@@ -573,34 +620,10 @@ namespace Wabbajack.Common
             new List<bool>().Do(_ => f());
         }
 
-        public static HttpResponseMessage GetSync(this HttpClient client, string url)
+        public static async Task<Stream> PostStream(this HttpClient client, string url, HttpContent content)
         {
-            var result = client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
-            result.Wait();
-            return result.Result;
-        }
-
-        public static string GetStringSync(this HttpClient client, string url)
-        {
-            var result = client.GetStringAsync(url);
-            result.Wait();
-            return result.Result;
-        }
-
-        public static Stream GetStreamSync(this HttpClient client, string url)
-        {
-            var result = client.GetStreamAsync(url);
-            result.Wait();
-            return result.Result;
-        }
-
-        public static Stream PostStreamSync(this HttpClient client, string url, HttpContent content)
-        {
-            var result = client.PostAsync(url, content);
-            result.Wait();
-            var stream = result.Result.Content.ReadAsStreamAsync();
-            stream.Wait();
-            return stream.Result;
+            var result = await client.PostAsync(url, content);
+            return await result.Content.ReadAsStreamAsync();
         }
 
         public static IEnumerable<T> DistinctBy<T, V>(this IEnumerable<T> vs, Func<T, V> select)
@@ -769,11 +792,11 @@ namespace Wabbajack.Common
             Log(s);
         }
 
-        private static long TestDiskSpeedInner(WorkQueue queue, string path)
+        private static async Task<long> TestDiskSpeedInner(WorkQueue queue, string path)
         {
             var startTime = DateTime.Now;
             var seconds = 2;
-            return Enumerable.Range(0, queue.ThreadCount)
+            var results = await Enumerable.Range(0, queue.ThreadCount)
                 .PMap(queue, idx =>
                 {
                     var random = new Random();
@@ -794,16 +817,17 @@ namespace Wabbajack.Common
                     }
                     File.Delete(file);
                     return size;
-                }).Sum() / seconds;
+                });
+            return results.Sum() / seconds;
         }
 
         private static Dictionary<string, long> _cachedDiskSpeeds = new Dictionary<string, long>();
-        public static long TestDiskSpeed(WorkQueue queue, string path)
+        public static async Task<long> TestDiskSpeed(WorkQueue queue, string path)
         {
             var driveName = Volume.GetUniqueVolumeNameForPath(path);
             if (_cachedDiskSpeeds.TryGetValue(driveName, out long speed))
                 return speed;
-            speed = TestDiskSpeedInner(queue, path);
+            speed = await TestDiskSpeedInner(queue, path);
             _cachedDiskSpeeds[driveName] = speed;
             return speed;
         }

Wabbajack.Common/WorkQueue.cs

@@ -5,27 +5,31 @@ using System.Linq;
 using System.Reactive.Linq;
 using System.Reactive.Subjects;
 using System.Threading;
+using System.Threading.Tasks;
 using Wabbajack.Common.StatusFeed;
 
 namespace Wabbajack.Common
 {
     public class WorkQueue : IDisposable
     {
-        internal BlockingCollection<Action>
-            Queue = new BlockingCollection<Action>(new ConcurrentStack<Action>());
+        internal BlockingCollection<Func<Task>>
+            Queue = new BlockingCollection<Func<Task>>(new ConcurrentStack<Func<Task>>());
 
-        public const int UnassignedCpuId = -1;
+        public const int UnassignedCpuId = 0;
 
-        [ThreadStatic] private static int _cpuId = UnassignedCpuId;
-        public static int CpuId => _cpuId;
+        private static readonly AsyncLocal<int> _cpuId = new AsyncLocal<int>();
+        public int CpuId => _cpuId.Value;
 
-        internal static bool WorkerThread => CurrentQueue != null;
-        [ThreadStatic] internal static WorkQueue CurrentQueue;
+        internal static bool WorkerThread => ThreadLocalCurrentQueue.Value != null;
+        internal static readonly ThreadLocal<WorkQueue> ThreadLocalCurrentQueue = new ThreadLocal<WorkQueue>();
+        internal static readonly AsyncLocal<WorkQueue> AsyncLocalCurrentQueue = new AsyncLocal<WorkQueue>();
 
         private readonly Subject<CPUStatus> _Status = new Subject<CPUStatus>();
         public IObservable<CPUStatus> Status => _Status;
 
-        public static List<Thread> Threads { get; private set; }
+        public List<Thread> Threads { get; private set; }
+
+        private CancellationTokenSource _cancel = new CancellationTokenSource();
 
         // This is currently a lie, as it wires to the Utils singleton stream This is still good to have, 
         // so that logic related to a single WorkQueue can subscribe to this dummy member so that If/when we 
@@ -40,10 +44,10 @@ namespace Wabbajack.Common
         private void StartThreads(int threadCount)
         {
             ThreadCount = threadCount;
-            Threads = Enumerable.Range(0, threadCount)
+            Threads = Enumerable.Range(1, threadCount)
                 .Select(idx =>
                 {
-                    var thread = new Thread(() => ThreadBody(idx));
+                    var thread = new Thread(() => ThreadBody(idx).Wait());
                     thread.Priority = ThreadPriority.BelowNormal;
                     thread.IsBackground = true;
                     thread.Name = string.Format("Wabbajack_Worker_{0}", idx);
@@ -54,16 +58,24 @@ namespace Wabbajack.Common
 
         public int ThreadCount { get; private set; }
 
-        private void ThreadBody(int idx)
+        private async Task ThreadBody(int idx)
         {
-            _cpuId = idx;
-            CurrentQueue = this;
+            _cpuId.Value = idx;
+            ThreadLocalCurrentQueue.Value = this;
+            AsyncLocalCurrentQueue.Value = this;
 
-            while (true)
+            try
+            {
+                while (true)
+                {
+                    Report("Waiting", 0, false);
+                    if (_cancel.IsCancellationRequested) return;
+                    var f = Queue.Take(_cancel.Token);
+                    await f();
+                }
+            }
+            catch (OperationCanceledException)
             {
-                Report("Waiting", 0, false);
-                var f = Queue.Take();
-                f();
             }
         }
 
@@ -75,25 +87,26 @@ namespace Wabbajack.Common
                     Progress = progress,
                     ProgressPercent = progress / 100f,
                     Msg = msg,
-                    ID = _cpuId,
+                    ID = _cpuId.Value,
                     IsWorking = isWorking
                 });
         }
 
-        public void QueueTask(Action a)
+        public void QueueTask(Func<Task> a)
         {
             Queue.Add(a);
         }
 
-        public void Shutdown()
-        {
-            Threads.Do(th => th.Abort());
-            Threads.Do(th => th.Join());
-        }
-
         public void Dispose()
         {
-            Shutdown();
+            _cancel.Cancel();
+            Threads.Do(th =>
+            {
+                if (th.ManagedThreadId != Thread.CurrentThread.ManagedThreadId)
+                {
+                    th.Join();
+                }
+            });
             Queue?.Dispose();
         }
     }

Wabbajack.Lib/ABatchProcessor.cs

@@ -14,12 +14,6 @@ namespace Wabbajack.Lib
     {
         public WorkQueue Queue { get; private set; }
 
-        public void Dispose()
-        {
-            Queue?.Shutdown();
-            _subs.Dispose();
-        }
-
         public Context VFS { get; private set; }
 
         protected StatusUpdateTracker UpdateTracker { get; private set; }
@@ -46,11 +40,10 @@ namespace Wabbajack.Lib
 
         private Subject<bool> _isRunning { get; } = new Subject<bool>();
         public IObservable<bool> IsRunning => _isRunning;
-        
-        private Thread _processorThread { get; set; }
 
         private int _configured;
         private int _started;
+        private readonly CancellationTokenSource _cancel = new CancellationTokenSource();
 
         private readonly CompositeDisposable _subs = new CompositeDisposable();
 
@@ -71,7 +64,7 @@ namespace Wabbajack.Lib
             VFS = new Context(Queue) { UpdateTracker = UpdateTracker };
         }
 
-        public static int RecommendQueueSize(string folder)
+        public static async Task<int> RecommendQueueSize(string folder)
         {
             if (!Directory.Exists(folder))
                 Directory.CreateDirectory(folder);
@@ -79,7 +72,7 @@ namespace Wabbajack.Lib
             using (var queue = new WorkQueue())
             {
                 Utils.Log($"Benchmarking {folder}");
-                var raw_speed = Utils.TestDiskSpeed(queue, folder);
+                var raw_speed = await Utils.TestDiskSpeed(queue, folder);
                 Utils.Log($"{raw_speed.ToFileSizeString()}/sec for {folder}");
                 int speed = (int)(raw_speed / 1024 / 1024);
 
@@ -88,7 +81,7 @@ namespace Wabbajack.Lib
             }
         }
 
-        protected abstract bool _Begin();
+        protected abstract Task<bool> _Begin(CancellationToken cancel);
         public Task<bool> Begin()
         {
             if (1 == Interlocked.CompareExchange(ref _started, 1, 1))
@@ -96,33 +89,24 @@ namespace Wabbajack.Lib
                 throw new InvalidDataException("Can't start the processor twice");
             }
 
-            _isRunning.OnNext(true);
-            var _tcs = new TaskCompletionSource<bool>();
-
-            _processorThread = new Thread(() =>
-            {
+            return Task.Run(async () =>
+            { 
                 try
                 {
-                    _tcs.SetResult(_Begin());
-                }
-                catch (Exception ex)
-                {
-                    _tcs.SetException(ex);
+                    _isRunning.OnNext(true);
+                    return await _Begin(_cancel.Token);
                 }
                 finally
                 {
                     _isRunning.OnNext(false);
                 }
             });
-            _processorThread.Priority = ThreadPriority.BelowNormal;
-            _processorThread.Start();
-            return _tcs.Task;
         }
 
-        public void Terminate()
+        public void Dispose()
         {
-            Queue?.Shutdown();
-            _processorThread?.Abort();
+            _cancel.Cancel();
+            Queue?.Dispose();
             _isRunning.OnNext(false);
         }
     }

Wabbajack.Lib/ACompiler.cs

@@ -5,6 +5,8 @@ using System.IO;
 using System.IO.Compression;
 using System.Linq;
 using System.Reactive.Subjects;
+using System.Threading.Tasks;
+using System.Threading;
 using CommonMark;
 using Wabbajack.Common;
 using Wabbajack.Lib.CompilationSteps;
@@ -188,7 +190,7 @@ namespace Wabbajack.Lib
                 + CommonMarkConverter.Convert(File.ReadAllText($"{ModList.Name}.md"));
         }
 
-        public void GatherArchives()
+        public async Task GatherArchives()
         {
             Info("Building a list of archives based on the files required");
 
@@ -200,21 +202,21 @@ namespace Wabbajack.Lib
                 .GroupBy(f => f.File.Hash)
                 .ToDictionary(f => f.Key, f => f.First());
 
-            SelectedArchives = shas.PMap(Queue, sha => ResolveArchive(sha, archives));
+            SelectedArchives = await shas.PMap(Queue, sha => ResolveArchive(sha, archives));
         }
 
-        public Archive ResolveArchive(string sha, IDictionary<string, IndexedArchive> archives)
+        public async Task<Archive> ResolveArchive(string sha, IDictionary<string, IndexedArchive> archives)
         {
             if (archives.TryGetValue(sha, out var found))
             {
-                return ResolveArchive(found);
+                return await ResolveArchive(found);
             }
 
             Error($"No match found for Archive sha: {sha} this shouldn't happen");
             return null;
         }
 
-        public Archive ResolveArchive(IndexedArchive archive)
+        public async Task<Archive> ResolveArchive(IndexedArchive archive)
         {
             if (archive.IniData == null)
                 Error(
@@ -235,19 +237,19 @@ namespace Wabbajack.Lib
 
             Info($"Checking link for {archive.Name}");
 
-            if (result.State != null && !result.State.Verify())
+            if (result.State != null && !await result.State.Verify())
                 Error(
                     $"Unable to resolve link for {archive.Name}. If this is hosted on the Nexus the file may have been removed.");
 
             return result;
         }
 
-        public Directive RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source)
+        public async Task<Directive> RunStack(IEnumerable<ICompilationStep> stack, RawSourceFile source)
         {
             Utils.Status($"Compiling {source.Path}");
             foreach (var step in stack)
             {
-                var result = step.Run(source);
+                var result = await step.Run(source);
                 if (result != null) return result;
             }
 

Wabbajack.Lib/AInstaller.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.IO.Compression;
 using System.Linq;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
 using Wabbajack.Lib.Downloaders;
@@ -85,7 +86,7 @@ namespace Wabbajack.Lib
         ///     We don't want to make the installer index all the archives, that's just a waste of time, so instead
         ///     we'll pass just enough information to VFS to let it know about the files we have.
         /// </summary>
-        public void PrimeVFS()
+        public async Task PrimeVFS()
         {
             VFS.AddKnown(HashedArchives.Select(a => new KnownFile
             {
@@ -99,7 +100,7 @@ namespace Wabbajack.Lib
                     .OfType<FromArchive>()
                     .Select(f => new KnownFile { Paths = f.ArchiveHashPath}));
 
-            VFS.BackfillMissing();
+            await VFS.BackfillMissing();
         }
 
         public void BuildFolderStructure()
@@ -115,7 +116,7 @@ namespace Wabbajack.Lib
                 });
         }
 
-        public void InstallArchives()
+        public async Task InstallArchives()
         {
             Info("Installing Archives");
             Info("Grouping Install Files");
@@ -129,10 +130,10 @@ namespace Wabbajack.Lib
                 .ToList();
 
             Info("Installing Archives");
-            archives.PMap(Queue, UpdateTracker,a => InstallArchive(a.Archive, a.AbsolutePath, grouped[a.Archive.Hash]));
+            await archives.PMap(Queue, UpdateTracker,a => InstallArchive(a.Archive, a.AbsolutePath, grouped[a.Archive.Hash]));
         }
 
-        private void InstallArchive(Archive archive, string absolutePath, IGrouping<string, FromArchive> grouping)
+        private async Task InstallArchive(Archive archive, string absolutePath, IGrouping<string, FromArchive> grouping)
         {
             Status($"Extracting {archive.Name}");
 
@@ -143,7 +144,7 @@ namespace Wabbajack.Lib
                 return g;
             }).ToList();
 
-            var onFinish = VFS.Stage(vFiles.Select(f => f.FromFile).Distinct());
+            var onFinish = await VFS.Stage(vFiles.Select(f => f.FromFile).Distinct());
 
 
             Status($"Copying files for {archive.Name}");
@@ -221,7 +222,7 @@ namespace Wabbajack.Lib
                 }
         }
 
-        public void DownloadArchives()
+        public async Task DownloadArchives()
         {
             var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
             Info($"Missing {missing.Count} archives");
@@ -230,25 +231,24 @@ namespace Wabbajack.Lib
 
             var dispatchers = missing.Select(m => m.State.GetDownloader()).Distinct();
 
-            foreach (var dispatcher in dispatchers)
-                dispatcher.Prepare();
+            await Task.WhenAll(dispatchers.Select(d => d.Prepare()));
             
-            DownloadMissingArchives(missing);
+            await DownloadMissingArchives(missing);
         }
 
-        private void DownloadMissingArchives(List<Archive> missing, bool download = true)
+        private async Task DownloadMissingArchives(List<Archive> missing, bool download = true)
         {
             if (download)
             {
                 foreach (var a in missing.Where(a => a.State.GetType() == typeof(ManualDownloader.State)))
                 {
                     var outputPath = Path.Combine(DownloadFolder, a.Name);
-                    a.State.Download(a, outputPath);
+                    await a.State.Download(a, outputPath);
                 }
             }
 
-            missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State))
-                .PMap(Queue, archive =>
+            await missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State))
+                .PMap(Queue, async archive =>
                 {
                     Info($"Downloading {archive.Name}");
                     var outputPath = Path.Combine(DownloadFolder, archive.Name);
@@ -257,16 +257,16 @@ namespace Wabbajack.Lib
                         if (outputPath.FileExists())
                             File.Delete(outputPath);
 
-                    return DownloadArchive(archive, download);
+                    return await DownloadArchive(archive, download);
                 });
         }
 
-        public bool DownloadArchive(Archive archive, bool download)
+        public async Task<bool> DownloadArchive(Archive archive, bool download)
         {
             try
             {
                 var path = Path.Combine(DownloadFolder, archive.Name);
-                archive.State.Download(archive, path);
+                await archive.State.Download(archive, path);
                 path.FileHashCached();
 
             }
@@ -280,11 +280,12 @@ namespace Wabbajack.Lib
             return false;
         }
 
-        public void HashArchives()
+        public async Task HashArchives()
         {
-            HashedArchives = Directory.EnumerateFiles(DownloadFolder)
+            var hashResults = await Directory.EnumerateFiles(DownloadFolder)
                 .Where(e => !e.EndsWith(Consts.HashFileExtension))
-                .PMap(Queue, e => (e.FileHashCached(), e))
+                .PMap(Queue, e => (e.FileHashCached(), e));
+            HashedArchives = hashResults
                 .OrderByDescending(e => File.GetLastWriteTime(e.Item2))
                 .GroupBy(e => e.Item1)
                 .Select(e => e.First())
@@ -322,11 +323,11 @@ namespace Wabbajack.Lib
             */
         }
 
-        public int RecommendQueueSize()
+        public async Task<int> RecommendQueueSize()
         {
-            var output_size = RecommendQueueSize(OutputFolder);
-            var download_size = RecommendQueueSize(DownloadFolder);
-            var scratch_size = RecommendQueueSize(Directory.GetCurrentDirectory());
+            var output_size = await RecommendQueueSize(OutputFolder);
+            var download_size = await RecommendQueueSize(DownloadFolder);
+            var scratch_size = await RecommendQueueSize(Directory.GetCurrentDirectory());
             var result =  Math.Min(output_size, Math.Min(download_size, scratch_size));
             Utils.Log($"Recommending a queue size of {result} based on disk performance and number of cores");
             return result;
@@ -337,13 +338,13 @@ namespace Wabbajack.Lib
         /// The user may already have some files in the OutputFolder. If so we can go through these and
         /// figure out which need to be updated, deleted, or left alone
         /// </summary>
-        public void OptimizeModlist()
+        public async Task OptimizeModlist()
         {
             Utils.Log("Optimizing Modlist directives");
             var indexed = ModList.Directives.ToDictionary(d => d.To);
 
             UpdateTracker.NextStep("Looking for files to delete");
-            Directory.EnumerateFiles(OutputFolder, "*", DirectoryEnumerationOptions.Recursive)
+            await Directory.EnumerateFiles(OutputFolder, "*", DirectoryEnumerationOptions.Recursive)
                 .PMap(Queue, UpdateTracker, f =>
                 {
                     var relative_to = f.RelativeTo(OutputFolder);
@@ -356,7 +357,7 @@ namespace Wabbajack.Lib
                 });
 
             UpdateTracker.NextStep("Looking for unmodified files");
-            indexed.Values.PMap(Queue, UpdateTracker, d =>
+            (await indexed.Values.PMap(Queue, UpdateTracker, d =>
             {
                 // Bit backwards, but we want to return null for 
                 // all files we *want* installed. We return the files
@@ -369,7 +370,8 @@ namespace Wabbajack.Lib
                 if (fi.Length != d.Size) return null;
                 
                 return path.FileHash() == d.Hash ? d : null;
-            }).Where(d => d != null)
+            }))
+              .Where(d => d != null)
               .Do(d => indexed.Remove(d.To));
 
             UpdateTracker.NextStep("Updating Modlist");

Wabbajack.Lib/CompilationSteps/ACompilationStep.cs

@@ -1,4 +1,6 @@
-namespace Wabbajack.Lib.CompilationSteps
+using System.Threading.Tasks;
+
+namespace Wabbajack.Lib.CompilationSteps
 {
     public abstract class ACompilationStep : ICompilationStep
     {
@@ -9,7 +11,7 @@
             _compiler = compiler;
         }
 
-        public abstract Directive Run(RawSourceFile source);
+        public abstract ValueTask<Directive> Run(RawSourceFile source);
         public abstract IState GetState();
     }
-}
+}

Wabbajack.Lib/CompilationSteps/DeconstructBSAs.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Compression.BSA;
 using Newtonsoft.Json;
@@ -49,7 +50,7 @@ namespace Wabbajack.Lib.CompilationSteps
             return new State();
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!Consts.SupportedBSAs.Contains(Path.GetExtension(source.Path).ToLower())) return null;
 
@@ -64,7 +65,7 @@ namespace Wabbajack.Lib.CompilationSteps
 
             var id = Guid.NewGuid().ToString();
 
-            var matches = sourceFiles.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e)
+            var matches = await sourceFiles.PMap(_mo2Compiler.Queue, e => _mo2Compiler.RunStack(stack, new RawSourceFile(e)
             {
                 Path = Path.Combine(Consts.BSACreationDir, id, e.Name)
             }));

Wabbajack.Lib/CompilationSteps/DirectMatch.cs

@@ -1,4 +1,5 @@
 using System.Linq;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 
@@ -10,7 +11,7 @@ namespace Wabbajack.Lib.CompilationSteps
         {
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!_compiler.IndexedFiles.TryGetValue(source.Hash, out var found)) return null;
             var result = source.EvolveTo<FromArchive>();
@@ -39,4 +40,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/DropAll.cs

@@ -1,4 +1,5 @@
-using Newtonsoft.Json;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
 using Wabbajack.Common;
 
 namespace Wabbajack.Lib.CompilationSteps
@@ -9,7 +10,7 @@ namespace Wabbajack.Lib.CompilationSteps
         {
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             var result = source.EvolveTo<NoMatch>();
             result.Reason = "No Match in Stack";
@@ -31,4 +32,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IStackStep.cs

@@ -1,8 +1,10 @@
-namespace Wabbajack.Lib.CompilationSteps
+using System.Threading.Tasks;
+
+namespace Wabbajack.Lib.CompilationSteps
 {
     public interface ICompilationStep
     {
-        Directive Run(RawSourceFile source);
+        ValueTask<Directive> Run(RawSourceFile source);
         IState GetState();
     }
 
@@ -10,4 +12,4 @@
     {
         ICompilationStep CreateStep(ACompiler compiler);
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IgnoreDisabledMods.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 using Wabbajack.Common;
@@ -25,7 +26,7 @@ namespace Wabbajack.Lib.CompilationSteps
                 .ToList();
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!source.Path.StartsWith("mods") || _allEnabledMods.Any(mod => source.Path.StartsWith(mod)))
                 return null;
@@ -63,4 +64,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IgnoreDisabledVortexMods.cs

@@ -1,4 +1,5 @@
-using Wabbajack.Common;
+using System.Threading.Tasks;
+using Wabbajack.Common;
 
 namespace Wabbajack.Lib.CompilationSteps
 {
@@ -11,7 +12,7 @@ namespace Wabbajack.Lib.CompilationSteps
             _vortexCompiler = (VortexCompiler) compiler;
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             var b = false;
             _vortexCompiler.ActiveArchives.Do(a =>

Wabbajack.Lib/CompilationSteps/IgnoreEndsWith.cs

@@ -1,4 +1,5 @@
-using Newtonsoft.Json;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
 
 namespace Wabbajack.Lib.CompilationSteps
 {
@@ -13,7 +14,7 @@ namespace Wabbajack.Lib.CompilationSteps
             _reason = $"Ignored because path ends with {postfix}";
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!source.Path.EndsWith(_postfix)) return null;
             var result = source.EvolveTo<IgnoredDirectly>();
@@ -46,4 +47,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs

@@ -1,4 +1,5 @@
-using Newtonsoft.Json;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
 using Wabbajack.Common;
 
 namespace Wabbajack.Lib.CompilationSteps
@@ -12,7 +13,7 @@ namespace Wabbajack.Lib.CompilationSteps
             _startDir = Consts.GameFolderFilesDir + "\\";
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!source.Path.StartsWith(_startDir)) return null;
             var i = source.EvolveTo<IgnoredDirectly>();
@@ -34,4 +35,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IgnorePathContains.cs

@@ -1,4 +1,5 @@
-using Newtonsoft.Json;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
 
 namespace Wabbajack.Lib.CompilationSteps
 {
@@ -13,7 +14,7 @@ namespace Wabbajack.Lib.CompilationSteps
             _reason = $"Ignored because path contains {_pattern}";
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!source.Path.Contains(_pattern)) return null;
             var result = source.EvolveTo<IgnoredDirectly>();
@@ -46,4 +47,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IgnoreRegex.cs

@@ -1,4 +1,5 @@
 using System.Text.RegularExpressions;
+using System.Threading.Tasks;
 using Newtonsoft.Json;
 
 namespace Wabbajack.Lib.CompilationSteps
@@ -16,7 +17,7 @@ namespace Wabbajack.Lib.CompilationSteps
             _regex = new Regex(pattern);
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!_regex.IsMatch(source.Path)) return null;
             var result = source.EvolveTo<IgnoredDirectly>();
@@ -49,4 +50,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IgnoreStartsWith.cs

@@ -1,4 +1,5 @@
-using Newtonsoft.Json;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
 
 namespace Wabbajack.Lib.CompilationSteps
 {
@@ -13,7 +14,7 @@ namespace Wabbajack.Lib.CompilationSteps
             _reason = string.Format("Ignored because path starts with {0}", _prefix);
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (source.Path.StartsWith(_prefix))
             {
@@ -50,4 +51,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IgnoreVortex.cs

@@ -1,4 +1,5 @@
 using System.IO;
+using System.Threading.Tasks;
 using Newtonsoft.Json;
 
 namespace Wabbajack.Lib.CompilationSteps
@@ -12,7 +13,7 @@ namespace Wabbajack.Lib.CompilationSteps
             _vortex = (VortexCompiler) compiler;
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (Path.GetDirectoryName(source.AbsolutePath) != _vortex.DownloadsFolder) return null;
             var result = source.EvolveTo<IgnoredDirectly>();

Wabbajack.Lib/CompilationSteps/IgnoreWabbajackInstallCruft.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 using Newtonsoft.Json;
 using Wabbajack.Common;
 
@@ -18,7 +19,7 @@ namespace Wabbajack.Lib.CompilationSteps
             };
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!_cruftFiles.Any(f => source.Path.StartsWith(f))) return null;
             var result = source.EvolveTo<IgnoredDirectly>();
@@ -40,4 +41,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IncludeAll.cs

@@ -1,4 +1,5 @@
-using Alphaleonis.Win32.Filesystem;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 
 namespace Wabbajack.Lib.CompilationSteps
@@ -9,7 +10,7 @@ namespace Wabbajack.Lib.CompilationSteps
         {
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             var inline = source.EvolveTo<InlineFile>();
             inline.SourceDataID = _compiler.IncludeFile(File.ReadAllBytes(source.AbsolutePath));
@@ -30,4 +31,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IncludeAllConfigs.cs

@@ -1,4 +1,5 @@
-using Alphaleonis.Win32.Filesystem;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 using Wabbajack.Common;
 
@@ -10,7 +11,7 @@ namespace Wabbajack.Lib.CompilationSteps
         {
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!Consts.ConfigFileExtensions.Contains(Path.GetExtension(source.Path))) return null;
             var result = source.EvolveTo<InlineFile>();
@@ -32,4 +33,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IncludeDummyESPs.cs

@@ -1,4 +1,5 @@
-using Alphaleonis.Win32.Filesystem;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 
 namespace Wabbajack.Lib.CompilationSteps
@@ -9,7 +10,7 @@ namespace Wabbajack.Lib.CompilationSteps
         {
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (Path.GetExtension(source.AbsolutePath) != ".esp" &&
                 Path.GetExtension(source.AbsolutePath) != ".esm") return null;
@@ -42,4 +43,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IncludeLOOTFiles.cs

@@ -1,4 +1,5 @@
-using Alphaleonis.Win32.Filesystem;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 using Wabbajack.Common;
 
@@ -13,7 +14,7 @@ namespace Wabbajack.Lib.CompilationSteps
             _prefix = Consts.LOOTFolderFilesDir + "\\";
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!source.Path.StartsWith(_prefix)) return null;
             var result = source.EvolveTo<InlineFile>();
@@ -35,4 +36,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IncludeModIniData.cs

@@ -1,4 +1,5 @@
-using Alphaleonis.Win32.Filesystem;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 
 namespace Wabbajack.Lib.CompilationSteps
@@ -9,7 +10,7 @@ namespace Wabbajack.Lib.CompilationSteps
         {
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!source.Path.StartsWith("mods\\") || !source.Path.EndsWith("\\meta.ini")) return null;
             var e = source.EvolveTo<InlineFile>();
@@ -31,4 +32,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IncludeOtherProfiles.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 
@@ -19,7 +20,7 @@ namespace Wabbajack.Lib.CompilationSteps
                 .ToList();
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!source.Path.StartsWith("profiles\\")) return null;
             if (_profiles.Any(profile => source.Path.StartsWith(profile))) return null;
@@ -42,4 +43,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IncludePatches.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 using Wabbajack.Common;
@@ -19,7 +20,7 @@ namespace Wabbajack.Lib.CompilationSteps
                 .ToDictionary(f => f.Key);
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!_indexed.TryGetValue(Path.GetFileName(source.File.Name.ToLower()), out var choices))
                 return null;

Wabbajack.Lib/CompilationSteps/IncludePropertyFiles.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 
@@ -12,7 +13,7 @@ namespace Wabbajack.Lib.CompilationSteps
         {
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             var files = new HashSet<string>
             {
@@ -52,4 +53,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IncludeRegex.cs

@@ -1,4 +1,5 @@
 using System.Text.RegularExpressions;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 
@@ -15,7 +16,7 @@ namespace Wabbajack.Lib.CompilationSteps
             _regex = new Regex(pattern);
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!_regex.IsMatch(source.Path)) return null;
 
@@ -49,4 +50,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IncludeSteamWorkshopItems.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Linq;
 using System.Text.RegularExpressions;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
 
@@ -16,7 +17,7 @@ namespace Wabbajack.Lib.CompilationSteps
             _game = steamGame;
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!_regex.IsMatch(source.Path)) 
                 return null;

Wabbajack.Lib/CompilationSteps/IncludeStubbedConfigfiles.cs

@@ -1,4 +1,5 @@
 using System.Text;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 using Wabbajack.Common;
@@ -14,7 +15,7 @@ namespace Wabbajack.Lib.CompilationSteps
             _mo2Compiler = (MO2Compiler) compiler;
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             return Consts.ConfigFileExtensions.Contains(Path.GetExtension(source.Path)) ? RemapFile(source) : null;
         }
@@ -58,4 +59,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IncludeTaggedMods.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 
@@ -24,7 +25,7 @@ namespace Wabbajack.Lib.CompilationSteps
             }).Select(kv => $"mods\\{kv.Key}\\");
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (!source.Path.StartsWith("mods")) return null;
             foreach (var modpath in _includeDirectly)
@@ -63,4 +64,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IncludeThisProfile.cs

@@ -1,6 +1,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Newtonsoft.Json;
 
@@ -17,7 +18,7 @@ namespace Wabbajack.Lib.CompilationSteps
             _correctProfiles = _mo2Compiler.SelectedProfiles.Select(p => Path.Combine("profiles", p) + "\\").ToList();
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             if (_correctProfiles.Any(p => source.Path.StartsWith(p)))
             {
@@ -56,4 +57,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/CompilationSteps/IncludeVortexDeployment.cs

@@ -1,4 +1,5 @@
 using System.IO;
+using System.Threading.Tasks;
 
 namespace Wabbajack.Lib.CompilationSteps
 {
@@ -8,9 +9,8 @@ namespace Wabbajack.Lib.CompilationSteps
         {
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
-
             if (!source.Path.EndsWith("vortex.deployment.msgpack") &&
                 !source.Path.EndsWith("\\vortex.deployment.json") && Path.GetExtension(source.Path) != ".meta") return null;
             var inline = source.EvolveTo<InlineFile>();

Wabbajack.Lib/CompilationSteps/PatchStockESMs.cs

@@ -1,4 +1,5 @@
 using System.IO;
+using System.Threading.Tasks;
 using Newtonsoft.Json;
 using Wabbajack.Common;
 using File = Alphaleonis.Win32.Filesystem.File;
@@ -15,7 +16,7 @@ namespace Wabbajack.Lib.CompilationSteps
             _mo2Compiler = (MO2Compiler) compiler;
         }
 
-        public override Directive Run(RawSourceFile source)
+        public override async ValueTask<Directive> Run(RawSourceFile source)
         {
             var filename = Path.GetFileName(source.Path);
             var gameFile = Path.Combine(_mo2Compiler.GamePath, "Data", filename);
@@ -53,4 +54,4 @@ namespace Wabbajack.Lib.CompilationSteps
             }
         }
     }
-}
+}

Wabbajack.Lib/Downloaders/AbstractDownloadState.cs

@@ -1,4 +1,5 @@
-using Alphaleonis.Win32.Filesystem;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Lib.Validation;
 
 namespace Wabbajack.Lib.Downloaders
@@ -19,18 +20,18 @@ namespace Wabbajack.Lib.Downloaders
         /// Downloads this file to the given destination location
         /// </summary>
         /// <param name="destination"></param>
-        public abstract void Download(Archive a, string destination);
+        public abstract Task Download(Archive a, string destination);
 
-        public void Download(string destination)
+        public async Task Download(string destination)
         {
-            Download(new Archive {Name = Path.GetFileName(destination)}, destination);
+            await Download(new Archive {Name = Path.GetFileName(destination)}, destination);
         }
 
         /// <summary>
         /// Returns true if this link is still valid
         /// </summary>
         /// <returns></returns>
-        public abstract bool Verify();
+        public abstract Task<bool> Verify();
 
         public abstract IDownloader GetDownloader();
 

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 using Wabbajack.Common;
 
 namespace Wabbajack.Lib.Downloaders
@@ -35,9 +36,10 @@ namespace Wabbajack.Lib.Downloaders
             return inst;
         }
 
-        public static AbstractDownloadState ResolveArchive(dynamic ini)
+        public static async Task<AbstractDownloadState> ResolveArchive(dynamic ini)
         {
-            return Downloaders.Select(d => d.GetDownloaderState(ini)).FirstOrDefault(result => result != null);
+            var states = await Task.WhenAll(Downloaders.Select(d => (Task<AbstractDownloadState>)d.GetDownloaderState(ini)));
+            return states.FirstOrDefault(result => result != null);
         }
 
         /// <summary>

Wabbajack.Lib/Downloaders/DropboxDownloader.cs

@@ -1,11 +1,12 @@
 using System;
+using System.Threading.Tasks;
 using System.Web;
 
 namespace Wabbajack.Lib.Downloaders
 {
     public class DropboxDownloader : IDownloader, IUrlDownloader
     {
-        public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
         {
             var urlstring = archiveINI?.General?.directURL;
             return GetDownloaderState(urlstring);
@@ -31,7 +32,7 @@ namespace Wabbajack.Lib.Downloaders
             };
         }
 
-        public void Prepare()
+        public async Task Prepare()
         {
         }
     }

Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs

@@ -13,7 +13,7 @@ namespace Wabbajack.Lib.Downloaders
 {
     public class GameFileSourceDownloader : IDownloader
     {
-        public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
         {
             var gameName = (string)archiveINI?.General?.gameName;
             var gameFile = (string)archiveINI?.General?.gameFile;
@@ -40,7 +40,7 @@ namespace Wabbajack.Lib.Downloaders
             };
         }
 
-        public void Prepare()
+        public async Task Prepare()
         {
         }
 
@@ -57,7 +57,7 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
-            public override void Download(Archive a, string destination)
+            public override async Task Download(Archive a, string destination)
             {
                 using(var src = File.OpenRead(SourcePath))
                 using (var dest = File.OpenWrite(destination))
@@ -67,7 +67,7 @@ namespace Wabbajack.Lib.Downloaders
                 }
             }
 
-            public override bool Verify()
+            public override async Task<bool> Verify()
             {
                 return File.Exists(SourcePath) && SourcePath.FileHashCached() == Hash;
             }

Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs

@@ -1,5 +1,6 @@
 using System.Net.Http;
 using System.Text.RegularExpressions;
+using System.Threading.Tasks;
 using Wabbajack.Common;
 using Wabbajack.Lib.Validation;
 
@@ -7,7 +8,7 @@ namespace Wabbajack.Lib.Downloaders
 {
     public class GoogleDriveDownloader : IDownloader, IUrlDownloader
     {
-        public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
         {
             var url = archiveINI?.General?.directURL;
             return GetDownloaderState(url);
@@ -28,7 +29,7 @@ namespace Wabbajack.Lib.Downloaders
             return null;
         }
 
-        public void Prepare()
+        public async Task Prepare()
         {
         }
 
@@ -40,16 +41,17 @@ namespace Wabbajack.Lib.Downloaders
                 return whitelist.GoogleIDs.Contains(Id);
             }
 
-            public override void Download(Archive a, string destination)
+            public override async Task Download(Archive a, string destination)
             {
-                ToHttpState().Download(a, destination);
+                var state = await ToHttpState();
+                await state.Download(a, destination);
             }
 
-            private HTTPDownloader.State ToHttpState()
+            private async Task<HTTPDownloader.State> ToHttpState()
             {
                 var initialURL = $"https://drive.google.com/uc?id={Id}&export=download";
                 var client = new HttpClient();
-                var result = client.GetStringSync(initialURL);
+                var result = await client.GetStringAsync(initialURL);
                 var regex = new Regex("(?<=/uc\\?export=download&amp;confirm=).*(?=;id=)");
                 var confirm = regex.Match(result);
                 var url = $"https://drive.google.com/uc?export=download&confirm={confirm}&id={Id}";
@@ -57,9 +59,10 @@ namespace Wabbajack.Lib.Downloaders
                 return httpState;
             }
 
-            public override bool Verify()
+            public override async Task<bool> Verify()
             {
-                return ToHttpState().Verify();
+                var state = await ToHttpState();
+                return await state.Verify();
             }
 
             public override IDownloader GetDownloader()

Wabbajack.Lib/Downloaders/HTTPDownloader.cs

@@ -4,6 +4,7 @@ using System.IO;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Threading.Tasks;
 using Ceras;
 using Wabbajack.Common;
 using Wabbajack.Lib.Validation;
@@ -14,11 +15,10 @@ namespace Wabbajack.Lib.Downloaders
     public class HTTPDownloader : IDownloader, IUrlDownloader
     {
 
-        public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
         {
             var url = archiveINI?.General?.directURL;
             return GetDownloaderState(url, archiveINI);
-
         }
 
         public AbstractDownloadState GetDownloaderState(string uri)
@@ -45,7 +45,7 @@ namespace Wabbajack.Lib.Downloaders
             return null;
         }
 
-        public void Prepare()
+        public async Task Prepare()
         {
         }
 
@@ -64,12 +64,12 @@ namespace Wabbajack.Lib.Downloaders
                 return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p));
             }
 
-            public override void Download(Archive a, string destination)
+            public override Task Download(Archive a, string destination)
             {
-                DoDownload(a, destination, true);
+                return DoDownload(a, destination, true);
             }
 
-            public bool DoDownload(Archive a, string destination, bool download)
+            public async Task<bool> DoDownload(Archive a, string destination, bool download)
             {
                 var client = Client ?? new HttpClient();
                 client.DefaultRequestHeaders.Add("User-Agent", Consts.UserAgent);
@@ -86,19 +86,17 @@ namespace Wabbajack.Lib.Downloaders
                 long totalRead = 0;
                 var bufferSize = 1024 * 32;
 
-                var response = client.GetSync(Url);
-                var stream = response.Content.ReadAsStreamAsync();
+                Utils.Status($"Starting Download {a?.Name ?? Url}", 0);
+                var response = await client.GetAsync(Url, HttpCompletionOption.ResponseHeadersRead);
+
+                Stream stream;
                 try
                 {
-                    stream.Wait();
+                    stream = await response.Content.ReadAsStreamAsync();
                 }
-                catch (Exception)
+                catch (Exception ex)
                 {
-                }
-
-                if (stream.IsFaulted || response.StatusCode != HttpStatusCode.OK)
-                {
-                    Utils.Error(stream.Exception, $"While downloading {Url}");
+                    Utils.Error(ex, $"While downloading {Url}");
                     return false;
                 }
 
@@ -117,7 +115,7 @@ namespace Wabbajack.Lib.Downloaders
                     Directory.CreateDirectory(fileInfo.Directory.FullName);
                 }
 
-                using (var webs = stream.Result)
+                using (var webs = stream)
                 using (var fs = File.OpenWrite(destination))
                 {
                     var buffer = new byte[bufferSize];
@@ -135,9 +133,9 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
-            public override bool Verify()
+            public override async Task<bool> Verify()
             {
-                return DoDownload(new Archive {Name = ""}, "", false);
+                return await DoDownload(new Archive {Name = ""}, "", false);
             }
 
             public override IDownloader GetDownloader()

Wabbajack.Lib/Downloaders/IDownloader.cs

@@ -1,13 +1,15 @@
-namespace Wabbajack.Lib.Downloaders
+using System.Threading.Tasks;
+
+namespace Wabbajack.Lib.Downloaders
 {
     public interface IDownloader
     {
-        AbstractDownloadState GetDownloaderState(dynamic archiveINI);
+        Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI);
         
         /// <summary>
         /// Called before any downloads are inacted by the installer;
         /// </summary>
-        void Prepare();
+        Task Prepare();
     }
 
 }

Wabbajack.Lib/Downloaders/LoversLabDownloader.cs

@@ -20,9 +20,8 @@ namespace Wabbajack.Lib.Downloaders
     {
         internal HttpClient _authedClient;
 
-        public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archive_ini)
         {
-
             Uri url = DownloaderUtils.GetDirectURL(archive_ini);
             if (url == null || url.Host != "www.loverslab.com" || !url.AbsolutePath.StartsWith("/files/file/")) return null;
             var id = HttpUtility.ParseQueryString(url.Query)["r"];
@@ -35,9 +34,9 @@ namespace Wabbajack.Lib.Downloaders
             };
         }
 
-        public void Prepare()
+        public async Task Prepare()
         {
-            _authedClient = GetAuthedClient().Result ?? throw new Exception("not logged into LL, TODO");
+            _authedClient = (await GetAuthedClient()) ?? throw new Exception("not logged into LL, TODO");
         }
 
         public static async Task<Helpers.Cookie[]> GetAndCacheLoversLabCookies(BaseCefBrowser browser, Action<string> updateStatus, CancellationToken cancel)
@@ -99,9 +98,9 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
-            public override void Download(Archive a, string destination)
+            public override async Task Download(Archive a, string destination)
             {
-                var stream = ResolveDownloadStream().Result;
+                var stream = await ResolveDownloadStream();
                 using (var file = File.OpenWrite(destination))
                 {
                     stream.CopyTo(file);
@@ -155,9 +154,9 @@ namespace Wabbajack.Lib.Downloaders
                 public int currentTime { get; set; }
             }
 
-            public override bool Verify()
+            public override async Task<bool> Verify()
             {
-                var stream = ResolveDownloadStream().Result;
+                var stream = await ResolveDownloadStream();
                 if (stream == null)
                 {
                     return false;
@@ -165,7 +164,6 @@ namespace Wabbajack.Lib.Downloaders
 
                 stream.Close();
                 return true;
-
             }
 
             public override IDownloader GetDownloader()

Wabbajack.Lib/Downloaders/MEGADownloader.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Threading.Tasks;
 using CG.Web.MegaApiClient;
 using Wabbajack.Common;
 
@@ -6,8 +7,7 @@ namespace Wabbajack.Lib.Downloaders
 {
     public class MegaDownloader : IDownloader, IUrlDownloader
     {
-
-        public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
         {
             var url = archiveINI?.General?.directURL;
             return GetDownloaderState(url);
@@ -20,13 +20,13 @@ namespace Wabbajack.Lib.Downloaders
             return null;
         }
 
-        public void Prepare()
+        public async Task Prepare()
         {
         }
 
         public class State : HTTPDownloader.State
         {
-            public override void Download(Archive a, string destination)
+            public override async Task Download(Archive a, string destination)
             {
                 var client = new MegaApiClient();
                 Utils.Status("Logging into MEGA (as anonymous)");
@@ -37,7 +37,7 @@ namespace Wabbajack.Lib.Downloaders
                 client.DownloadFile(fileLink, destination);
             }
 
-            public override bool Verify()
+            public override async Task<bool> Verify()
             {
                 var client = new MegaApiClient();
                 Utils.Status("Logging into MEGA (as anonymous)");

Wabbajack.Lib/Downloaders/ManualDownloader.cs

@@ -4,6 +4,7 @@ using System.IO;
 using System.Linq;
 using System.Reactive.Linq;
 using System.Reactive.Subjects;
+using System.Threading.Tasks;
 using Syroot.Windows.IO;
 using Wabbajack.Common;
 using Wabbajack.Lib.Validation;
@@ -58,13 +59,13 @@ namespace Wabbajack.Lib.Downloaders
             }
         }
 
-        public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
         {
             var url = archiveINI?.General?.manualURL;
             return url != null ? new State { Url = url} : null;
         }
 
-        public void Prepare()
+        public async Task Prepare()
         {
         }
 
@@ -76,7 +77,7 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
-            public override void Download(Archive a, string destination)
+            public override async Task Download(Archive a, string destination)
             {
                 var downloader = (ManualDownloader)GetDownloader();
                 var absPath = Path.Combine(downloader._downloadfolder.Path, a.Name);
@@ -107,7 +108,7 @@ namespace Wabbajack.Lib.Downloaders
                 }
             }
 
-            public override bool Verify()
+            public override async Task<bool> Verify()
             {
                 return true;
             }

Wabbajack.Lib/Downloaders/MediaFireDownloader.cs

@@ -9,7 +9,7 @@ namespace Wabbajack.Lib.Downloaders
 {
     public class MediaFireDownloader : IUrlDownloader
     {
-        public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
         {
             Uri url = DownloaderUtils.GetDirectURL(archiveINI);
             if (url == null || url.Host != "www.mediafire.com") return null;
@@ -29,14 +29,15 @@ namespace Wabbajack.Lib.Downloaders
                 return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p));
             }
 
-            public override void Download(Archive a, string destination)
+            public override async Task Download(Archive a, string destination)
             {
-                Resolve().Result.Download(a, destination);
+                var result = await Resolve();
+                await result.Download(a, destination);
             }
 
-            public override bool Verify()
+            public override async Task<bool> Verify()
             {
-                return Resolve().Result != null;
+                return await Resolve() != null;
             }
 
             private async Task<HTTPDownloader.State> Resolve()
@@ -69,7 +70,7 @@ namespace Wabbajack.Lib.Downloaders
 
         }
 
-        public void Prepare()
+        public async Task Prepare()
         {
         }
 

Wabbajack.Lib/Downloaders/ModDBDownloader.cs

@@ -1,5 +1,6 @@
 using System.Net.Http;
 using System.Text.RegularExpressions;
+using System.Threading.Tasks;
 using Wabbajack.Common;
 using Wabbajack.Lib.Validation;
 
@@ -7,7 +8,7 @@ namespace Wabbajack.Lib.Downloaders
 {
     public class ModDBDownloader : IDownloader, IUrlDownloader
     {
-        public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
         {
             var url = archiveINI?.General?.directURL;
             return GetDownloaderState(url);
@@ -26,7 +27,7 @@ namespace Wabbajack.Lib.Downloaders
             return null;
         }
 
-        public void Prepare()
+        public async Task Prepare()
         {
         }
 
@@ -39,26 +40,26 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
-            public override void Download(Archive a, string destination)
+            public override async Task Download(Archive a, string destination)
             {
-                var newURL = GetDownloadUrl();
-                new HTTPDownloader.State {Url = newURL}.Download(a, destination);
+                var newURL = await GetDownloadUrl();
+                await new HTTPDownloader.State {Url = newURL}.Download(a, destination);
             }
 
-            private string GetDownloadUrl()
+            private async Task<string> GetDownloadUrl()
             {
                 var client = new HttpClient();
-                var result = client.GetStringSync(Url);
+                var result = await client.GetStringAsync(Url);
                 var regex = new Regex("https:\\/\\/www\\.moddb\\.com\\/downloads\\/mirror\\/.*(?=\\\")");
                 var match = regex.Match(result);
                 var newURL = match.Value;
                 return newURL;
             }
 
-            public override bool Verify()
+            public override async Task<bool> Verify()
             {
-                var newURL = GetDownloadUrl();
-                return new HTTPDownloader.State { Url = newURL }.Verify();
+                var newURL = await GetDownloadUrl();
+                return await new HTTPDownloader.State { Url = newURL }.Verify();
             }
 
             public override IDownloader GetDownloader()

Wabbajack.Lib/Downloaders/NexusDownloader.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Linq;
+using System.Threading.Tasks;
 using Wabbajack.Common;
 using Wabbajack.Common.StatusFeed.Errors;
 using Wabbajack.Lib.NexusApi;
@@ -9,7 +10,7 @@ namespace Wabbajack.Lib.Downloaders
 {
     public class NexusDownloader : IDownloader
     {
-        public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
         {
             var general = archiveINI?.General;
 
@@ -18,7 +19,8 @@ namespace Wabbajack.Lib.Downloaders
                 var name = (string)general.gameName;
                 var gameMeta = GameRegistry.GetByMO2ArchiveName(name);
                 var game = gameMeta != null ? GameRegistry.GetByMO2ArchiveName(name).Game : GameRegistry.GetByNexusName(name).Game;
-                var info = new NexusApiClient().GetModInfo(game, general.modID);
+                var client = await NexusApiClient.Get();
+                var info = await client.GetModInfo(game, general.modID);
                 return new State
                 {
                     GameName = general.gameName,
@@ -40,10 +42,10 @@ namespace Wabbajack.Lib.Downloaders
             return null;
         }
 
-        public void Prepare()
+        public async Task Prepare()
         {
-            var client = new NexusApiClient();
-            var status = client.GetUserStatus();
+            var client = await NexusApiClient.Get();
+            var status = await client.GetUserStatus();
             if (!client.IsAuthenticated)
             {
                 Utils.ErrorThrow(new UnconvertedError($"Authenticating for the Nexus failed. A nexus account is required to automatically download mods."));
@@ -51,7 +53,7 @@ namespace Wabbajack.Lib.Downloaders
             }
 
             if (status.is_premium) return;
-            Utils.ErrorThrow(new UnconvertedError($"Automated installs with Wabbajack requires a premium nexus account. {client.Username} is not a premium account."));
+            Utils.ErrorThrow(new UnconvertedError($"Automated installs with Wabbajack requires a premium nexus account. {await client.Username()} is not a premium account."));
         }
 
         public class State : AbstractDownloadState
@@ -75,12 +77,13 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
-            public override void Download(Archive a, string destination)
+            public override async Task Download(Archive a, string destination)
             {
                 string url;
                 try
                 {
-                    url = new NexusApiClient().GetNexusDownloadLink(this, false);
+                    var client = await NexusApiClient.Get();
+                    url = await client.GetNexusDownloadLink(this, false);
                 }
                 catch (Exception ex)
                 {
@@ -90,14 +93,14 @@ namespace Wabbajack.Lib.Downloaders
 
                 Utils.Log($"Downloading Nexus Archive - {a.Name} - {GameName} - {ModID} - {FileID}");
 
-                new HTTPDownloader.State
+                await new HTTPDownloader.State
                 {
                     Url = url
                 }.Download(a, destination);
 
             }
 
-            public override bool Verify()
+            public override async Task<bool> Verify()
             {
                 try
                 {
@@ -109,7 +112,8 @@ namespace Wabbajack.Lib.Downloaders
                     if (!int.TryParse(ModID, out var modID))
                         return false;
 
-                    var modFiles = new NexusApiClient().GetModFiles(game, modID);
+                    var client = await NexusApiClient.Get();
+                    var modFiles = await client.GetModFiles(game, modID);
 
                     if (!ulong.TryParse(FileID, out var fileID))
                         return false;

Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs

@@ -15,7 +15,7 @@ namespace Wabbajack.Lib.Downloaders
     {
         private SteamWorkshopItem _item;
 
-        public AbstractDownloadState GetDownloaderState(dynamic archiveINI)
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
         {
             var id = archiveINI?.General?.itemID;
             var steamID = archiveINI?.General?.steamID;
@@ -29,7 +29,7 @@ namespace Wabbajack.Lib.Downloaders
             return new State {Item = _item};
         }
 
-        public void Prepare()
+        public async Task Prepare()
         {
         }
 
@@ -46,7 +46,7 @@ namespace Wabbajack.Lib.Downloaders
                 return true;
             }
 
-            public override void Download(Archive a, string destination)
+            public override async Task Download(Archive a, string destination)
             {
                 var currentLib = "";
                 SteamHandler.Instance.InstallFolders.Where(f => f.Contains(Item.Game.InstallDir)).Do(s => currentLib = s);
@@ -77,7 +77,7 @@ namespace Wabbajack.Lib.Downloaders
                 }
             }
 
-            public override bool Verify()
+            public override async Task<bool> Verify()
             {
                 //TODO: find a way to verify steam workshop items
                 throw new NotImplementedException();

Wabbajack.Lib/IBatchProcessor.cs

@@ -36,12 +36,5 @@ namespace Wabbajack.Lib
         /// Begin processing
         /// </summary>
         Task<bool> Begin();
-
-        /// <summary>
-        /// Terminate any processing currently in progress by the processor. The processor should be disposed of
-        /// after calling this function as processing cannot be resumed and the tasks may be half completed.
-        /// Should only be called while IsRunning = true;
-        /// </summary>
-        void Terminate();
     }
 }

Wabbajack.Lib/MO2Compiler.cs

@@ -4,6 +4,7 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
@@ -77,8 +78,9 @@ namespace Wabbajack.Lib
 
         public HashSet<string> SelectedProfiles { get; set; } = new HashSet<string>();
 
-        protected override bool _Begin()
+        protected override async Task<bool> _Begin(CancellationToken cancel)
         {
+            if (cancel.IsCancellationRequested) return false;
             ConfigureProcessor(19);
             UpdateTracker.Reset();
             UpdateTracker.NextStep("Gathering information");
@@ -90,7 +92,8 @@ namespace Wabbajack.Lib
 
             Info("Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p)));
 
-            VFS.IntegrateFromFile(_vfsCacheName);
+            if (cancel.IsCancellationRequested) return false;
+            await VFS.IntegrateFromFile(_vfsCacheName);
 
             var roots = new List<string>()
             {
@@ -108,8 +111,9 @@ namespace Wabbajack.Lib
             }
             UpdateTracker.NextStep("Indexing folders");
 
-            VFS.AddRoots(roots);
-            VFS.WriteToFile(_vfsCacheName);
+            if (cancel.IsCancellationRequested) return false;
+            await VFS.AddRoots(roots);
+            await VFS.WriteToFile(_vfsCacheName);
             
             if (Directory.Exists(lootPath))
             {
@@ -119,17 +123,21 @@ namespace Wabbajack.Lib
                         { Path = Path.Combine(Consts.LOOTFolderFilesDir, p.RelativeTo(lootPath)) });
             }
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Cleaning output folder");
             if (Directory.Exists(ModListOutputFolder))
                 Utils.DeleteDirectory(ModListOutputFolder);
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Inferring metas for game file downloads");
-            InferMetas();
+            await InferMetas();
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Reindexing downloads after meta inferring");
-            VFS.AddRoot(MO2DownloadsFolder);
-            VFS.WriteToFile(_vfsCacheName);
+            await VFS.AddRoot(MO2DownloadsFolder);
+            await VFS.WriteToFile(_vfsCacheName);
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Pre-validating Archives");
 
             IndexedArchives = Directory.EnumerateFiles(MO2DownloadsFolder)
@@ -143,7 +151,7 @@ namespace Wabbajack.Lib
                 })
                 .ToList();
 
-            CleanInvalidArchives();
+           await CleanInvalidArchives();
 
             UpdateTracker.NextStep("Finding Install Files");
             Directory.CreateDirectory(ModListOutputFolder);
@@ -177,6 +185,7 @@ namespace Wabbajack.Lib
 
             Info($"Found {AllFiles.Count} files to build into mod list");
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Verifying destinations");
 
             var dups = AllFiles.GroupBy(f => f.Path)
@@ -195,6 +204,7 @@ namespace Wabbajack.Lib
             ExtraFiles = new ConcurrentBag<Directive>();
 
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Loading INIs");
 
             ModInis = Directory.EnumerateDirectories(Path.Combine(MO2Folder, "mods"))
@@ -209,15 +219,15 @@ namespace Wabbajack.Lib
                 .Where(f => f.Item2 != null)
                 .ToDictionary(f => f.Item1, f => f.Item2);
 
+            if (cancel.IsCancellationRequested) return false;
             var stack = MakeStack();
-
             UpdateTracker.NextStep("Running Compilation Stack");
-            var results = AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f)).ToList();
+            var results = await AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f));
 
             // Add the extra files that were generated by the stack
-
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep($"Adding {ExtraFiles.Count} that were generated by the stack");
-            results = results.Concat(ExtraFiles).ToList();
+            results = results.Concat(ExtraFiles).ToArray();
 
             var nomatch = results.OfType<NoMatch>();
             Info($"No match for {nomatch.Count()} files");
@@ -242,8 +252,8 @@ namespace Wabbajack.Lib
 
             if (IndexedArchives.Any(a => a.IniData?.General?.gameName != null))
             {
-                var nexusClient = new NexusApiClient();
-                if (!nexusClient.IsPremium) Error($"User {nexusClient.Username} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue");
+                var nexusClient = await NexusApiClient.Get();
+                if (!(await nexusClient.IsPremium())) Error($"User {(await nexusClient.Username())} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue");
 
             }
 
@@ -252,11 +262,11 @@ namespace Wabbajack.Lib
 
 
             UpdateTracker.NextStep("Gathering Archives");
-            GatherArchives();
+            await GatherArchives();
             UpdateTracker.NextStep("Including Archive Metadata");
-            IncludeArchiveMetadata();
+            await IncludeArchiveMetadata();
             UpdateTracker.NextStep("Building Patches");
-            BuildPatches();
+            await BuildPatches();
 
             ModList = new ModList
             {
@@ -275,7 +285,7 @@ namespace Wabbajack.Lib
 
             UpdateTracker.NextStep("Running Validation");
 
-            ValidateModlist.RunValidation(ModList);
+            await ValidateModlist.RunValidation(Queue, ModList);
             UpdateTracker.NextStep("Generating Report");
 
             GenerateReport();
@@ -292,20 +302,20 @@ namespace Wabbajack.Lib
             return true;
         }
 
-        private void CleanInvalidArchives()
+        private async Task CleanInvalidArchives()
         {
-            var remove = IndexedArchives.PMap(Queue, a =>
+            var remove = (await IndexedArchives.PMap(Queue, async a =>
             {
                 try
                 {
-                    ResolveArchive(a);
+                    await ResolveArchive(a);
                     return null;
                 }
                 catch
                 {
                     return a;
                 }
-            }).Where(a => a != null).ToHashSet();
+            })).Where(a => a != null).ToHashSet();
 
             if (remove.Count == 0)
                 return;
@@ -316,7 +326,7 @@ namespace Wabbajack.Lib
             IndexedArchives.RemoveAll(a => remove.Contains(a));
         }
 
-        private void InferMetas()
+        private async Task InferMetas()
         {
             var to_find = Directory.EnumerateFiles(MO2DownloadsFolder)
                 .Where(f => !f.EndsWith(".meta") && !f.EndsWith(Consts.HashFileExtension))
@@ -332,7 +342,7 @@ namespace Wabbajack.Lib
                 .GroupBy(f => (Path.GetFileName(f.name), new FileInfo(f.name).Length))
                 .ToDictionary(f => f.Key);
 
-            to_find.PMap(Queue, f =>
+            await to_find.PMap(Queue, f =>
             {
                 var vf = VFS.Index.ByFullPath[f];
                 if (!game_files.TryGetValue((Path.GetFileName(f), vf.Size), out var found))
@@ -353,10 +363,10 @@ namespace Wabbajack.Lib
         }
 
 
-        private void IncludeArchiveMetadata()
+        private async Task IncludeArchiveMetadata()
         {
             Utils.Log($"Including {SelectedArchives.Count} .meta files for downloads");
-            SelectedArchives.Do(a =>
+            await SelectedArchives.PMap(Queue, a =>
             {
                 var source = Path.Combine(MO2DownloadsFolder, a.Name + ".meta");
                 InstallDirectives.Add(new ArchiveMeta()
@@ -384,7 +394,7 @@ namespace Wabbajack.Lib
         /// <summary>
         ///     Fills in the Patch fields in files that require them
         /// </summary>
-        private void BuildPatches()
+        private async Task BuildPatches()
         {
             Info("Gathering patch files");
 
@@ -403,21 +413,21 @@ namespace Wabbajack.Lib
 
             Info($"Patching building patches from {groups.Count} archives");
             var absolutePaths = AllFiles.ToDictionary(e => e.Path, e => e.AbsolutePath);
-            groups.PMap(Queue, group => BuildArchivePatches(group.Key, group, absolutePaths));
+            await groups.PMap(Queue, group => BuildArchivePatches(group.Key, group, absolutePaths));
 
             if (InstallDirectives.OfType<PatchedFromArchive>().FirstOrDefault(f => f.PatchID == null) != null)
                 Error("Missing patches after generation, this should not happen");
         }
 
-        private void BuildArchivePatches(string archiveSha, IEnumerable<PatchedFromArchive> group,
+        private async Task BuildArchivePatches(string archiveSha, IEnumerable<PatchedFromArchive> group,
             Dictionary<string, string> absolutePaths)
         {
-            using (var files = VFS.StageWith(group.Select(g => VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath))))
+            using (var files = await VFS.StageWith(group.Select(g => VFS.Index.FileForArchiveHashPath(g.ArchiveHashPath))))
             {
                 var byPath = files.GroupBy(f => string.Join("|", f.FilesInFullPath.Skip(1).Select(i => i.Name)))
                     .ToDictionary(f => f.Key, f => f.First());
                 // Now Create the patches
-                group.PMap(Queue, entry =>
+                await group.PMap(Queue, entry =>
                 {
                     Info($"Patching {entry.To}");
                     Status($"Patching {entry.To}");

Wabbajack.Lib/MO2Installer.cs

@@ -2,6 +2,8 @@
 using System.IO;
 using System.Linq;
 using System.Text;
+using System.Threading.Tasks;
+using System.Threading;
 using System.Windows;
 using Alphaleonis.Win32.Filesystem;
 using IniParser;
@@ -34,9 +36,10 @@ namespace Wabbajack.Lib
         {
         }
 
-        protected override bool _Begin()
+        protected override async Task<bool> _Begin(CancellationToken cancel)
         {
-            ConfigureProcessor(18, RecommendQueueSize());
+            if (cancel.IsCancellationRequested) return false;
+            ConfigureProcessor(18, await RecommendQueueSize());
             var game = ModList.GameType.MetaData();
 
             if (GameFolder == null)
@@ -52,35 +55,41 @@ namespace Wabbajack.Lib
                 return false;
             }
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Validating Game ESMs");
             ValidateGameESMs();
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Validating Modlist");
-            ValidateModlist.RunValidation(ModList);
+            await ValidateModlist.RunValidation(Queue, ModList);
 
             Directory.CreateDirectory(OutputFolder);
             Directory.CreateDirectory(DownloadFolder);
 
             if (Directory.Exists(Path.Combine(OutputFolder, "mods")) && WarnOnOverwrite)
             {
-                if (Utils.Log(new ConfirmUpdateOfExistingInstall { ModListName = ModList.Name, OutputFolder = OutputFolder}).Task.Result == ConfirmUpdateOfExistingInstall.Choice.Abort)
+                if ((await Utils.Log(new ConfirmUpdateOfExistingInstall { ModListName = ModList.Name, OutputFolder = OutputFolder }).Task) == ConfirmUpdateOfExistingInstall.Choice.Abort)
                 {
                     Utils.Log("Existing installation at the request of the user, existing mods folder found.");
                     return false;
                 }
             }
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Optimizing Modlist");
-            OptimizeModlist();
+            await OptimizeModlist();
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Hashing Archives");
-            HashArchives();
+            await HashArchives();
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Downloading Missing Archives");
-            DownloadArchives();
+            await DownloadArchives();
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Hashing Remaining Archives");
-            HashArchives();
+            await HashArchives();
 
             var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
             if (missing.Count > 0)
@@ -93,26 +102,33 @@ namespace Wabbajack.Lib
                     Error("Cannot continue, was unable to download one or more archives");
             }
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Priming VFS");
-            PrimeVFS();
+            await PrimeVFS();
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Building Folder Structure");
             BuildFolderStructure();
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Installing Archives");
-            InstallArchives();
+            await InstallArchives();
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Installing Included files");
-            InstallIncludedFiles();
+            await InstallIncludedFiles();
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Installing Archive Metas");
-            InstallIncludedDownloadMetas();
+            await InstallIncludedDownloadMetas();
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Building BSAs");
-            BuildBSAs();
+            await BuildBSAs();
 
+            if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Generating Merges");
-            zEditIntegration.GenerateMerges(this);
+            await zEditIntegration.GenerateMerges(this);
 
             UpdateTracker.NextStep("Updating System-specific ini settings");
             SetScreenSizeInPrefs();
@@ -122,9 +138,9 @@ namespace Wabbajack.Lib
         }
 
 
-        private void InstallIncludedDownloadMetas()
+        private async Task InstallIncludedDownloadMetas()
         {
-            ModList.Directives
+            await ModList.Directives
                    .OfType<ArchiveMeta>()
                    .PMap(Queue, directive =>
                    {
@@ -150,7 +166,7 @@ namespace Wabbajack.Lib
             }
         }
 
-        private void AskToEndorse()
+        private async Task AskToEndorse()
         {
             var mods = ModList.Archives
                 .Select(m => m.State)
@@ -177,27 +193,28 @@ namespace Wabbajack.Lib
                 mods[b] = tmp;
             }
 
-            mods.PMap(Queue, mod =>
+            await mods.PMap(Queue, async mod =>
             {
-                var er = new NexusApiClient().EndorseMod(mod);
+                var client = await NexusApiClient.Get();
+                var er = await client.EndorseMod(mod);
                 Utils.Log($"Endorsed {mod.GameName} - {mod.ModID} - Result: {er.message}");
             });
             Info("Done! You may now exit the application!");
         }
 
-        private void BuildBSAs()
+        private async Task BuildBSAs()
         {
             var bsas = ModList.Directives.OfType<CreateBSA>().ToList();
             Info($"Building {bsas.Count} bsa files");
 
-            bsas.Do(bsa =>
+            foreach (var bsa in bsas)
             {
                 Status($"Building {bsa.To}");
                 var sourceDir = Path.Combine(OutputFolder, Consts.BSACreationDir, bsa.TempID);
 
                 using (var a = bsa.State.MakeBuilder())
                 {
-                    bsa.FileStates.PMap(Queue, state =>
+                    await bsa.FileStates.PMap(Queue, state =>
                     {
                         Status($"Adding {state.Path} to BSA");
                         using (var fs = File.OpenRead(Path.Combine(sourceDir, state.Path)))
@@ -209,8 +226,7 @@ namespace Wabbajack.Lib
                     Info($"Writing {bsa.To}");
                     a.Build(Path.Combine(OutputFolder, bsa.To));
                 }
-            });
-
+            }
 
             var bsaDir = Path.Combine(OutputFolder, Consts.BSACreationDir);
             if (Directory.Exists(bsaDir))
@@ -220,10 +236,10 @@ namespace Wabbajack.Lib
             }
         }
 
-        private void InstallIncludedFiles()
+        private async Task InstallIncludedFiles()
         {
             Info("Writing inline files");
-            ModList.Directives
+            await ModList.Directives
                 .OfType<InlineFile>()
                 .PMap(Queue, directive =>
                 {

Wabbajack.Lib/ModListRegistry/ModListMetadata.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Net.Http;
+using System.Threading.Tasks;
 using System.Windows.Media.Imaging;
 using Newtonsoft.Json;
 using Wabbajack.Common;
@@ -55,11 +56,11 @@ namespace Wabbajack.Lib.ModListRegistry
 
 
 
-        public static List<ModlistMetadata> LoadFromGithub()
+        public static async Task<List<ModlistMetadata>> LoadFromGithub()
         {
             var client = new HttpClient();
             Utils.Log("Loading ModLists from Github");
-            var result = client.GetStringSync(Consts.ModlistMetadataURL); 
+            var result = await client.GetStringAsync(Consts.ModlistMetadataURL); 
             return result.FromJSONString<List<ModlistMetadata>>();
         }
 

Wabbajack.Lib/NexusApi/NexusApi.cs

@@ -32,22 +32,17 @@ namespace Wabbajack.Lib.NexusApi
         private static readonly string API_KEY_CACHE_FILE = "nexus.key_cache";
         private static string _additionalEntropy = "vtP2HF6ezg";
 
-        private readonly HttpClient _httpClient;
-
-        public HttpClient HttpClient => _httpClient;
-
+        public HttpClient HttpClient { get; } = new HttpClient();
 
         #region Authentication
 
-        private readonly string _apiKey;
+        public string ApiKey { get; }
 
-        public string ApiKey => _apiKey;
+        public bool IsAuthenticated => ApiKey != null;
 
-        public bool IsAuthenticated => _apiKey != null;
+        private Task<UserStatus> _userStatus;
 
-        private UserStatus _userStatus;
-
-        public UserStatus UserStatus
+        public Task<UserStatus> UserStatus
         {
             get
             {
@@ -57,15 +52,19 @@ namespace Wabbajack.Lib.NexusApi
             }
         }
 
-        public bool IsPremium => IsAuthenticated && UserStatus.is_premium;
-
-        public string Username => UserStatus?.name;
-
-
-        private static object _getAPIKeyLock = new object();
-        private static string GetApiKey()
+        public async Task<bool> IsPremium()
         {
-            lock (_getAPIKeyLock)
+            return IsAuthenticated && (await UserStatus).is_premium;
+        }
+
+        public async Task<string> Username() => (await UserStatus).name;
+
+        private static SemaphoreSlim _getAPIKeyLock = new SemaphoreSlim(1, 1);
+        private static async Task<string> GetApiKey()
+        {
+            await _getAPIKeyLock.WaitAsync();
+
+            try
             {
                 // Clean up old location
                 if (File.Exists(API_KEY_CACHE_FILE))
@@ -87,10 +86,14 @@ namespace Wabbajack.Lib.NexusApi
                     return env_key;
                 }
 
-                var result = Utils.Log(new RequestNexusAuthorization()).Task.Result;
+                var result = await Utils.Log(new RequestNexusAuthorization()).Task;
                 result.ToEcryptedJson("nexusapikey");
                 return result;
             }
+            finally
+            {
+                _getAPIKeyLock.Release();
+            }
         }
 
         class RefererHandler : RequestHandler
@@ -153,10 +156,10 @@ namespace Wabbajack.Lib.NexusApi
             }
         }
 
-        public UserStatus GetUserStatus()
+        public async Task<UserStatus> GetUserStatus()
         {
             var url = "https://api.nexusmods.com/v1/users/validate.json";
-            return Get<UserStatus>(url);
+            return await Get<UserStatus>(url);
         }
 
         #endregion
@@ -213,16 +216,14 @@ namespace Wabbajack.Lib.NexusApi
 
         #endregion
 
-
-        public NexusApiClient(string apiKey = null)
+        private NexusApiClient(string apiKey = null)
         {
-            _apiKey = apiKey ?? GetApiKey();
-            _httpClient = new HttpClient();
+            ApiKey = apiKey;
 
             // set default headers for all requests to the Nexus API
-            var headers = _httpClient.DefaultRequestHeaders;
+            var headers = HttpClient.DefaultRequestHeaders;
             headers.Add("User-Agent", Consts.UserAgent);
-            headers.Add("apikey", _apiKey);
+            headers.Add("apikey", ApiKey);
             headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
             headers.Add("Application-Name", Consts.AppName);
             headers.Add("Application-Version", $"{Assembly.GetEntryAssembly()?.GetName()?.Version ?? new Version(0, 1)}");
@@ -231,24 +232,24 @@ namespace Wabbajack.Lib.NexusApi
                 Directory.CreateDirectory(Consts.NexusCacheDirectory);
         }
 
-        private T Get<T>(string url)
+        public static async Task<NexusApiClient> Get(string apiKey = null)
         {
-            Task<HttpResponseMessage> responseTask = _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
-            responseTask.Wait();
+            apiKey = apiKey ?? await GetApiKey();
+            return new NexusApiClient(apiKey);
+        }
 
-            var response = responseTask.Result;
+        private async Task<T> Get<T>(string url)
+        {
+            var response = await HttpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
             UpdateRemaining(response);
 
-            var contentTask = response.Content.ReadAsStreamAsync();
-            contentTask.Wait();
-
-            using (var stream = contentTask.Result)
+            using (var stream = await response.Content.ReadAsStreamAsync())
             {
                 return stream.FromJSON<T>();
             }
         }
 
-        private T GetCached<T>(string url)
+        private async Task<T> GetCached<T>(string url)
         {
             var code = Encoding.UTF8.GetBytes(url).ToHex() + ".json";
 
@@ -263,7 +264,7 @@ namespace Wabbajack.Lib.NexusApi
                     return cache_file.FromJSON<T>();
                 }
 
-                var result = Get<T>(url);
+                var result = await Get<T>(url);
                 if (result != null)
                     result.ToJSON(cache_file);
                 return result;
@@ -271,27 +272,33 @@ namespace Wabbajack.Lib.NexusApi
 
             try
             {
-                return Get<T>(Consts.WabbajackCacheLocation + code);
+                return await Get<T>(Consts.WabbajackCacheLocation + code);
             }
             catch (Exception)
             {
-                return Get<T>(url);
+                return await Get<T>(url);
             }
 
         }
 
-        public string GetNexusDownloadLink(NexusDownloader.State archive, bool cache = false)
+        public async Task<string> GetNexusDownloadLink(NexusDownloader.State archive, bool cache = false)
         {
-            if (cache && TryGetCachedLink(archive, out var result))
-                return result;
+            if (cache)
+            {
+                var result = await TryGetCachedLink(archive);
+                if (result.Succeeded)
+                {
+                    return result.Value;
+                }
+            }
 
             ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
 
             var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(archive.GameName)}/mods/{archive.ModID}/files/{archive.FileID}/download_link.json";
-            return Get<List<DownloadLink>>(url).First().URI;
+            return (await Get<List<DownloadLink>>(url)).First().URI;
         }
 
-        private bool TryGetCachedLink(NexusDownloader.State archive, out string result)
+        private async Task<GetResponse<string>> TryGetCachedLink(NexusDownloader.State archive)
         {
             if (!Directory.Exists(Consts.NexusCacheDirectory))
                 Directory.CreateDirectory(Consts.NexusCacheDirectory);
@@ -300,19 +307,18 @@ namespace Wabbajack.Lib.NexusApi
             if (!File.Exists(path) || (DateTime.Now - new FileInfo(path).LastWriteTime).TotalHours > 24)
             {
                 File.Delete(path);
-                result = GetNexusDownloadLink(archive);
+                var result = await GetNexusDownloadLink(archive);
                 File.WriteAllText(path, result);
-                return true;
+                return GetResponse<string>.Succeed(result);
             }
 
-            result = File.ReadAllText(path);
-            return true;
+            return GetResponse<string>.Succeed(File.ReadAllText(path));
         }
 
-        public NexusFileInfo GetFileInfo(NexusDownloader.State mod)
+        public async Task<NexusFileInfo> GetFileInfo(NexusDownloader.State mod)
         {
             var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(mod.GameName)}/mods/{mod.ModID}/files/{mod.FileID}.json";
-            return GetCached<NexusFileInfo>(url);
+            return await GetCached<NexusFileInfo>(url);
         }
 
         public class GetModFilesResponse
@@ -320,32 +326,32 @@ namespace Wabbajack.Lib.NexusApi
             public List<NexusFileInfo> files;
         }
 
-        public GetModFilesResponse GetModFiles(Game game, int modid)
+        public async Task<GetModFilesResponse> GetModFiles(Game game, int modid)
         {
             var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modid}/files.json";
-            return GetCached<GetModFilesResponse>(url);
+            return await GetCached<GetModFilesResponse>(url);
         }
 
-        public List<MD5Response> GetModInfoFromMD5(Game game, string md5Hash)
+        public async Task<List<MD5Response>> GetModInfoFromMD5(Game game, string md5Hash)
         {
             var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/md5_search/{md5Hash}.json";
-            return Get<List<MD5Response>>(url);
+            return await Get<List<MD5Response>>(url);
         }
 
-        public ModInfo GetModInfo(Game game, string modId)
+        public async Task<ModInfo> GetModInfo(Game game, string modId)
         {
             var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modId}.json";
-            return GetCached<ModInfo>(url);
+            return await GetCached<ModInfo>(url);
         }
 
-        public EndorsementResponse EndorseMod(NexusDownloader.State mod)
+        public async Task<EndorsementResponse> EndorseMod(NexusDownloader.State mod)
         {
             Utils.Status($"Endorsing ${mod.GameName} - ${mod.ModID}");
             var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(mod.GameName)}/mods/{mod.ModID}/endorse.json";
 
             var content = new FormUrlEncodedContent(new Dictionary<string, string> { { "version", mod.Version } });
 
-            using (var stream = _httpClient.PostStreamSync(url, content))
+            using (var stream = await HttpClient.PostStream(url, content))
             {
                 return stream.FromJSON<EndorsementResponse>();
             }
@@ -386,27 +392,33 @@ namespace Wabbajack.Lib.NexusApi
             set => _localCacheDir = value;
         }
 
-        public void ClearUpdatedModsInCache()
+
+        public async Task ClearUpdatedModsInCache()
         {
             if (!UseLocalCache) return;
 
-            var purge = GameRegistry.Games.Values
+            var gameTasks = GameRegistry.Games.Values
                 .Where(game => game.NexusName != null)
-                .Select(game => new
+                .Select(async game =>
                 {
-                    game = game,
-                    mods = Get<List<UpdatedMod>>(
-                        $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m")
+                    return (game,
+                        mods: await Get<List<UpdatedMod>>(
+                            $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m"));
                 })
-                .SelectMany(r => r.mods.Select(mod => new {game = r.game, 
-                                                           mod = mod}))
+                .Select(async rTask =>
+                {
+                    var (game, mods) = await rTask;
+                    return mods.Select(mod => new { game = game, mod = mod });
+                });
+            var purge = (await Task.WhenAll(gameTasks))
+                .SelectMany(i => i)
                 .ToList();
 
             Utils.Log($"Found {purge.Count} updated mods in the last month");
             using (var queue = new WorkQueue())
             {
-                var to_purge = Directory.EnumerateFiles(LocalCacheDir, "*.json")
-                    .PMap(queue,f =>
+                var to_purge = (await Directory.EnumerateFiles(LocalCacheDir, "*.json")
+                    .PMap(queue, f =>
                     {
                         Utils.Status("Cleaning Nexus cache for");
                         var uri = new Uri(Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(f).FromHex()));
@@ -420,24 +432,24 @@ namespace Wabbajack.Lib.NexusApi
                             return (should_remove, f);
                         }
 
+                        // ToDo
+                        // Can improve to not read the entire file to see if it starts with null
                         if (File.ReadAllText(f).StartsWith("null"))
                             return (true, f);
 
                         return (false, f);
-                    })
+                    }))
                     .Where(p => p.Item1)
                     .ToList();
 
                 Utils.Log($"Purging {to_purge.Count} cache entries");
-                to_purge.PMap(queue, f =>
+                await to_purge.PMap(queue, f =>
                 {
                     var uri = new Uri(Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(f.f).FromHex()));
                     Utils.Log($"Purging {uri}");
                     File.Delete(f.f);
                 });
             }
-
         }
     }
-
 }

Wabbajack.Lib/UI/UIUtils.cs

@@ -12,54 +12,6 @@ namespace Wabbajack.Lib
 {
     public static class UIUtils
     {
-
-        public static string ShowFolderSelectionDialog(string prompt)
-        {
-            if (System.Windows.Application.Current.Dispatcher.Thread != Thread.CurrentThread)
-            {
-                var task = new TaskCompletionSource<string>();
-                System.Windows.Application.Current.Dispatcher.Invoke(() =>
-                {
-                    try
-                    {
-                        task.SetResult(ShowFolderSelectionDialog(prompt));
-                    }
-                    catch (Exception ex)
-                    {
-                        task.SetException(ex);
-                    }
-                });
-                task.Task.Wait();
-                if (task.Task.IsFaulted)
-                    throw task.Task.Exception;
-                return task.Task.Result;
-            }
-
-
-            var dlg = new CommonOpenFileDialog();
-            dlg.Title = prompt;
-            dlg.IsFolderPicker = true;
-            dlg.InitialDirectory = Assembly.GetEntryAssembly().Location;
-
-            dlg.AddToMostRecentlyUsedList = false;
-            dlg.AllowNonFileSystemItems = false;
-            dlg.DefaultDirectory = Assembly.GetEntryAssembly().Location;
-            dlg.EnsureFileExists = true;
-            dlg.EnsurePathExists = true;
-            dlg.EnsureReadOnly = false;
-            dlg.EnsureValidNames = true;
-            dlg.Multiselect = false;
-            dlg.ShowPlacesList = true;
-
-            if (dlg.ShowDialog() == CommonFileDialogResult.Ok)
-            {
-                return dlg.FileName;
-                // Do something with selected folder string
-            }
-
-            return null;
-        }
-
         public static BitmapImage BitmapImageFromResource(string name) => BitmapImageFromStream(System.Windows.Application.GetResourceStream(new Uri("pack://application:,,,/Wabbajack;component/" + name)).Stream);
 
         public static BitmapImage BitmapImageFromStream(Stream stream)

Wabbajack.Lib/Validation/ValidateModlist.cs

@@ -3,6 +3,7 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net.Http;
+using System.Threading.Tasks;
 using Wabbajack.Common;
 using Wabbajack.Lib.Downloaders;
 using Path = Alphaleonis.Win32.Filesystem.Path;
@@ -17,9 +18,14 @@ namespace Wabbajack.Lib.Validation
     {
         public Dictionary<string, Author> AuthorPermissions { get; set; } = new Dictionary<string, Author>();
 
-        private WorkQueue Queue = new WorkQueue();
+        private readonly WorkQueue _queue;
         public ServerWhitelist ServerWhitelist { get; set; } = new ServerWhitelist();
 
+        public ValidateModlist(WorkQueue workQueue)
+        {
+            _queue = workQueue;
+        }
+
         public void LoadAuthorPermissionsFromString(string s)
         {
             AuthorPermissions = s.FromYaml<Dictionary<string, Author>>();
@@ -30,18 +36,18 @@ namespace Wabbajack.Lib.Validation
             ServerWhitelist = s.FromYaml<ServerWhitelist>();
         }
 
-        public void LoadListsFromGithub()
+        public async Task LoadListsFromGithub()
         {
             var client = new HttpClient();
             Utils.Log("Loading Nexus Mod Permissions");
-            using (var result = client.GetStreamSync(Consts.ModPermissionsURL))
+            using (var result = await client.GetStreamAsync(Consts.ModPermissionsURL))
             {
                 AuthorPermissions = result.FromYaml<Dictionary<string, Author>>();
                 Utils.Log($"Loaded permissions for {AuthorPermissions.Count} authors");
             }
 
             Utils.Log("Loading Server Whitelist");
-            using (var result = client.GetStreamSync(Consts.ServerWhitelistURL))
+            using (var result = await client.GetStreamAsync(Consts.ServerWhitelistURL))
             {
                 ServerWhitelist = result.FromYaml<ServerWhitelist>();
                 Utils.Log($"Loaded permissions for {ServerWhitelist.AllowedPrefixes.Count} servers and {ServerWhitelist.GoogleIDs.Count} GDrive files");
@@ -49,14 +55,14 @@ namespace Wabbajack.Lib.Validation
 
         }
 
-        public static void RunValidation(ModList modlist)
+        public static async Task RunValidation(WorkQueue queue, ModList modlist)
         {
-            var validator = new ValidateModlist();
+            var validator = new ValidateModlist(queue);
 
-            validator.LoadListsFromGithub();
+            await validator.LoadListsFromGithub();
 
             Utils.Log("Running validation checks");
-            var errors = validator.Validate(modlist);
+            var errors = await validator.Validate(modlist);
             errors.Do(e => Utils.Log(e));
             if (errors.Count() > 0)
             {
@@ -97,18 +103,18 @@ namespace Wabbajack.Lib.Validation
             };
         }
 
-        public IEnumerable<string> Validate(ModList modlist)
+        public async Task<IEnumerable<string>> Validate(ModList modlist)
         {
             ConcurrentStack<string> ValidationErrors = new ConcurrentStack<string>();
             
-            var nexus_mod_permissions = modlist.Archives
+            var nexus_mod_permissions = (await modlist.Archives
                 .Where(a => a.State is NexusDownloader.State)
-                .PMap(Queue, a => (a.Hash, FilePermissions((NexusDownloader.State)a.State), a))
+                .PMap(_queue, a => (a.Hash, FilePermissions((NexusDownloader.State)a.State), a)))
                 .ToDictionary(a => a.Hash, a => new { permissions = a.Item2, archive = a.a });
 
-            modlist.Directives
+            await modlist.Directives
                 .OfType<PatchedFromArchive>()
-                .PMap(Queue, p =>
+                .PMap(_queue, p =>
                 {
                     if (nexus_mod_permissions.TryGetValue(p.ArchiveHashPath[0], out var archive))
                     {
@@ -125,9 +131,9 @@ namespace Wabbajack.Lib.Validation
                     }
                 });
 
-            modlist.Directives
+            await modlist.Directives
                 .OfType<FromArchive>()
-                .PMap(Queue,p =>
+                .PMap(_queue, p =>
                 {
                     if (nexus_mod_permissions.TryGetValue(p.ArchiveHashPath[0], out var archive))
                     {

Wabbajack.Lib/VortexCompiler.cs

@@ -4,6 +4,8 @@ using System.IO;
 using System.Linq;
 using System.Security.Cryptography;
 using System.Text;
+using System.Threading.Tasks;
+using System.Threading;
 using DynamicData;
 using Microsoft.WindowsAPICodePack.Shell;
 using Newtonsoft.Json;
@@ -68,31 +70,38 @@ namespace Wabbajack.Lib
 
             ActiveArchives = new List<string>();
         }
-
-        protected override bool _Begin()
+        
+        protected override async Task<bool> _Begin(CancellationToken cancel)
         {
+            if (cancel.IsCancellationRequested) return false;
+
             ConfigureProcessor(10);
             if (string.IsNullOrEmpty(ModListName))
                 ModListName = $"Vortex ModList for {Game.ToString()}";
 
             Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at  {DownloadsFolder}.");
 
+            if (cancel.IsCancellationRequested) return false;
             ParseDeploymentFile();
 
+            if (cancel.IsCancellationRequested) return false;
             Info("Starting pre-compilation steps");
-            CreateMetaFiles();
+            await CreateMetaFiles();
 
+            if (cancel.IsCancellationRequested) return false;
             Info($"Indexing {StagingFolder}");
-            VFS.AddRoot(StagingFolder);
+            await VFS.AddRoot(StagingFolder);
 
             Info($"Indexing {GamePath}");
-            VFS.AddRoot(GamePath);
+            await VFS.AddRoot(GamePath);
 
             Info($"Indexing {DownloadsFolder}");
-            VFS.AddRoot(DownloadsFolder);
+            await VFS.AddRoot(DownloadsFolder);
 
-            AddExternalFolder();
+            if (cancel.IsCancellationRequested) return false;
+            await AddExternalFolder();
 
+            if (cancel.IsCancellationRequested) return false;
             Info("Cleaning output folder");
             if (Directory.Exists(ModListOutputFolder)) Utils.DeleteDirectory(ModListOutputFolder);
             Directory.CreateDirectory(ModListOutputFolder);
@@ -138,6 +147,7 @@ namespace Wabbajack.Lib
 
             Info($"Found {AllFiles.Count} files to build into mod list");
 
+            if (cancel.IsCancellationRequested) return false;
             Info("Verifying destinations");
             var duplicates = AllFiles.GroupBy(f => f.Path)
                 .Where(fs => fs.Count() > 1)
@@ -197,7 +207,7 @@ namespace Wabbajack.Lib
             var stack = MakeStack();
 
             Info("Running Compilation Stack");
-            var results = AllFiles.PMap(Queue, f => RunStack(stack.Where(s => s != null), f)).ToList();
+            var results = await AllFiles.PMap(Queue, f => RunStack(stack.Where(s => s != null), f));
 
             IEnumerable<NoMatch> noMatch = results.OfType<NoMatch>().ToList();
             Info($"No match for {noMatch.Count()} files");
@@ -228,7 +238,8 @@ namespace Wabbajack.Lib
             }
             */
 
-            GatherArchives();
+            if (cancel.IsCancellationRequested) return false;
+            await GatherArchives();
 
             ModList = new ModList
             {
@@ -298,27 +309,28 @@ namespace Wabbajack.Lib
         /// <summary>
         /// Some have mods outside their game folder located
         /// </summary>
-        private void AddExternalFolder()
+        private async Task AddExternalFolder()
         {
             var currentGame = Game.MetaData();
             if (currentGame.AdditionalFolders == null || currentGame.AdditionalFolders.Count == 0) return;
-            currentGame.AdditionalFolders.Do(f =>
+            foreach (var f in currentGame.AdditionalFolders)
             {
                 var path = f.Replace("%documents%", KnownFolders.Documents.Path);
                 if (!Directory.Exists(path)) return;
                 Info($"Indexing {path}");
-                VFS.AddRoot(path);
-            });
+                await VFS.AddRoot(path);
+            }
         }
 
-        private void CreateMetaFiles()
+        private async Task CreateMetaFiles()
         {
             Utils.Log("Getting Nexus api_key, please click authorize if a browser window appears");
-            var nexusClient = new NexusApiClient();
+            var nexusClient = await NexusApiClient.Get();
 
-            Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly)
+            await Task.WhenAll(
+                Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly)
                 .Where(File.Exists)
-                .Do(f =>
+                .Select(async f =>
                 {
                     if (Path.GetExtension(f) != ".meta" && Path.GetExtension(f) != ".xxHash" && !File.Exists($"{f}.meta") && ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)))
                     {
@@ -340,7 +352,7 @@ namespace Wabbajack.Lib
                             Utils.Log($"Hash is {hash}");
                         }
 
-                        var md5Response = nexusClient.GetModInfoFromMD5(Game, hash);
+                        var md5Response = await nexusClient.GetModInfoFromMD5(Game, hash);
                         if (md5Response.Count >= 1)
                         {
                             var modInfo = md5Response[0].mod;
@@ -385,7 +397,7 @@ namespace Wabbajack.Lib
                             ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
                         }
                     }
-                });
+                }));
 
             Utils.Log($"Checking for Steam Workshop Items...");
             if (!_isSteamGame || _steamGame == null || _steamGame.WorkshopItems.Count <= 0)

Wabbajack.Lib/VortexInstaller.cs

@@ -2,6 +2,8 @@
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Threading.Tasks;
+using System.Threading;
 using System.Windows;
 using Wabbajack.Common;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
@@ -32,21 +34,29 @@ namespace Wabbajack.Lib
             GameInfo = ModList.GameType.MetaData();
         }
 
-        protected override bool _Begin()
+        protected override async Task<bool> _Begin(CancellationToken cancel)
         {
+            if (cancel.IsCancellationRequested) return false;
             MessageBox.Show(
                 "Vortex Support is still experimental and may produce unexpected results. " +
                 "If anything fails go to the special vortex support channels on the discord. @erri120#2285 " +
                 "for support.", "Warning",
                 MessageBoxButton.OK);
 
-            ConfigureProcessor(10, RecommendQueueSize());
+            if (cancel.IsCancellationRequested) return false;
+            ConfigureProcessor(10, await RecommendQueueSize());
             Directory.CreateDirectory(DownloadFolder);
 
-            HashArchives();
-            DownloadArchives();
-            HashArchives();
+            if (cancel.IsCancellationRequested) return false;
+            await HashArchives();
 
+            if (cancel.IsCancellationRequested) return false;
+            await DownloadArchives();
+
+            if (cancel.IsCancellationRequested) return false;
+            await HashArchives();
+
+            if (cancel.IsCancellationRequested) return false;
             var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
             if (missing.Count > 0)
             {
@@ -58,20 +68,29 @@ namespace Wabbajack.Lib
                     Error("Cannot continue, was unable to download one or more archives");
             }
 
-            PrimeVFS();
+            await PrimeVFS();
 
+            if (cancel.IsCancellationRequested) return false;
             BuildFolderStructure();
-            InstallArchives();
-            InstallIncludedFiles();
-            InstallManualGameFiles();
-            InstallSteamWorkshopItems();
+
+            if (cancel.IsCancellationRequested) return false;
+            await InstallArchives();
+
+            if (cancel.IsCancellationRequested) return false;
+            await InstallIncludedFiles();
+
+            if (cancel.IsCancellationRequested) return false;
+            await InstallManualGameFiles();
+
+            if (cancel.IsCancellationRequested) return false;
+            await InstallSteamWorkshopItems();
             //InstallIncludedDownloadMetas();
 
             Info("Installation complete! You may exit the program.");
             return true;
         }
 
-        private void InstallManualGameFiles()
+        private async Task InstallManualGameFiles()
         {
             if (!ModList.Directives.Any(d => d.To.StartsWith(Consts.ManualGameFilesDir)))
                 return;
@@ -96,7 +115,7 @@ namespace Wabbajack.Lib
                 return;
             }
 
-            Directory.EnumerateDirectories(manualFilesDir).PMap(Queue, dir =>
+            await Directory.EnumerateDirectories(manualFilesDir).PMap(Queue, dir =>
             {
                 var dirInfo = new DirectoryInfo(dir);
                 dirInfo.GetDirectories("*", SearchOption.AllDirectories).Do(d =>
@@ -122,7 +141,7 @@ namespace Wabbajack.Lib
             });
         }
 
-        private void InstallSteamWorkshopItems()
+        private async Task InstallSteamWorkshopItems()
         {
             //var currentLib = "";
             SteamGame currentSteamGame = null;
@@ -143,7 +162,7 @@ namespace Wabbajack.Lib
             if (result != MessageBoxResult.Yes)
                 return;
 
-            ModList.Directives.OfType<SteamMeta>()
+            await ModList.Directives.OfType<SteamMeta>()
                 .PMap(Queue, item =>
                 {
                     Status("Extracting Steam meta file to temp folder");
@@ -167,10 +186,10 @@ namespace Wabbajack.Lib
                 });
         }
 
-        private void InstallIncludedFiles()
+        private async Task InstallIncludedFiles()
         {
             Info("Writing inline files");
-            ModList.Directives.OfType<InlineFile>()
+            await ModList.Directives.OfType<InlineFile>()
                 .PMap(Queue,directive =>
                 {
                     if (directive.To.EndsWith(".meta"))

Wabbajack.Lib/zEditIntegration.cs

@@ -9,6 +9,7 @@ using Wabbajack.Lib.CompilationSteps;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
 using File = Alphaleonis.Win32.Filesystem.File;
 using Path = Alphaleonis.Win32.Filesystem.Path;
+using System.Threading.Tasks;
 
 namespace Wabbajack.Lib
 {
@@ -70,7 +71,7 @@ namespace Wabbajack.Lib
                         m => m.First());
             }
 
-            public override Directive Run(RawSourceFile source)
+            public override async ValueTask<Directive> Run(RawSourceFile source)
             {
                 if (!_mergesIndexed.TryGetValue(source.AbsolutePath, out var merge)) return null;
                 var result = source.EvolveTo<MergedPatch>();
@@ -160,9 +161,9 @@ namespace Wabbajack.Lib
             }
         }
 
-        public static void GenerateMerges(MO2Installer installer)
+        public static async Task GenerateMerges(MO2Installer installer)
         {
-            installer.ModList
+            await installer.ModList
                 .Directives
                 .OfType<MergedPatch>()
                 .PMap(installer.Queue, m =>

Wabbajack.Test.ListValidation/ListValidation.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Threading.Tasks;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Wabbajack.Common;
 using Wabbajack.Lib;
@@ -15,11 +16,11 @@ namespace Wabbajack.Test.ListValidation
     public class ListValidation
     {
         [ClassInitialize]
-        public static void SetupNexus(TestContext context)
+        public static async Task SetupNexus(TestContext context)
         {
             Utils.LogMessages.Subscribe(m => context.WriteLine(m.ToString()));
-            var api = new NexusApiClient();
-            api.ClearUpdatedModsInCache();
+            var api = await NexusApiClient.Get();
+            await api.ClearUpdatedModsInCache();
         }
 
         private WorkQueue Queue { get; set; }
@@ -34,7 +35,7 @@ namespace Wabbajack.Test.ListValidation
         [TestCleanup]
         public void Cleanup()
         {
-            Queue.Shutdown();
+            Queue.Dispose();
             Queue = null;
         }
 
@@ -44,7 +45,7 @@ namespace Wabbajack.Test.ListValidation
         [TestCategory("ListValidation")]
         [DataTestMethod]
         [DynamicData(nameof(GetModLists), DynamicDataSourceType.Method)]
-        public void ValidateModLists(string name, ModlistMetadata list)
+        public async Task ValidateModLists(string name, ModlistMetadata list)
         {
             Log($"Testing {list.Links.MachineURL} - {list.Title}");
             var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ".wabbajack");
@@ -53,7 +54,7 @@ namespace Wabbajack.Test.ListValidation
             {
                 var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
                 Log($"Downloading {list.Links.MachineURL} - {list.Title}");
-                state.Download(modlist_path);
+                await state.Download(modlist_path);
             }
             else
             {
@@ -67,12 +68,12 @@ namespace Wabbajack.Test.ListValidation
 
             Log($"{installer.Archives.Count} archives to validate");
 
-            var invalids = installer.Archives
-                .PMap(Queue,archive =>
+            var invalids = (await installer.Archives
+                .PMap(Queue, async archive =>
                 {
                     Log($"Validating: {archive.Name}");
-                    return new {archive, is_valid = archive.State.Verify()};
-                })
+                    return new {archive, is_valid = await archive.State.Verify()};
+                }))
                 .Where(a => !a.is_valid)
                 .ToList();
 
@@ -92,9 +93,9 @@ namespace Wabbajack.Test.ListValidation
             TestContext.WriteLine(msg);
         }
 
-        public static IEnumerable<object[]> GetModLists()
+        public static async Task<IEnumerable<object[]>> GetModLists()
         {
-            return ModlistMetadata.LoadFromGithub().Select(l => new object[] {l.Title, l});
+            return (await ModlistMetadata.LoadFromGithub()).Select(l => new object[] {l.Title, l});
         }
     }
 }

Wabbajack.Test/ACompilerTest.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Threading.Tasks;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Wabbajack.Common;
 using Wabbajack.Lib;
@@ -30,20 +31,20 @@ namespace Wabbajack.Test
             utils.Dispose();
         }
 
-        protected MO2Compiler ConfigureAndRunCompiler(string profile)
+        protected async Task<MO2Compiler> ConfigureAndRunCompiler(string profile)
         {
             var compiler = new MO2Compiler(
                 mo2Folder: utils.MO2Folder,
                 mo2Profile: profile,
                 outputFile: profile + ExtensionManager.Extension);
             compiler.ShowReportWhenFinished = false;
-            Assert.IsTrue(compiler.Begin().Result);
+            Assert.IsTrue(await compiler.Begin());
             return compiler;
         }
 
-        protected ModList CompileAndInstall(string profile)
+        protected async Task<ModList> CompileAndInstall(string profile)
         {
-            var compiler = ConfigureAndRunCompiler(profile);
+            var compiler = await ConfigureAndRunCompiler(profile);
             Install(compiler);
             return compiler.ModList;
         }

Wabbajack.Test/AVortexCompilerTest.cs

@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Threading.Tasks;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Wabbajack.Common;
 using Wabbajack.Lib;
@@ -31,13 +32,13 @@ namespace Wabbajack.Test
             utils.Dispose();
         }
 
-        protected VortexCompiler ConfigureAndRunCompiler()
+        protected async Task<VortexCompiler> ConfigureAndRunCompiler()
         {
             var vortexCompiler = MakeCompiler();
             vortexCompiler.DownloadsFolder = utils.DownloadsFolder;
             vortexCompiler.StagingFolder = utils.InstallFolder;
             Directory.CreateDirectory(utils.InstallFolder);
-            Assert.IsTrue(vortexCompiler.Begin().Result);
+            Assert.IsTrue(await vortexCompiler.Begin());
             return vortexCompiler;
         }
 
@@ -52,9 +53,9 @@ namespace Wabbajack.Test
                 outputFile: $"test{ExtensionManager.Extension}");
         }
 
-        protected ModList CompileAndInstall()
+        protected async Task<ModList> CompileAndInstall()
         {
-            var vortexCompiler = ConfigureAndRunCompiler();
+            var vortexCompiler = await ConfigureAndRunCompiler();
             Install(vortexCompiler);
             return vortexCompiler.ModList;
         }

Wabbajack.Test/CompilationStackTests.cs

@@ -1,4 +1,5 @@
-using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System.Threading.Tasks;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Wabbajack.Lib.CompilationSteps;
 
 namespace Wabbajack.Test
@@ -7,13 +8,13 @@ namespace Wabbajack.Test
     public class CompilationStackTests : ACompilerTest
     {
         [TestMethod]
-        public void TestStackSerialization()
+        public async Task TestStackSerialization()
         {
             var profile = utils.AddProfile();
             var mod = utils.AddMod("test");
 
             utils.Configure();
-            var compiler = ConfigureAndRunCompiler(profile);
+            var compiler = await ConfigureAndRunCompiler(profile);
             var stack = compiler.MakeStack();
 
             var serialized = Serialization.Serialize(stack);

Wabbajack.Test/ContentRightsManagementTests.cs

@@ -6,6 +6,8 @@ using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib;
 using Wabbajack.Lib.Validation;
 using Game = Wabbajack.Common.Game;
+using Wabbajack.Common;
+using System.Threading.Tasks;
 
 namespace Wabbajack.Test
 {
@@ -47,9 +49,12 @@ namespace Wabbajack.Test
         [TestInitialize]
         public void TestSetup()
         {
-            validate = new ValidateModlist();
-            validate.LoadAuthorPermissionsFromString(permissions);
-            validate.LoadServerWhitelist(server_whitelist);
+            using (var workQueue = new WorkQueue())
+            {
+                validate = new ValidateModlist(workQueue);
+                validate.LoadAuthorPermissionsFromString(permissions);
+                validate.LoadServerWhitelist(server_whitelist);
+            }
         }
 
         [TestMethod]
@@ -123,7 +128,7 @@ namespace Wabbajack.Test
 
 
         [TestMethod]
-        public void TestModValidation()
+        public async Task TestModValidation()
         {
             var modlist = new ModList
             {
@@ -156,7 +161,7 @@ namespace Wabbajack.Test
             IEnumerable<string> errors;
 
             // No errors, simple archive extraction
-            errors = validate.Validate(modlist);
+            errors = await validate.Validate(modlist);
             Assert.AreEqual(errors.Count(), 0);
 
 
@@ -167,7 +172,7 @@ namespace Wabbajack.Test
                 ArchiveHashPath = new[] {"DEADBEEF", "foo\\bar\\baz.pex"},
             };
 
-            errors = validate.Validate(modlist);
+            errors = await validate.Validate(modlist);
             Assert.AreEqual(errors.Count(), 1);
 
             // Error due to extracted BSA file
@@ -176,7 +181,7 @@ namespace Wabbajack.Test
                 ArchiveHashPath = new[] { "DEADBEEF", "foo.bsa", "foo\\bar\\baz.dds" },
             };
 
-            errors = validate.Validate(modlist);
+            errors = await validate.Validate(modlist);
             Assert.AreEqual(errors.Count(), 1);
 
             // No error since we're just installing the .bsa, not extracting it
@@ -185,7 +190,7 @@ namespace Wabbajack.Test
                 ArchiveHashPath = new[] { "DEADBEEF", "foo.bsa"},
             };
 
-            errors = validate.Validate(modlist);
+            errors = await validate.Validate(modlist);
             Assert.AreEqual(0, errors.Count());
             
             // Error due to game conversion
@@ -194,7 +199,7 @@ namespace Wabbajack.Test
             {
                 ArchiveHashPath = new[] { "DEADBEEF", "foo\\bar\\baz.dds" },
             };
-            errors = validate.Validate(modlist);
+            errors = await validate.Validate(modlist);
             Assert.AreEqual(errors.Count(), 1);
 
             // Error due to file downloaded from 3rd party
@@ -204,7 +209,7 @@ namespace Wabbajack.Test
                 State = new HTTPDownloader.State() { Url = "https://somebadplace.com" },
                 Hash = "DEADBEEF"
             };
-            errors = validate.Validate(modlist);
+            errors = await validate.Validate(modlist);
             Assert.AreEqual(1, errors.Count());
 
             // Ok due to file downloaded from whitelisted 3rd party
@@ -214,7 +219,7 @@ namespace Wabbajack.Test
                 State = new HTTPDownloader.State { Url = "https://somegoodplace.com/baz.7z" },
                 Hash = "DEADBEEF"
             };
-            errors = validate.Validate(modlist);
+            errors = await validate.Validate(modlist);
             Assert.AreEqual(0, errors.Count());
 
 
@@ -225,7 +230,7 @@ namespace Wabbajack.Test
                 State = new GoogleDriveDownloader.State { Id = "bleg"},
                 Hash = "DEADBEEF"
             };
-            errors = validate.Validate(modlist);
+            errors = await validate.Validate(modlist);
             Assert.AreEqual(errors.Count(), 1);
 
             // Ok due to file downloaded from good google site
@@ -235,17 +240,18 @@ namespace Wabbajack.Test
                 State = new GoogleDriveDownloader.State { Id = "googleDEADBEEF" },
                 Hash = "DEADBEEF"
             };
-            errors = validate.Validate(modlist);
+            errors = await validate.Validate(modlist);
             Assert.AreEqual(0, errors.Count());
 
         }
 
         [TestMethod]
-        public void CanLoadFromGithub()
+        public async Task CanLoadFromGithub()
         {
-            new ValidateModlist().LoadListsFromGithub();
+            using (var workQueue = new WorkQueue())
+            {
+                await new ValidateModlist(workQueue).LoadListsFromGithub();
+            }
         }
     }
-
-
 }

Wabbajack.Test/DownloaderTests.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Threading.Tasks;
 using System.Reactive.Disposables;
 using System.Reactive.Linq;
 using Alphaleonis.Win32.Filesystem;
@@ -39,7 +40,7 @@ namespace Wabbajack.Test
         }
 
         [TestMethod]
-        public void MegaDownload()
+        public async Task MegaDownload()
         {
             var ini = @"[General]
                         directURL=https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k";
@@ -56,13 +57,13 @@ namespace Wabbajack.Test
 
 
             var converted = state.ViaJSON();
-            Assert.IsTrue(converted.Verify());
+            Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
             Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List<string>{"https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k" } }));
             Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>{ "blerg" }}));
 
-            converted.Download(new Archive {Name = "MEGA Test.txt"}, filename);
+            await converted.Download(new Archive {Name = "MEGA Test.txt"}, filename);
 
             Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
 
@@ -70,7 +71,7 @@ namespace Wabbajack.Test
         }
 
         [TestMethod]
-        public void DropboxTests()
+        public async Task DropboxTests()
         {
             var ini = @"[General]
                         directURL=https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?dl=0";
@@ -86,13 +87,13 @@ namespace Wabbajack.Test
                 ((HTTPDownloader.State)url_state).Url);
 
             var converted = state.ViaJSON();
-            Assert.IsTrue(converted.Verify());
+            Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
             Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?" } }));
             Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "blerg" } }));
 
-            converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
+            await converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
 
             Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
 
@@ -100,7 +101,7 @@ namespace Wabbajack.Test
         }
 
         [TestMethod]
-        public void GoogleDriveTests()
+        public async Task GoogleDriveTests()
         {
             var ini = @"[General]
                         directURL=https://drive.google.com/file/d/1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_/view?usp=sharing";
@@ -116,13 +117,13 @@ namespace Wabbajack.Test
                 ((GoogleDriveDownloader.State)url_state).Id);
 
             var converted = state.ViaJSON();
-            Assert.IsTrue(converted.Verify());
+            Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
             Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List<string> { "1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_" } }));
             Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List<string>()}));
 
-            converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
+            await converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
 
             Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
 
@@ -130,7 +131,7 @@ namespace Wabbajack.Test
         }
 
         [TestMethod]
-        public void HttpDownload()
+        public async Task HttpDownload()
         {
             var ini = @"[General]
                         directURL=http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt";
@@ -145,13 +146,13 @@ namespace Wabbajack.Test
                 ((HTTPDownloader.State)url_state).Url);
 
             var converted = state.ViaJSON();
-            Assert.IsTrue(converted.Verify());
+            Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
             Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "http://build.wabbajack.org/" } }));
             Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
 
-            converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
+            await converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
 
             Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
 
@@ -159,7 +160,7 @@ namespace Wabbajack.Test
         }
 
         [TestMethod]
-        public void ManualDownload()
+        public async Task ManualDownload()
         {
             var ini = @"[General]
                         manualURL=http://build.wabbajack.org/WABBAJACK_TEST_FILE.zip";
@@ -169,12 +170,12 @@ namespace Wabbajack.Test
             Assert.IsNotNull(state);
 
             var converted = state.ViaJSON();
-            Assert.IsTrue(converted.Verify());
+            Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
             Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "http://build.wabbajack.org/" } }));
 
-            converted.Download(new Archive { Name = "WABBAJACK_TEST_FILE.zip", Size = 20, Hash = "eSIyd+KOG3s="}, filename);
+            await converted.Download(new Archive { Name = "WABBAJACK_TEST_FILE.zip", Size = 20, Hash = "eSIyd+KOG3s="}, filename);
 
             Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
 
@@ -182,7 +183,7 @@ namespace Wabbajack.Test
         }
 
         [TestMethod]
-        public void MediaFireDownload()
+        public async Task MediaFireDownload()
         {
             var ini = @"[General]
                     directURL=http://www.mediafire.com/file/agiqzm1xwebczpx/WABBAJACK_TEST_FILE.txt";
@@ -198,21 +199,21 @@ namespace Wabbajack.Test
                 ((MediaFireDownloader.State) url_state).Url);
 
             var converted = state.ViaJSON();
-            Assert.IsTrue(converted.Verify());
+            Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
             Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist
                 {AllowedPrefixes = new List<string> {"http://www.mediafire.com/file/agiqzm1xwebczpx/"}}));
             Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List<string>()}));
 
-            converted.Download(new Archive {Name = "Media Fire Test.txt"}, filename);
+            await converted.Download(new Archive {Name = "Media Fire Test.txt"}, filename);
 
             Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
 
         }
 
         [TestMethod]
-        public void NexusDownload()
+        public async Task NexusDownload()
         {
             var old_val = NexusApiClient.UseLocalCache;
             try
@@ -229,14 +230,14 @@ namespace Wabbajack.Test
 
 
                 var converted = state.ViaJSON();
-                Assert.IsTrue(converted.Verify());
+                Assert.IsTrue(await converted.Verify());
                 // Exercise the cache code
-                Assert.IsTrue(converted.Verify());
+                Assert.IsTrue(await converted.Verify());
                 var filename = Guid.NewGuid().ToString();
 
                 Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> () }));
 
-                converted.Download(new Archive { Name = "SkyUI.7z" }, filename);
+                await converted.Download(new Archive { Name = "SkyUI.7z" }, filename);
 
                 Assert.AreEqual(filename.FileHash(), "dF2yafV2Oks=");
 
@@ -248,7 +249,7 @@ namespace Wabbajack.Test
         }
 
         [TestMethod]
-        public void ModDbTests()
+        public async Task ModDbTests()
         {
             var ini = @"[General]
                         directURL=https://www.moddb.com/downloads/start/124908?referer=https%3A%2F%2Fwww.moddb.com%2Fmods%2Fautopause";
@@ -264,20 +265,20 @@ namespace Wabbajack.Test
                 ((ModDBDownloader.State)url_state).Url);
 
             var converted = state.ViaJSON();
-            Assert.IsTrue(converted.Verify());
+            Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
             Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
 
-            converted.Download(new Archive { Name = "moddbtest.7z" }, filename);
+            await converted.Download(new Archive { Name = "moddbtest.7z" }, filename);
 
             Assert.AreEqual("2lZt+1h6wxM=", filename.FileHash());
         }
 
         [TestMethod]
-        public void LoversLabDownload()
+        public async Task LoversLabDownload()
         {
-            DownloadDispatcher.GetInstance<LoversLabDownloader>().Prepare();
+            await DownloadDispatcher.GetInstance<LoversLabDownloader>().Prepare();
             var ini = @"[General]
                         directURL=https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1";
 
@@ -290,12 +291,12 @@ namespace Wabbajack.Test
                 ((HTTPDownloader.State)url_state).Url);
                 */
             var converted = state.ViaJSON();
-            Assert.IsTrue(converted.Verify());
+            Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
             Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
 
-            converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
+            await converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
 
             Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
 
@@ -303,9 +304,9 @@ namespace Wabbajack.Test
         }
 
         [TestMethod]
-        public void GameFileSourceDownload()
+        public async Task GameFileSourceDownload()
         {
-            DownloadDispatcher.GetInstance<LoversLabDownloader>().Prepare();
+            await DownloadDispatcher.GetInstance<LoversLabDownloader>().Prepare();
             var ini = $@"[General]
                         gameName={Game.SkyrimSpecialEdition.MetaData().MO2ArchiveName}
                         gameFile=Data/Update.esm";
@@ -315,12 +316,12 @@ namespace Wabbajack.Test
             Assert.IsNotNull(state);
 
             var converted = state.ViaJSON();
-            Assert.IsTrue(converted.Verify());
+            Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
             Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
 
-            converted.Download(new Archive { Name = "Update.esm" }, filename);
+            await converted.Download(new Archive { Name = "Update.esm" }, filename);
 
             Assert.AreEqual("/DLG/LjdGXI=", Utils.FileHash(filename));
             CollectionAssert.AreEqual(File.ReadAllBytes(Path.Combine(Game.SkyrimSpecialEdition.MetaData().GameLocation(), "Data/Update.esm")), File.ReadAllBytes(filename));

Wabbajack.Test/EndToEndTests.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Wabbajack.Common;
@@ -39,16 +40,16 @@ namespace Wabbajack.Test
         [TestCleanup]
         public void Cleanup()
         {
-            Queue.Shutdown();
+            Queue.Dispose();
         }
 
         [TestMethod]
-        public void CreateModlist()
+        public async Task CreateModlist()
         {
             var profile = utils.AddProfile("Default");
             var mod = utils.AddMod();
 
-            DownloadAndInstall(
+            await DownloadAndInstall(
                 "https://github.com/ModOrganizer2/modorganizer/releases/download/v2.2.1/Mod.Organizer.2.2.1.7z",
                 "Mod.Organizer.2.2.1.7z",
                 utils.MO2Folder);
@@ -59,7 +60,7 @@ namespace Wabbajack.Test
                     "directURL=https://github.com/ModOrganizer2/modorganizer/releases/download/v2.2.1/Mod.Organizer.2.2.1.7z"
                 });
 
-            DownloadAndInstall(Game.SkyrimSpecialEdition, 12604, "SkyUI");
+            await DownloadAndInstall(Game.SkyrimSpecialEdition, 12604, "SkyUI");
 
             utils.Configure();
 
@@ -77,17 +78,17 @@ namespace Wabbajack.Test
                 outputFile: profile + ExtensionManager.Extension);
             compiler.MO2DownloadsFolder = Path.Combine(utils.DownloadsFolder);
             compiler.ShowReportWhenFinished = false;
-            Assert.IsTrue(compiler.Begin().Result);
+            Assert.IsTrue(await compiler.Begin());
 
         }
 
-        private void DownloadAndInstall(string url, string filename, string mod_name = null)
+        private async Task DownloadAndInstall(string url, string filename, string mod_name = null)
         {
             var src = Path.Combine(DOWNLOAD_FOLDER, filename);
             if (!File.Exists(src))
             {
                 var state = DownloadDispatcher.ResolveArchive(url);
-                state.Download(new Archive() { Name = "Unknown"}, src);
+                await state.Download(new Archive() { Name = "Unknown"}, src);
             }
 
             if (!Directory.Exists(utils.DownloadsFolder))
@@ -97,15 +98,16 @@ namespace Wabbajack.Test
 
             File.Copy(src, Path.Combine(utils.DownloadsFolder, filename));
 
-            FileExtractor.ExtractAll(Queue, src,
+            await FileExtractor.ExtractAll(Queue, src,
                 mod_name == null ? utils.MO2Folder : Path.Combine(utils.ModsFolder, mod_name));
         }
 
-        private void DownloadAndInstall(Game game, int modid, string mod_name)
+        private async Task DownloadAndInstall(Game game, int modid, string mod_name)
         {
             utils.AddMod(mod_name);
-            var client = new NexusApiClient();
-            var file = client.GetModFiles(game, modid).files.First(f => f.is_primary);
+            var client = await NexusApiClient.Get();
+            var resp = await client.GetModFiles(game, modid);
+            var file = resp.files.First(f => f.is_primary);
             var src = Path.Combine(DOWNLOAD_FOLDER, file.file_name);
 
             var ini = string.Join("\n",
@@ -132,14 +134,14 @@ namespace Wabbajack.Test
             var dest = Path.Combine(utils.DownloadsFolder, file.file_name);
             File.Copy(src, dest);
 
-            FileExtractor.ExtractAll(Queue, src, Path.Combine(utils.ModsFolder, mod_name));
+            await FileExtractor.ExtractAll(Queue, src, Path.Combine(utils.ModsFolder, mod_name));
 
             File.WriteAllText(dest + ".meta", ini);
         }
 
-        private ModList CompileAndInstall(string profile)
+        private async Task<ModList> CompileAndInstall(string profile)
         {
-            var compiler = ConfigureAndRunCompiler(profile);
+            var compiler = await ConfigureAndRunCompiler(profile);
             Install(compiler);
             return compiler.ModList;
         }
@@ -156,14 +158,14 @@ namespace Wabbajack.Test
             installer.Begin().Wait();
         }
 
-        private MO2Compiler ConfigureAndRunCompiler(string profile)
+        private async Task<MO2Compiler> ConfigureAndRunCompiler(string profile)
         {
             var compiler = new MO2Compiler(
                 mo2Folder: utils.MO2Folder,
                 mo2Profile: profile,
                 outputFile: profile + ExtensionManager.Extension);
             compiler.ShowReportWhenFinished = false;
-            Assert.IsTrue(compiler.Begin().Result);
+            Assert.IsTrue(await compiler.Begin());
             return compiler;
         }
     }

Wabbajack.Test/ModlistMetadataTests.cs

@@ -1,4 +1,5 @@
 using System.Linq;
+using System.Threading.Tasks;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.ModListRegistry;
@@ -9,22 +10,22 @@ namespace Wabbajack.Test
     public class ModlistMetadataTests
     {
         [TestMethod]
-        public void TestLoadingModlists()
+        public async Task TestLoadingModlists()
         {
-            var modlists = ModlistMetadata.LoadFromGithub();
+            var modlists = await ModlistMetadata.LoadFromGithub();
             Assert.IsTrue(modlists.Count > 0);
         }
 
         [TestMethod]
-        public void VerifyLogoURLs()
+        public async Task VerifyLogoURLs()
         {
-            var modlists = ModlistMetadata.LoadFromGithub();
+            var modlists = await ModlistMetadata.LoadFromGithub();
 
             foreach (var modlist in modlists.Select(m => m.Links))
             {
                 var logo_state = DownloadDispatcher.ResolveArchive(modlist.ImageUri);
                 Assert.IsNotNull(logo_state);
-                Assert.IsTrue(logo_state.Verify(), $"{modlist.ImageUri} is not valid");
+                Assert.IsTrue(await logo_state.Verify(), $"{modlist.ImageUri} is not valid");
             }
         }
     }

Wabbajack.Test/SanityTests.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Wabbajack.Common;
 using Wabbajack.Lib;
@@ -14,7 +15,7 @@ namespace Wabbajack.Test
     public class SanityTests : ACompilerTest
     {
         [TestMethod]
-        public void TestDirectMatch()
+        public async Task TestDirectMatch()
         {
 
             var profile = utils.AddProfile();
@@ -26,13 +27,13 @@ namespace Wabbajack.Test
             utils.AddManualDownload(
                 new Dictionary<string, byte[]> {{"/baz/biz.pex", File.ReadAllBytes(test_pex)}});
 
-            CompileAndInstall(profile);
+            await CompileAndInstall(profile);
 
             utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex");
         }
 
         [TestMethod]
-        public void TestDuplicateFilesAreCopied()
+        public async Task TestDuplicateFilesAreCopied()
         {
 
             var profile = utils.AddProfile();
@@ -47,14 +48,14 @@ namespace Wabbajack.Test
             utils.AddManualDownload(
                 new Dictionary<string, byte[]> { { "/baz/biz.pex", File.ReadAllBytes(test_pex) } });
 
-            CompileAndInstall(profile);
+            await CompileAndInstall(profile);
 
             utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex");
             utils.VerifyInstalledFile(mod, @"Data\scripts\test.pex.copy");
         }
 
         [TestMethod]
-        public void TestUpdating()
+        public async Task TestUpdating()
         {
 
             var profile = utils.AddProfile();
@@ -73,7 +74,7 @@ namespace Wabbajack.Test
                     { "/baz/modified.pex", File.ReadAllBytes(modified) },
                 });
 
-            CompileAndInstall(profile);
+            await CompileAndInstall(profile);
 
             utils.VerifyInstalledFile(mod, @"Data\scripts\unchanged.pex");
             utils.VerifyInstalledFile(mod, @"Data\scripts\deleted.pex");
@@ -95,7 +96,7 @@ namespace Wabbajack.Test
 
             Assert.IsTrue(File.Exists(extra_path));
 
-            CompileAndInstall(profile);
+            await CompileAndInstall(profile);
 
             utils.VerifyInstalledFile(mod, @"Data\scripts\unchanged.pex");
             utils.VerifyInstalledFile(mod, @"Data\scripts\deleted.pex");
@@ -108,7 +109,7 @@ namespace Wabbajack.Test
 
 
         [TestMethod]
-        public void CleanedESMTest()
+        public async Task CleanedESMTest()
         {
             var profile = utils.AddProfile();
             var mod = utils.AddMod("Cleaned ESMs");
@@ -123,7 +124,7 @@ namespace Wabbajack.Test
 
             utils.VerifyInstalledFile(mod, @"Update.esm");
 
-            var compiler = ConfigureAndRunCompiler(profile);
+            var compiler = await ConfigureAndRunCompiler(profile);
 
             // Update the file and verify that it throws an error.
             utils.GenerateRandomFileData(game_file, 20);
@@ -155,7 +156,7 @@ namespace Wabbajack.Test
         }
 
         [TestMethod]
-        public void UnmodifiedInlinedFilesArePulledFromArchives()
+        public async Task UnmodifiedInlinedFilesArePulledFromArchives()
         {
             var profile = utils.AddProfile();
             var mod = utils.AddMod();
@@ -165,7 +166,7 @@ namespace Wabbajack.Test
             utils.AddManualDownload(
                 new Dictionary<string, byte[]> { { "/baz/biz.pex", File.ReadAllBytes(ini) } });
 
-            var modlist = CompileAndInstall(profile);
+            var modlist = await CompileAndInstall(profile);
             var directive = modlist.Directives.Where(m => m.To == $"mods\\{mod}\\foo.ini").FirstOrDefault();
 
             Assert.IsNotNull(directive);
@@ -173,7 +174,7 @@ namespace Wabbajack.Test
         }
 
         [TestMethod]
-        public void ModifiedIniFilesArePatchedAgainstFileWithSameName()
+        public async Task ModifiedIniFilesArePatchedAgainstFileWithSameName()
         {
             var profile = utils.AddProfile();
             var mod = utils.AddMod();
@@ -195,7 +196,7 @@ namespace Wabbajack.Test
             // Modify after creating mod archive in the downloads folder
             File.WriteAllText(ini, "Wabbajack, Wabbajack, Wabbajack!");
 
-            var modlist = CompileAndInstall(profile);
+            var modlist = await CompileAndInstall(profile);
             var directive = modlist.Directives.Where(m => m.To == $"mods\\{mod}\\foo.ini").FirstOrDefault();
 
             Assert.IsNotNull(directive);

Wabbajack.Test/ZEditIntegrationTests.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Wabbajack.Common;
@@ -11,7 +12,7 @@ namespace Wabbajack.Test
     public class zEditIntegrationTests : ACompilerTest
     {
         [TestMethod]
-        public void CanCreatezEditPatches()
+        public async Task CanCreatezEditPatches()
         {
             var profile = utils.AddProfile();
             var moda = utils.AddMod();
@@ -72,7 +73,7 @@ namespace Wabbajack.Test
 
                 });
 
-            var modlist = CompileAndInstall(profile);
+            var modlist = await CompileAndInstall(profile);
             var directive = modlist.Directives.Where(m => m.To == $"mods\\{moddest}\\merged.esp").FirstOrDefault();
 
             Assert.IsNotNull(directive);

Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs

@@ -31,10 +31,10 @@ namespace Wabbajack.VirtualFileSystem.Test
         }
 
         [TestMethod]
-        public void FilesAreIndexed()
+        public async Task FilesAreIndexed()
         {
             AddFile("test.txt", "This is a test");
-            AddTestRoot();
+            await AddTestRoot();
 
             var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
             Assert.IsNotNull(file);
@@ -43,11 +43,11 @@ namespace Wabbajack.VirtualFileSystem.Test
             Assert.AreEqual(file.Hash, "qX0GZvIaTKM=");
         }
 
-        private void AddTestRoot()
+        private async Task AddTestRoot()
         {
-            context.AddRoot(VFS_TEST_DIR_FULL);
-            context.WriteToFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
-            context.IntegrateFromFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
+            await context.AddRoot(VFS_TEST_DIR_FULL);
+            await context.WriteToFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
+            await context.IntegrateFromFile(Path.Combine(VFS_TEST_DIR_FULL, "vfs_cache.bin"));
         }
 
 
@@ -56,7 +56,7 @@ namespace Wabbajack.VirtualFileSystem.Test
         {
             AddFile("archive/test.txt", "This is a test");
             ZipUpFolder("archive", "test.zip");
-            AddTestRoot();
+            await AddTestRoot();
 
             var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip");
             var file = context.Index.ByFullPath[abs_path];
@@ -79,7 +79,7 @@ namespace Wabbajack.VirtualFileSystem.Test
             ZipUpFolder("archive", "test.zip");
 
             AddFile("test.txt", "This is a test");
-            AddTestRoot();
+            await AddTestRoot();
 
 
             var files = context.Index.ByHash["qX0GZvIaTKM="];
@@ -90,7 +90,7 @@ namespace Wabbajack.VirtualFileSystem.Test
         public async Task DeletedFilesAreRemoved()
         {
             AddFile("test.txt", "This is a test");
-            AddTestRoot();
+            await AddTestRoot();
 
             var file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
             Assert.IsNotNull(file);
@@ -100,21 +100,21 @@ namespace Wabbajack.VirtualFileSystem.Test
 
             File.Delete(Path.Combine(VFS_TEST_DIR_FULL, "test.txt"));
 
-            AddTestRoot();
+            await AddTestRoot();
 
             CollectionAssert.DoesNotContain(context.Index.ByFullPath, Path.Combine(VFS_TEST_DIR_FULL, "test.txt"));
         }
 
         [TestMethod]
-        public void UnmodifiedFilesAreNotReIndexed()
+        public async Task UnmodifiedFilesAreNotReIndexed()
         {
             AddFile("test.txt", "This is a test");
-            AddTestRoot();
+            await AddTestRoot();
 
             var old_file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
             var old_time = old_file.LastAnalyzed;
 
-            AddTestRoot();
+            await AddTestRoot();
 
             var new_file = context.Index.ByFullPath[Path.Combine(VFS_TEST_DIR_FULL, "test.txt")];
 
@@ -122,23 +122,23 @@ namespace Wabbajack.VirtualFileSystem.Test
         }
 
         [TestMethod]
-        public void CanStageSimpleArchives()
+        public async Task CanStageSimpleArchives()
         {
             AddFile("archive/test.txt", "This is a test");
             ZipUpFolder("archive", "test.zip");
-            AddTestRoot();
+            await AddTestRoot();
 
             var abs_path = Path.Combine(VFS_TEST_DIR_FULL, "test.zip");
             var file = context.Index.ByFullPath[abs_path + "|test.txt"];
 
-            var cleanup = context.Stage(new List<VirtualFile> {file});
+            var cleanup = await context.Stage(new List<VirtualFile> {file});
             Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));
 
             cleanup();
         }
 
         [TestMethod]
-        public void CanStageNestedArchives()
+        public async Task CanStageNestedArchives()
         {
             AddFile("archive/test.txt", "This is a test");
             ZipUpFolder("archive", "test.zip");
@@ -148,11 +148,11 @@ namespace Wabbajack.VirtualFileSystem.Test
                 Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir\nested.zip"));
             ZipUpFolder("archive", "test.zip");
 
-            AddTestRoot();
+            await AddTestRoot();
 
             var files = context.Index.ByHash["qX0GZvIaTKM="];
 
-            var cleanup = context.Stage(files);
+            var cleanup = await context.Stage(files);
 
             foreach (var file in files)
                 Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));
@@ -161,7 +161,7 @@ namespace Wabbajack.VirtualFileSystem.Test
         }
 
         [TestMethod]
-        public void CanRequestPortableFileTrees()
+        public async Task CanRequestPortableFileTrees()
         {
             AddFile("archive/test.txt", "This is a test");
             ZipUpFolder("archive", "test.zip");
@@ -171,7 +171,7 @@ namespace Wabbajack.VirtualFileSystem.Test
                 Path.Combine(VFS_TEST_DIR_FULL, @"archive\other\dir\nested.zip"));
             ZipUpFolder("archive", "test.zip");
 
-            AddTestRoot();
+            await AddTestRoot();
 
             var files = context.Index.ByHash["qX0GZvIaTKM="];
             var archive = context.Index.ByRootPath[Path.Combine(VFS_TEST_DIR_FULL, "test.zip")];
@@ -180,12 +180,12 @@ namespace Wabbajack.VirtualFileSystem.Test
 
             var new_context = new Context(Queue);
 
-            new_context.IntegrateFromPortable(state,
+            await new_context.IntegrateFromPortable(state,
                 new Dictionary<string, string> {{archive.Hash, archive.FullPath}});
 
             var new_files = new_context.Index.ByHash["qX0GZvIaTKM="];
 
-            var close = new_context.Stage(new_files);
+            var close = await new_context.Stage(new_files);
 
             foreach (var file in new_files)
                 Assert.AreEqual("This is a test", File.ReadAllText(file.StagedPath));

Wabbajack.VirtualFileSystem/Context.cs

@@ -46,7 +46,7 @@ namespace Wabbajack.VirtualFileSystem
             return new TemporaryDirectory(Path.Combine(_stagingFolder, Guid.NewGuid().ToString()));
         }
 
-        public IndexRoot AddRoot(string root)
+        public async Task<IndexRoot> AddRoot(string root)
         {
             if (!Path.IsPathRooted(root))
                 throw new InvalidDataException($"Path is not absolute: {root}");
@@ -59,8 +59,8 @@ namespace Wabbajack.VirtualFileSystem
             
             var results = Channel.Create(1024, ProgressUpdater<VirtualFile>($"Indexing {root}", filesToIndex.Count));
 
-            var allFiles= filesToIndex
-                            .PMap(Queue, f =>
+            var allFiles = await filesToIndex
+                            .PMap(Queue, async f =>
                             {
                                 if (byPath.TryGetValue(f, out var found))
                                 {
@@ -69,10 +69,10 @@ namespace Wabbajack.VirtualFileSystem
                                         return found;
                                 }
 
-                                return VirtualFile.Analyze(this, null, f, f);
+                                return await VirtualFile.Analyze(this, null, f, f);
                             });
 
-            var newIndex = IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
+            var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
 
             lock (this)
             {
@@ -82,7 +82,7 @@ namespace Wabbajack.VirtualFileSystem
             return newIndex;
         }
 
-        public IndexRoot AddRoots(List<string> roots)
+        public async Task<IndexRoot> AddRoots(List<string> roots)
         {
             if (!roots.All(p => Path.IsPathRooted(p)))
                 throw new InvalidDataException($"Paths are not absolute");
@@ -95,8 +95,8 @@ namespace Wabbajack.VirtualFileSystem
 
             var results = Channel.Create(1024, ProgressUpdater<VirtualFile>($"Indexing roots", filesToIndex.Count));
 
-            var allFiles = filesToIndex
-                .PMap(Queue, f =>
+            var allFiles = await filesToIndex
+                .PMap(Queue, async f =>
                 {
                     if (byPath.TryGetValue(f, out var found))
                     {
@@ -105,10 +105,10 @@ namespace Wabbajack.VirtualFileSystem
                             return found;
                     }
 
-                    return VirtualFile.Analyze(this, null, f, f);
+                    return await VirtualFile.Analyze(this, null, f, f);
                 });
 
-            var newIndex = IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
+            var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
 
             lock (this)
             {
@@ -137,7 +137,7 @@ namespace Wabbajack.VirtualFileSystem
             });
         }
 
-        public void WriteToFile(string filename)
+        public async Task WriteToFile(string filename)
         {
             using (var fs = File.OpenWrite(filename))
             using (var bw = new BinaryWriter(fs, Encoding.UTF8, true))
@@ -148,13 +148,13 @@ namespace Wabbajack.VirtualFileSystem
                 bw.Write(FileVersion);
                 bw.Write((ulong) Index.AllFiles.Count);
 
-                Index.AllFiles
+                (await Index.AllFiles
                     .PMap(Queue, f =>
                     {
                         var ms = new MemoryStream();
                         f.Write(ms);
                         return ms;
-                    })
+                    }))
                     .Do(ms =>
                     {
                         var size = ms.Position;
@@ -166,7 +166,7 @@ namespace Wabbajack.VirtualFileSystem
             }
         }
 
-        public void IntegrateFromFile(string filename)
+        public async Task IntegrateFromFile(string filename)
         {
             try
             {
@@ -188,7 +188,7 @@ namespace Wabbajack.VirtualFileSystem
                             br.BaseStream.Read(bytes, 0, (int) size);
                             return VirtualFile.Read(this, bytes);
                         }).ToList();
-                    var newIndex = Index.Integrate(files);
+                    var newIndex = await Index.Integrate(files);
                     lock (this)
                     {
                         Index = newIndex;
@@ -202,7 +202,7 @@ namespace Wabbajack.VirtualFileSystem
             }
         }
 
-        public Action Stage(IEnumerable<VirtualFile> files)
+        public async Task<Action> Stage(IEnumerable<VirtualFile> files)
         {
             var grouped = files.SelectMany(f => f.FilesInFullPath)
                 .Distinct()
@@ -216,7 +216,7 @@ namespace Wabbajack.VirtualFileSystem
             foreach (var group in grouped)
             {
                 var tmpPath = Path.Combine(_stagingFolder, Guid.NewGuid().ToString());
-                FileExtractor.ExtractAll(Queue, group.Key.StagedPath, tmpPath);
+                await FileExtractor.ExtractAll(Queue, group.Key.StagedPath, tmpPath);
                 paths.Add(tmpPath);
                 foreach (var file in group)
                     file.StagedPath = Path.Combine(tmpPath, file.Name);
@@ -245,23 +245,23 @@ namespace Wabbajack.VirtualFileSystem
                 }).ToList();
         }
 
-        public void IntegrateFromPortable(List<PortableFile> state, Dictionary<string, string> links)
+        public async Task IntegrateFromPortable(List<PortableFile> state, Dictionary<string, string> links)
         {
             var indexedState = state.GroupBy(f => f.ParentHash)
                 .ToDictionary(f => f.Key ?? "", f => (IEnumerable<PortableFile>) f);
-            var parents = indexedState[""]
+            var parents = await indexedState[""]
                 .PMap(Queue,f => VirtualFile.CreateFromPortable(this, indexedState, links, f));
 
-            var newIndex = Index.Integrate(parents);
+            var newIndex = await Index.Integrate(parents);
             lock (this)
             {
                 Index = newIndex;
             }
         }
 
-        public DisposableList<VirtualFile> StageWith(IEnumerable<VirtualFile> files)
+        public async Task<DisposableList<VirtualFile>> StageWith(IEnumerable<VirtualFile> files)
         {
-            return new DisposableList<VirtualFile>(Stage(files), files);
+            return new DisposableList<VirtualFile>(await Stage(files), files);
         }
 
 
@@ -273,7 +273,7 @@ namespace Wabbajack.VirtualFileSystem
             _knownFiles.AddRange(known);
         }
 
-        public void BackfillMissing()
+        public async Task BackfillMissing()
         {
             var newFiles = _knownFiles.Where(f => f.Paths.Length == 1)
                                        .GroupBy(f => f.Hash)
@@ -307,7 +307,7 @@ namespace Wabbajack.VirtualFileSystem
             }
             _knownFiles.Where(f => f.Paths.Length > 1).Do(BackFillOne);
 
-            var newIndex = Index.Integrate(newFiles.Values.ToList());
+            var newIndex = await Index.Integrate(newFiles.Values.ToList());
 
             lock (this)
                 Index = newIndex;
@@ -373,7 +373,7 @@ namespace Wabbajack.VirtualFileSystem
         public ImmutableDictionary<string, ImmutableStack<VirtualFile>> ByName { get; set; }
         public ImmutableDictionary<string, VirtualFile> ByRootPath { get; }
 
-        public IndexRoot Integrate(ICollection<VirtualFile> files)
+        public async Task<IndexRoot> Integrate(ICollection<VirtualFile> files)
         {
             Utils.Log($"Integrating {files.Count} files");
             var allFiles = AllFiles.Concat(files).GroupBy(f => f.Name).Select(g => g.Last()).ToImmutableList();
@@ -391,10 +391,10 @@ namespace Wabbajack.VirtualFileSystem
             var byRootPath = Task.Run(() => allFiles.ToImmutableDictionary(f => f.Name));
 
             var result = new IndexRoot(allFiles,
-                byFullPath.Result,
-                byHash.Result,
-                byRootPath.Result,
-                byName.Result);
+                await byFullPath,
+                await byHash,
+                await byRootPath,
+                await byName);
             Utils.Log($"Done integrating");
             return result;
         }

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -4,6 +4,7 @@ using System.Collections.Immutable;
 using System.IO;
 using System.Linq;
 using System.Text;
+using System.Threading.Tasks;
 using Wabbajack.Common;
 using Wabbajack.Common.CSP;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
@@ -128,7 +129,7 @@ namespace Wabbajack.VirtualFileSystem
             }
         }
 
-        public static VirtualFile Analyze(Context context, VirtualFile parent, string abs_path,
+        public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, string abs_path,
             string rel_path)
         {
             var fi = new FileInfo(abs_path);
@@ -148,11 +149,12 @@ namespace Wabbajack.VirtualFileSystem
 
                 using (var tempFolder = context.GetTemporaryFolder())
                 {
-                    FileExtractor.ExtractAll(context.Queue, abs_path, tempFolder.FullName);
+                    await FileExtractor.ExtractAll(context.Queue, abs_path, tempFolder.FullName);
 
-                    self.Children = Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories)
-                                        .PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName)))
-                                        .ToImmutableList();
+                    var list = await Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories)
+                                        .PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName)));
+
+                    self.Children = list.ToImmutableList();
                 }
 
             }

Wabbajack/View Models/ModListGalleryVM.cs

@@ -36,9 +36,9 @@ namespace Wabbajack
             RefreshCommand.StartingExecution()
                 .StartWith(Unit.Default)
                 .ObserveOn(RxApp.TaskpoolScheduler)
-                .Select(_ =>
+                .SelectTask(async _ =>
                 {
-                    return ModlistMetadata.LoadFromGithub()
+                    return (await ModlistMetadata.LoadFromGithub())
                         .AsObservableChangeSet(x => x.DownloadMetadata?.Hash ?? $"Fallback{missingHashFallbackCounter++}");
                 })
                 .Switch()

Wabbajack/View Models/ModListMetadataVM.cs

@@ -81,10 +81,10 @@ namespace Wabbajack
             var sub = queue.Status.Select(i => i.ProgressPercent)
                 .Subscribe(percent => ProgressPercent = percent);
             TaskCompletionSource<bool> tcs = new TaskCompletionSource<bool>();
-            queue.QueueTask(() =>
+            queue.QueueTask(async () =>
             {
                 var downloader = DownloadDispatcher.ResolveArchive(Metadata.Links.Download);
-                downloader.Download(new Archive{ Name = Metadata.Title, Size = Metadata.DownloadMetadata?.Size ?? 0}, Location);
+                await downloader.Download(new Archive{ Name = Metadata.Title, Size = Metadata.DownloadMetadata?.Size ?? 0}, Location);
                 Location.FileHashCached();
                 sub.Dispose();
                 tcs.SetResult(true);