From 91ddb3a01f917ba8b050847e6263982a8aaaa628 Mon Sep 17 00:00:00 2001
From: Timothy Baldridge <tbaldridge@gmail.com>
Date: Fri, 13 Dec 2019 14:03:16 -0700
Subject: [PATCH 01/19] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ecc3d301..cf69bced 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@ Wabbajack is an automated ModList installer that can recreate contents of a fold
 ## Social Links
 
 - [wabbajack.org](https://www.wabbajack.org) The official Wabbajack website where you can find instructions and a [Gallery](https://www.wabbajack.org/gallery) for some ModLists.
-- [Discord](https://discord.gg/zgbrkmA) The official Wabbajack discord for instructions, ModLists, support or friendly chatting with fellow modders.
+- [Discord](https://discord.gg/wabbajack) The official Wabbajack discord for instructions, ModLists, support or friendly chatting with fellow modders.
 - [Patreon](https://www.patreon.com/user?u=11907933) contains update posts and keeps the [Code Signing Certificate](https://www.digicert.com/code-signing/) alive.
 
 ## Supported Games and Mod Manager

From c070c2431f7ccfbfcc1ccab6a60d4269be61ed3f Mon Sep 17 00:00:00 2001
From: halgari <tbaldridge@gmail.com>
Date: Fri, 13 Dec 2019 14:39:13 -0700
Subject: [PATCH 02/19] Fix end-to-end test

---
 Wabbajack.Test/EndToEndTests.cs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Wabbajack.Test/EndToEndTests.cs b/Wabbajack.Test/EndToEndTests.cs
index b666d40d..d7cd7cb9 100644
--- a/Wabbajack.Test/EndToEndTests.cs
+++ b/Wabbajack.Test/EndToEndTests.cs
@@ -122,7 +122,7 @@ namespace Wabbajack.Test
             if (!File.Exists(src))
             {
 
-                var state = DownloadDispatcher.ResolveArchive(ini.LoadIniString());
+                var state = await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
                 state.Download(src);
             }
 

From cd441560df8bf4dcfe7d7a75ab0084ab9d2414ca Mon Sep 17 00:00:00 2001
From: halgari <tbaldridge@gmail.com>
Date: Fri, 13 Dec 2019 14:48:40 -0700
Subject: [PATCH 03/19] Another missing await

---
 Wabbajack.Test/EndToEndTests.cs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Wabbajack.Test/EndToEndTests.cs b/Wabbajack.Test/EndToEndTests.cs
index d7cd7cb9..af0b1f81 100644
--- a/Wabbajack.Test/EndToEndTests.cs
+++ b/Wabbajack.Test/EndToEndTests.cs
@@ -122,8 +122,8 @@ namespace Wabbajack.Test
             if (!File.Exists(src))
             {
 
-                var state = await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
-                state.Download(src);
+                var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
+                await state.Download(src);
             }
 
             if (!Directory.Exists(utils.DownloadsFolder))

From f60c1e3c8a64af65401cefc32c3ce5a3fd089038 Mon Sep 17 00:00:00 2001
From: Timothy Baldridge <tbaldridge@gmail.com>
Date: Fri, 13 Dec 2019 15:05:13 -0700
Subject: [PATCH 04/19] disable failing VT upload

---
 azure-pipelines.yml | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 4d136ae2..40573a09 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -57,24 +57,24 @@ steps:
 - task: CmdLine@2
   inputs:
     script: 'pip install requests'
-- task: PythonScript@0
-  condition: eq(variables['Build.SourceBranch'], 'refs/heads/master')
-  inputs:
-    scriptSource: 'inline'
-    script: |
-      import requests, sys
-      
-      url = 'https://www.virustotal.com/vtapi/v2/file/scan'
-      
-      params = {'apikey': sys.argv[1]}
-      
-      files = {'file': ('Wabbajack.exe', open(sys.argv[2], 'rb'))}
-      
-      response = requests.post(url, files=files, params=params)
-      
-      print(response.json())
-    arguments: '$(VirusTotalAPIKey) $(System.DefaultWorkingDirectory)/Wabbajack/bin/x64/Debug/Wabbajack.exe'
-
+#- task: PythonScript@0
+#  condition: eq(variables['Build.SourceBranch'], 'refs/heads/master')
+#  inputs:
+#    scriptSource: 'inline'
+#    script: |
+#      import requests, sys
+#      
+#      url = 'https://www.virustotal.com/vtapi/v2/file/scan'
+#      
+#      params = {'apikey': sys.argv[1]}
+#      
+#      files = {'file': ('Wabbajack.exe', open(sys.argv[2], 'rb'))}
+#      
+#      response = requests.post(url, files=files, params=params)
+#      
+#      print(response.json())
+#    arguments: '$(VirusTotalAPIKey) $(System.DefaultWorkingDirectory)/Wabbajack/bin/x64/Debug/Wabbajack.exe'
+#
 - task: PublishBuildArtifacts@1
   condition: eq(variables['Build.SourceBranch'], 'refs/heads/master')
   inputs:

From 1dc161890f80ebed7d5f439ec5f02ee3c8dd30ba Mon Sep 17 00:00:00 2001
From: halgari <tbaldridge@gmail.com>
Date: Fri, 13 Dec 2019 15:07:58 -0700
Subject: [PATCH 05/19] Run tests in parallel

---
 .runsettings  | 9 +++++++++
 Wabbajack.sln | 1 +
 2 files changed, 10 insertions(+)
 create mode 100644 .runsettings

diff --git a/.runsettings b/.runsettings
new file mode 100644
index 00000000..3a1dfb82
--- /dev/null
+++ b/.runsettings
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <MSTest>
+    <Parallelize>
+      <Workers>0</Workers>
+      <Scope>MethodLevel</Scope>
+    </Parallelize>
+  </MSTest>
+</RunSettings>
\ No newline at end of file
diff --git a/Wabbajack.sln b/Wabbajack.sln
index 419521c1..430adcd9 100644
--- a/Wabbajack.sln
+++ b/Wabbajack.sln
@@ -12,6 +12,7 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{4EDEF6CC-2F5C-439B-BEAF-9D03895099F1}"
 	ProjectSection(SolutionItems) = preProject
 		.editorconfig = .editorconfig
+		.runsettings = .runsettings
 		CHANGELOG.md = CHANGELOG.md
 		LICENSE.txt = LICENSE.txt
 		NexusPage.html = NexusPage.html

From 0464f1b43c8e9c8f5896fb254af27319ff3dd05b Mon Sep 17 00:00:00 2001
From: halgari <tbaldridge@gmail.com>
Date: Fri, 13 Dec 2019 22:46:20 -0700
Subject: [PATCH 06/19] Integrated list validation into the caching HTTP
 server.

---
 .../ListValidationService.cs                  | 193 ++++++++++++++++++
 Wabbajack.CacheServer/NexusCacheModule.cs     |   5 +-
 Wabbajack.CacheServer/Program.cs              |   3 +-
 Wabbajack.CacheServer/Server.cs               |   1 +
 .../Wabbajack.CacheServer.csproj              |   3 +-
 Wabbajack.Common/WorkQueue.cs                 |  18 +-
 Wabbajack.Lib/Downloaders/NexusDownloader.cs  |  38 +++-
 Wabbajack.Lib/NexusApi/NexusApi.cs            |   5 +-
 8 files changed, 244 insertions(+), 22 deletions(-)
 create mode 100644 Wabbajack.CacheServer/ListValidationService.cs

diff --git a/Wabbajack.CacheServer/ListValidationService.cs b/Wabbajack.CacheServer/ListValidationService.cs
new file mode 100644
index 00000000..d87a4c8c
--- /dev/null
+++ b/Wabbajack.CacheServer/ListValidationService.cs
@@ -0,0 +1,193 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using Nancy;
+using Nancy.Responses;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.ModListRegistry;
+
+namespace Wabbajack.CacheServer
+{
+    public class ListValidationService : NancyModule
+    {
+        public class ModListStatus
+        {
+            public string Name;
+            public DateTime Checked = DateTime.Now;
+            public List<(Archive archive, bool)> Archives { get; set; }
+            public DownloadMetadata DownloadMetaData { get; set; }
+            public bool HasFailures { get; set; }
+        }
+
+        public class ModlistSummary
+        {
+            public string Name;
+            public DateTime Checked;
+            public int Failed;
+            public int Passed;
+            public string Link => $"/lists/status/{Name}.json";
+            public string Report => $"/lists/status/{Name}.html";
+        }
+
+        public static Dictionary<string, ModListStatus> ModLists { get; set; }
+
+        public ListValidationService() : base("/lists")
+        {
+            Get("/status", HandleGetLists);
+            Get("/status/{Name}.json", HandleGetListJson);
+            Get("/status/{Name}.html", HandleGetListHtml);
+        }
+
+        private object HandleGetLists(object arg)
+        {
+            var summaries = ModLists.Values.Select(m => new ModlistSummary
+            {
+                Name = m.Name,
+                Checked = m.Checked,
+                Failed = m.Archives.Count(a => a.Item2),
+                Passed = m.Archives.Count(a => !a.Item2),
+            }).ToList();
+            return summaries.ToJSON();
+        }
+
+        public class ArchiveSummary
+        {
+            public string Name;
+            public AbstractDownloadState State;
+        }
+        public class DetailedSummary
+        {
+            public string Name;
+            public DateTime Checked;
+            public List<ArchiveSummary> Failed;
+            public List<ArchiveSummary> Passed;
+
+        }
+        private object HandleGetListJson(dynamic arg)
+        {
+            var lst = ModLists[(string)arg.Name];
+            var summary = new DetailedSummary
+            {
+                Name = lst.Name,
+                Checked = lst.Checked,
+                Failed = lst.Archives.Where(a => a.Item2)
+                    .Select(a => new ArchiveSummary {Name = a.archive.Name, State = a.archive.State}).ToList(),
+                Passed = lst.Archives.Where(a => !a.Item2)
+                    .Select(a => new ArchiveSummary { Name = a.archive.Name, State = a.archive.State }).ToList(),
+            };
+            return summary.ToJSON();
+        }
+
+        private object HandleGetListHtml(dynamic arg)
+        {
+            var lst = ModLists[(string)arg.Name];
+            var sb = new StringBuilder();
+
+            sb.Append("<html><body>");
+            sb.Append($"<h2>{lst.Name} - {lst.Checked}</h2>");
+
+            var failed_list = lst.Archives.Where(a => a.Item2).ToList();
+            sb.Append($"<h3>Failed ({failed_list.Count}):</h3>");
+            sb.Append("<ul>");
+            foreach (var archive in failed_list)
+            {
+                sb.Append($"<li>{archive.archive.Name}</li>");
+            }
+            sb.Append("</ul>");
+
+            var pased_list = lst.Archives.Where(a => !a.Item2).ToList();
+            sb.Append($"<h3>Passed ({pased_list.Count}):</h3>");
+            sb.Append("<ul>");
+            foreach (var archive in pased_list.OrderBy(f => f.archive.Name))
+            {
+                sb.Append($"<li>{archive.archive.Name}</li>");
+            }
+            sb.Append("</ul>");
+
+            sb.Append("</body></html>");
+            var response = (Response)sb.ToString();
+            response.ContentType = "text/html";
+            return response;
+        }
+
+        public static void Start()
+        {
+            var tsk = ValidateLists();
+        }
+        public static async Task ValidateLists()
+        {
+            Utils.Log("Starting Modlist Validation");
+            var modlists = await ModlistMetadata.LoadFromGithub();
+
+            var statuses = new Dictionary<string, ModListStatus>();
+
+            using (var queue = new WorkQueue())
+            {
+                foreach (var list in modlists)
+                {
+                    var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ".wabbajack");
+
+                    if (list.NeedsDownload(modlist_path))
+                    {
+                        var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
+                        Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
+                        await state.Download(modlist_path);
+                    }
+                    else
+                    {
+                        Utils.Log($"No changes detected from downloaded modlist");
+                    }
+
+
+                    Utils.Log($"Loading {modlist_path}");
+
+                    var installer = AInstaller.LoadFromFile(modlist_path);
+
+                    Utils.Log($"{installer.Archives.Count} archives to validate");
+ 
+                    DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
+
+                    var validated = (await installer.Archives
+                            .PMap(queue, async archive =>
+                            {
+                                Utils.Log($"Validating: {archive.Name}");
+                                bool is_failed;
+                                try
+                                {
+                                    is_failed = !(await archive.State.Verify());
+                                }
+                                catch (Exception)
+                                {
+                                    is_failed = false;
+                                }
+
+                                return (archive, is_failed);
+                            }))
+                        .ToList();
+
+
+                    var status = new ModListStatus
+                    {
+                        Name = list.Title, 
+                        Archives = validated.OrderBy(v => v.archive.Name).ToList(),
+                        DownloadMetaData = list.DownloadMetadata,
+                        HasFailures = validated.Any(v => v.is_failed)
+                    };
+
+                    statuses.Add(status.Name, status);
+                 }
+            }
+
+            Utils.Log($"Done validating {statuses.Count} lists");
+            ModLists = statuses;
+        }
+    }
+}
+
diff --git a/Wabbajack.CacheServer/NexusCacheModule.cs b/Wabbajack.CacheServer/NexusCacheModule.cs
index b6b59366..7e001ace 100644
--- a/Wabbajack.CacheServer/NexusCacheModule.cs
+++ b/Wabbajack.CacheServer/NexusCacheModule.cs
@@ -18,9 +18,6 @@ namespace Wabbajack.CacheServer
 
         public NexusCacheModule() : base("/")
         {
-            // ToDo
-            // Handle what to do with the fact that lots of these are now a tasks
-            throw new NotImplementedException("Unsure if following functions still work when taking in a Task");
             Get("/v1/games/{GameName}/mods/{ModID}/files/{FileID}.json", HandleFileID);
             Get("/v1/games/{GameName}/mods/{ModID}/files.json", HandleGetFiles);
             Get("/v1/games/{GameName}/mods/{ModID}.json", HandleModInfo);
@@ -29,7 +26,7 @@ namespace Wabbajack.CacheServer
             Get("/nexus_api_cache/update", UpdateCache);
         }
 
-        private async Task<object> UpdateCache(object arg)
+        public async Task<object> UpdateCache(object arg)
         {
             var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
             await api.ClearUpdatedModsInCache();
diff --git a/Wabbajack.CacheServer/Program.cs b/Wabbajack.CacheServer/Program.cs
index 9a1039b4..5e0b4930 100644
--- a/Wabbajack.CacheServer/Program.cs
+++ b/Wabbajack.CacheServer/Program.cs
@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Nancy.Hosting.Self;
 using Wabbajack.Common;
 
 namespace Wabbajack.CacheServer
@@ -12,9 +13,9 @@ namespace Wabbajack.CacheServer
         static void Main(string[] args)
         {
             Utils.LogMessages.Subscribe(Console.WriteLine);
-
             using (var server = new Server("http://localhost:8080"))
             {
+                ListValidationService.Start();
                 server.Start();
                 Console.ReadLine();
             }
diff --git a/Wabbajack.CacheServer/Server.cs b/Wabbajack.CacheServer/Server.cs
index ea2e2222..038abb87 100644
--- a/Wabbajack.CacheServer/Server.cs
+++ b/Wabbajack.CacheServer/Server.cs
@@ -18,6 +18,7 @@ namespace Wabbajack.CacheServer
         {
             Address = address;
             _config = new HostConfiguration();
+            _config.MaximumConnectionCount = 24;
             //_config.UrlReservations.CreateAutomatically = true;
             _config.RewriteLocalhost = true;
             _server = new NancyHost(_config, new Uri(address));
diff --git a/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj b/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj
index dc19922f..76ab8e3f 100644
--- a/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj
+++ b/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj
@@ -15,7 +15,7 @@
     <TargetFrameworkProfile />
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
-    <PlatformTarget>AnyCPU</PlatformTarget>
+    <PlatformTarget>x64</PlatformTarget>
     <DebugSymbols>true</DebugSymbols>
     <DebugType>full</DebugType>
     <Optimize>false</Optimize>
@@ -73,6 +73,7 @@
     <Reference Include="WindowsBase" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="ListValidationService.cs" />
     <Compile Include="NexusCacheModule.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
diff --git a/Wabbajack.Common/WorkQueue.cs b/Wabbajack.Common/WorkQueue.cs
index a6648ad4..53ee0784 100644
--- a/Wabbajack.Common/WorkQueue.cs
+++ b/Wabbajack.Common/WorkQueue.cs
@@ -70,7 +70,16 @@ namespace Wabbajack.Common
                 {
                     Report("Waiting", 0, false);
                     if (_cancel.IsCancellationRequested) return;
-                    var f = Queue.Take(_cancel.Token);
+                    Func<Task> f;
+                    try
+                    {
+                        f = Queue.Take(_cancel.Token);
+                    }
+                    catch (Exception)
+                    {
+                        throw new OperationCanceledException();
+                    }
+
                     await f();
                 }
             }
@@ -100,13 +109,6 @@ namespace Wabbajack.Common
         public void Dispose()
         {
             _cancel.Cancel();
-            Threads.Do(th =>
-            {
-                if (th.ManagedThreadId != Thread.CurrentThread.ManagedThreadId)
-                {
-                    th.Join();
-                }
-            });
             Queue?.Dispose();
         }
     }
diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs
index fe32ccd9..f94550f0 100644
--- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs
+++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Wabbajack.Common;
 using Wabbajack.Common.StatusFeed.Errors;
@@ -10,6 +11,11 @@ namespace Wabbajack.Lib.Downloaders
 {
     public class NexusDownloader : IDownloader
     {
+        private bool _prepared;
+        private SemaphoreSlim _lock = new SemaphoreSlim(1);
+        private UserStatus _status;
+        private NexusApiClient _client;
+
         public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
         {
             var general = archiveINI?.General;
@@ -44,16 +50,34 @@ namespace Wabbajack.Lib.Downloaders
 
         public async Task Prepare()
         {
-            var client = await NexusApiClient.Get();
-            var status = await client.GetUserStatus();
-            if (!client.IsAuthenticated)
+            if (!_prepared)
             {
-                Utils.ErrorThrow(new UnconvertedError($"Authenticating for the Nexus failed. A nexus account is required to automatically download mods."));
-                return;
+                await _lock.WaitAsync();
+                try
+                {
+                    // Could have become prepared while we waited for the lock
+                    if (!_prepared)
+                    {
+                        _client = await NexusApiClient.Get();
+                        _status = await _client.GetUserStatus();
+                        if (!_client.IsAuthenticated)
+                        {
+                            Utils.ErrorThrow(new UnconvertedError(
+                                $"Authenticating for the Nexus failed. A nexus account is required to automatically download mods."));
+                            return;
+                        }
+                    }
+                }
+                finally
+                {
+                    _lock.Release();
+                }
             }
 
-            if (status.is_premium) return;
-            Utils.ErrorThrow(new UnconvertedError($"Automated installs with Wabbajack requires a premium nexus account. {await client.Username()} is not a premium account."));
+            _prepared = true;
+
+            if (_status.is_premium) return;
+            Utils.ErrorThrow(new UnconvertedError($"Automated installs with Wabbajack requires a premium nexus account. {await _client.Username()} is not a premium account."));
         }
 
         public class State : AbstractDownloadState
diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs
index 1bee610e..958dbe44 100644
--- a/Wabbajack.Lib/NexusApi/NexusApi.cs
+++ b/Wabbajack.Lib/NexusApi/NexusApi.cs
@@ -329,7 +329,10 @@ namespace Wabbajack.Lib.NexusApi
         public async Task<GetModFilesResponse> GetModFiles(Game game, int modid)
         {
             var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modid}/files.json";
-            return await GetCached<GetModFilesResponse>(url);
+            var result = await GetCached<GetModFilesResponse>(url);
+            if (result.files == null)
+                throw new InvalidOperationException("Got Null data from the Nexus while finding mod files");
+            return result;
         }
 
         public async Task<List<MD5Response>> GetModInfoFromMD5(Game game, string md5Hash)

From dcf0113c17c66119d2a41b5dfdc67af77f07a021 Mon Sep 17 00:00:00 2001
From: erri120 <erri120@protonmail.com>
Date: Fri, 13 Dec 2019 14:31:19 +0100
Subject: [PATCH 07/19] Disabled manual game files

---
 Wabbajack.Common/Consts.cs                    |  2 +-
 Wabbajack.Lib/CerasConfig.cs                  |  2 +-
 .../Downloaders/DownloadDispatcher.cs         |  2 +-
 .../Downloaders/GameFileSourceDownloader.cs   | 86 -------------------
 .../Downloaders/SteamWorkshopDownloader.cs    |  2 -
 Wabbajack.Lib/VortexCompiler.cs               | 10 +--
 Wabbajack.Lib/Wabbajack.Lib.csproj            |  1 -
 7 files changed, 8 insertions(+), 97 deletions(-)
 delete mode 100644 Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs

diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs
index c839b5f0..7b943710 100644
--- a/Wabbajack.Common/Consts.cs
+++ b/Wabbajack.Common/Consts.cs
@@ -33,7 +33,7 @@ namespace Wabbajack.Common
         public static string WABBAJACK_INCLUDE = "WABBAJACK_INCLUDE";
         public static string WABBAJACK_ALWAYS_ENABLE = "WABBAJACK_ALWAYS_ENABLE";
         public static string WABBAJACK_NOMATCH_INCLUDE = "WABBAJACK_NOMATCH_INCLUDE";
-        public static string WABBAJACK_VORTEX_MANUAL = "WABBAJACK_VORTEX_MANUAL";
+        //public static string WABBAJACK_VORTEX_MANUAL = "WABBAJACK_VORTEX_MANUAL";
 
         public static string GAME_PATH_MAGIC_BACK = "{--||GAME_PATH_MAGIC_BACK||--}";
         public static string GAME_PATH_MAGIC_DOUBLE_BACK = "{--||GAME_PATH_MAGIC_DOUBLE_BACK||--}";
diff --git a/Wabbajack.Lib/CerasConfig.cs b/Wabbajack.Lib/CerasConfig.cs
index 290bc843..56bd2efc 100644
--- a/Wabbajack.Lib/CerasConfig.cs
+++ b/Wabbajack.Lib/CerasConfig.cs
@@ -24,7 +24,7 @@ namespace Wabbajack.Lib
                 typeof(BSAStateObject), typeof(BSAFileStateObject), typeof(BA2StateObject), typeof(BA2DX10EntryState),
                 typeof(BA2FileEntryState), typeof(MediaFireDownloader.State), typeof(ArchiveMeta),
                 typeof(PropertyFile), typeof(SteamMeta), typeof(SteamWorkshopDownloader), typeof(SteamWorkshopDownloader.State),
-                typeof(LoversLabDownloader.State), typeof(GameFileSourceDownloader.State)
+                typeof(LoversLabDownloader.State), //typeof(GameFileSourceDownloader.State)
 
             }
         };
diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
index f2524f95..b69a55b6 100644
--- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
+++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
@@ -10,7 +10,7 @@ namespace Wabbajack.Lib.Downloaders
     {
         public static readonly List<IDownloader> Downloaders = new List<IDownloader>()
         {
-            new GameFileSourceDownloader(),
+            //new GameFileSourceDownloader(),
             new MegaDownloader(),
             new DropboxDownloader(),
             new GoogleDriveDownloader(),
diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
deleted file mode 100644
index 8f389fbd..00000000
--- a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
+++ /dev/null
@@ -1,86 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using Alphaleonis.Win32.Filesystem;
-using Wabbajack.Common;
-using Wabbajack.Lib.Validation;
-using File = Alphaleonis.Win32.Filesystem.File;
-using Game = Wabbajack.Common.Game;
-
-namespace Wabbajack.Lib.Downloaders
-{
-    public class GameFileSourceDownloader : IDownloader
-    {
-        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
-        {
-            var gameName = (string)archiveINI?.General?.gameName;
-            var gameFile = (string)archiveINI?.General?.gameFile;
-
-            if (gameFile == null || gameFile == null)
-                return null;
-
-            var game = GameRegistry.GetByMO2ArchiveName(gameName);
-            if (game == null) return null;
-
-            var path = game.GameLocation();
-            var filePath = Path.Combine(path, gameFile);
-            
-            if (!File.Exists(filePath))
-                return null;
-
-            var hash = filePath.FileHashCached();
-
-            return new State
-            {
-                Game = GameRegistry.GetByMO2ArchiveName(gameName).Game, 
-                GameFile = gameFile,
-                Hash = hash,
-            };
-        }
-
-        public async Task Prepare()
-        {
-        }
-
-        public class State : AbstractDownloadState
-        {
-            public Game Game { get; set; }
-            public string GameFile { get; set; }
-            public string Hash { get; set; }
-
-            internal string SourcePath => Path.Combine(Game.MetaData().GameLocation(), GameFile);
-
-            public override bool IsWhitelisted(ServerWhitelist whitelist)
-            {
-                return true;
-            }
-
-            public override async Task Download(Archive a, string destination)
-            {
-                using(var src = File.OpenRead(SourcePath))
-                using (var dest = File.OpenWrite(destination))
-                {
-                    var size = new FileInfo(SourcePath).Length;
-                    src.CopyToWithStatus(size, dest, "Copying from Game folder");
-                }
-            }
-
-            public override async Task<bool> Verify()
-            {
-                return File.Exists(SourcePath) && SourcePath.FileHashCached() == Hash;
-            }
-
-            public override IDownloader GetDownloader()
-            {
-                return DownloadDispatcher.GetInstance<GameFileSourceDownloader>();
-            }
-
-            public override string GetReportEntry(Archive a)
-            {
-                return $"* Game File {Game} - {GameFile}";
-            }
-        }
-    }
-}
diff --git a/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs b/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs
index 503e8563..73bea8a6 100644
--- a/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs
+++ b/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs
@@ -1,9 +1,7 @@
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Wabbajack.Common;
diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs
index dde32891..4906465f 100644
--- a/Wabbajack.Lib/VortexCompiler.cs
+++ b/Wabbajack.Lib/VortexCompiler.cs
@@ -162,7 +162,7 @@ namespace Wabbajack.Lib
                 Error($"Found {duplicates.Count} duplicates, exiting");
             }
 
-            for (var i = 0; i < AllFiles.Count; i++)
+            /*for (var i = 0; i < AllFiles.Count; i++)
             {
                 var f = AllFiles[i];
                 if (!f.Path.StartsWith(Consts.GameFolderFilesDir) || !IndexedFiles.ContainsKey(f.Hash))
@@ -202,7 +202,7 @@ namespace Wabbajack.Lib
                 AllFiles.RemoveAt(i);
                 AllFiles.Insert(i, replace);
                 //AllFiles.Replace(f, replace);
-            }
+            }*/
 
             var stack = MakeStack();
 
@@ -384,11 +384,11 @@ namespace Wabbajack.Lib
                                 if (line.Contains("tag="))
                                     tag = line.Substring("tag=".Length);
 
-                                if (tag != Consts.WABBAJACK_VORTEX_MANUAL)
+                                //if (tag != Consts.WABBAJACK_VORTEX_MANUAL)
                                     return;
 
-                                Utils.Log($"File {f} contains the {Consts.WABBAJACK_VORTEX_MANUAL} tag, adding to ActiveArchives");
-                                ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
+                                //Utils.Log($"File {f} contains the {Consts.WABBAJACK_VORTEX_MANUAL} tag, adding to ActiveArchives");
+                                //ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
                             });
                         }
                         else
diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj
index faa70202..6694591b 100644
--- a/Wabbajack.Lib/Wabbajack.Lib.csproj
+++ b/Wabbajack.Lib/Wabbajack.Lib.csproj
@@ -116,7 +116,6 @@
     <Compile Include="CompilationSteps\IStackStep.cs" />
     <Compile Include="CompilationSteps\PatchStockESMs.cs" />
     <Compile Include="CompilationSteps\Serialization.cs" />
-    <Compile Include="Downloaders\GameFileSourceDownloader.cs" />
     <Compile Include="Downloaders\LoversLabDownloader.cs" />
     <Compile Include="Downloaders\SteamWorkshopDownloader.cs" />
     <Compile Include="LibCefHelpers\Init.cs" />

From 5bc6f44aeb34fa09c876f37c8c3b92a6e70c5bd1 Mon Sep 17 00:00:00 2001
From: erri120 <erri120@protonmail.com>
Date: Fri, 13 Dec 2019 15:34:18 +0100
Subject: [PATCH 08/19] Redid the entire Vortex Compiler

---
 Wabbajack.Lib/VortexCompiler.cs | 316 +++++++++++---------------------
 1 file changed, 107 insertions(+), 209 deletions(-)

diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs
index 4906465f..4824f1d2 100644
--- a/Wabbajack.Lib/VortexCompiler.cs
+++ b/Wabbajack.Lib/VortexCompiler.cs
@@ -4,15 +4,18 @@ using System.IO;
 using System.Linq;
 using System.Security.Cryptography;
 using System.Text;
-using System.Threading.Tasks;
 using System.Threading;
-using DynamicData;
+using System.Threading.Tasks;
 using Microsoft.WindowsAPICodePack.Shell;
 using Newtonsoft.Json;
 using Wabbajack.Common;
 using Wabbajack.Lib.CompilationSteps;
 using Wabbajack.Lib.NexusApi;
-using File = Alphaleonis.Win32.Filesystem.File;
+using Wabbajack.Lib.Validation;
+using Directory = Alphaleonis.Win32.Filesystem.Directory;
+using File = System.IO.File;
+using Game = Wabbajack.Common.Game;
+using Path = Alphaleonis.Win32.Filesystem.Path;
 
 namespace Wabbajack.Lib
 {
@@ -29,90 +32,84 @@ namespace Wabbajack.Lib
         public Game Game { get; }
         public string GameName { get; }
 
+        public bool IgnoreMissingFiles { get; set; }
+
         public string VortexFolder { get; set; }
         public string StagingFolder { get; set; }
         public string DownloadsFolder { get; set; }
 
-        public bool IgnoreMissingFiles { get; set; }
-
         public override ModManager ModManager => ModManager.Vortex;
         public override string GamePath { get; }
-        public override string ModListOutputFolder { get; }
+        public override string ModListOutputFolder => "output_folder";
         public override string ModListOutputFile { get; }
 
         public const string StagingMarkerName = "__vortex_staging_folder";
         public const string DownloadMarkerName = "__vortex_downloads_folder";
 
-        private bool _isSteamGame;
-        private SteamGame _steamGame;
-        private bool _hasSteamWorkshopItems;
-
-        public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder, string stagingFolder, string outputFile)
+        public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder,
+            string stagingFolder, string outputFile)
         {
             Game = game;
-
             GamePath = gamePath;
-            GameName = game.MetaData().NexusName;
             VortexFolder = vortexFolder;
             DownloadsFolder = downloadsFolder;
             StagingFolder = stagingFolder;
-            ModListOutputFolder = "output_folder";
             ModListOutputFile = outputFile;
 
-            // there can be max one game after filtering
-            SteamHandler.Instance.Games.Where(g => g.Game != null && g.Game == game).Do(g =>
+            if (string.IsNullOrEmpty(ModListName))
             {
-                _isSteamGame = true;
-                _steamGame = g;
-                SteamHandler.Instance.LoadWorkshopItems(_steamGame);
-                _hasSteamWorkshopItems = _steamGame.WorkshopItems.Count > 0;
-            });
+                ModListName = $"Vortex ModList for {Game.ToString()}";
+                ModListOutputFile = $"{ModListName}{ExtensionManager.Extension}";
+            }
+
+            GameName = Game.MetaData().NexusName;
 
             ActiveArchives = new List<string>();
         }
-        
+
         protected override async Task<bool> _Begin(CancellationToken cancel)
         {
             if (cancel.IsCancellationRequested) return false;
 
-            ConfigureProcessor(10);
-            if (string.IsNullOrEmpty(ModListName))
-                ModListName = $"Vortex ModList for {Game.ToString()}";
+            Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}.");
 
-            Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at  {DownloadsFolder}.");
+            ConfigureProcessor(12);
+            UpdateTracker.Reset();
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Parsing deployment file");
             ParseDeploymentFile();
 
             if (cancel.IsCancellationRequested) return false;
-            Info("Starting pre-compilation steps");
+            UpdateTracker.NextStep("Creating metas for archives");
             await CreateMetaFiles();
 
             if (cancel.IsCancellationRequested) return false;
-            Info($"Indexing {StagingFolder}");
-            await VFS.AddRoot(StagingFolder);
+            await VFS.IntegrateFromFile(_vfsCacheName);
 
-            Info($"Indexing {GamePath}");
-            await VFS.AddRoot(GamePath);
-
-            Info($"Indexing {DownloadsFolder}");
-            await VFS.AddRoot(DownloadsFolder);
+            var roots = new List<string> {StagingFolder, GamePath, DownloadsFolder};
+            AddExternalFolder(ref roots);
 
             if (cancel.IsCancellationRequested) return false;
-            await AddExternalFolder();
+            UpdateTracker.NextStep("Indexing folders");
+            await VFS.AddRoots(roots);
+            await VFS.WriteToFile(_vfsCacheName);
 
             if (cancel.IsCancellationRequested) return false;
-            Info("Cleaning output folder");
-            if (Directory.Exists(ModListOutputFolder)) Utils.DeleteDirectory(ModListOutputFolder);
+            UpdateTracker.NextStep("Cleaning output folder");
+            if (Directory.Exists(ModListOutputFolder))
+                Utils.DeleteDirectory(ModListOutputFolder);
+
             Directory.CreateDirectory(ModListOutputFolder);
-            
+
+            UpdateTracker.NextStep("Finding Install Files");
             var vortexStagingFiles = Directory.EnumerateFiles(StagingFolder, "*", SearchOption.AllDirectories)
                 .Where(p => p.FileExists() && p != StagingMarkerName)
                 .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
                     {Path = p.RelativeTo(StagingFolder)});
             
             var vortexDownloads = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.AllDirectories)
-                .Where(p => p.FileExists())
+                .Where(p => p.FileExists() && p != DownloadMarkerName)
                 .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
                     {Path = p.RelativeTo(DownloadsFolder)});
 
@@ -139,7 +136,6 @@ namespace Wabbajack.Lib
                 .GroupBy(f => f.Hash)
                 .ToDictionary(f => f.Key, f => f.AsEnumerable());
 
-            Info("Searching for mod files");
             AllFiles = vortexStagingFiles.Concat(vortexDownloads)
                 .Concat(gameFiles)
                 .DistinctBy(f => f.Path)
@@ -148,7 +144,8 @@ namespace Wabbajack.Lib
             Info($"Found {AllFiles.Count} files to build into mod list");
 
             if (cancel.IsCancellationRequested) return false;
-            Info("Verifying destinations");
+            UpdateTracker.NextStep("Verifying destinations");
+
             var duplicates = AllFiles.GroupBy(f => f.Path)
                 .Where(fs => fs.Count() > 1)
                 .Select(fs =>
@@ -162,52 +159,10 @@ namespace Wabbajack.Lib
                 Error($"Found {duplicates.Count} duplicates, exiting");
             }
 
-            /*for (var i = 0; i < AllFiles.Count; i++)
-            {
-                var f = AllFiles[i];
-                if (!f.Path.StartsWith(Consts.GameFolderFilesDir) || !IndexedFiles.ContainsKey(f.Hash))
-                    continue;
-
-                if (!IndexedFiles.TryGetValue(f.Hash, out var value))
-                    continue;
-
-                var element = value.ElementAt(0);
-
-                if (!f.Path.Contains(element.Name))
-                    continue;
-
-                IndexedArchive targetArchive = null;
-                IndexedArchives.Where(a => a.File.Children.Contains(element)).Do(a => targetArchive = a);
-
-                if (targetArchive == null)
-                    continue;
-                
-                if(targetArchive.IniData?.General?.tag == null || targetArchive.IniData?.General?.tag != Consts.WABBAJACK_VORTEX_MANUAL)
-                    continue;
-
-                #if DEBUG
-                Utils.Log($"Double hash for: {f.AbsolutePath}");
-                #endif
-
-                var replace = f;
-                var name = replace.File.Name;
-                var archiveName = targetArchive.Name;
-                var elementPath = element.FullPath.Substring(element.FullPath.IndexOf('|')+1);
-                var gameToFile = name.Substring(GamePath.Length + 1).Replace(elementPath, "");
-                if (gameToFile.EndsWith("\\"))
-                    gameToFile = gameToFile.Substring(0, gameToFile.Length - 1);
-                //replace.Path = replace.Path.Replace(Consts.GameFolderFilesDir, Consts.ManualGameFilesDir);
-                replace.Path = Path.Combine(Consts.ManualGameFilesDir, archiveName, gameToFile, elementPath);
-                //replace.Path = Path.Combine(Consts.ManualGameFilesDir, element.FullPath.Substring(DownloadsFolder.Length + 1).Replace('|', '\\'));
-                AllFiles.RemoveAt(i);
-                AllFiles.Insert(i, replace);
-                //AllFiles.Replace(f, replace);
-            }*/
-
+            if (cancel.IsCancellationRequested) return false;
             var stack = MakeStack();
-
-            Info("Running Compilation Stack");
-            var results = await AllFiles.PMap(Queue, f => RunStack(stack.Where(s => s != null), f));
+            UpdateTracker.NextStep("Running Compilation Stack");
+            var results = await AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f));
 
             IEnumerable<NoMatch> noMatch = results.OfType<NoMatch>().ToList();
             Info($"No match for {noMatch.Count()} files");
@@ -228,17 +183,17 @@ namespace Wabbajack.Lib
 
             InstallDirectives = results.Where(i => !(i is IgnoredDirectly)).ToList();
 
-            // TODO: nexus stuff
-            /*Info("Getting Nexus api_key, please click authorize if a browser window appears");
+            Info("Getting Nexus api_key, please click authorize if a browser window appears");
+
             if (IndexedArchives.Any(a => a.IniData?.General?.gameName != null))
             {
-                var nexusClient = new NexusApiClient();
-                if (!nexusClient.IsPremium) Error($"User {nexusClient.Username} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue");
+                var nexusClient = await NexusApiClient.Get();
+                if (!await nexusClient.IsPremium()) Error($"User {await nexusClient.Username()} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue");
 
             }
-            */
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Gathering Archives");
             await GatherArchives();
 
             ModList = new ModList
@@ -254,16 +209,46 @@ namespace Wabbajack.Lib
                 Directives = InstallDirectives,
                 GameType = Game
             };
-            
+
+            UpdateTracker.NextStep("Running Validation");
+            await ValidateModlist.RunValidation(Queue, ModList);
+
+            UpdateTracker.NextStep("Generating Report");
             GenerateReport();
+
+            UpdateTracker.NextStep("Exporting ModList");
             ExportModList();
 
-            Info("Done Building ModList");
+            ResetMembers();
 
             ShowReport();
+
+            UpdateTracker.NextStep("Done Building ModList");
+
             return true;
         }
 
+        /// <summary>
+        ///     Clear references to lists that hold a lot of data.
+        /// </summary>
+        private void ResetMembers()
+        {
+            AllFiles = null;
+            InstallDirectives = null;
+            SelectedArchives = null;
+        }
+
+        private void AddExternalFolder(ref List<string> roots)
+        {
+            var currentGame = Game.MetaData();
+            if (currentGame.AdditionalFolders == null || currentGame.AdditionalFolders.Count == 0) return;
+            foreach (var path in currentGame.AdditionalFolders.Select(f => f.Replace("%documents%", KnownFolders.Documents.Path)))
+            {
+                if (!Directory.Exists(path)) return;
+                roots.Add(path);
+            }
+        }
+
         private void ParseDeploymentFile()
         {
             Info("Searching for vortex.deployment.json...");
@@ -280,10 +265,10 @@ namespace Wabbajack.Lib
 
             if (string.IsNullOrEmpty(deploymentFile))
             {
-                Info("vortex.deployment.json not found!");
+                Error("vortex.deployment.json not found!");
                 return;
             }
-            Info("vortex.deployment.json found at "+deploymentFile);
+            Info($"vortex.deployment.json found at {deploymentFile}");
 
             Info("Parsing vortex.deployment.json...");
             try
@@ -306,127 +291,42 @@ namespace Wabbajack.Lib
             });
         }
 
-        /// <summary>
-        /// Some have mods outside their game folder located
-        /// </summary>
-        private async Task AddExternalFolder()
-        {
-            var currentGame = Game.MetaData();
-            if (currentGame.AdditionalFolders == null || currentGame.AdditionalFolders.Count == 0) return;
-            foreach (var f in currentGame.AdditionalFolders)
-            {
-                var path = f.Replace("%documents%", KnownFolders.Documents.Path);
-                if (!Directory.Exists(path)) return;
-                Info($"Indexing {path}");
-                await VFS.AddRoot(path);
-            }
-        }
-
         private async Task CreateMetaFiles()
         {
             Utils.Log("Getting Nexus api_key, please click authorize if a browser window appears");
             var nexusClient = await NexusApiClient.Get();
 
-            await Task.WhenAll(
-                Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly)
-                .Where(File.Exists)
-                .Select(async f =>
-                {
-                    if (Path.GetExtension(f) != ".meta" && Path.GetExtension(f) != ".xxHash" && !File.Exists($"{f}.meta") && ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)))
-                    {
-                        Utils.Log($"Trying to create meta file for {Path.GetFileName(f)}");
-                        var metaString = "[General]\n" +
-                                         "repository=Nexus\n" +
-                                         "installed=true\n" +
-                                         "uninstalled=false\n" +
-                                         "paused=false\n" +
-                                         "removed=false\n" +
-                                         $"gameName={GameName}\n";
-                        string hash;
-                        using(var md5 = MD5.Create())
-                        using (var stream = File.OpenRead(f))
-                        {
-                            Utils.Log($"Calculating hash for {Path.GetFileName(f)}");
-                            var cH = md5.ComputeHash(stream);
-                            hash = BitConverter.ToString(cH).Replace("-", "").ToLowerInvariant();
-                            Utils.Log($"Hash is {hash}");
-                        }
+            var archives = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly).Where(f =>
+                File.Exists(f) && Path.GetExtension(f) != ".meta" && Path.GetExtension(f) != ".xxHash" &&
+                !File.Exists($"{f}.meta") && ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)));
 
-                        var md5Response = await nexusClient.GetModInfoFromMD5(Game, hash);
-                        if (md5Response.Count >= 1)
-                        {
-                            var modInfo = md5Response[0].mod;
-                            metaString += $"modID={modInfo.mod_id}\n" +
-                                          $"modName={modInfo.name}\nfileID={md5Response[0].file_details.file_id}";
-                            File.WriteAllText(f+".meta",metaString, Encoding.UTF8);
-                        }
-                        else
-                        {
-                            Error("Error while getting information from NexusMods via MD5 hash!");
-                        }
-                    }
-                    else
-                    {
-                        if (Path.GetExtension(f) != ".meta" ||
-                            ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)))
-                            return;
-
-                        Utils.Log($"File {f} is not in ActiveArchives");
-                        var lines = File.ReadAllLines(f);
-                        if (lines.Length == 0 || !lines.Any(line => line.Contains("directURL=")))
-                        {
-                            if (lines.Length == 0)
-                                return;
-
-                            lines.Do(line =>
-                            {
-                                var tag = "";
-                                if (line.Contains("tag="))
-                                    tag = line.Substring("tag=".Length);
-
-                                //if (tag != Consts.WABBAJACK_VORTEX_MANUAL)
-                                    return;
-
-                                //Utils.Log($"File {f} contains the {Consts.WABBAJACK_VORTEX_MANUAL} tag, adding to ActiveArchives");
-                                //ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
-                            });
-                        }
-                        else
-                        {
-                            Utils.Log($"File {f} appears to not come from the Nexus, adding to ActiveArchives");
-                            ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
-                        }
-                    }
-                }));
-
-            Utils.Log($"Checking for Steam Workshop Items...");
-            if (!_isSteamGame || _steamGame == null || _steamGame.WorkshopItems.Count <= 0)
-                return;
-
-            _steamGame.WorkshopItems.Do(item =>
+            await archives.PMap(Queue, async f =>
             {
-                var filePath = Path.Combine(DownloadsFolder, $"steamWorkshopItem_{item.ItemID}.meta");
-                if (File.Exists(filePath))
+                Info($"Creating meta file for {Path.GetFileName(f)}");
+                var metaString = "[General]\n" +
+                                 "repository=Nexus\n" +
+                                 $"gameName={GameName}\n";
+                string hash;
+                using(var md5 = MD5.Create())
+                using (var stream = File.OpenRead(f))
                 {
-                    Utils.Log($"File {filePath} already exists, skipping...");
-                    return;
+                    Info($"Calculating hash for {Path.GetFileName(f)}");
+                    var cH = md5.ComputeHash(stream);
+                    hash = BitConverter.ToString(cH).Replace("-", "").ToLowerInvariant();
+                    Info($"Hash is {hash}");
                 }
 
-                Utils.Log($"Creating meta file for {item.ItemID}");
-                var metaString = "[General]\n" +
-                                 "repository=Steam\n" +
-                                 "installed=true\n" +
-                                 $"gameName={GameName}\n" +
-                                 $"steamID={_steamGame.AppId}\n" +
-                                 $"itemID={item.ItemID}\n" +
-                                 $"itemSize={item.Size}\n";
-                try
+                var md5Response = await nexusClient.GetModInfoFromMD5(Game, hash);
+                if (md5Response.Count >= 1)
                 {
-                    File.WriteAllText(filePath, metaString);
+                    var modInfo = md5Response[0].mod;
+                    metaString += $"modID={modInfo.mod_id}\n" +
+                                  $"modName={modInfo.name}\nfileID={md5Response[0].file_details.file_id}";
+                    File.WriteAllText(f+".meta",metaString, Encoding.UTF8);
                 }
-                catch (Exception e)
+                else
                 {
-                    Utils.Error(e, $"Exception while writing to disk at {filePath}");
+                    Error("Error while getting information from NexusMods via MD5 hash!");
                 }
             });
         }
@@ -440,20 +340,18 @@ namespace Wabbajack.Lib
 
             var stack = MakeStack();
 
-            File.WriteAllText(Path.Combine(s, "_current_compilation_stack.yml"),
-                Serialization.Serialize(stack));
+            var compilationSteps = stack.ToList();
+            File.WriteAllText(Path.Combine(s, "_current_compilation_stack.yml"), Serialization.Serialize(compilationSteps));
 
-            return stack;
+            return compilationSteps;
         }
 
         public override IEnumerable<ICompilationStep> MakeStack()
         {
-            Utils.Log("Generating compilation stack");
+            Info("Generating compilation stack");
             return new List<ICompilationStep>
             {
                 new IncludePropertyFiles(this),
-                new IncludeSteamWorkshopItems(this, _steamGame),
-                _hasSteamWorkshopItems ? new IncludeRegex(this, "^steamWorkshopItem_\\d*\\.meta$") : null,
                 new IgnoreDisabledVortexMods(this),
                 new IncludeVortexDeployment(this),
                 new IgnoreVortex(this),

From 758df21d913c0603c8cd07c1dcc44bf8bada3bf9 Mon Sep 17 00:00:00 2001
From: erri120 <erri120@protonmail.com>
Date: Fri, 13 Dec 2019 15:39:46 +0100
Subject: [PATCH 09/19] Added version info to created meta file

---
 Wabbajack.Lib/VortexCompiler.cs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs
index 4824f1d2..78b7c289 100644
--- a/Wabbajack.Lib/VortexCompiler.cs
+++ b/Wabbajack.Lib/VortexCompiler.cs
@@ -321,7 +321,9 @@ namespace Wabbajack.Lib
                 {
                     var modInfo = md5Response[0].mod;
                     metaString += $"modID={modInfo.mod_id}\n" +
-                                  $"modName={modInfo.name}\nfileID={md5Response[0].file_details.file_id}";
+                                  $"modName={modInfo.name}\n" +
+                                  $"fileID={md5Response[0].file_details.file_id}\n" +
+                                  $"version={md5Response[0].file_details.version}\n";
                     File.WriteAllText(f+".meta",metaString, Encoding.UTF8);
                 }
                 else

From f6d51b4e3351310d1ee513a8dfa33e806df439ec Mon Sep 17 00:00:00 2001
From: erri120 <erri120@protonmail.com>
Date: Fri, 13 Dec 2019 16:01:26 +0100
Subject: [PATCH 10/19] Re-added SteamWorkshop items, changed ReportBuilder to
 include those items

---
 Wabbajack.Lib/ReportBuilder.cs  |  9 +++++--
 Wabbajack.Lib/VortexCompiler.cs | 48 ++++++++++++++++++++++++++++++++-
 2 files changed, 54 insertions(+), 3 deletions(-)

diff --git a/Wabbajack.Lib/ReportBuilder.cs b/Wabbajack.Lib/ReportBuilder.cs
index a721527c..b39a465e 100644
--- a/Wabbajack.Lib/ReportBuilder.cs
+++ b/Wabbajack.Lib/ReportBuilder.cs
@@ -64,8 +64,12 @@ namespace Wabbajack.Lib
                         .Do(NoWrapText);
             }
 
+            var archiveCount = lst.Archives.Count + lst.Directives.Count(d => d is SteamMeta);
+            var totalSize = lst.Archives.Sum(a => a.Size);
+            totalSize += lst.Directives.Where(d => d is SteamMeta).Cast<SteamMeta>().Sum(s => s.Size);
+
             Text(
-                $"#### Download Summary ({lst.Archives.Count} archives - {lst.Archives.Sum(a => a.Size).ToFileSizeString()})");
+                $"#### Download Summary ({archiveCount} archives - {totalSize.ToFileSizeString()})");
             foreach (var archive in SortArchives(lst.Archives))
             {
                 var hash = archive.Hash.FromBase64().ToHex();
@@ -78,7 +82,8 @@ namespace Wabbajack.Lib
                 if (f is SteamMeta s)
                 {
                     var link = $"https://steamcommunity.com/sharedfiles/filedetails/?id={s.ItemID}";
-                    NoWrapText($"* Steam Workshop Item: [{s.ItemID}]({link}) | Size: {s.Size}");
+                    var size = ((long)s.Size).ToFileSizeString();
+                    NoWrapText($"* Steam Workshop Item: [{s.ItemID}]({link}) | Size: {size}");
                 }
             });
 
diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs
index 78b7c289..c331f271 100644
--- a/Wabbajack.Lib/VortexCompiler.cs
+++ b/Wabbajack.Lib/VortexCompiler.cs
@@ -46,6 +46,10 @@ namespace Wabbajack.Lib
         public const string StagingMarkerName = "__vortex_staging_folder";
         public const string DownloadMarkerName = "__vortex_downloads_folder";
 
+        private bool _isSteamGame;
+        private SteamGame _steamGame;
+        private bool _hasSteamWorkshopItems;
+
         public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder,
             string stagingFolder, string outputFile)
         {
@@ -65,6 +69,14 @@ namespace Wabbajack.Lib
             GameName = Game.MetaData().NexusName;
 
             ActiveArchives = new List<string>();
+
+            SteamHandler.Instance.Games.Where(g => g.Game != null && g.Game == game).Do(g =>
+            {
+                _isSteamGame = true;
+                _steamGame = g;
+                SteamHandler.Instance.LoadWorkshopItems(_steamGame);
+                _hasSteamWorkshopItems = _steamGame.WorkshopItems.Count > 0;
+            });
         }
 
         protected override async Task<bool> _Begin(CancellationToken cancel)
@@ -162,7 +174,7 @@ namespace Wabbajack.Lib
             if (cancel.IsCancellationRequested) return false;
             var stack = MakeStack();
             UpdateTracker.NextStep("Running Compilation Stack");
-            var results = await AllFiles.PMap(Queue, UpdateTracker, f => RunStack(stack, f));
+            var results = await AllFiles.PMap(Queue, f => RunStack(stack.Where(s => s != null), f));
 
             IEnumerable<NoMatch> noMatch = results.OfType<NoMatch>().ToList();
             Info($"No match for {noMatch.Count()} files");
@@ -331,6 +343,36 @@ namespace Wabbajack.Lib
                     Error("Error while getting information from NexusMods via MD5 hash!");
                 }
             });
+
+            Info($"Checking for Steam Workshop Items...");
+            if (!_isSteamGame || _steamGame == null || !_hasSteamWorkshopItems)
+                return;
+
+            _steamGame.WorkshopItems.Do(item =>
+            {
+                var filePath = Path.Combine(DownloadsFolder, $"steamWorkshopItem_{item.ItemID}.meta");
+                if (File.Exists(filePath))
+                {
+                    Utils.Log($"File {filePath} already exists, skipping...");
+                    return;
+                }
+
+                Utils.Log($"Creating meta file for {item.ItemID}");
+                var metaString = "[General]\n" +
+                                 "repository=Steam\n" +
+                                 $"gameName={GameName}\n" +
+                                 $"steamID={_steamGame.AppId}\n" +
+                                 $"itemID={item.ItemID}\n" +
+                                 $"itemSize={item.Size}\n";
+                try
+                {
+                    File.WriteAllText(filePath, metaString);
+                }
+                catch (Exception e)
+                {
+                    Utils.Error(e, $"Exception while writing to disk at {filePath}");
+                }
+            });
         }
 
         public override IEnumerable<ICompilationStep> GetStack()
@@ -354,6 +396,10 @@ namespace Wabbajack.Lib
             return new List<ICompilationStep>
             {
                 new IncludePropertyFiles(this),
+
+                new IncludeSteamWorkshopItems(this, _steamGame),
+                _hasSteamWorkshopItems ? new IncludeRegex(this, "^steamWorkshopItem_\\d*\\.meta$") : null,
+
                 new IgnoreDisabledVortexMods(this),
                 new IncludeVortexDeployment(this),
                 new IgnoreVortex(this),

From 4c637fe7b170dfa646c12aac35996f65b69dc917 Mon Sep 17 00:00:00 2001
From: erri120 <erri120@protonmail.com>
Date: Fri, 13 Dec 2019 16:10:24 +0100
Subject: [PATCH 11/19] Updated the VortexInstaller

---
 Wabbajack.Lib/VortexInstaller.cs | 42 ++++++++++++++++++++++++++------
 1 file changed, 34 insertions(+), 8 deletions(-)

diff --git a/Wabbajack.Lib/VortexInstaller.cs b/Wabbajack.Lib/VortexInstaller.cs
index 0a17c2a2..aa2465cf 100644
--- a/Wabbajack.Lib/VortexInstaller.cs
+++ b/Wabbajack.Lib/VortexInstaller.cs
@@ -19,12 +19,14 @@ namespace Wabbajack.Lib
 
         public override ModManager ModManager => ModManager.Vortex;
 
+        public string GameFolder { get; set; }
+
         public VortexInstaller(string archive, ModList modList, string outputFolder, string downloadFolder)
             : base(
-                  archive: archive,
-                  modList: modList,
-                  outputFolder: outputFolder,
-                  downloadFolder: downloadFolder)
+                archive,
+                modList,
+                outputFolder,
+                downloadFolder)
         {
             #if DEBUG
             // TODO: only for testing
@@ -43,17 +45,33 @@ namespace Wabbajack.Lib
                 "for support.", "Warning",
                 MessageBoxButton.OK);
 
+            if (GameFolder == null)
+                GameFolder = GameInfo.GameLocation();
+
+            if (GameFolder == null)
+            {
+                MessageBox.Show(
+                    $"In order to do a proper install Wabbajack needs to know where your {GameInfo.NexusName} folder resides. We tried looking the" +
+                    "game location up but were unable to find it, please make sure you launch the game once before running this installer. ",
+                    "Could not find game location", MessageBoxButton.OK);
+                Error("Exiting because we couldn't find the game folder.");
+                return false;
+            }
+
             if (cancel.IsCancellationRequested) return false;
-            ConfigureProcessor(10, await RecommendQueueSize());
+            ConfigureProcessor(9, await RecommendQueueSize());
             Directory.CreateDirectory(DownloadFolder);
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Hashing Archives");
             await HashArchives();
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Downloading Missing Archives");
             await DownloadArchives();
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Hashing Remaining Archives");
             await HashArchives();
 
             if (cancel.IsCancellationRequested) return false;
@@ -68,25 +86,33 @@ namespace Wabbajack.Lib
                     Error("Cannot continue, was unable to download one or more archives");
             }
 
+            if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Priming VFS");
             await PrimeVFS();
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Building Folder Structure");
             BuildFolderStructure();
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Installing Archives");
             await InstallArchives();
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Installing Included files");
             await InstallIncludedFiles();
 
-            if (cancel.IsCancellationRequested) return false;
-            await InstallManualGameFiles();
+            /*if (cancel.IsCancellationRequested) return false;
+            await InstallManualGameFiles();*/
 
             if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Installing SteamWorkshopItems");
             await InstallSteamWorkshopItems();
+            
+            
             //InstallIncludedDownloadMetas();
 
-            Info("Installation complete! You may exit the program.");
+            UpdateTracker.NextStep("Installation complete! You may exit the program.");
             return true;
         }
 

From 04584720a7ca5bf8f21c84a6448b5ec5a4b9d2d8 Mon Sep 17 00:00:00 2001
From: erri120 <erri120@protonmail.com>
Date: Sat, 14 Dec 2019 12:10:22 +0100
Subject: [PATCH 12/19] Re-added manual files

---
 Wabbajack.Common/Consts.cs                    |  2 +-
 Wabbajack.Lib/CerasConfig.cs                  |  2 +-
 .../Downloaders/GameFileSourceDownloader.cs   | 82 +++++++++++++++++++
 Wabbajack.Lib/VortexCompiler.cs               | 74 +++++++++++++++++
 Wabbajack.Lib/Wabbajack.Lib.csproj            |  1 +
 5 files changed, 159 insertions(+), 2 deletions(-)
 create mode 100644 Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs

diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs
index 7b943710..c839b5f0 100644
--- a/Wabbajack.Common/Consts.cs
+++ b/Wabbajack.Common/Consts.cs
@@ -33,7 +33,7 @@ namespace Wabbajack.Common
         public static string WABBAJACK_INCLUDE = "WABBAJACK_INCLUDE";
         public static string WABBAJACK_ALWAYS_ENABLE = "WABBAJACK_ALWAYS_ENABLE";
         public static string WABBAJACK_NOMATCH_INCLUDE = "WABBAJACK_NOMATCH_INCLUDE";
-        //public static string WABBAJACK_VORTEX_MANUAL = "WABBAJACK_VORTEX_MANUAL";
+        public static string WABBAJACK_VORTEX_MANUAL = "WABBAJACK_VORTEX_MANUAL";
 
         public static string GAME_PATH_MAGIC_BACK = "{--||GAME_PATH_MAGIC_BACK||--}";
         public static string GAME_PATH_MAGIC_DOUBLE_BACK = "{--||GAME_PATH_MAGIC_DOUBLE_BACK||--}";
diff --git a/Wabbajack.Lib/CerasConfig.cs b/Wabbajack.Lib/CerasConfig.cs
index 56bd2efc..290bc843 100644
--- a/Wabbajack.Lib/CerasConfig.cs
+++ b/Wabbajack.Lib/CerasConfig.cs
@@ -24,7 +24,7 @@ namespace Wabbajack.Lib
                 typeof(BSAStateObject), typeof(BSAFileStateObject), typeof(BA2StateObject), typeof(BA2DX10EntryState),
                 typeof(BA2FileEntryState), typeof(MediaFireDownloader.State), typeof(ArchiveMeta),
                 typeof(PropertyFile), typeof(SteamMeta), typeof(SteamWorkshopDownloader), typeof(SteamWorkshopDownloader.State),
-                typeof(LoversLabDownloader.State), //typeof(GameFileSourceDownloader.State)
+                typeof(LoversLabDownloader.State), typeof(GameFileSourceDownloader.State)
 
             }
         };
diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
new file mode 100644
index 00000000..f7b02721
--- /dev/null
+++ b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
@@ -0,0 +1,82 @@
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using Wabbajack.Common;
+using Wabbajack.Lib.Validation;
+using File = Alphaleonis.Win32.Filesystem.File;
+using Game = Wabbajack.Common.Game;
+
+namespace Wabbajack.Lib.Downloaders
+{
+    public class GameFileSourceDownloader : IDownloader
+    {
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
+        {
+            var gameName = (string)archiveINI?.General?.gameName;
+            var gameFile = (string)archiveINI?.General?.gameFile;
+
+            if (gameFile == null || gameFile == null)
+                return null;
+
+            var game = GameRegistry.GetByMO2ArchiveName(gameName);
+            if (game == null) return null;
+
+            var path = game.GameLocation();
+            var filePath = Path.Combine(path, gameFile);
+
+            if (!File.Exists(filePath))
+                return null;
+
+            var hash = filePath.FileHashCached();
+
+            return new State
+            {
+                Game = GameRegistry.GetByMO2ArchiveName(gameName).Game, 
+                GameFile = gameFile,
+                Hash = hash,
+            };
+        }
+
+        public async Task Prepare()
+        {
+        }
+
+        public class State : AbstractDownloadState
+        {
+            public Game Game { get; set; }
+            public string GameFile { get; set; }
+            public string Hash { get; set; }
+
+            internal string SourcePath => Path.Combine(Game.MetaData().GameLocation(), GameFile);
+
+            public override bool IsWhitelisted(ServerWhitelist whitelist)
+            {
+                return true;
+            }
+
+            public override async Task Download(Archive a, string destination)
+            {
+                using(var src = File.OpenRead(SourcePath))
+                using (var dest = File.OpenWrite(destination))
+                {
+                    var size = new FileInfo(SourcePath).Length;
+                    src.CopyToWithStatus(size, dest, "Copying from Game folder");
+                }
+            }
+
+            public override async Task<bool> Verify()
+            {
+                return File.Exists(SourcePath) && SourcePath.FileHashCached() == Hash;
+            }
+
+            public override IDownloader GetDownloader()
+            {
+                return DownloadDispatcher.GetInstance<GameFileSourceDownloader>();
+            }
+
+            public override string GetReportEntry(Archive a)
+            {
+                return $"* Game File {Game} - {GameFile}";
+            }
+        }
+    }
+}
diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs
index c331f271..992ef0e0 100644
--- a/Wabbajack.Lib/VortexCompiler.cs
+++ b/Wabbajack.Lib/VortexCompiler.cs
@@ -171,6 +171,48 @@ namespace Wabbajack.Lib
                 Error($"Found {duplicates.Count} duplicates, exiting");
             }
 
+            for (var i = 0; i < AllFiles.Count; i++)
+            {
+                var f = AllFiles[i];
+                if (!f.Path.StartsWith(Consts.GameFolderFilesDir) || !IndexedFiles.ContainsKey(f.Hash))
+                    continue;
+
+                if (!IndexedFiles.TryGetValue(f.Hash, out var value))
+                    continue;
+
+                var element = value.ElementAt(0);
+
+                if (!f.Path.Contains(element.Name))
+                    continue;
+
+                IndexedArchive targetArchive = null;
+                IndexedArchives.Where(a => a.File.Children.Contains(element)).Do(a => targetArchive = a);
+
+                if (targetArchive == null)
+                    continue;
+                
+                if(targetArchive.IniData?.General?.tag == null || targetArchive.IniData?.General?.tag != Consts.WABBAJACK_VORTEX_MANUAL)
+                    continue;
+
+#if DEBUG
+                Utils.Log($"Double hash for: {f.AbsolutePath}");
+#endif
+
+                var replace = f;
+                var name = replace.File.Name;
+                var archiveName = targetArchive.Name;
+                var elementPath = element.FullPath.Substring(element.FullPath.IndexOf('|')+1);
+                var gameToFile = name.Substring(GamePath.Length + 1).Replace(elementPath, "");
+                if (gameToFile.EndsWith("\\"))
+                    gameToFile = gameToFile.Substring(0, gameToFile.Length - 1);
+                //replace.Path = replace.Path.Replace(Consts.GameFolderFilesDir, Consts.ManualGameFilesDir);
+                replace.Path = Path.Combine(Consts.ManualGameFilesDir, archiveName, gameToFile, elementPath);
+                //replace.Path = Path.Combine(Consts.ManualGameFilesDir, element.FullPath.Substring(DownloadsFolder.Length + 1).Replace('|', '\\'));
+                AllFiles.RemoveAt(i);
+                AllFiles.Insert(i, replace);
+                //AllFiles.Replace(f, replace);
+            }
+
             if (cancel.IsCancellationRequested) return false;
             var stack = MakeStack();
             UpdateTracker.NextStep("Running Compilation Stack");
@@ -344,6 +386,38 @@ namespace Wabbajack.Lib
                 }
             });
 
+            var otherFiles = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly).Where(f =>
+                Path.GetExtension(f) == ".meta" && !ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)));
+
+            await otherFiles.PMap(Queue, async f =>
+            {
+                Info($"File {f} is not in ActiveArchives");
+                var lines = File.ReadAllLines(f);
+                if (lines.Length == 0 || !lines.Any(line => lines.Contains("directURL=")))
+                {
+                    if (lines.Length == 0)
+                        return;
+
+                    lines.Do(line =>
+                    {
+                        var tag = "";
+                        if (line.Contains("tag="))
+                            tag = line.Substring("tag=".Length);
+
+                        if (tag != Consts.WABBAJACK_VORTEX_MANUAL)
+                            return;
+
+                        Info($"File {f} contains the {Consts.WABBAJACK_VORTEX_MANUAL} tag, adding to ActiveArchives");
+                        ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
+                    });
+                }
+                else
+                {
+                    Info($"File {f} appears to not be from the Nexus, adding to ActiveArchives");
+                    ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
+                }
+            });
+
             Info($"Checking for Steam Workshop Items...");
             if (!_isSteamGame || _steamGame == null || !_hasSteamWorkshopItems)
                 return;
diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj
index 6694591b..faa70202 100644
--- a/Wabbajack.Lib/Wabbajack.Lib.csproj
+++ b/Wabbajack.Lib/Wabbajack.Lib.csproj
@@ -116,6 +116,7 @@
     <Compile Include="CompilationSteps\IStackStep.cs" />
     <Compile Include="CompilationSteps\PatchStockESMs.cs" />
     <Compile Include="CompilationSteps\Serialization.cs" />
+    <Compile Include="Downloaders\GameFileSourceDownloader.cs" />
     <Compile Include="Downloaders\LoversLabDownloader.cs" />
     <Compile Include="Downloaders\SteamWorkshopDownloader.cs" />
     <Compile Include="LibCefHelpers\Init.cs" />

From 5ad86f1bddbdac55da19f249203e4a97fc41ecdb Mon Sep 17 00:00:00 2001
From: erri120 <erri120@protonmail.com>
Date: Sat, 14 Dec 2019 15:08:22 +0000
Subject: [PATCH 13/19] Fixed manual files not being installed

---
 Wabbajack.Lib/VortexInstaller.cs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Wabbajack.Lib/VortexInstaller.cs b/Wabbajack.Lib/VortexInstaller.cs
index aa2465cf..863bf882 100644
--- a/Wabbajack.Lib/VortexInstaller.cs
+++ b/Wabbajack.Lib/VortexInstaller.cs
@@ -59,7 +59,7 @@ namespace Wabbajack.Lib
             }
 
             if (cancel.IsCancellationRequested) return false;
-            ConfigureProcessor(9, await RecommendQueueSize());
+            ConfigureProcessor(10, await RecommendQueueSize());
             Directory.CreateDirectory(DownloadFolder);
 
             if (cancel.IsCancellationRequested) return false;
@@ -102,8 +102,9 @@ namespace Wabbajack.Lib
             UpdateTracker.NextStep("Installing Included files");
             await InstallIncludedFiles();
 
-            /*if (cancel.IsCancellationRequested) return false;
-            await InstallManualGameFiles();*/
+            if (cancel.IsCancellationRequested) return false;
+            UpdateTracker.NextStep("Installing Manual files");
+            await InstallManualGameFiles();
 
             if (cancel.IsCancellationRequested) return false;
             UpdateTracker.NextStep("Installing SteamWorkshopItems");

From 39e5aca68edab423feedef272f569911be346e62 Mon Sep 17 00:00:00 2001
From: halgari <tbaldridge@gmail.com>
Date: Sat, 14 Dec 2019 09:33:05 -0700
Subject: [PATCH 14/19] Add cache clearing and check links every two hours

---
 .../ListValidationService.cs                  | 24 ++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/Wabbajack.CacheServer/ListValidationService.cs b/Wabbajack.CacheServer/ListValidationService.cs
index d87a4c8c..f6c16cc9 100644
--- a/Wabbajack.CacheServer/ListValidationService.cs
+++ b/Wabbajack.CacheServer/ListValidationService.cs
@@ -3,7 +3,9 @@ using System.Collections;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Linq;
+using System.Net.Http;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Nancy;
@@ -119,10 +121,30 @@ namespace Wabbajack.CacheServer
 
         public static void Start()
         {
-            var tsk = ValidateLists();
+            new Thread(() =>
+            {
+                while (true)
+                {
+                    try
+                    {
+                        ValidateLists().Wait();
+                    }
+                    catch (Exception ex)
+                    {
+                        Utils.Log(ex.ToString());
+                    }
+
+                    // Sleep for two hours
+                    Thread.Sleep(1000 * 60 * 60 * 2);
+                }
+            }).Start();
         }
         public static async Task ValidateLists()
         {
+            Utils.Log("Cleaning Nexus Cache");
+            var client = new HttpClient();
+            await client.GetAsync("http://build.wabbajack.org/nexus_api_cache/update");
+
             Utils.Log("Starting Modlist Validation");
             var modlists = await ModlistMetadata.LoadFromGithub();
 

From 7fa7f42fc36f6ced7e90bc680a2d208e6b02570c Mon Sep 17 00:00:00 2001
From: halgari <tbaldridge@gmail.com>
Date: Sat, 14 Dec 2019 10:10:06 -0700
Subject: [PATCH 15/19] bump version number

---
 CHANGELOG.md                         | 5 +++++
 Wabbajack/Properties/AssemblyInfo.cs | 4 ++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a726f29f..4caa0f4c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
 ### Changelog
 
+#### Version - 1.0 beta 5 - 12/14/2019
+* Added LoversLab download support
+* Nexus and LL logins now happen via a in-ap browser
+* Several UI enhancements
+
 #### Version - 1.0 beta 4 - 12/3/2019
 * Several crash and bug fixes
 
diff --git a/Wabbajack/Properties/AssemblyInfo.cs b/Wabbajack/Properties/AssemblyInfo.cs
index 8228e28f..7a8ab008 100644
--- a/Wabbajack/Properties/AssemblyInfo.cs
+++ b/Wabbajack/Properties/AssemblyInfo.cs
@@ -49,5 +49,5 @@ using System.Windows;
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.0.9.0")]
-[assembly: AssemblyFileVersion("1.0.9.0")]
+[assembly: AssemblyVersion("1.0.10.0")]
+[assembly: AssemblyFileVersion("1.0.10.0")]

From 1058a5a3c40b38ccaad42c89deb3ac5a406789fc Mon Sep 17 00:00:00 2001
From: Timothy Baldridge <tbaldridge@gmail.com>
Date: Sat, 14 Dec 2019 10:30:52 -0700
Subject: [PATCH 16/19] Revert "Vortex Redone"

---
 .../Downloaders/DownloadDispatcher.cs         |   2 +-
 .../Downloaders/GameFileSourceDownloader.cs   |   8 +-
 .../Downloaders/SteamWorkshopDownloader.cs    |   2 +
 Wabbajack.Lib/ReportBuilder.cs                |   9 +-
 Wabbajack.Lib/VortexCompiler.cs               | 290 ++++++++----------
 Wabbajack.Lib/VortexInstaller.cs              |  37 +--
 6 files changed, 151 insertions(+), 197 deletions(-)

diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
index b69a55b6..f2524f95 100644
--- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
+++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
@@ -10,7 +10,7 @@ namespace Wabbajack.Lib.Downloaders
     {
         public static readonly List<IDownloader> Downloaders = new List<IDownloader>()
         {
-            //new GameFileSourceDownloader(),
+            new GameFileSourceDownloader(),
             new MegaDownloader(),
             new DropboxDownloader(),
             new GoogleDriveDownloader(),
diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
index f7b02721..8f389fbd 100644
--- a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
+++ b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
@@ -1,4 +1,8 @@
-using System.Threading.Tasks;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
 using Wabbajack.Lib.Validation;
@@ -22,7 +26,7 @@ namespace Wabbajack.Lib.Downloaders
 
             var path = game.GameLocation();
             var filePath = Path.Combine(path, gameFile);
-
+            
             if (!File.Exists(filePath))
                 return null;
 
diff --git a/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs b/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs
index 73bea8a6..503e8563 100644
--- a/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs
+++ b/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs
@@ -1,7 +1,9 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Wabbajack.Common;
diff --git a/Wabbajack.Lib/ReportBuilder.cs b/Wabbajack.Lib/ReportBuilder.cs
index b39a465e..a721527c 100644
--- a/Wabbajack.Lib/ReportBuilder.cs
+++ b/Wabbajack.Lib/ReportBuilder.cs
@@ -64,12 +64,8 @@ namespace Wabbajack.Lib
                         .Do(NoWrapText);
             }
 
-            var archiveCount = lst.Archives.Count + lst.Directives.Count(d => d is SteamMeta);
-            var totalSize = lst.Archives.Sum(a => a.Size);
-            totalSize += lst.Directives.Where(d => d is SteamMeta).Cast<SteamMeta>().Sum(s => s.Size);
-
             Text(
-                $"#### Download Summary ({archiveCount} archives - {totalSize.ToFileSizeString()})");
+                $"#### Download Summary ({lst.Archives.Count} archives - {lst.Archives.Sum(a => a.Size).ToFileSizeString()})");
             foreach (var archive in SortArchives(lst.Archives))
             {
                 var hash = archive.Hash.FromBase64().ToHex();
@@ -82,8 +78,7 @@ namespace Wabbajack.Lib
                 if (f is SteamMeta s)
                 {
                     var link = $"https://steamcommunity.com/sharedfiles/filedetails/?id={s.ItemID}";
-                    var size = ((long)s.Size).ToFileSizeString();
-                    NoWrapText($"* Steam Workshop Item: [{s.ItemID}]({link}) | Size: {size}");
+                    NoWrapText($"* Steam Workshop Item: [{s.ItemID}]({link}) | Size: {s.Size}");
                 }
             });
 
diff --git a/Wabbajack.Lib/VortexCompiler.cs b/Wabbajack.Lib/VortexCompiler.cs
index 992ef0e0..dde32891 100644
--- a/Wabbajack.Lib/VortexCompiler.cs
+++ b/Wabbajack.Lib/VortexCompiler.cs
@@ -4,18 +4,15 @@ using System.IO;
 using System.Linq;
 using System.Security.Cryptography;
 using System.Text;
-using System.Threading;
 using System.Threading.Tasks;
+using System.Threading;
+using DynamicData;
 using Microsoft.WindowsAPICodePack.Shell;
 using Newtonsoft.Json;
 using Wabbajack.Common;
 using Wabbajack.Lib.CompilationSteps;
 using Wabbajack.Lib.NexusApi;
-using Wabbajack.Lib.Validation;
-using Directory = Alphaleonis.Win32.Filesystem.Directory;
-using File = System.IO.File;
-using Game = Wabbajack.Common.Game;
-using Path = Alphaleonis.Win32.Filesystem.Path;
+using File = Alphaleonis.Win32.Filesystem.File;
 
 namespace Wabbajack.Lib
 {
@@ -32,15 +29,15 @@ namespace Wabbajack.Lib
         public Game Game { get; }
         public string GameName { get; }
 
-        public bool IgnoreMissingFiles { get; set; }
-
         public string VortexFolder { get; set; }
         public string StagingFolder { get; set; }
         public string DownloadsFolder { get; set; }
 
+        public bool IgnoreMissingFiles { get; set; }
+
         public override ModManager ModManager => ModManager.Vortex;
         public override string GamePath { get; }
-        public override string ModListOutputFolder => "output_folder";
+        public override string ModListOutputFolder { get; }
         public override string ModListOutputFile { get; }
 
         public const string StagingMarkerName = "__vortex_staging_folder";
@@ -50,26 +47,19 @@ namespace Wabbajack.Lib
         private SteamGame _steamGame;
         private bool _hasSteamWorkshopItems;
 
-        public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder,
-            string stagingFolder, string outputFile)
+        public VortexCompiler(Game game, string gamePath, string vortexFolder, string downloadsFolder, string stagingFolder, string outputFile)
         {
             Game = game;
+
             GamePath = gamePath;
+            GameName = game.MetaData().NexusName;
             VortexFolder = vortexFolder;
             DownloadsFolder = downloadsFolder;
             StagingFolder = stagingFolder;
+            ModListOutputFolder = "output_folder";
             ModListOutputFile = outputFile;
 
-            if (string.IsNullOrEmpty(ModListName))
-            {
-                ModListName = $"Vortex ModList for {Game.ToString()}";
-                ModListOutputFile = $"{ModListName}{ExtensionManager.Extension}";
-            }
-
-            GameName = Game.MetaData().NexusName;
-
-            ActiveArchives = new List<string>();
-
+            // there can be max one game after filtering
             SteamHandler.Instance.Games.Where(g => g.Game != null && g.Game == game).Do(g =>
             {
                 _isSteamGame = true;
@@ -77,51 +67,52 @@ namespace Wabbajack.Lib
                 SteamHandler.Instance.LoadWorkshopItems(_steamGame);
                 _hasSteamWorkshopItems = _steamGame.WorkshopItems.Count > 0;
             });
-        }
 
+            ActiveArchives = new List<string>();
+        }
+        
         protected override async Task<bool> _Begin(CancellationToken cancel)
         {
             if (cancel.IsCancellationRequested) return false;
 
-            Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at {DownloadsFolder}.");
+            ConfigureProcessor(10);
+            if (string.IsNullOrEmpty(ModListName))
+                ModListName = $"Vortex ModList for {Game.ToString()}";
 
-            ConfigureProcessor(12);
-            UpdateTracker.Reset();
+            Info($"Starting Vortex compilation for {GameName} at {GamePath} with staging folder at {StagingFolder} and downloads folder at  {DownloadsFolder}.");
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Parsing deployment file");
             ParseDeploymentFile();
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Creating metas for archives");
+            Info("Starting pre-compilation steps");
             await CreateMetaFiles();
 
             if (cancel.IsCancellationRequested) return false;
-            await VFS.IntegrateFromFile(_vfsCacheName);
+            Info($"Indexing {StagingFolder}");
+            await VFS.AddRoot(StagingFolder);
 
-            var roots = new List<string> {StagingFolder, GamePath, DownloadsFolder};
-            AddExternalFolder(ref roots);
+            Info($"Indexing {GamePath}");
+            await VFS.AddRoot(GamePath);
+
+            Info($"Indexing {DownloadsFolder}");
+            await VFS.AddRoot(DownloadsFolder);
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Indexing folders");
-            await VFS.AddRoots(roots);
-            await VFS.WriteToFile(_vfsCacheName);
+            await AddExternalFolder();
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Cleaning output folder");
-            if (Directory.Exists(ModListOutputFolder))
-                Utils.DeleteDirectory(ModListOutputFolder);
-
+            Info("Cleaning output folder");
+            if (Directory.Exists(ModListOutputFolder)) Utils.DeleteDirectory(ModListOutputFolder);
             Directory.CreateDirectory(ModListOutputFolder);
-
-            UpdateTracker.NextStep("Finding Install Files");
+            
             var vortexStagingFiles = Directory.EnumerateFiles(StagingFolder, "*", SearchOption.AllDirectories)
                 .Where(p => p.FileExists() && p != StagingMarkerName)
                 .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
                     {Path = p.RelativeTo(StagingFolder)});
             
             var vortexDownloads = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.AllDirectories)
-                .Where(p => p.FileExists() && p != DownloadMarkerName)
+                .Where(p => p.FileExists())
                 .Select(p => new RawSourceFile(VFS.Index.ByRootPath[p])
                     {Path = p.RelativeTo(DownloadsFolder)});
 
@@ -148,6 +139,7 @@ namespace Wabbajack.Lib
                 .GroupBy(f => f.Hash)
                 .ToDictionary(f => f.Key, f => f.AsEnumerable());
 
+            Info("Searching for mod files");
             AllFiles = vortexStagingFiles.Concat(vortexDownloads)
                 .Concat(gameFiles)
                 .DistinctBy(f => f.Path)
@@ -156,8 +148,7 @@ namespace Wabbajack.Lib
             Info($"Found {AllFiles.Count} files to build into mod list");
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Verifying destinations");
-
+            Info("Verifying destinations");
             var duplicates = AllFiles.GroupBy(f => f.Path)
                 .Where(fs => fs.Count() > 1)
                 .Select(fs =>
@@ -194,9 +185,9 @@ namespace Wabbajack.Lib
                 if(targetArchive.IniData?.General?.tag == null || targetArchive.IniData?.General?.tag != Consts.WABBAJACK_VORTEX_MANUAL)
                     continue;
 
-#if DEBUG
+                #if DEBUG
                 Utils.Log($"Double hash for: {f.AbsolutePath}");
-#endif
+                #endif
 
                 var replace = f;
                 var name = replace.File.Name;
@@ -213,9 +204,9 @@ namespace Wabbajack.Lib
                 //AllFiles.Replace(f, replace);
             }
 
-            if (cancel.IsCancellationRequested) return false;
             var stack = MakeStack();
-            UpdateTracker.NextStep("Running Compilation Stack");
+
+            Info("Running Compilation Stack");
             var results = await AllFiles.PMap(Queue, f => RunStack(stack.Where(s => s != null), f));
 
             IEnumerable<NoMatch> noMatch = results.OfType<NoMatch>().ToList();
@@ -237,17 +228,17 @@ namespace Wabbajack.Lib
 
             InstallDirectives = results.Where(i => !(i is IgnoredDirectly)).ToList();
 
-            Info("Getting Nexus api_key, please click authorize if a browser window appears");
-
+            // TODO: nexus stuff
+            /*Info("Getting Nexus api_key, please click authorize if a browser window appears");
             if (IndexedArchives.Any(a => a.IniData?.General?.gameName != null))
             {
-                var nexusClient = await NexusApiClient.Get();
-                if (!await nexusClient.IsPremium()) Error($"User {await nexusClient.Username()} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue");
+                var nexusClient = new NexusApiClient();
+                if (!nexusClient.IsPremium) Error($"User {nexusClient.Username} is not a premium Nexus user, so we cannot access the necessary API calls, cannot continue");
 
             }
+            */
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Gathering Archives");
             await GatherArchives();
 
             ModList = new ModList
@@ -263,46 +254,16 @@ namespace Wabbajack.Lib
                 Directives = InstallDirectives,
                 GameType = Game
             };
-
-            UpdateTracker.NextStep("Running Validation");
-            await ValidateModlist.RunValidation(Queue, ModList);
-
-            UpdateTracker.NextStep("Generating Report");
+            
             GenerateReport();
-
-            UpdateTracker.NextStep("Exporting ModList");
             ExportModList();
 
-            ResetMembers();
+            Info("Done Building ModList");
 
             ShowReport();
-
-            UpdateTracker.NextStep("Done Building ModList");
-
             return true;
         }
 
-        /// <summary>
-        ///     Clear references to lists that hold a lot of data.
-        /// </summary>
-        private void ResetMembers()
-        {
-            AllFiles = null;
-            InstallDirectives = null;
-            SelectedArchives = null;
-        }
-
-        private void AddExternalFolder(ref List<string> roots)
-        {
-            var currentGame = Game.MetaData();
-            if (currentGame.AdditionalFolders == null || currentGame.AdditionalFolders.Count == 0) return;
-            foreach (var path in currentGame.AdditionalFolders.Select(f => f.Replace("%documents%", KnownFolders.Documents.Path)))
-            {
-                if (!Directory.Exists(path)) return;
-                roots.Add(path);
-            }
-        }
-
         private void ParseDeploymentFile()
         {
             Info("Searching for vortex.deployment.json...");
@@ -319,10 +280,10 @@ namespace Wabbajack.Lib
 
             if (string.IsNullOrEmpty(deploymentFile))
             {
-                Error("vortex.deployment.json not found!");
+                Info("vortex.deployment.json not found!");
                 return;
             }
-            Info($"vortex.deployment.json found at {deploymentFile}");
+            Info("vortex.deployment.json found at "+deploymentFile);
 
             Info("Parsing vortex.deployment.json...");
             try
@@ -345,81 +306,101 @@ namespace Wabbajack.Lib
             });
         }
 
+        /// <summary>
+        /// Some have mods outside their game folder located
+        /// </summary>
+        private async Task AddExternalFolder()
+        {
+            var currentGame = Game.MetaData();
+            if (currentGame.AdditionalFolders == null || currentGame.AdditionalFolders.Count == 0) return;
+            foreach (var f in currentGame.AdditionalFolders)
+            {
+                var path = f.Replace("%documents%", KnownFolders.Documents.Path);
+                if (!Directory.Exists(path)) return;
+                Info($"Indexing {path}");
+                await VFS.AddRoot(path);
+            }
+        }
+
         private async Task CreateMetaFiles()
         {
             Utils.Log("Getting Nexus api_key, please click authorize if a browser window appears");
             var nexusClient = await NexusApiClient.Get();
 
-            var archives = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly).Where(f =>
-                File.Exists(f) && Path.GetExtension(f) != ".meta" && Path.GetExtension(f) != ".xxHash" &&
-                !File.Exists($"{f}.meta") && ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)));
-
-            await archives.PMap(Queue, async f =>
-            {
-                Info($"Creating meta file for {Path.GetFileName(f)}");
-                var metaString = "[General]\n" +
-                                 "repository=Nexus\n" +
-                                 $"gameName={GameName}\n";
-                string hash;
-                using(var md5 = MD5.Create())
-                using (var stream = File.OpenRead(f))
+            await Task.WhenAll(
+                Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly)
+                .Where(File.Exists)
+                .Select(async f =>
                 {
-                    Info($"Calculating hash for {Path.GetFileName(f)}");
-                    var cH = md5.ComputeHash(stream);
-                    hash = BitConverter.ToString(cH).Replace("-", "").ToLowerInvariant();
-                    Info($"Hash is {hash}");
-                }
-
-                var md5Response = await nexusClient.GetModInfoFromMD5(Game, hash);
-                if (md5Response.Count >= 1)
-                {
-                    var modInfo = md5Response[0].mod;
-                    metaString += $"modID={modInfo.mod_id}\n" +
-                                  $"modName={modInfo.name}\n" +
-                                  $"fileID={md5Response[0].file_details.file_id}\n" +
-                                  $"version={md5Response[0].file_details.version}\n";
-                    File.WriteAllText(f+".meta",metaString, Encoding.UTF8);
-                }
-                else
-                {
-                    Error("Error while getting information from NexusMods via MD5 hash!");
-                }
-            });
-
-            var otherFiles = Directory.EnumerateFiles(DownloadsFolder, "*", SearchOption.TopDirectoryOnly).Where(f =>
-                Path.GetExtension(f) == ".meta" && !ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)));
-
-            await otherFiles.PMap(Queue, async f =>
-            {
-                Info($"File {f} is not in ActiveArchives");
-                var lines = File.ReadAllLines(f);
-                if (lines.Length == 0 || !lines.Any(line => lines.Contains("directURL=")))
-                {
-                    if (lines.Length == 0)
-                        return;
-
-                    lines.Do(line =>
+                    if (Path.GetExtension(f) != ".meta" && Path.GetExtension(f) != ".xxHash" && !File.Exists($"{f}.meta") && ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)))
                     {
-                        var tag = "";
-                        if (line.Contains("tag="))
-                            tag = line.Substring("tag=".Length);
+                        Utils.Log($"Trying to create meta file for {Path.GetFileName(f)}");
+                        var metaString = "[General]\n" +
+                                         "repository=Nexus\n" +
+                                         "installed=true\n" +
+                                         "uninstalled=false\n" +
+                                         "paused=false\n" +
+                                         "removed=false\n" +
+                                         $"gameName={GameName}\n";
+                        string hash;
+                        using(var md5 = MD5.Create())
+                        using (var stream = File.OpenRead(f))
+                        {
+                            Utils.Log($"Calculating hash for {Path.GetFileName(f)}");
+                            var cH = md5.ComputeHash(stream);
+                            hash = BitConverter.ToString(cH).Replace("-", "").ToLowerInvariant();
+                            Utils.Log($"Hash is {hash}");
+                        }
 
-                        if (tag != Consts.WABBAJACK_VORTEX_MANUAL)
+                        var md5Response = await nexusClient.GetModInfoFromMD5(Game, hash);
+                        if (md5Response.Count >= 1)
+                        {
+                            var modInfo = md5Response[0].mod;
+                            metaString += $"modID={modInfo.mod_id}\n" +
+                                          $"modName={modInfo.name}\nfileID={md5Response[0].file_details.file_id}";
+                            File.WriteAllText(f+".meta",metaString, Encoding.UTF8);
+                        }
+                        else
+                        {
+                            Error("Error while getting information from NexusMods via MD5 hash!");
+                        }
+                    }
+                    else
+                    {
+                        if (Path.GetExtension(f) != ".meta" ||
+                            ActiveArchives.Contains(Path.GetFileNameWithoutExtension(f)))
                             return;
 
-                        Info($"File {f} contains the {Consts.WABBAJACK_VORTEX_MANUAL} tag, adding to ActiveArchives");
-                        ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
-                    });
-                }
-                else
-                {
-                    Info($"File {f} appears to not be from the Nexus, adding to ActiveArchives");
-                    ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
-                }
-            });
+                        Utils.Log($"File {f} is not in ActiveArchives");
+                        var lines = File.ReadAllLines(f);
+                        if (lines.Length == 0 || !lines.Any(line => line.Contains("directURL=")))
+                        {
+                            if (lines.Length == 0)
+                                return;
 
-            Info($"Checking for Steam Workshop Items...");
-            if (!_isSteamGame || _steamGame == null || !_hasSteamWorkshopItems)
+                            lines.Do(line =>
+                            {
+                                var tag = "";
+                                if (line.Contains("tag="))
+                                    tag = line.Substring("tag=".Length);
+
+                                if (tag != Consts.WABBAJACK_VORTEX_MANUAL)
+                                    return;
+
+                                Utils.Log($"File {f} contains the {Consts.WABBAJACK_VORTEX_MANUAL} tag, adding to ActiveArchives");
+                                ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
+                            });
+                        }
+                        else
+                        {
+                            Utils.Log($"File {f} appears to not come from the Nexus, adding to ActiveArchives");
+                            ActiveArchives.Add(Path.GetFileNameWithoutExtension(f));
+                        }
+                    }
+                }));
+
+            Utils.Log($"Checking for Steam Workshop Items...");
+            if (!_isSteamGame || _steamGame == null || _steamGame.WorkshopItems.Count <= 0)
                 return;
 
             _steamGame.WorkshopItems.Do(item =>
@@ -434,6 +415,7 @@ namespace Wabbajack.Lib
                 Utils.Log($"Creating meta file for {item.ItemID}");
                 var metaString = "[General]\n" +
                                  "repository=Steam\n" +
+                                 "installed=true\n" +
                                  $"gameName={GameName}\n" +
                                  $"steamID={_steamGame.AppId}\n" +
                                  $"itemID={item.ItemID}\n" +
@@ -458,22 +440,20 @@ namespace Wabbajack.Lib
 
             var stack = MakeStack();
 
-            var compilationSteps = stack.ToList();
-            File.WriteAllText(Path.Combine(s, "_current_compilation_stack.yml"), Serialization.Serialize(compilationSteps));
+            File.WriteAllText(Path.Combine(s, "_current_compilation_stack.yml"),
+                Serialization.Serialize(stack));
 
-            return compilationSteps;
+            return stack;
         }
 
         public override IEnumerable<ICompilationStep> MakeStack()
         {
-            Info("Generating compilation stack");
+            Utils.Log("Generating compilation stack");
             return new List<ICompilationStep>
             {
                 new IncludePropertyFiles(this),
-
                 new IncludeSteamWorkshopItems(this, _steamGame),
                 _hasSteamWorkshopItems ? new IncludeRegex(this, "^steamWorkshopItem_\\d*\\.meta$") : null,
-
                 new IgnoreDisabledVortexMods(this),
                 new IncludeVortexDeployment(this),
                 new IgnoreVortex(this),
diff --git a/Wabbajack.Lib/VortexInstaller.cs b/Wabbajack.Lib/VortexInstaller.cs
index 863bf882..0a17c2a2 100644
--- a/Wabbajack.Lib/VortexInstaller.cs
+++ b/Wabbajack.Lib/VortexInstaller.cs
@@ -19,14 +19,12 @@ namespace Wabbajack.Lib
 
         public override ModManager ModManager => ModManager.Vortex;
 
-        public string GameFolder { get; set; }
-
         public VortexInstaller(string archive, ModList modList, string outputFolder, string downloadFolder)
             : base(
-                archive,
-                modList,
-                outputFolder,
-                downloadFolder)
+                  archive: archive,
+                  modList: modList,
+                  outputFolder: outputFolder,
+                  downloadFolder: downloadFolder)
         {
             #if DEBUG
             // TODO: only for testing
@@ -45,33 +43,17 @@ namespace Wabbajack.Lib
                 "for support.", "Warning",
                 MessageBoxButton.OK);
 
-            if (GameFolder == null)
-                GameFolder = GameInfo.GameLocation();
-
-            if (GameFolder == null)
-            {
-                MessageBox.Show(
-                    $"In order to do a proper install Wabbajack needs to know where your {GameInfo.NexusName} folder resides. We tried looking the" +
-                    "game location up but were unable to find it, please make sure you launch the game once before running this installer. ",
-                    "Could not find game location", MessageBoxButton.OK);
-                Error("Exiting because we couldn't find the game folder.");
-                return false;
-            }
-
             if (cancel.IsCancellationRequested) return false;
             ConfigureProcessor(10, await RecommendQueueSize());
             Directory.CreateDirectory(DownloadFolder);
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Hashing Archives");
             await HashArchives();
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Downloading Missing Archives");
             await DownloadArchives();
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Hashing Remaining Archives");
             await HashArchives();
 
             if (cancel.IsCancellationRequested) return false;
@@ -86,34 +68,25 @@ namespace Wabbajack.Lib
                     Error("Cannot continue, was unable to download one or more archives");
             }
 
-            if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Priming VFS");
             await PrimeVFS();
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Building Folder Structure");
             BuildFolderStructure();
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Installing Archives");
             await InstallArchives();
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Installing Included files");
             await InstallIncludedFiles();
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Installing Manual files");
             await InstallManualGameFiles();
 
             if (cancel.IsCancellationRequested) return false;
-            UpdateTracker.NextStep("Installing SteamWorkshopItems");
             await InstallSteamWorkshopItems();
-            
-            
             //InstallIncludedDownloadMetas();
 
-            UpdateTracker.NextStep("Installation complete! You may exit the program.");
+            Info("Installation complete! You may exit the program.");
             return true;
         }
 

From 86d91a3e4b1646f1c6ac2b96eec03398c0af637d Mon Sep 17 00:00:00 2001
From: erri120 <erri120@protonmail.com>
Date: Sat, 14 Dec 2019 22:34:55 +0100
Subject: [PATCH 17/19] Added safety checks to the SteamHandler

---
 Wabbajack.Common/SteamHandler.cs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Wabbajack.Common/SteamHandler.cs b/Wabbajack.Common/SteamHandler.cs
index 35934868..d69bfbb5 100644
--- a/Wabbajack.Common/SteamHandler.cs
+++ b/Wabbajack.Common/SteamHandler.cs
@@ -92,7 +92,8 @@ namespace Wabbajack.Common
                 if (!l.Contains("BaseInstallFolder_")) return;
                 var s = GetVdfValue(l);
                 s = Path.Combine(s, "steamapps");
-                paths.Add(s);
+                if(Directory.Exists(s))
+                    paths.Add(s);
             });
 
             // Default path in the Steam folder isn't in the configs
@@ -111,7 +112,7 @@ namespace Wabbajack.Common
 
             InstallFolders.Do(p =>
             {
-                Directory.EnumerateFiles(p, "*.acf", SearchOption.TopDirectoryOnly).Do(f =>
+                Directory.EnumerateFiles(p, "*.acf", SearchOption.TopDirectoryOnly).Where(File.Exists).Do(f =>
                 {
                     var steamGame = new SteamGame();
                     var valid = false;
@@ -157,7 +158,7 @@ namespace Wabbajack.Common
                 if(!Directory.Exists(workshop))
                     return;
 
-                Directory.EnumerateFiles(workshop, "*.acf", SearchOption.TopDirectoryOnly).Do(f =>
+                Directory.EnumerateFiles(workshop, "*.acf", SearchOption.TopDirectoryOnly).Where(File.Exists).Do(f =>
                 {
                     if (Path.GetFileName(f)  != $"appworkshop_{game.AppId}.acf")
                         return;

From e13a57009a0a58a8e986eb02c8ac72ca3389bb61 Mon Sep 17 00:00:00 2001
From: erri120 <erri120@protonmail.com>
Date: Sat, 14 Dec 2019 22:40:10 +0100
Subject: [PATCH 18/19] Added ExtensionManager safety checks

---
 Wabbajack.Common/ExtensionManager.cs | 13 +++++++------
 Wabbajack/Views/MainWindow.xaml.cs   | 13 +++++++++++--
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/Wabbajack.Common/ExtensionManager.cs b/Wabbajack.Common/ExtensionManager.cs
index 34651409..1f0f5075 100644
--- a/Wabbajack.Common/ExtensionManager.cs
+++ b/Wabbajack.Common/ExtensionManager.cs
@@ -33,25 +33,26 @@ namespace Wabbajack.Common
             var progIDKey = Registry.CurrentUser.OpenSubKey(ProgIDPath);
             var tempKey = progIDKey?.OpenSubKey("shell\\open\\command");
             if (progIDKey == null || tempKey == null) return true;
-            return tempKey.GetValue("").ToString().Equals($"\"{appPath}\" -i \"%1\"");
+            var value = tempKey.GetValue("");
+            return value == null || value.ToString().Equals($"\"{appPath}\" -i \"%1\"");
         }
 
         public static bool IsAssociated()
         {
             var progIDKey = Registry.CurrentUser.OpenSubKey(ProgIDPath);
             var extKey = Registry.CurrentUser.OpenSubKey(ExtPath);
-            return (progIDKey != null && extKey != null);
+            return progIDKey != null && extKey != null;
         }
 
         public static void Associate(string appPath)
         {
             var progIDKey = Registry.CurrentUser.CreateSubKey(ProgIDPath, RegistryKeyPermissionCheck.ReadWriteSubTree);
-            foreach (KeyValuePair<string, string> entry in ProgIDList)
+            foreach (var entry in ProgIDList)
             {
                 if (entry.Key.Contains("\\"))
                 {
-                    var tempKey = progIDKey.CreateSubKey(entry.Key);
-                    tempKey.SetValue("", entry.Value.Replace("{appPath}", appPath));
+                    var tempKey = progIDKey?.CreateSubKey(entry.Key);
+                    tempKey?.SetValue("", entry.Value.Replace("{appPath}", appPath));
                 }
                 else
                 {
@@ -60,7 +61,7 @@ namespace Wabbajack.Common
             }
 
             var extKey = Registry.CurrentUser.CreateSubKey(ExtPath, RegistryKeyPermissionCheck.ReadWriteSubTree);
-            foreach (KeyValuePair<string, string> entry in ExtList)
+            foreach (var entry in ExtList)
             {
                 extKey?.SetValue(entry.Key, entry.Value);
             }
diff --git a/Wabbajack/Views/MainWindow.xaml.cs b/Wabbajack/Views/MainWindow.xaml.cs
index bf7cbe01..9c9139aa 100644
--- a/Wabbajack/Views/MainWindow.xaml.cs
+++ b/Wabbajack/Views/MainWindow.xaml.cs
@@ -4,6 +4,7 @@ using System.Windows;
 using MahApps.Metro.Controls;
 using Wabbajack.Common;
 using Application = System.Windows.Application;
+using Utils = Wabbajack.Common.Utils;
 
 namespace Wabbajack
 {
@@ -25,10 +26,18 @@ namespace Wabbajack
             };
 
             var appPath = System.Reflection.Assembly.GetExecutingAssembly().Location;
-            if (!ExtensionManager.IsAssociated() || ExtensionManager.NeedsUpdating(appPath))
+            try
             {
-                ExtensionManager.Associate(appPath);
+                if (!ExtensionManager.IsAssociated() || ExtensionManager.NeedsUpdating(appPath))
+                {
+                    ExtensionManager.Associate(appPath);
+                }
             }
+            catch (Exception e)
+            {
+                Utils.Log($"ExtensionManager had an exception:\n{e}");
+            }
+            
 
             Wabbajack.Common.Utils.Log($"Wabbajack Build - {ThisAssembly.Git.Sha}");
 

From 3c21dfd7818d706c851b6dd38df2dfbc1fc0ead6 Mon Sep 17 00:00:00 2001
From: erri120 <erri120@protonmail.com>
Date: Sat, 14 Dec 2019 22:43:47 +0100
Subject: [PATCH 19/19] Added check for InstallDir

---
 Wabbajack.Common/SteamHandler.cs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/Wabbajack.Common/SteamHandler.cs b/Wabbajack.Common/SteamHandler.cs
index d69bfbb5..9fe0c695 100644
--- a/Wabbajack.Common/SteamHandler.cs
+++ b/Wabbajack.Common/SteamHandler.cs
@@ -97,7 +97,8 @@ namespace Wabbajack.Common
             });
 
             // Default path in the Steam folder isn't in the configs
-            paths.Add(Path.Combine(SteamPath, "steamapps"));
+            if(Directory.Exists(Path.Combine(SteamPath, "steamapps")))
+                paths.Add(Path.Combine(SteamPath, "steamapps"));
 
             InstallFolders = paths;
         }
@@ -123,8 +124,11 @@ namespace Wabbajack.Common
                                 return;
                         if(l.Contains("\"name\""))
                             steamGame.Name = GetVdfValue(l);
-                        if(l.Contains("\"installdir\""))
-                            steamGame.InstallDir = Path.Combine(p, "common", GetVdfValue(l));
+                        if (l.Contains("\"installdir\""))
+                        {
+                            var path = Path.Combine(p, "common", GetVdfValue(l));
+                            steamGame.InstallDir = Directory.Exists(path) ? path : null;
+                        }
 
                         if (steamGame.AppId != 0 && !string.IsNullOrWhiteSpace(steamGame.Name) &&
                             !string.IsNullOrWhiteSpace(steamGame.InstallDir))