Merge remote-tracking branch 'wabbajack-tools/master' into polish-and-fixes

.runsettings

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <MSTest>
+    <Parallelize>
+      <Workers>0</Workers>
+      <Scope>MethodLevel</Scope>
+    </Parallelize>
+  </MSTest>
+</RunSettings>

CHANGELOG.md

@@ -1,5 +1,10 @@
 ### Changelog
 
+#### Version - 1.0 beta 5 - 12/14/2019
+* Added LoversLab download support
+* Nexus and LL logins now happen via a in-ap browser
+* Several UI enhancements
+
 #### Version - 1.0 beta 4 - 12/3/2019
 * Several crash and bug fixes
 

README.md

@@ -7,7 +7,7 @@ Wabbajack is an automated ModList installer that can recreate contents of a fold
 ## Social Links
 
 - [wabbajack.org](https://www.wabbajack.org) The official Wabbajack website where you can find instructions and a [Gallery](https://www.wabbajack.org/gallery) for some ModLists.
-- [Discord](https://discord.gg/zgbrkmA) The official Wabbajack discord for instructions, ModLists, support or friendly chatting with fellow modders.
+- [Discord](https://discord.gg/wabbajack) The official Wabbajack discord for instructions, ModLists, support or friendly chatting with fellow modders.
 - [Patreon](https://www.patreon.com/user?u=11907933) contains update posts and keeps the [Code Signing Certificate](https://www.digicert.com/code-signing/) alive.
 
 ## Supported Games and Mod Manager

Wabbajack.CacheServer/ListValidationService.cs

@@ -0,0 +1,215 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Net.Http;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using Nancy;
+using Nancy.Responses;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.ModListRegistry;
+
+namespace Wabbajack.CacheServer
+{
+    public class ListValidationService : NancyModule
+    {
+        public class ModListStatus
+        {
+            public string Name;
+            public DateTime Checked = DateTime.Now;
+            public List<(Archive archive, bool)> Archives { get; set; }
+            public DownloadMetadata DownloadMetaData { get; set; }
+            public bool HasFailures { get; set; }
+        }
+
+        public class ModlistSummary
+        {
+            public string Name;
+            public DateTime Checked;
+            public int Failed;
+            public int Passed;
+            public string Link => $"/lists/status/{Name}.json";
+            public string Report => $"/lists/status/{Name}.html";
+        }
+
+        public static Dictionary<string, ModListStatus> ModLists { get; set; }
+
+        public ListValidationService() : base("/lists")
+        {
+            Get("/status", HandleGetLists);
+            Get("/status/{Name}.json", HandleGetListJson);
+            Get("/status/{Name}.html", HandleGetListHtml);
+        }
+
+        private object HandleGetLists(object arg)
+        {
+            var summaries = ModLists.Values.Select(m => new ModlistSummary
+            {
+                Name = m.Name,
+                Checked = m.Checked,
+                Failed = m.Archives.Count(a => a.Item2),
+                Passed = m.Archives.Count(a => !a.Item2),
+            }).ToList();
+            return summaries.ToJSON();
+        }
+
+        public class ArchiveSummary
+        {
+            public string Name;
+            public AbstractDownloadState State;
+        }
+        public class DetailedSummary
+        {
+            public string Name;
+            public DateTime Checked;
+            public List<ArchiveSummary> Failed;
+            public List<ArchiveSummary> Passed;
+
+        }
+        private object HandleGetListJson(dynamic arg)
+        {
+            var lst = ModLists[(string)arg.Name];
+            var summary = new DetailedSummary
+            {
+                Name = lst.Name,
+                Checked = lst.Checked,
+                Failed = lst.Archives.Where(a => a.Item2)
+                    .Select(a => new ArchiveSummary {Name = a.archive.Name, State = a.archive.State}).ToList(),
+                Passed = lst.Archives.Where(a => !a.Item2)
+                    .Select(a => new ArchiveSummary { Name = a.archive.Name, State = a.archive.State }).ToList(),
+            };
+            return summary.ToJSON();
+        }
+
+        private object HandleGetListHtml(dynamic arg)
+        {
+            var lst = ModLists[(string)arg.Name];
+            var sb = new StringBuilder();
+
+            sb.Append("<html><body>");
+            sb.Append($"<h2>{lst.Name} - {lst.Checked}</h2>");
+
+            var failed_list = lst.Archives.Where(a => a.Item2).ToList();
+            sb.Append($"<h3>Failed ({failed_list.Count}):</h3>");
+            sb.Append("<ul>");
+            foreach (var archive in failed_list)
+            {
+                sb.Append($"<li>{archive.archive.Name}</li>");
+            }
+            sb.Append("</ul>");
+
+            var pased_list = lst.Archives.Where(a => !a.Item2).ToList();
+            sb.Append($"<h3>Passed ({pased_list.Count}):</h3>");
+            sb.Append("<ul>");
+            foreach (var archive in pased_list.OrderBy(f => f.archive.Name))
+            {
+                sb.Append($"<li>{archive.archive.Name}</li>");
+            }
+            sb.Append("</ul>");
+
+            sb.Append("</body></html>");
+            var response = (Response)sb.ToString();
+            response.ContentType = "text/html";
+            return response;
+        }
+
+        public static void Start()
+        {
+            new Thread(() =>
+            {
+                while (true)
+                {
+                    try
+                    {
+                        ValidateLists().Wait();
+                    }
+                    catch (Exception ex)
+                    {
+                        Utils.Log(ex.ToString());
+                    }
+
+                    // Sleep for two hours
+                    Thread.Sleep(1000 * 60 * 60 * 2);
+                }
+            }).Start();
+        }
+        public static async Task ValidateLists()
+        {
+            Utils.Log("Cleaning Nexus Cache");
+            var client = new HttpClient();
+            await client.GetAsync("http://build.wabbajack.org/nexus_api_cache/update");
+
+            Utils.Log("Starting Modlist Validation");
+            var modlists = await ModlistMetadata.LoadFromGithub();
+
+            var statuses = new Dictionary<string, ModListStatus>();
+
+            using (var queue = new WorkQueue())
+            {
+                foreach (var list in modlists)
+                {
+                    var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ".wabbajack");
+
+                    if (list.NeedsDownload(modlist_path))
+                    {
+                        var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
+                        Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
+                        await state.Download(modlist_path);
+                    }
+                    else
+                    {
+                        Utils.Log($"No changes detected from downloaded modlist");
+                    }
+
+
+                    Utils.Log($"Loading {modlist_path}");
+
+                    var installer = AInstaller.LoadFromFile(modlist_path);
+
+                    Utils.Log($"{installer.Archives.Count} archives to validate");
+ 
+                    DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
+
+                    var validated = (await installer.Archives
+                            .PMap(queue, async archive =>
+                            {
+                                Utils.Log($"Validating: {archive.Name}");
+                                bool is_failed;
+                                try
+                                {
+                                    is_failed = !(await archive.State.Verify());
+                                }
+                                catch (Exception)
+                                {
+                                    is_failed = false;
+                                }
+
+                                return (archive, is_failed);
+                            }))
+                        .ToList();
+
+
+                    var status = new ModListStatus
+                    {
+                        Name = list.Title, 
+                        Archives = validated.OrderBy(v => v.archive.Name).ToList(),
+                        DownloadMetaData = list.DownloadMetadata,
+                        HasFailures = validated.Any(v => v.is_failed)
+                    };
+
+                    statuses.Add(status.Name, status);
+                 }
+            }
+
+            Utils.Log($"Done validating {statuses.Count} lists");
+            ModLists = statuses;
+        }
+    }
+}
+

Wabbajack.CacheServer/NexusCacheModule.cs

@@ -18,9 +18,6 @@ namespace Wabbajack.CacheServer
 
         public NexusCacheModule() : base("/")
         {
-            // ToDo
-            // Handle what to do with the fact that lots of these are now a tasks
-            throw new NotImplementedException("Unsure if following functions still work when taking in a Task");
             Get("/v1/games/{GameName}/mods/{ModID}/files/{FileID}.json", HandleFileID);
             Get("/v1/games/{GameName}/mods/{ModID}/files.json", HandleGetFiles);
             Get("/v1/games/{GameName}/mods/{ModID}.json", HandleModInfo);
@@ -29,7 +26,7 @@ namespace Wabbajack.CacheServer
             Get("/nexus_api_cache/update", UpdateCache);
         }
 
-        private async Task<object> UpdateCache(object arg)
+        public async Task<object> UpdateCache(object arg)
         {
             var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
             await api.ClearUpdatedModsInCache();

Wabbajack.CacheServer/Program.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Nancy.Hosting.Self;
 using Wabbajack.Common;
 
 namespace Wabbajack.CacheServer
@@ -12,9 +13,9 @@ namespace Wabbajack.CacheServer
         static void Main(string[] args)
         {
             Utils.LogMessages.Subscribe(Console.WriteLine);
-
             using (var server = new Server("http://localhost:8080"))
             {
+                ListValidationService.Start();
                 server.Start();
                 Console.ReadLine();
             }

Wabbajack.CacheServer/Server.cs

@@ -18,6 +18,7 @@ namespace Wabbajack.CacheServer
         {
             Address = address;
             _config = new HostConfiguration();
+            _config.MaximumConnectionCount = 24;
             //_config.UrlReservations.CreateAutomatically = true;
             _config.RewriteLocalhost = true;
             _server = new NancyHost(_config, new Uri(address));

Wabbajack.CacheServer/Wabbajack.CacheServer.csproj

@@ -15,7 +15,7 @@
     <TargetFrameworkProfile />
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
-    <PlatformTarget>AnyCPU</PlatformTarget>
+    <PlatformTarget>x64</PlatformTarget>
     <DebugSymbols>true</DebugSymbols>
     <DebugType>full</DebugType>
     <Optimize>false</Optimize>
@@ -73,6 +73,7 @@
     <Reference Include="WindowsBase" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="ListValidationService.cs" />
     <Compile Include="NexusCacheModule.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />

Wabbajack.Common/ExtensionManager.cs

@@ -33,25 +33,26 @@ namespace Wabbajack.Common
             var progIDKey = Registry.CurrentUser.OpenSubKey(ProgIDPath);
             var tempKey = progIDKey?.OpenSubKey("shell\\open\\command");
             if (progIDKey == null || tempKey == null) return true;
-            return tempKey.GetValue("").ToString().Equals($"\"{appPath}\" -i \"%1\"");
+            var value = tempKey.GetValue("");
+            return value == null || value.ToString().Equals($"\"{appPath}\" -i \"%1\"");
         }
 
         public static bool IsAssociated()
         {
             var progIDKey = Registry.CurrentUser.OpenSubKey(ProgIDPath);
             var extKey = Registry.CurrentUser.OpenSubKey(ExtPath);
-            return (progIDKey != null && extKey != null);
+            return progIDKey != null && extKey != null;
         }
 
         public static void Associate(string appPath)
         {
             var progIDKey = Registry.CurrentUser.CreateSubKey(ProgIDPath, RegistryKeyPermissionCheck.ReadWriteSubTree);
-            foreach (KeyValuePair<string, string> entry in ProgIDList)
+            foreach (var entry in ProgIDList)
             {
                 if (entry.Key.Contains("\\"))
                 {
-                    var tempKey = progIDKey.CreateSubKey(entry.Key);
-                    tempKey.SetValue("", entry.Value.Replace("{appPath}", appPath));
+                    var tempKey = progIDKey?.CreateSubKey(entry.Key);
+                    tempKey?.SetValue("", entry.Value.Replace("{appPath}", appPath));
                 }
                 else
                 {
@@ -60,7 +61,7 @@ namespace Wabbajack.Common
             }
 
             var extKey = Registry.CurrentUser.CreateSubKey(ExtPath, RegistryKeyPermissionCheck.ReadWriteSubTree);
-            foreach (KeyValuePair<string, string> entry in ExtList)
+            foreach (var entry in ExtList)
             {
                 extKey?.SetValue(entry.Key, entry.Value);
             }

Wabbajack.Common/SteamHandler.cs

@@ -92,11 +92,13 @@ namespace Wabbajack.Common
                 if (!l.Contains("BaseInstallFolder_")) return;
                 var s = GetVdfValue(l);
                 s = Path.Combine(s, "steamapps");
-                paths.Add(s);
+                if(Directory.Exists(s))
+                    paths.Add(s);
             });
 
             // Default path in the Steam folder isn't in the configs
-            paths.Add(Path.Combine(SteamPath, "steamapps"));
+            if(Directory.Exists(Path.Combine(SteamPath, "steamapps")))
+                paths.Add(Path.Combine(SteamPath, "steamapps"));
 
             InstallFolders = paths;
         }
@@ -111,7 +113,7 @@ namespace Wabbajack.Common
 
             InstallFolders.Do(p =>
             {
-                Directory.EnumerateFiles(p, "*.acf", SearchOption.TopDirectoryOnly).Do(f =>
+                Directory.EnumerateFiles(p, "*.acf", SearchOption.TopDirectoryOnly).Where(File.Exists).Do(f =>
                 {
                     var steamGame = new SteamGame();
                     var valid = false;
@@ -122,8 +124,11 @@ namespace Wabbajack.Common
                                 return;
                         if(l.Contains("\"name\""))
                             steamGame.Name = GetVdfValue(l);
-                        if(l.Contains("\"installdir\""))
-                            steamGame.InstallDir = Path.Combine(p, "common", GetVdfValue(l));
+                        if (l.Contains("\"installdir\""))
+                        {
+                            var path = Path.Combine(p, "common", GetVdfValue(l));
+                            steamGame.InstallDir = Directory.Exists(path) ? path : null;
+                        }
 
                         if (steamGame.AppId != 0 && !string.IsNullOrWhiteSpace(steamGame.Name) &&
                             !string.IsNullOrWhiteSpace(steamGame.InstallDir))
@@ -157,7 +162,7 @@ namespace Wabbajack.Common
                 if(!Directory.Exists(workshop))
                     return;
 
-                Directory.EnumerateFiles(workshop, "*.acf", SearchOption.TopDirectoryOnly).Do(f =>
+                Directory.EnumerateFiles(workshop, "*.acf", SearchOption.TopDirectoryOnly).Where(File.Exists).Do(f =>
                 {
                     if (Path.GetFileName(f)  != $"appworkshop_{game.AppId}.acf")
                         return;

Wabbajack.Common/WorkQueue.cs

@@ -70,7 +70,16 @@ namespace Wabbajack.Common
                 {
                     Report("Waiting", 0, false);
                     if (_cancel.IsCancellationRequested) return;
-                    var f = Queue.Take(_cancel.Token);
+                    Func<Task> f;
+                    try
+                    {
+                        f = Queue.Take(_cancel.Token);
+                    }
+                    catch (Exception)
+                    {
+                        throw new OperationCanceledException();
+                    }
+
                     await f();
                 }
             }
@@ -100,13 +109,6 @@ namespace Wabbajack.Common
         public void Dispose()
         {
             _cancel.Cancel();
-            Threads.Do(th =>
-            {
-                if (th.ManagedThreadId != Thread.CurrentThread.ManagedThreadId)
-                {
-                    th.Join();
-                }
-            });
             Queue?.Dispose();
         }
     }

Wabbajack.Lib/Downloaders/NexusDownloader.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Wabbajack.Common;
 using Wabbajack.Common.StatusFeed.Errors;
@@ -10,6 +11,11 @@ namespace Wabbajack.Lib.Downloaders
 {
     public class NexusDownloader : IDownloader
     {
+        private bool _prepared;
+        private SemaphoreSlim _lock = new SemaphoreSlim(1);
+        private UserStatus _status;
+        private NexusApiClient _client;
+
         public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
         {
             var general = archiveINI?.General;
@@ -44,16 +50,34 @@ namespace Wabbajack.Lib.Downloaders
 
         public async Task Prepare()
         {
-            var client = await NexusApiClient.Get();
-            var status = await client.GetUserStatus();
-            if (!client.IsAuthenticated)
+            if (!_prepared)
             {
-                Utils.ErrorThrow(new UnconvertedError($"Authenticating for the Nexus failed. A nexus account is required to automatically download mods."));
-                return;
+                await _lock.WaitAsync();
+                try
+                {
+                    // Could have become prepared while we waited for the lock
+                    if (!_prepared)
+                    {
+                        _client = await NexusApiClient.Get();
+                        _status = await _client.GetUserStatus();
+                        if (!_client.IsAuthenticated)
+                        {
+                            Utils.ErrorThrow(new UnconvertedError(
+                                $"Authenticating for the Nexus failed. A nexus account is required to automatically download mods."));
+                            return;
+                        }
+                    }
+                }
+                finally
+                {
+                    _lock.Release();
+                }
             }
 
-            if (status.is_premium) return;
-            Utils.ErrorThrow(new UnconvertedError($"Automated installs with Wabbajack requires a premium nexus account. {await client.Username()} is not a premium account."));
+            _prepared = true;
+
+            if (_status.is_premium) return;
+            Utils.ErrorThrow(new UnconvertedError($"Automated installs with Wabbajack requires a premium nexus account. {await _client.Username()} is not a premium account."));
         }
 
         public class State : AbstractDownloadState

Wabbajack.Lib/NexusApi/NexusApi.cs

@@ -329,7 +329,10 @@ namespace Wabbajack.Lib.NexusApi
         public async Task<GetModFilesResponse> GetModFiles(Game game, int modid)
         {
             var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modid}/files.json";
-            return await GetCached<GetModFilesResponse>(url);
+            var result = await GetCached<GetModFilesResponse>(url);
+            if (result.files == null)
+                throw new InvalidOperationException("Got Null data from the Nexus while finding mod files");
+            return result;
         }
 
         public async Task<List<MD5Response>> GetModInfoFromMD5(Game game, string md5Hash)

Wabbajack.Test/EndToEndTests.cs

@@ -122,8 +122,8 @@ namespace Wabbajack.Test
             if (!File.Exists(src))
             {
 
-                var state = DownloadDispatcher.ResolveArchive(ini.LoadIniString());
-                state.Download(src);
+                var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
+                await state.Download(src);
             }
 
             if (!Directory.Exists(utils.DownloadsFolder))

Wabbajack.sln

@@ -12,6 +12,7 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{4EDEF6CC-2F5C-439B-BEAF-9D03895099F1}"
 	ProjectSection(SolutionItems) = preProject
 		.editorconfig = .editorconfig
+		.runsettings = .runsettings
 		CHANGELOG.md = CHANGELOG.md
 		LICENSE.txt = LICENSE.txt
 		NexusPage.html = NexusPage.html

Wabbajack/Properties/AssemblyInfo.cs

@@ -49,5 +49,5 @@ using System.Windows;
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.0.9.0")]
-[assembly: AssemblyFileVersion("1.0.9.0")]
+[assembly: AssemblyVersion("1.0.10.0")]
+[assembly: AssemblyFileVersion("1.0.10.0")]

Wabbajack/Views/MainWindow.xaml.cs

@@ -4,6 +4,7 @@ using System.Windows;
 using MahApps.Metro.Controls;
 using Wabbajack.Common;
 using Application = System.Windows.Application;
+using Utils = Wabbajack.Common.Utils;
 
 namespace Wabbajack
 {
@@ -25,10 +26,18 @@ namespace Wabbajack
             };
 
             var appPath = System.Reflection.Assembly.GetExecutingAssembly().Location;
-            if (!ExtensionManager.IsAssociated() || ExtensionManager.NeedsUpdating(appPath))
+            try
             {
-                ExtensionManager.Associate(appPath);
+                if (!ExtensionManager.IsAssociated() || ExtensionManager.NeedsUpdating(appPath))
+                {
+                    ExtensionManager.Associate(appPath);
+                }
             }
+            catch (Exception e)
+            {
+                Utils.Log($"ExtensionManager had an exception:\n{e}");
+            }
+            
 
             Wabbajack.Common.Utils.Log($"Wabbajack Build - {ThisAssembly.Git.Sha}");
 

azure-pipelines.yml

@@ -57,24 +57,24 @@ steps:
 - task: CmdLine@2
   inputs:
     script: 'pip install requests'
-- task: PythonScript@0
-  condition: eq(variables['Build.SourceBranch'], 'refs/heads/master')
-  inputs:
-    scriptSource: 'inline'
-    script: |
-      import requests, sys
-      
-      url = 'https://www.virustotal.com/vtapi/v2/file/scan'
-      
-      params = {'apikey': sys.argv[1]}
-      
-      files = {'file': ('Wabbajack.exe', open(sys.argv[2], 'rb'))}
-      
-      response = requests.post(url, files=files, params=params)
-      
-      print(response.json())
-    arguments: '$(VirusTotalAPIKey) $(System.DefaultWorkingDirectory)/Wabbajack/bin/x64/Debug/Wabbajack.exe'
-
+#- task: PythonScript@0
+#  condition: eq(variables['Build.SourceBranch'], 'refs/heads/master')
+#  inputs:
+#    scriptSource: 'inline'
+#    script: |
+#      import requests, sys
+#      
+#      url = 'https://www.virustotal.com/vtapi/v2/file/scan'
+#      
+#      params = {'apikey': sys.argv[1]}
+#      
+#      files = {'file': ('Wabbajack.exe', open(sys.argv[2], 'rb'))}
+#      
+#      response = requests.post(url, files=files, params=params)
+#      
+#      print(response.json())
+#    arguments: '$(VirusTotalAPIKey) $(System.DefaultWorkingDirectory)/Wabbajack/bin/x64/Debug/Wabbajack.exe'
+#
 - task: PublishBuildArtifacts@1
   condition: eq(variables['Build.SourceBranch'], 'refs/heads/master')
   inputs: