Merge branch 'master' of https://github.com/wabbajack-tools/wabbajack

CHANGELOG.md

@@ -1,5 +1,37 @@
 ### Changelog
 
+#### Version - 2.3.5.1 - 12/23/2020
+* HOTFIX : Recover from errors in the EGS location detector
+
+#### Version - 2.3.5.0 - 12/16/2020
+* Fix tesall.ru download support
+* Implement MechWarrior 5 support as a native compiler game
+* Make the title in the WJ gallery (in app) optional for games that want the title to be in the splash screen
+* Worked a few kinks out of the native game compiler
+
+#### Version - 2.3.4.3 - 12/6/2020
+* Disable the back button during install/compilation
+
+#### Version - 2.3.4.2 - 11/24/2020
+* Add Support for Kingdom Come : Deliverance (via MO2)
+* Several other small bug fixes and deps updates
+
+#### Version - 2.3.4.1 - 11/15/2020
+* Tell the mod updater to use the existing Nexus Client instead of creating a new one
+
+#### Version - 2.3.4.0 - 11/15/2020
+* Removed the internal Manifest Viewer, you can still view the Manifest of any Modlist on the website
+* Improved Nexus warnings about being low on API calls
+* Added marker for "utility modlists" we will expand on this feature further in later releases
+
+#### Version - 2.3.3.0 - 11/5/2020
+* Game file hashes are now stored on Github instead of on the build server
+* Added CLI Verb to produce these hash files for the Github repo
+* When a user runs out of Nexus API calls we no longer bombard the Nexus with download attempts
+* Check API limits before attempting a modlist download
+* Logger is less chatty about recoverable download errors
+* Display integer progress values during install so users know how far along in the process they are #issue-1156
+
 #### Version - 2.3.2.0 - 11/2/2020
 * 7Zip errors now re-hash the extracted file to check for file corruption issues. Should provide
 better feedback in cases that a file is modified after being downloaded (perhaps by a disk failure)

Compression.BSA/Compression.BSA.csproj

@@ -21,8 +21,8 @@
     <ItemGroup>
       <PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
       <PackageReference Include="K4os.Compression.LZ4.Streams" Version="1.2.6" />
-      <PackageReference Include="SharpZipLib" Version="1.3.0" />
-      <PackageReference Include="System.Text.Encoding.CodePages" Version="5.0.0-preview.8.20407.11" />
+      <PackageReference Include="SharpZipLib" Version="1.3.1" />
+      <PackageReference Include="System.Text.Encoding.CodePages" Version="5.0.0" />
     </ItemGroup>
     <ItemGroup>
       <Folder Include="Properties\" />

README.md

@@ -192,12 +192,13 @@ Example structure for a good README:
 Some Modlists that host all their stuff on GitHub:
 
 - the OG GitHub Modlist: [Lotus](https://github.com/erri120/lotus)
+- [Keizaal](https://github.com/PierreDespereaux/Keizaal)
 - [Living Skyrim](https://github.com/ForgottenGlory/Living-Skyrim-2)
 - [Eldersouls](https://github.com/jdsmith2816/eldersouls)
 - [Total Visual Overhaul](https://github.com/NotTotal/Total-Visual-Overhaul)
 - [Serenity](https://github.com/ixanza/serenity)
 - [MOISE](https://github.com/ForgottenGlory/MOISE)
-- [Cupid](https://github.com/ForgottenGlory/MOISE)
+- [Dungeons & Deviousness](https://github.com/ForgottenGlory/Dungeons-Deviousness)
 
 ### Meta Files
 

Wabbajack.CLI/OptionsDefinition.cs

@@ -30,7 +30,12 @@ namespace Wabbajack.CLI
             typeof(HashVariants),
             typeof(ParseMeta),
             typeof(NoPatch),
-            typeof(NexusPermissions)
+            typeof(NexusPermissions),
+            typeof(ExportServerGameFiles),
+            typeof(HashGamefiles),
+            typeof(Backup),
+            typeof(Restore),
+            typeof(PurgeArchive)
         };
     }
 }

Wabbajack.CLI/Verbs/Backup.cs

@@ -0,0 +1,97 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using CommandLine;
+using Wabbajack.Common;
+using Wabbajack.Common.IO;
+
+namespace Wabbajack.CLI.Verbs
+{
+    
+    [Verb("backup", HelpText = @"Copy all encrypted and personal info into a location for transferring to a new machine", Hidden = true)]
+    public class Backup : AVerb
+    {
+        public static Dictionary<string, bool> Keys = new Dictionary<string, bool>()
+        {
+            {"bunnycdn", true},
+            {"nexusapikey", true},
+            {"nexus-cookies", true},
+            {"x-metrics-key", true},
+            {"author-api-key.txt", false}
+        };
+        
+        [Option('o', "output", Required = true, HelpText = @"Output folder for the decrypted data")]
+        public string Output { get; set; } = "";
+
+        public AbsolutePath OutputPath => (AbsolutePath)Output;
+        protected override async Task<ExitCode> Run()
+        {
+            foreach(var (name, encrypted) in Keys)
+            {
+                byte[] data;
+                var src = name.RelativeTo(Consts.LocalAppDataPath);
+                if (!src.Exists)
+                {
+                    Console.WriteLine($"{name} doesn't exist, skipping");
+                    continue;
+                }
+
+                if (encrypted)
+                {
+                    data = await Utils.FromEncryptedData(name);
+                }
+                else
+                {
+                    data = await src.ReadAllBytesAsync();
+                }
+
+                await name.RelativeTo(OutputPath).WriteAllBytesAsync(data);
+            }
+
+            return ExitCode.Ok;
+        }
+    }
+    
+    [Verb("restore", HelpText = @"Copy all encrypted and personal info into a location for transferring to a new machine", Hidden = true)]
+    public class Restore : AVerb
+    {
+        public static Dictionary<string, bool> Keys = new Dictionary<string, bool>()
+        {
+            {"bunnycdn", true},
+            {"nexusapikey", true},
+            {"nexus-cookies", true},
+            {"x-metrics-key", true},
+            {"author-api-key.txt", false}
+        };
+        
+        [Option('i', "input", Required = true, HelpText = @"Input folder for the decrypted data")]
+        public string Input { get; set; } = "";
+
+        public AbsolutePath InputPath => (AbsolutePath)Input;
+        protected override async Task<ExitCode> Run()
+        {
+            foreach(var (name, encrypted) in Keys)
+            {
+                var src = name.RelativeTo(InputPath);
+                if (!src.Exists) 
+                {
+                    Console.WriteLine($"{name} doesn't exist, skipping");
+                    continue;
+                }
+
+                var data = await src.ReadAllBytesAsync(); 
+
+                if (encrypted)
+                {
+                    await data.ToEcryptedData(name);
+                }
+                else
+                {
+                    await name.RelativeTo(Consts.LocalAppDataPath).WriteAllBytesAsync(data);
+                }
+            }
+
+            return ExitCode.Ok;
+        }
+    }
+}

Wabbajack.CLI/Verbs/ExportServerGameFiles.cs

@@ -0,0 +1,32 @@
+using System.Linq;
+using System.Threading.Tasks;
+using CommandLine;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+
+namespace Wabbajack.CLI.Verbs
+{
+    [Verb("export-server-game-files", HelpText = "Exports all the game file data from the server to the output folder")]
+    public class ExportServerGameFiles : AVerb
+    {
+        [Option('o', "output", Required = true, HelpText = @"Output folder in which the files will be placed")]
+        public string OutputFolder { get; set; } = "";
+
+        private AbsolutePath _outputFolder => (AbsolutePath)OutputFolder;
+
+        protected override async Task<ExitCode> Run()
+        {
+            var games = await ClientAPI.GetServerGamesAndVersions();
+            foreach (var (game, version) in games)
+            {
+                Utils.Log($"Exporting {game} {version}");
+                var file = _outputFolder.Combine(game.ToString(), version).WithExtension(new Extension(".json"));
+                file.Parent.CreateDirectory();
+                var files = await ClientAPI.GetGameFilesFromServer(game, version);
+                await files.ToJsonAsync(file, prettyPrint:true);
+            }
+
+            return ExitCode.Ok;
+        }
+    }
+}

Wabbajack.CLI/Verbs/HashGameFiles.cs

@@ -0,0 +1,59 @@
+using System.Linq;
+using System.Threading.Tasks;
+using CommandLine;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+
+namespace Wabbajack.CLI.Verbs
+{
+    [Verb("hash-game-files", HelpText = "Hashes a game's files for inclusion in the public github repo")]
+    public class HashGamefiles : AVerb
+    {
+        [Option('o', "output", Required = true, HelpText = @"Output folder in which the file will be placed")]
+        public string OutputFolder { get; set; } = "";
+
+        private AbsolutePath _outputFolder => (AbsolutePath)OutputFolder;
+        
+        [Option('g', "game", Required = true, HelpText = @"WJ Game to index")]
+        public string Game { get; set; } = "";
+
+        private Game _game => GameRegistry.GetByFuzzyName(Game).Game;
+
+        protected override async Task<ExitCode> Run()
+        {
+            var version = _game.MetaData().InstalledVersion;
+            var file = _outputFolder.Combine(_game.ToString(), version).WithExtension(new Extension(".json"));
+            file.Parent.CreateDirectory();
+
+            using var queue = new WorkQueue();
+            var gameLocation = _game.MetaData().GameLocation();
+            
+            Utils.Log($"Hashing files for {_game} {version}");
+            
+            var indexed = await gameLocation
+                .EnumerateFiles()
+                .PMap(queue, async f =>
+                {
+                    var hash = await f.FileHashCachedAsync();
+                    return new Archive(new GameFileSourceDownloader.State
+                    {
+                        Game = _game, 
+                        GameFile = f.RelativeTo(gameLocation), 
+                        Hash = hash, 
+                        GameVersion = version
+                    })
+                    {
+                        Name = f.FileName.ToString(),
+                        Hash = hash,
+                        Size = f.Size
+                    };
+
+                });
+
+            Utils.Log($"Found and hashed {indexed.Length} files");
+            await indexed.ToJsonAsync(file, prettyPrint: true);
+            return ExitCode.Ok;
+        }
+    }
+}

Wabbajack.CLI/Verbs/PurgeArchive.cs

@@ -0,0 +1,65 @@
+using System.IO.Compression;
+using System.Linq;
+using System.Threading.Tasks;
+using CommandLine;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+
+namespace Wabbajack.CLI.Verbs
+{
+    [Verb("purge-archive", HelpText = "Purges an archive and all directives from a .wabbajack file")]
+    public class PurgeArchive : AVerb
+    {
+        [Option('i', "input", Required = true, HelpText = "Input .wabbajack file")]
+        public string Input { get; set; } = "";
+        private AbsolutePath _Input => (AbsolutePath)Input;
+
+        [Option('o', "output", Required = true, HelpText = "Output .wabbajack file")]
+        public string Output { get; set; } = "";
+        private AbsolutePath _Output => (AbsolutePath)Output;
+
+        [Option('h', "hash", Required = true, HelpText = "Hash to purge")]
+        public string ArchiveHash { get; set; } = "";
+        private Hash _Hash => Hash.Interpret(ArchiveHash);
+
+        protected override async Task<ExitCode> Run()
+        {
+            Utils.Log("Copying .wabbajack file");
+            await _Input.CopyToAsync(_Output);
+
+            Utils.Log("Loading modlist");
+
+            await using var fs = await _Output.OpenWrite();
+            using var ar = new ZipArchive(fs, ZipArchiveMode.Update);
+            ModList modlist;
+            await using (var entry = ar.Entries.First(e => e.Name == "modlist").Open())
+            {
+                modlist = entry.FromJson<ModList>();
+            }
+
+            Utils.Log("Purging archives");
+            modlist.Archives = modlist.Archives.Where(a => a.Hash != _Hash).ToList();
+            modlist.Directives = modlist.Directives.Select(d =>
+            {
+                if (d is FromArchive a)
+                {
+                    if (a.ArchiveHashPath.BaseHash == _Hash) return (false, d);
+                }
+                return (true, d);
+            }).Where(d => d.Item1)
+                .Select(d => d.d)
+                .ToList();
+
+            Utils.Log("Writing modlist");
+            
+            await using (var entry = ar.Entries.First(e => e.Name == "modlist").Open())
+            {
+                entry.SetLength(0);
+                entry.Position = 0;
+                modlist.ToJson(entry);
+            }
+
+            return ExitCode.Ok;
+        }
+    }
+}

Wabbajack.CLI/Wabbajack.CLI.csproj

@@ -6,8 +6,8 @@
         <AssemblyName>wabbajack-cli</AssemblyName>
         <Company>Wabbajack</Company>
         <Platforms>x64</Platforms>
-        <AssemblyVersion>2.3.2.0</AssemblyVersion>
-        <FileVersion>2.3.2.0</FileVersion>
+        <AssemblyVersion>2.3.5.1</AssemblyVersion>
+        <FileVersion>2.3.5.1</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>An automated ModList installer</Description>
         <PublishReadyToRun>true</PublishReadyToRun>        
@@ -19,9 +19,9 @@
     <ItemGroup>
       <PackageReference Include="CommandLineParser" Version="2.9.0-preview1" />
       <PackageReference Include="F23.StringSimilarity" Version="3.1.0" />
-      <PackageReference Include="Markdig" Version="0.22.0" />
-      <PackageReference Include="System.Reactive" Version="4.4.1" />
-      <PackageReference Include="System.Reactive.Linq" Version="4.4.1" />
+      <PackageReference Include="Markdig" Version="0.22.1" />
+      <PackageReference Include="System.Reactive" Version="5.0.0" />
+      <PackageReference Include="System.Reactive.Linq" Version="5.0.0" />
     </ItemGroup>
 
     <ItemGroup>

Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj

@@ -6,7 +6,7 @@
         <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
     </PropertyGroup>
     <ItemGroup>
-      <PackageReference Include="System.Reactive" Version="4.4.1" />
+      <PackageReference Include="System.Reactive" Version="5.0.0" />
       <PackageReference Include="System.Threading.Tasks.Extensions" Version="4.6.0-preview.18571.3" />
     </ItemGroup>
     <ItemGroup>

Wabbajack.Common/AsyncBlockingCollection.cs

@@ -30,7 +30,7 @@ namespace Wabbajack.Common
                 }
 
                 if (DateTime.Now - startTime > timeout || token.IsCancellationRequested || isDisposed)
-                    return (false, default);
+                    return (false, default)!;
                 await Task.Delay(100);
             }
         }

Wabbajack.Common/Consts.cs

@@ -84,6 +84,7 @@ namespace Wabbajack.Common
 
         public static string ServerWhitelistURL = "https://raw.githubusercontent.com/wabbajack-tools/opt-out-lists/master/ServerWhitelist.yml";
         public static string ModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/modlists.json";
+        public static string UtilityModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/utility_modlists.json";
         public static string UnlistedModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/unlisted_modlists.json";
         public static string ModlistSummaryURL = "http://build.wabbajack.org/lists/status.json";
         public static string UserAgent

Wabbajack.Common/Error States/GetResponse.cs

@@ -33,7 +33,7 @@ namespace Wabbajack
             string? reason = null,
             Exception? ex = null)
         {
-            Value = val;
+            Value = val!;
             Succeeded = succeeded;
             _reason = reason ?? string.Empty;
             Exception = ex;
@@ -128,7 +128,7 @@ namespace Wabbajack
 
         public static GetResponse<T> Create(bool successful, T val = default(T), string? reason = null)
         {
-            return new GetResponse<T>(successful, val, reason);
+            return new GetResponse<T>(successful, val!, reason);
         }
         #endregion
     }

Wabbajack.Common/Extensions/RxExt.cs

@@ -103,7 +103,7 @@ namespace Wabbajack
                     {
                         // We have another value that came in to fire.
                         // Reregister for callback
-                        dueTimeDisposable.Disposable = scheduler.Schedule(interval, internalCallback);
+                        dueTimeDisposable.Disposable = scheduler!.Schedule(interval, internalCallback);
                         o.OnNext(value!);
                         value = default;
                         hasValue = false;
@@ -204,7 +204,7 @@ namespace Wabbajack
                 var prev = prevStorage;
                 prevStorage = i;
                 return (prev, i);
-            });
+            })!;
         }
 
         public static IObservable<T> DelayInitial<T>(this IObservable<T> source, TimeSpan delay, IScheduler scheduler)

Wabbajack.Common/GameMetaData.cs

@@ -4,6 +4,7 @@ using System.ComponentModel;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
+using System.Net.Http.Headers;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
 using Wabbajack.Common.StoreHandlers;
@@ -37,7 +38,9 @@ namespace Wabbajack.Common
         Dishonored,
         Witcher3,
         [Description("Stardew Valley")]
-        StardewValley
+        StardewValley,
+        KingdomComeDeliverance,
+        MechWarrior5Mercenaries
     }
 
     public static class GameExtensions
@@ -66,6 +69,8 @@ namespace Wabbajack.Common
 
         // to get gog ids: https://www.gogdb.org
         public List<int>? GOGIDs { get; internal set; }
+        
+        public List<string> EpicGameStoreIDs { get; internal set; } = new List<string>();
 
         // to get BethNet IDs: check the registry
         public int BethNetID { get; internal set; }
@@ -98,7 +103,16 @@ namespace Wabbajack.Common
                 if (MainExecutable == null)
                     throw new NotImplementedException();
 
-                return FileVersionInfo.GetVersionInfo((string)gameLoc.Combine(MainExecutable)).ProductVersion;
+                var info = FileVersionInfo.GetVersionInfo((string)gameLoc.Combine(MainExecutable));
+                var version =  info.ProductVersion;
+                if (string.IsNullOrWhiteSpace(version))
+                {
+                    version =
+                        $"{info.ProductMajorPart}.{info.ProductMinorPart}.{info.ProductBuildPart}.{info.ProductPrivatePart}";
+                    return version;
+                }
+
+                return version;
             }
         }
 
@@ -490,6 +504,40 @@ namespace Wabbajack.Common
                     },
                     MainExecutable = "Stardew Valley.exe"
                 }
+            },
+            {
+                Game.KingdomComeDeliverance, new GameMetaData
+                {
+                    Game = Game.KingdomComeDeliverance,
+                    NexusName = "kingdomcomedeliverance",
+                    MO2Name = "Kingdom Come: Deliverance",
+                    MO2ArchiveName = "kingdomcomedeliverance",
+                    NexusGameId = 2298,
+                    SteamIDs = new List<int>{379430},
+                    IsGenericMO2Plugin = true,
+                    RequiredFiles = new List<string>
+                    {
+                        @"bin\Win64\KingdomCome.exe"
+                    },
+                    MainExecutable = @"bin\Win64\KingdomCome.exe"
+                }
+            },
+            {
+                Game.MechWarrior5Mercenaries, new GameMetaData
+                {
+                    Game = Game.MechWarrior5Mercenaries,
+                    NexusName = "mechwarrior5mercenaries",
+                    MO2Name = "Mechwarrior 5: Mercenaries",
+                    MO2ArchiveName = "mechwarrior5mercenaries",
+                    NexusGameId = 3099,
+                    EpicGameStoreIDs = new List<string> {"9fd39d8ac72946a2a10a887ce86e6c35"},
+                    IsGenericMO2Plugin = true,
+                    RequiredFiles = new List<string>
+                    {
+                        @"MW5Mercs\Binaries\Win64\MechWarrior-Win64-Shipping.exe"
+                    },
+                    MainExecutable = @"MW5Mercs\Binaries\Win64\MechWarrior-Win64-Shipping.exe"
+                }
             }
         };
 

Wabbajack.Common/Json.cs

@@ -36,6 +36,15 @@ namespace Wabbajack.Common
                 Converters = Converters,
                 DateTimeZoneHandling = DateTimeZoneHandling.Utc
             };
+        
+        public static JsonSerializerSettings JsonSettingsPretty  =>
+            new JsonSerializerSettings {
+                TypeNameHandling = TypeNameHandling.Objects,
+                SerializationBinder = new JsonNameSerializationBinder(),
+                Converters = Converters,
+                DateTimeZoneHandling = DateTimeZoneHandling.Utc,
+                Formatting = Formatting.Indented
+            };
 
         public static JsonSerializerSettings GenericJsonSettings =>
             new JsonSerializerSettings
@@ -51,18 +60,27 @@ namespace Wabbajack.Common
             File.WriteAllText(filename, JsonConvert.SerializeObject(obj, Formatting.Indented, JsonSettings));
         }
         
-        public static void ToJson<T>(this T obj, Stream stream, bool useGenericSettings = false)
+        public static void ToJson<T>(this T obj, Stream stream, bool useGenericSettings = false, bool prettyPrint = false)
         {
             using var tw = new StreamWriter(stream, Encoding.UTF8, bufferSize: 1024, leaveOpen: true);
             using var writer = new JsonTextWriter(tw);
-            var ser = JsonSerializer.Create(useGenericSettings ? GenericJsonSettings : JsonSettings);
+
+            JsonSerializerSettings settings = (useGenericSettings, prettyPrint) switch
+            {
+                (true, true) => GenericJsonSettings,
+                (false, true) => JsonSettingsPretty,
+                (false, false) => JsonSettings,
+                (true, false) => GenericJsonSettings
+            };
+
+            var ser = JsonSerializer.Create(settings);
             ser.Serialize(writer, obj);
         }
         
-        public static async ValueTask ToJsonAsync<T>(this T obj, AbsolutePath path, bool useGenericSettings = false)
+        public static async ValueTask ToJsonAsync<T>(this T obj, AbsolutePath path, bool useGenericSettings = false, bool prettyPrint = false)
         {
             await using var fs = await path.Create();
-            obj.ToJson(fs, useGenericSettings);
+            obj.ToJson(fs, useGenericSettings, prettyPrint: prettyPrint);
         }
 
         public static string ToJson<T>(this T obj, bool useGenericSettings = false)

Wabbajack.Common/Paths/AbsolutePath.cs

@@ -95,7 +95,7 @@ namespace Wabbajack.Common
         public ValueTask<FileStream> OpenWrite()
         {
             var path = _path;
-            return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () => File.OpenWrite(path));
+            return CircuitBreaker.WithAutoRetryAsync<FileStream, IOException>(async () => File.Open(path, FileMode.OpenOrCreate, FileAccess.ReadWrite));
         }
 
         public async Task WriteAllTextAsync(string text)
@@ -275,7 +275,7 @@ namespace Wabbajack.Common
 
         public bool InFolder(AbsolutePath folder)
         {
-            return _path.StartsWith(folder._path + Path.DirectorySeparator);
+            return _path.StartsWith(folder._path + Path.DirectorySeparator, StringComparison.OrdinalIgnoreCase);
         }
 
         public async Task<byte[]> ReadAllBytesAsync()
@@ -386,7 +386,7 @@ namespace Wabbajack.Common
 
         public int CompareTo(AbsolutePath other)
         {
-            return string.Compare(_path, other._path, StringComparison.Ordinal);
+            return string.Compare(_path, other._path, StringComparison.OrdinalIgnoreCase);
         }
 
         public string ReadAllText()

Wabbajack.Common/Paths/RelativePath.cs

@@ -136,12 +136,12 @@ namespace Wabbajack.Common
 
         public bool StartsWith(string s)
         {
-            return _path.StartsWith(s);
+            return _path.StartsWith(s, StringComparison.OrdinalIgnoreCase);
         }
         
         public bool StartsWith(RelativePath s)
         {
-            return _path.StartsWith(s._path);
+            return _path.StartsWith(s._path, StringComparison.OrdinalIgnoreCase);
         }
 
         public RelativePath Combine(params RelativePath[] paths )
@@ -156,7 +156,7 @@ namespace Wabbajack.Common
 
         public int CompareTo(RelativePath other)
         {
-            return string.Compare(_path, other._path, StringComparison.Ordinal);
+            return string.Compare(_path, other._path, StringComparison.OrdinalIgnoreCase);
         }
     }
 }

Wabbajack.Common/StatusUpdateTracker.cs

@@ -7,7 +7,7 @@ namespace Wabbajack.Common
     public class StatusUpdateTracker
     {
         private Subject<string> _stepName = new Subject<string>();
-        public IObservable<string> StepName => _stepName;
+        public IObservable<string> StepName => _stepName.Debounce(TimeSpan.FromMilliseconds(100));
 
         private Subject<int> _step = new Subject<int>();
         public IObservable<int> Step => _step;
@@ -20,6 +20,7 @@ namespace Wabbajack.Common
 
         private int _internalCurrentStep;
         private int _internalMaxStep;
+        private string _currentStepName = "";
 
         public StatusUpdateTracker(int maxStep)
         {
@@ -33,10 +34,11 @@ namespace Wabbajack.Common
 
         public void NextStep(string name)
         {
+            _currentStepName = name;
             _internalCurrentStep += 1;
             Utils.Log(name);
             _step.OnNext(_internalCurrentStep);
-            _stepName.OnNext(name);
+            _stepName.OnNext($"({_internalCurrentStep}/{_internalMaxStep}) {_currentStepName}");
             MakeUpdate(Percent.Zero);
         }
 
@@ -62,6 +64,7 @@ namespace Wabbajack.Common
         public void MakeUpdate(int max, int curr)
         {
             MakeUpdate(Percent.FactoryPutInRange(curr, max == 0 ? 1 : max));
+            _stepName.OnNext($"({_internalCurrentStep}/{_internalMaxStep}) {_currentStepName}, {curr} of {max}");
         }
     }
 

Wabbajack.Common/StoreHandlers/EpicGameStoreHandler.cs

@@ -0,0 +1,86 @@
+using System;
+using System.Linq;
+using Microsoft.Win32;
+
+namespace Wabbajack.Common.StoreHandlers
+{
+    public class EpicGameStoreHandler : AStoreHandler
+    {
+        public override StoreType Type { get; internal set; }
+
+        public string BaseRegKey = @"SOFTWARE\Epic Games\EOS";
+        public override bool Init()
+        {
+            return true;
+        }
+
+        public override bool LoadAllGames()
+        {
+            try
+            {
+                using var eosKey = Registry.CurrentUser.OpenSubKey(BaseRegKey);
+                if (eosKey == null)
+                {
+                    Utils.Log("Epic Game Store is not installed");
+                    return false;
+                }
+
+                var name = eosKey.GetValue("ModSdkMetadataDir");
+                if (name == null)
+                {
+                    Utils.Log("Registry key entry does not exist for Epic Game store");
+                    return false;
+                }
+
+                var byID = GameRegistry.Games.SelectMany(g => g.Value.EpicGameStoreIDs
+                        .Select(id => (id, g.Value.Game)))
+                    .GroupBy(t => t.id)
+                    .ToDictionary(t => t.Key, t => t.First().Game);
+
+                foreach (var itm in ((AbsolutePath)(string)(name!)).EnumerateFiles(false, "*.item"))
+                {
+                    var item = itm.FromJson<EpicGameItem>();
+                    Utils.Log($"Found Epic Game Store Game: {item.DisplayName} at {item.InstallLocation}");
+
+                    if (byID.TryGetValue(item.CatalogItemId, out var game))
+                    {
+                        Games.Add(new EpicStoreGame(game, item));
+                    }
+
+                }
+            }
+            catch (NullReferenceException ex)
+            {
+                Utils.Log("Epic Game Store is does not appear to be installed");
+                return false;
+            }
+
+            return true;
+        }
+
+        public class EpicStoreGame : AStoreGame
+        {
+            public EpicStoreGame(Game game, EpicGameItem item)
+            {
+                Type = StoreType.EpicGameStore;
+                Game = game;
+                Path = (AbsolutePath)item.InstallLocation;
+                Name = game.MetaData().HumanFriendlyGameName;
+                
+            }
+
+            public override Game Game { get; internal set; }
+            public override StoreType Type { get; internal set; }
+        }
+
+        public class EpicGameItem
+        {
+            public string DisplayName { get; set; } = "";
+            public string InstallationGuid { get; set; } = "";
+            public string CatalogItemId { get; set; } = "";
+            public string CatalogNamespace { get; set; } = "";
+            public string InstallSessionId { get; set; } = "";
+            public string InstallLocation { get; set; } = "";
+        }
+    }
+}

Wabbajack.Common/StoreHandlers/StoreHandler.cs

@@ -9,7 +9,8 @@ namespace Wabbajack.Common.StoreHandlers
     {
         STEAM,
         GOG,
-        BethNet
+        BethNet,
+        EpicGameStore
     }
 
     public class StoreHandler
@@ -25,6 +26,9 @@ namespace Wabbajack.Common.StoreHandlers
 
         private static readonly Lazy<BethNetHandler> _bethNetHandler = new Lazy<BethNetHandler>(() => new BethNetHandler());
         public BethNetHandler BethNetHandler = _bethNetHandler.Value;
+        
+        private static readonly Lazy<EpicGameStoreHandler> _epicGameStoreHandler = new Lazy<EpicGameStoreHandler>(() => new EpicGameStoreHandler());
+        public EpicGameStoreHandler EpicGameStoreHandler = _epicGameStoreHandler.Value;
 
         public List<AStoreGame> StoreGames;
 
@@ -67,6 +71,18 @@ namespace Wabbajack.Common.StoreHandlers
             {
                 Utils.Error(new StoreException("Could not Init the BethNetHandler, check previous error messages!"));
             }
+            
+            if (EpicGameStoreHandler.Init())
+            {
+                if (EpicGameStoreHandler.LoadAllGames())
+                    StoreGames.AddRange(EpicGameStoreHandler.Games);
+                else
+                    Utils.Error(new StoreException("Could not load all Games from the EpicGameStoreHandler, check previous error messages!"));
+            }
+            else
+            {
+                Utils.Error(new StoreException("Could not Init the EpicGameStoreHandler, check previous error messages!"));
+            }
         }
 
         public AbsolutePath? TryGetGamePath(Game game)

Wabbajack.Common/Utils.cs

@@ -866,7 +866,7 @@ namespace Wabbajack.Common
 
         public static bool IsInPath(this string path, string parent)
         {
-            return path.ToLower().TrimEnd('\\').StartsWith(parent.ToLower().TrimEnd('\\') + "\\");
+            return path.ToLower().TrimEnd('\\').StartsWith(parent.ToLower().TrimEnd('\\') + "\\", StringComparison.OrdinalIgnoreCase);
         }
         
         public static async Task CopyToLimitAsync(this Stream frm, Stream tw, long limit)

Wabbajack.Common/Wabbajack.Common.csproj

@@ -49,19 +49,19 @@
     </ItemGroup>
     <ItemGroup>
         <PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
-        <PackageReference Include="HtmlAgilityPack" Version="1.11.26" />
+        <PackageReference Include="HtmlAgilityPack" Version="1.11.28" />
         <PackageReference Include="ini-parser-netstandard" Version="2.5.2" />
-        <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0-preview.8.20407.11" />
+        <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
         <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
         <PackageReference Include="Octodiff" Version="1.2.1" />
         <PackageReference Include="RocksDbNative" Version="6.2.2" />
         <PackageReference Include="RocksDbSharp" Version="6.2.2" />
-        <PackageReference Include="SharpZipLib" Version="1.3.0" />
+        <PackageReference Include="SharpZipLib" Version="1.3.1" />
         <PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
         <PackageReference Include="System.Net.Http" Version="4.3.4" />
-        <PackageReference Include="System.Reactive" Version="4.4.1" />
-        <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="5.0.0-preview.8.20407.11" />
-        <PackageReference Include="System.Security.Principal.Windows" Version="5.0.0-preview.8.20407.11" />
+        <PackageReference Include="System.Reactive" Version="5.0.0" />
+        <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="5.0.0" />
+        <PackageReference Include="System.Security.Principal.Windows" Version="5.0.0" />
         <PackageReference Include="YamlDotNet" Version="8.1.2" />
     </ItemGroup>
     <ItemGroup>

Wabbajack.Launcher/Wabbajack.Launcher.csproj

@@ -4,8 +4,8 @@
         <OutputType>WinExe</OutputType>
         <TargetFramework>netcoreapp3.1</TargetFramework>
         <UseWPF>true</UseWPF>
-        <AssemblyVersion>2.3.2.0</AssemblyVersion>
-        <FileVersion>2.3.2.0</FileVersion>
+        <AssemblyVersion>2.3.5.1</AssemblyVersion>
+        <FileVersion>2.3.5.1</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Application Launcher</Description>
         <PublishReadyToRun>true</PublishReadyToRun>

Wabbajack.Lib/AInstaller.cs

@@ -181,7 +181,7 @@ namespace Wabbajack.Lib
                         await file.To.RelativeTo(OutputFolder).Compact(FileCompaction.Algorithm.XPRESS16K);
                     }
                 }
-            }, tempFolder: OutputFolder);
+            }, tempFolder: OutputFolder, updateTracker: UpdateTracker);
         }
 
         public async Task DownloadArchives()
@@ -195,6 +195,12 @@ namespace Wabbajack.Lib
 
             await Task.WhenAll(dispatchers.Select(d => d.Prepare()));
 
+            var nexusDownloader = dispatchers.OfType<NexusDownloader>().FirstOrDefault();
+            if (nexusDownloader != null && !await nexusDownloader.HaveEnoughAPICalls(missing))
+            {
+                throw new Exception($"Not enough Nexus API calls to download this list, please try again after midnight GMT when your API limits reset");
+            }
+
             await DownloadMissingArchives(missing);
         }
 
@@ -202,6 +208,7 @@ namespace Wabbajack.Lib
         {
             if (download)
             {
+                var result = SendDownloadMetrics(missing);
                 foreach (var a in missing.Where(a => a.State.GetType() == typeof(ManualDownloader.State)))
                 {
                     var outputPath = DownloadFolder.Combine(a.Name);
@@ -210,8 +217,9 @@ namespace Wabbajack.Lib
             }
 
             DesiredThreads.OnNext(DownloadThreads);
+
             await missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State))
-                .PMap(Queue, async archive =>
+                .PMap(Queue, UpdateTracker, async archive =>
                 {
                     Info($"Downloading {archive.Name}");
                     var outputPath = DownloadFolder.Combine(archive.Name);
@@ -235,6 +243,15 @@ namespace Wabbajack.Lib
 
         }
 
+        private async Task SendDownloadMetrics(List<Archive> missing)
+        {
+            var grouped = missing.GroupBy(m => m.State.GetType());
+            foreach (var group in grouped)
+            {
+                await Metrics.Send($"downloading_{group.Key.FullName!.Split(".").Last().Split("+").First()}", group.Sum(g => g.Size).ToString());
+            }
+        }
+
         public async Task<bool> DownloadArchive(Archive archive, bool download, AbsolutePath? destination = null)
         {
             try
@@ -268,7 +285,7 @@ namespace Wabbajack.Lib
             
             var hashResults = await 
                 toHash
-                .PMap(Queue, async e => (await e.FileHashCachedAsync(), e)); 
+                .PMap(Queue, UpdateTracker,async e => (await e.FileHashCachedAsync(), e)); 
             
             HashedArchives.SetTo(hashResults
                 .OrderByDescending(e => e.Item2.LastModified)
@@ -385,9 +402,6 @@ namespace Wabbajack.Lib
                 var path = OutputFolder.Combine(d.To);
                 if (!existingfiles.Contains(path)) return null;
 
-                if (path.Size != d.Size) return null;
-                Status($"Optimizing {d.To}");
-                
                 return await path.FileHashCachedAsync() == d.Hash ? d : null;
             }))
               .Do(d =>

Wabbajack.Lib/ClientAPI.cs

@@ -157,9 +157,7 @@ using Wabbajack.Lib.Downloaders;
                 return new Archive[0];
             var client = await GetClient();
             var metaData = game.MetaData();
-            var results =
-                await client.GetJsonAsync<Archive[]>(
-                    $"{Consts.WabbajackBuildServerUri}game_files/{game}/{metaData.InstalledVersion}");
+            var results = await GetGameFilesFromGithub(game, metaData.InstalledVersion);
 
             return (await results.PMap(queue, async file => (await file.State.Verify(file), file))).Where(f => f.Item1)
                 .Select(f =>
@@ -169,6 +167,22 @@ using Wabbajack.Lib.Downloaders;
                 })
                 .ToArray();
         }
+        
+        public static async Task<Archive[]> GetGameFilesFromGithub(Game game, string version)
+        {
+            var url =
+                $"https://raw.githubusercontent.com/wabbajack-tools/indexed-game-files/master/{game}/{version}.json";
+            Utils.Log($"Loading game file definition from {url}");
+            var client = await GetClient();
+            return await client.GetJsonAsync<Archive[]>(url);
+        }
+
+        public static async Task<Archive[]> GetGameFilesFromServer(Game game, string version)
+        {
+            var client = await GetClient();
+            return await client.GetJsonAsync<Archive[]>(
+                $"{Consts.WabbajackBuildServerUri}game_files/{game}/{version}");
+        }
 
         public static async Task<AbstractDownloadState?> InferDownloadState(Hash hash)
         {
@@ -250,7 +264,7 @@ using Wabbajack.Lib.Downloaders;
                     await client.GetStringAsync($"{Consts.WabbajackBuildServerUri}mirror/{archiveHash.ToHex()}");
                 return new Uri(result);
             }
-            catch (HttpException ex)
+            catch (HttpException)
             {
                 return null;
             }
@@ -262,5 +276,14 @@ using Wabbajack.Lib.Downloaders;
             return await client.GetJsonAsync<Helpers.Cookie[]>(
                 $"{Consts.WabbajackBuildServerUri}site-integration/auth-info/{key}");
         }
+
+        public static async Task<IEnumerable<(Game, string)>> GetServerGamesAndVersions()
+        {
+            var client = await GetClient();
+            var results =
+                await client.GetJsonAsync<(Game, string)[]>(
+                    $"{Consts.WabbajackBuildServerUri}game_files");
+            return results;
+        }
     }
 }

Wabbajack.Lib/CompilationSteps/IgnoreGameFiles.cs

@@ -15,7 +15,7 @@ namespace Wabbajack.Lib.CompilationSteps
 
         public override async ValueTask<Directive?> Run(RawSourceFile source)
         {
-            if (!((string)source.Path).StartsWith(_startDir)) return null;
+            if (!((string)source.Path).StartsWith(_startDir, System.StringComparison.OrdinalIgnoreCase)) return null;
             var i = source.EvolveTo<IgnoredDirectly>();
             i.Reason = "Default game file";
             return i;

Wabbajack.Lib/CompilationSteps/IncludePatches.cs

@@ -16,7 +16,6 @@ namespace Wabbajack.Lib.CompilationSteps
         private readonly Dictionary<RelativePath, IGrouping<RelativePath, VirtualFile>> _indexed;
         private VirtualFile? _bsa;
         private Dictionary<RelativePath, IEnumerable<VirtualFile>> _indexedByName;
-        private ACompiler _compiler;
         private bool _isGenericGame;
 
         public IncludePatches(ACompiler compiler, VirtualFile? constructingFromBSA = null) : base(compiler)

Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs

@@ -68,6 +68,16 @@ namespace Wabbajack.Lib.Downloaders
                     IsAttachment = true
                 };
             }
+            
+            
+            if (url.PathAndQuery.StartsWith("/files/download/") && long.TryParse(url.PathAndQuery.Split("/").Last(), out var fileId))
+            {
+                return new TState
+                {
+                    FullURL = url.ToString(),
+                    IsAttachment = true
+                };
+            }
            
             if (!url.PathAndQuery.StartsWith("/files/file/"))
             {
@@ -76,6 +86,7 @@ namespace Wabbajack.Lib.Downloaders
                 absolute = false;
             }
 
+
             var id = HttpUtility.ParseQueryString(url.Query)["r"];
             var file = absolute
                 ? url.AbsolutePath.Split('/').Last(s => s != "")

Wabbajack.Lib/Downloaders/NexusDownloader.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.ComponentModel.DataAnnotations;
 using System.Linq;
 using System.Reactive;
@@ -142,6 +143,16 @@ namespace Wabbajack.Lib.Downloaders
             }
         }
 
+        public async Task<bool> HaveEnoughAPICalls(IEnumerable<Archive> archives)
+        {
+            if (await Client!.IsPremium())
+                return true;
+            
+            var count = archives.Select(a => a.State).OfType<State>().Count();
+
+            return count < Client!.RemainingAPICalls;
+        }
+
         [JsonName("NexusDownloader")]
         public class State : AbstractDownloadState, IMetaState, IUpgradingState
         {
@@ -189,14 +200,16 @@ namespace Wabbajack.Lib.Downloaders
                     var client = await NexusApiClient.Get();
                     url = await client.GetNexusDownloadLink(this);
                 }
-                catch (Exception ex)
+                catch (NexusAPIQuotaExceeded ex)
+                {
+                    Utils.Log(ex.ExtendedDescription);
+                    throw;
+                }
+                catch (Exception)
                 {
-                    Utils.Log($"{a.Name} - Error getting Nexus download URL - {ex.Message}");
                     return false;
                 }
 
-                Utils.Log($"Downloading Nexus Archive - {a.Name} - {Game} - {ModID} - {FileID}");
-
                 return await new HTTPDownloader.State(url).Download(a, destination);
             }
 
@@ -239,7 +252,11 @@ namespace Wabbajack.Lib.Downloaders
 
             public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
             {
-                var client = await NexusApiClient.Get();
+                var client = DownloadDispatcher.GetInstance<NexusDownloader>().Client ?? await NexusApiClient.Get();
+                await client.IsPremium();
+                
+                if (client.RemainingAPICalls <= 0)
+                    throw new NexusAPIQuotaExceeded();
 
                 var mod = await client.GetModInfo(Game, ModID);
                 if (!mod.available) 
@@ -274,6 +291,7 @@ namespace Wabbajack.Lib.Downloaders
                     return (newArchive, new TempFile());
                 }
 
+                Utils.Log($"Downloading possible upgrade {newArchive.State.PrimaryKeyString}");
                 var tempFile = new TempFile();
                  
                 await newArchive.State.Download(newArchive, tempFile.Path);
@@ -281,6 +299,8 @@ namespace Wabbajack.Lib.Downloaders
                 newArchive.Size = tempFile.Path.Size;
                 newArchive.Hash = await tempFile.Path.FileHashAsync();
 
+                Utils.Log($"Possible upgrade {newArchive.State.PrimaryKeyString} downloaded");
+
                 return (newArchive, tempFile);
             }
 

Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs

@@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders
         {
             {"wabbajack.b-cdn.net", "authored-files.wabbajack.org"},
             {"wabbajack-mirror.b-cdn.net", "mirror.wabbajack.org"},
-            {"wabbajack-patches.b-cdn-net", "patches.wabbajack.org"},
+            {"wabbajack-patches.b-cdn.net", "patches.wabbajack.org"},
             {"wabbajacktest.b-cdn.net", "test-files.wabbajack.org"}
         };
 

Wabbajack.Lib/Extensions/ReactiveUIExt.cs

@@ -105,7 +105,7 @@ namespace Wabbajack
         {
             return source
                 .ToProperty(vm, property, initialValue, deferSubscription, RxApp.MainThreadScheduler)
-                .DisposeWith(vm.CompositeDisposable);
+                .DisposeWith(vm.CompositeDisposable)!;
         }
 
         public static void ToGuiProperty<TRet>(
@@ -116,7 +116,7 @@ namespace Wabbajack
             TRet initialValue = default,
             bool deferSubscription = false)
         {
-            source.ToProperty(vm, property, out result, initialValue, deferSubscription, RxApp.MainThreadScheduler)
+            source.ToProperty(vm, property, out result!, initialValue, deferSubscription, RxApp.MainThreadScheduler)
                 .DisposeWith(vm.CompositeDisposable);
         }
 

Wabbajack.Lib/MO2Compiler.cs

@@ -15,8 +15,6 @@ namespace Wabbajack.Lib
 {
     public class MO2Compiler : ACompiler
     {
-        private AbsolutePath _mo2DownloadsFolder;
-        
         public MO2Compiler(AbsolutePath sourcePath, AbsolutePath downloadsPath, string mo2Profile, AbsolutePath outputFile)
             : base(21, mo2Profile, sourcePath, downloadsPath, outputFile)
         {
@@ -202,7 +200,7 @@ namespace Wabbajack.Lib
             // Find all Downloads
             IndexedArchives = (await DownloadsPath.EnumerateFiles()
                 .Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
-                .PMap(Queue,
+                .PMap(Queue, UpdateTracker,
                     async f => new IndexedArchive(VFS.Index.ByRootPath[f])
                     {
                         Name = (string)f.FileName,

Wabbajack.Lib/MO2Installer.cs

@@ -257,7 +257,7 @@ namespace Wabbajack.Lib
         private async Task InstallIncludedDownloadMetas()
         {
             await ModList.Archives
-                   .PMap(Queue, async archive =>
+                   .PMap(Queue, UpdateTracker, async archive =>
                    {
                        if (HashedArchives.TryGetValue(archive.Hash, out var paths))
                        {
@@ -313,7 +313,7 @@ namespace Wabbajack.Lib
                 var bsaSize = bsa.FileStates.Select(state => sourceDir.Combine(state.Path).Size).Sum();
 
                 await using var a = await bsa.State.MakeBuilder(bsaSize);
-                var streams = await bsa.FileStates.PMap(Queue, async state =>
+                var streams = await bsa.FileStates.PMap(Queue, UpdateTracker, async state =>
                 {
                     Status($"Adding {state.Path} to BSA");
                     var fs = await sourceDir.Combine(state.Path).OpenRead();
@@ -344,7 +344,7 @@ namespace Wabbajack.Lib
             Info("Writing inline files");
             await ModList.Directives
                 .OfType<InlineFile>()
-                .PMap(Queue, async directive =>
+                .PMap(Queue, UpdateTracker, async directive =>
                 {
                     Status($"Writing included file {directive.To}");
                     var outPath = OutputFolder.Combine(directive.To);

Wabbajack.Lib/ModListRegistry/ModListMetadata.cs

@@ -34,6 +34,15 @@ namespace Wabbajack.Lib.ModListRegistry
 
         [JsonProperty("nsfw")]
         public bool NSFW { get; set; }
+        
+        [JsonProperty("utility_list")]
+        public bool UtilityList { get; set; }
+        
+        [JsonProperty("image_contains_title")]
+        public bool ImageContainsTitle { get; set; }
+
+        [JsonProperty("force_down")]
+        public bool ForceDown { get; set; }
 
         [JsonProperty("links")]
         public LinksObject Links { get; set; } = new LinksObject();
@@ -65,9 +74,11 @@ namespace Wabbajack.Lib.ModListRegistry
             var client = new Http.Client();
             Utils.Log("Loading ModLists from GitHub");
             var metadataResult = client.GetStringAsync(Consts.ModlistMetadataURL);
+            var utilityResult = client.GetStringAsync(Consts.UtilityModlistMetadataURL);
             var summaryResult = client.GetStringAsync(Consts.ModlistSummaryURL);
 
             var metadata = (await metadataResult).FromJsonString<List<ModlistMetadata>>();
+            metadata = metadata.Concat((await utilityResult).FromJsonString<List<ModlistMetadata>>()).ToList();
             try
             {
                 var summaries = (await summaryResult).FromJsonString<List<ModListSummary>>().ToDictionary(d => d.MachineURL);

Wabbajack.Lib/NexusApi/INexusApi.cs

@@ -13,5 +13,7 @@ namespace Wabbajack.Lib.NexusApi
         public Task<UserStatus> GetUserStatus();
         public Task<bool> IsPremium();
         public bool IsAuthenticated { get; }
+        
+        public int RemainingAPICalls { get; }
     }
 }

Wabbajack.Lib/NexusApi/NexusApi.cs

@@ -27,6 +27,7 @@ namespace Wabbajack.Lib.NexusApi
         public static string? ApiKey { get; set; }
 
         public bool IsAuthenticated => ApiKey != null;
+        public int RemainingAPICalls => Math.Max(HourlyRemaining, DailyRemaining);
 
         private Task<UserStatus>? _userStatus;
         public Task<UserStatus> UserStatus
@@ -70,7 +71,7 @@ namespace Wabbajack.Lib.NexusApi
                 {
                     return env_key;
                 }
-
+                
                 return await RequestAndCacheAPIKey();
             }
         }
@@ -154,7 +155,12 @@ namespace Wabbajack.Lib.NexusApi
         public async Task<UserStatus> GetUserStatus()
         {
             var url = "https://api.nexusmods.com/v1/users/validate.json";
-            return await Get<UserStatus>(url);
+            var result = await Get<UserStatus>(url);
+
+            Utils.Log($"Logged into the nexus as {result.name}");
+            Utils.Log($"Nexus calls remaining: {DailyRemaining} daily, {HourlyRemaining} hourly");
+
+            return result;
         }
 
         public async Task<(int, int)> GetRemainingApiCalls()
@@ -213,24 +219,12 @@ namespace Wabbajack.Lib.NexusApi
 
         }
 
-
         protected virtual async Task UpdateRemaining(HttpResponseMessage response)
         {
             try
             {
-                var oldDaily = _dailyRemaining;
-                var oldHourly = _hourlyRemaining;
-                var dailyRemaining = int.Parse(response.Headers.GetValues("x-rl-daily-remaining").First());
-                var hourlyRemaining = int.Parse(response.Headers.GetValues("x-rl-hourly-remaining").First());
-
-                lock (RemainingLock)
-                {
-                    _dailyRemaining = Math.Min(_dailyRemaining, dailyRemaining);
-                    _hourlyRemaining = Math.Min(_hourlyRemaining, hourlyRemaining);
-                }
-                
-                if (oldDaily != _dailyRemaining || oldHourly != _hourlyRemaining) 
-                    Utils.Log($"Nexus requests remaining: {_dailyRemaining} daily - {_hourlyRemaining} hourly");
+                _dailyRemaining = int.Parse(response.Headers.GetValues("x-rl-daily-remaining").First());
+                _hourlyRemaining = int.Parse(response.Headers.GetValues("x-rl-hourly-remaining").First());
 
                 this.RaisePropertyChanged(nameof(DailyRemaining));
                 this.RaisePropertyChanged(nameof(HourlyRemaining));
@@ -317,25 +311,22 @@ namespace Wabbajack.Lib.NexusApi
             var info = await GetModInfo(archive.Game, archive.ModID);
             if (!info.available)
                 throw new Exception("Mod unavailable");
-            
-            var url = $"https://api.nexusmods.com/v1/games/{archive.Game.MetaData().NexusName}/mods/{archive.ModID}/files/{archive.FileID}/download_link.json";
-            try
+
+            if (await IsPremium())
             {
+                if (HourlyRemaining <= 0 && DailyRemaining <= 0)
+                {
+                    throw new NexusAPIQuotaExceeded();
+                }
+
+                var url =
+                    $"https://api.nexusmods.com/v1/games/{archive.Game.MetaData().NexusName}/mods/{archive.ModID}/files/{archive.FileID}/download_link.json";
                 return (await Get<List<DownloadLink>>(url)).First().URI;
             }
-            catch (HttpException ex)
-            {
-
-                
-                if (ex.Code != 403 || await IsPremium())
-                {
-                    throw;
-                }
-            }
 
             try
             {
-                Utils.Log($"Requesting manual download for {archive.Name}");
+                Utils.Log($"Requesting manual download for {archive.Name} {archive.PrimaryKeyString}");
                 return (await Utils.Log(await ManuallyDownloadNexusFile.Create(archive)).Task).ToString();
             }
             catch (TaskCanceledException ex)

Wabbajack.Lib/StatusMessages/NexusAPIQuotaExceeded.cs

@@ -0,0 +1,12 @@
+using Wabbajack.Common.StatusFeed;
+
+namespace Wabbajack.Lib
+{
+    public class NexusAPIQuotaExceeded : AErrorMessage
+    {
+        public override string ShortDescription => $"You have exceeded your Nexus API limit for the day";
+
+        public override string ExtendedDescription =>
+            "You have exceeded your Nexus API limit for the day, please try again after midnight GMT";
+    }
+}

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -8,10 +8,10 @@
     </PropertyGroup>
     <ItemGroup>
       <PackageReference Include="CefSharp.Common">
-        <Version>85.3.130</Version>
+        <Version>86.0.241</Version>
       </PackageReference>
       <PackageReference Include="CefSharp.OffScreen">
-        <Version>85.3.130</Version>
+        <Version>86.0.241</Version>
       </PackageReference>
       <PackageReference Include="F23.StringSimilarity">
         <Version>3.1.0</Version>
@@ -25,7 +25,7 @@
         <Version>2.2.2.1</Version>
       </PackageReference>
       <PackageReference Include="HtmlAgilityPack">
-        <Version>1.11.26</Version>
+        <Version>1.11.28</Version>
       </PackageReference>
       <PackageReference Include="MegaApiClient">
         <Version>1.8.2</Version>
@@ -46,16 +46,16 @@
         <Version>0.26.0</Version>
       </PackageReference>
       <PackageReference Include="System.Collections.Immutable">
-        <Version>5.0.0-preview.8.20407.11</Version>
+        <Version>5.0.0</Version>
       </PackageReference>
       <PackageReference Include="System.Drawing.Common">
-        <Version>5.0.0-preview.8.20407.11</Version>
+        <Version>5.0.0</Version>
       </PackageReference>
       <PackageReference Include="System.Net.Http">
         <Version>4.3.4</Version>
       </PackageReference>
       <PackageReference Include="System.ServiceModel.Syndication">
-        <Version>5.0.0-preview.8.20407.11</Version>
+        <Version>5.0.0</Version>
       </PackageReference>
       <PackageReference Include="WebSocketSharp-netstandard">
         <Version>1.0.1</Version>

Wabbajack.Server.Test/DiscordFrontentTests.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Wabbajack.BuildServer.Test;
+using Wabbajack.Server.Services;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Wabbajack.Server.Test
+{
+    public class DiscordFrontentTests: ABuildServerSystemTest
+    {
+        public DiscordFrontentTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
+        {
+        }
+
+        [Fact]
+        public async Task CanLogIn()
+        {
+            var frontend = Fixture.GetService<DiscordFrontend>();
+            frontend.Start();
+        }
+        
+    }
+}

Wabbajack.Server.Test/ModListValidationTests.cs

@@ -189,6 +189,9 @@ namespace Wabbajack.BuildServer.Test
             
             var statusRss = await _client.GetHtmlAsync(MakeURL("lists/status/test_list/broken.rss"));
             Assert.Equal(failed, statusRss.DocumentNode.SelectNodes("//item")?.Count ?? 0);
+
+            var heartBeat = await _client.GetHtmlAsync(MakeURL("heartbeat/report"));
+            Assert.Contains(heartBeat.DocumentNode.Descendants(), c => c.InnerText.StartsWith("test_list"));
         }
 
         

Wabbajack.Server/Controllers/GameFiles.cs

@@ -67,6 +67,14 @@ namespace Wabbajack.BuildServer.Controllers
             return Ok(files.ToJson());
         }
 
+        [Authorize(Roles = "User")]
+        [HttpGet]
+        public async Task<IActionResult> GetAllGames()
+        {
+            var registeredGames = await _sql.GetAllRegisteredGames();
+            return Ok(registeredGames.ToArray().ToJson());
+        }
+
 
 
     }

Wabbajack.Server/Controllers/Heartbeat.cs

@@ -1,12 +1,16 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
+using System.Net;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
+using Nettle;
 using Wabbajack.Common.StatusFeed;
 using Wabbajack.Server;
 using Wabbajack.Server.DataLayer;
 using Wabbajack.Server.DTOs;
+using Wabbajack.Server.Services;
 
 namespace Wabbajack.BuildServer.Controllers
 {
@@ -19,12 +23,18 @@ namespace Wabbajack.BuildServer.Controllers
             
         }
         private static DateTime _startTime;
+        
+        private QuickSync _quickSync;
+        private ListValidator _listValidator;
 
-        public Heartbeat(ILogger<Heartbeat> logger, SqlService sql, GlobalInformation globalInformation)
+
+        public Heartbeat(ILogger<Heartbeat> logger, SqlService sql, GlobalInformation globalInformation, QuickSync quickSync, ListValidator listValidator)
         {
             _globalInformation = globalInformation;
             _sql = sql;
             _logger = logger;
+            _quickSync = quickSync;
+            _listValidator = listValidator;
         }
 
         private const int MAX_LOG_SIZE = 128;
@@ -52,6 +62,53 @@ namespace Wabbajack.BuildServer.Controllers
                 LastNexusUpdate = _globalInformation.TimeSinceLastNexusSync,
             });
         }
+        
+        private static readonly Func<object, string> HandleGetReport = NettleEngine.GetCompiler().Compile(@"
+            <html><body>
+                <h2>Server Status</h2>
+
+                <h3>Service Overview ({{services.Length}}):</h3>
+                <ul>
+                {{each $.services }}
+                {{if $.IsLate}}
+                <li><b>{{$.Name}} - {{$.Time}} - {{$.MaxTime}}</b></li>
+                {{else}}
+                <li>{{$.Name}} - {{$.Time}} - {{$.MaxTime}}</li>
+                {{/if}}
+                {{/each}}
+                </ul>
+
+
+                <h3>Lists ({{lists.Length}}):</h3>
+                <ul>
+                {{each $.lists }}
+                <li><a href='/lists/status/{{$.Name}}.html'>{{$.Name}}</a> - {{$.Time}}</li>
+                {{/each}}
+                </ul>
+            </body></html>
+        ");
+
+        [HttpGet("report")]
+        public async Task<ContentResult> Report()
+        {
+            var response = HandleGetReport(new
+            {
+                services = (await _quickSync.Report())
+                    .Select(s => new {Name = s.Key.Name, Time = s.Value.LastRunTime, MaxTime = s.Value.Delay, IsLate = s.Value.LastRunTime > s.Value.Delay})
+                    .OrderBy(s => s.Name)
+                    .ToArray(),
+                lists = _listValidator.ValidationInfo.Select(s => new {Name = s.Key, Time = s.Value.ValidationTime})
+                    .OrderBy(l => l.Name)
+                    .ToArray()
+            });
+            return new ContentResult
+            {
+                ContentType = "text/html",
+                StatusCode = (int) HttpStatusCode.OK,
+                Content = response
+            };
+            
+        }
 
 
 

Wabbajack.Server/Controllers/Metrics.cs

@@ -43,13 +43,23 @@ namespace Wabbajack.BuildServer.Controllers
         [ResponseCache(Duration = 60 * 60)]
         public async Task<IActionResult> MetricsReport(string subject)
         {
-            var results = (await _sql.MetricsReport(subject))
+            var metrics = (await _sql.MetricsReport(subject)).ToList();
+            var labels = metrics.GroupBy(m => m.Date)
+                .OrderBy(m => m.Key)
+                .Select(m => m.Key)
+                .ToArray();
+            var labelStrings = labels.Select(l => l.ToString("MM-dd-yyy")).ToList();
+            var results = metrics
                 .GroupBy(m => m.Subject)
-                .Select(g => new MetricResult
+                .Select(g =>
                 {
-                    SeriesName = g.Key,
-                    Labels = g.Select(m => m.Date.ToString(CultureInfo.InvariantCulture)).ToList(),
-                    Values = g.Select(m => m.Count).ToList()
+                    var indexed = g.ToDictionary(m => m.Date, m => m.Count);
+                    return new MetricResult
+                    {
+                        SeriesName = g.Key,
+                        Labels = labelStrings,
+                        Values = labels.Select(l => indexed.TryGetValue(l, out var found) ? found : 0).ToList()
+                    };
                 });
             return Ok(results.ToList());
         }

Wabbajack.Server/DataLayer/ArchiveDownloads.cs

@@ -255,5 +255,16 @@ namespace Wabbajack.Server.DataLayer
 
             return files.ToArray();
         }
+
+        public async Task<IEnumerable<(Game, string)>> GetAllRegisteredGames()
+        {
+            await using var conn = await Open();
+            var pks = (await conn.QueryAsync<string>(
+                    @"SELECT PrimaryKeyString FROM dbo.ArchiveDownloads WHERE PrimaryKeyString like 'GameFileSourceDownloader+State|%'")
+                );
+            return pks.Select(p => p.Split("|"))
+                .Select(t => (GameRegistry.GetByFuzzyName(t[1]).Game, t[2]))
+                .Distinct();
+        }
     }
 }

Wabbajack.Server/DataLayer/Metrics.cs

@@ -30,23 +30,24 @@ namespace Wabbajack.Server.DataLayer
         {
             await using var conn = await Open();
             return (await conn.QueryAsync<AggregateMetric>(@"
-                        SELECT d.Date, d.GroupingSubject as Subject, Count(*) as Count FROM 
-                        (select DISTINCT CONVERT(date, Timestamp) as Date, GroupingSubject, Action, MetricsKey from dbo.Metrics) m
-                        RIGHT OUTER JOIN
-                        (SELECT CONVERT(date, DATEADD(DAY, number + 1, dbo.MinMetricDate())) as Date, GroupingSubject, Action
-                        FROM master..spt_values
-                        CROSS JOIN (
-                          SELECT DISTINCT GroupingSubject, Action FROM dbo.Metrics 
-                          WHERE MetricsKey is not null 
-                          AND Subject != 'Default'
-                          AND TRY_CONVERT(uniqueidentifier, Subject) is null) as keys
-                        WHERE type = 'P'
-                        AND DATEADD(DAY, number+1, dbo.MinMetricDate()) <= dbo.MaxMetricDate()) as d
-                        ON m.Date = d.Date AND m.GroupingSubject = d.GroupingSubject AND m.Action = d.Action
-                        WHERE d.Action = @action
-                        AND d.Date >= DATEADD(month, -1, GETUTCDATE())
-                        group by d.Date, d.GroupingSubject, d.Action
-                        ORDER BY d.Date, d.GroupingSubject, d.Action", new {Action = action}))
+                select
+                datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)) as Date, 
+                GroupingSubject as Subject,
+                count(*) as Count 
+                from dbo.metrics where 
+                Action = @Action
+                AND GroupingSubject in  (select DISTINCT GroupingSubject from dbo.Metrics
+                 WHERE action = @Action
+                 AND MetricsKey is not null 
+                AND Subject != 'Default'
+                AND Subject != 'untitled'
+                AND TRY_CONVERT(uniqueidentifier, Subject) is null
+                 AND Timestamp >= DATEADD(DAY, -1, GETUTCDATE()))
+                group by
+                 datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)),
+                 GroupingSubject
+                 Order by  datefromparts(datepart(YEAR,Timestamp), datepart(MONTH,Timestamp), datepart(DAY,Timestamp)) asc", 
+                    new {Action = action}))
                 .ToList();
         }
 

Wabbajack.Server/Services/AbstractService.cs

@@ -10,14 +10,27 @@ namespace Wabbajack.Server.Services
     {
         public void Start();
     }
+
+    public interface IReportingService
+    {
+        public TimeSpan Delay { get; }
+        public DateTime LastStart { get; }
+        public DateTime LastEnd { get; }
+
+        
+    }
     
-    public abstract class AbstractService<TP, TR> : IStartable
+    public abstract class AbstractService<TP, TR> : IStartable, IReportingService
     {
         protected AppSettings _settings;
         private TimeSpan _delay;
         protected ILogger<TP> _logger;
         protected QuickSync _quickSync;
 
+        public TimeSpan Delay => _delay;
+        public DateTime LastStart { get; private set; }
+        public DateTime LastEnd { get; private set; }
+
         public AbstractService(ILogger<TP> logger, AppSettings settings, QuickSync quickSync, TimeSpan delay)
         {
             _settings = settings;
@@ -40,7 +53,7 @@ namespace Wabbajack.Server.Services
                 Task.Run(async () =>
                 {
                     await Setup();
-
+                    await _quickSync.Register(this);
                     
                     while (true)
                     {
@@ -48,7 +61,9 @@ namespace Wabbajack.Server.Services
                         try
                         {
                             _logger.LogInformation($"Running: {GetType().Name}");
+                            LastStart = DateTime.UtcNow;
                             await Execute();
+                            LastEnd = DateTime.UtcNow;
                         }
                         catch (Exception ex)
                         {

Wabbajack.Server/Services/DiscordFrontend.cs

@@ -0,0 +1,141 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Discord;
+using Discord.WebSocket;
+using Microsoft.Extensions.Logging;
+using OMODFramework;
+using Wabbajack.BuildServer;
+using Wabbajack.Common;
+using Wabbajack.Server.DataLayer;
+using Utils = Wabbajack.Common.Utils;
+
+namespace Wabbajack.Server.Services
+{
+    public class DiscordFrontend : IStartable
+    {
+        private ILogger<DiscordFrontend> _logger;
+        private AppSettings _settings;
+        private QuickSync _quickSync;
+        private DiscordSocketClient _client;
+        private SqlService _sql;
+
+        public DiscordFrontend(ILogger<DiscordFrontend> logger, AppSettings settings, QuickSync quickSync, SqlService sql)
+        {
+            _logger = logger;
+            _settings = settings;
+            _quickSync = quickSync;
+            
+            _client = new DiscordSocketClient();
+
+            _client.Log += LogAsync;
+            _client.Ready += ReadyAsync;
+            _client.MessageReceived += MessageReceivedAsync;
+
+            _sql = sql;
+        }
+
+        private async Task MessageReceivedAsync(SocketMessage arg)
+        {
+            _logger.LogInformation(arg.Content);
+            if (arg.Content.StartsWith("!dervenin"))
+            {
+                var parts = arg.Content.Split(" ", StringSplitOptions.RemoveEmptyEntries);
+                if (parts[0] != "!dervenin")
+                    return;
+                
+                if (parts.Length == 1)
+                {
+                    await ReplyTo(arg, "Wat?");
+                }
+
+                if (parts[1] == "purge-nexus-cache")
+                {
+                    if (parts.Length != 3)
+                    {
+                        await ReplyTo(arg, "Welp you did that wrong, gotta give me a mod-id or url");
+                        return;
+                    }
+                    await PurgeNexusCache(arg, parts[2]);
+                }
+                else if (parts[1] == "cyberpunk")
+                {
+                    var random = new Random();
+                    var releaseDate = new DateTime(2020, 12, 10, 0, 0, 0, DateTimeKind.Utc);
+                    var r = releaseDate - DateTime.UtcNow;
+                    if (r < TimeSpan.Zero)
+                    {
+                        await ReplyTo(arg, "It's out, what are you doing here?");
+                    }
+                    else
+                    {
+                        var msgs = (await "cyberpunk_message.txt".RelativeTo(AbsolutePath.EntryPoint)
+                            .ReadAllLinesAsync()).ToArray();
+                        var msg = msgs[random.Next(0, msgs.Length)];
+                        var fullmsg = String.Format(msg,
+                            $"{r.Days} days, {r.Hours} hours, {r.Minutes} minutes, {r.Seconds} seconds");
+                        await ReplyTo(arg, fullmsg);
+                    }
+                }
+            }
+        }
+
+        private async Task PurgeNexusCache(SocketMessage arg, string mod)
+        {
+            if (Uri.TryCreate(mod, UriKind.Absolute, out var url))
+            {
+                mod = Enumerable.Last(url.AbsolutePath.Split("/", StringSplitOptions.RemoveEmptyEntries));
+            }
+            
+            if (int.TryParse(mod, out var mod_id))
+            {
+                await _sql.PurgeNexusCache(mod_id);
+                await _quickSync.Notify<ListValidator>();
+                await ReplyTo(arg, $"It is done, {mod_id} has been purged, list validation has been triggered");
+            }
+        }
+
+        private async Task ReplyTo(SocketMessage socketMessage, string message)
+        {
+            await socketMessage.Channel.SendMessageAsync(message);
+        }
+
+        private async Task ReadyAsync()
+        {
+        }
+
+        private async Task LogAsync(LogMessage arg)
+        {
+            switch (arg.Severity)
+            {
+                case LogSeverity.Info:
+                    _logger.LogInformation(arg.Message);
+                    break;
+                case LogSeverity.Warning:
+                    _logger.LogWarning(arg.Message);
+                    break;
+                case LogSeverity.Critical:
+                    _logger.LogCritical(arg.Message);
+                    break;
+                case LogSeverity.Error:
+                    _logger.LogError(arg.Exception, arg.Message);
+                    break;
+                case LogSeverity.Verbose:
+                    _logger.LogTrace(arg.Message);
+                    break;
+                case LogSeverity.Debug:
+                    _logger.LogDebug(arg.Message);
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException();
+            }
+        }
+
+        public void Start()
+        {
+            _client.LoginAsync(TokenType.Bot, Utils.FromEncryptedJson<string>("discord-key").Result).Wait();
+            _client.StartAsync().Wait();
+        }
+        
+    }
+}

Wabbajack.Server/Services/ListValidator.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
@@ -27,8 +28,9 @@ namespace Wabbajack.Server.Services
         private NexusKeyMaintainance _nexus;
         private ArchiveMaintainer _archives;
 
-        public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries { get; private set; } =
-            new (ModListSummary Summary, DetailedStatus Detailed)[0];
+        public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries => ValidationInfo.Values.Select(e => (e.Summary, e.Detailed));
+        
+        public ConcurrentDictionary<string, (ModListSummary Summary, DetailedStatus Detailed, TimeSpan ValidationTime)> ValidationInfo = new ConcurrentDictionary<string, (ModListSummary Summary, DetailedStatus Detailed, TimeSpan ValidationTime)>();
 
 
         public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives, QuickSync quickSync) 
@@ -50,14 +52,21 @@ namespace Wabbajack.Server.Services
             var stopwatch = new Stopwatch();
             stopwatch.Start();
 
-            var results = await data.ModLists.PMap(queue, async metadata =>
+            var results = await data.ModLists.Where(m => !m.ForceDown).PMap(queue, async metadata =>
             {
+                var timer = new Stopwatch();
+                timer.Start();
                 var oldSummary =
                     oldSummaries.FirstOrDefault(s => s.Summary.MachineURL == metadata.Links.MachineURL);
                 
                 var listArchives = await _sql.ModListArchives(metadata.Links.MachineURL);
                 var archives = await listArchives.PMap(queue, async archive =>
                 {
+                    if (timer.Elapsed > Delay)
+                    {
+                        return (archive, ArchiveStatus.InValid);
+                    }
+
                     try
                     {
                         var (_, result) = await ValidateArchive(data, archive);
@@ -109,6 +118,24 @@ namespace Wabbajack.Server.Services
                     }).ToList()
                 };
 
+                if (timer.Elapsed > Delay)
+                {
+                    await _discord.Send(Channel.Ham,
+                        new DiscordMessage
+                        {
+                            Embeds = new[]
+                            {
+                                new DiscordEmbed
+                                {
+                                    Title =
+                                        $"Failing {summary.Name} (`{summary.MachineURL}`) because the max validation time expired",
+                                    Url = new Uri(
+                                        $"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html")
+                                }
+                            }
+                        });
+                }
+
                 if (oldSummary != default && oldSummary.Summary.Failed != summary.Failed)
                 {
                     _logger.Log(LogLevel.Information, $"Number of failures {oldSummary.Summary.Failed} -> {summary.Failed}");
@@ -151,9 +178,14 @@ namespace Wabbajack.Server.Services
 
                 }
                 
+                timer.Stop();
+                
+
+                
+                ValidationInfo[summary.MachineURL] = (summary, detailed, timer.Elapsed);
+                
                 return (summary, detailed);
             });
-            Summaries = results;
             
             stopwatch.Stop();
             _logger.LogInformation($"Finished Validation in {stopwatch.Elapsed}");

Wabbajack.Server/Services/NexusKeyMaintainance.cs

@@ -25,7 +25,15 @@ namespace Wabbajack.Server.Services
             var keys = await _sql.GetNexusApiKeysWithCounts(1500);
             foreach (var key in keys.Where(k => k.Key != _selfKey))
             {
-                return new TrackingClient(_sql, key);
+                var client = new TrackingClient(_sql, key);
+                if (!await client.IsPremium())
+                {
+                    _logger.LogWarning($"Purging non premium key");
+                    await _sql.DeleteNexusAPIKey(key.Key);
+                    continue;
+                }
+
+                return client;
             }
 
             return await NexusApiClient.Get();

Wabbajack.Server/Services/QuickSync.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
@@ -11,6 +12,7 @@ namespace Wabbajack.Server.Services
     public class QuickSync
     {
         private Dictionary<Type, CancellationTokenSource> _syncs = new Dictionary<Type, CancellationTokenSource>();
+        private Dictionary<Type, IReportingService> _services = new Dictionary<Type, IReportingService>();
         private AsyncLock _lock = new AsyncLock();
         private ILogger<QuickSync> _logger;
 
@@ -19,6 +21,20 @@ namespace Wabbajack.Server.Services
             _logger = logger;
         }
 
+        public async Task<Dictionary<Type, (TimeSpan Delay, TimeSpan LastRunTime)>> Report()
+        {
+            using var _ = await _lock.WaitAsync();
+            return _services.ToDictionary(s => s.Key,
+                s => (s.Value.Delay, DateTime.UtcNow - s.Value.LastEnd));
+        }
+
+        public async Task Register<T>(T service)
+        where T : IReportingService
+        {
+            using var _ = await _lock.WaitAsync();
+            _services[service.GetType()] = service;
+        }
+
         public async Task<CancellationToken> GetToken<T>()
         {
             using var _ = await _lock.WaitAsync();

Wabbajack.Server/Services/Watchdog.cs

@@ -0,0 +1,33 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer;
+using Wabbajack.Server.DTOs;
+
+namespace Wabbajack.Server.Services
+{
+    public class Watchdog : AbstractService<Watchdog, int>
+    {
+        private DiscordWebHook _discord;
+
+        public Watchdog(ILogger<Watchdog> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discordWebHook) : base(logger, settings, quickSync, TimeSpan.FromMinutes(5))
+        {
+            _discord = discordWebHook;
+        }
+
+        public override async Task<int> Execute()
+        {
+            var report = await _quickSync.Report();
+            foreach (var service in report)
+            {
+                if (service.Value.LastRunTime != default && service.Value.LastRunTime >= service.Value.Delay * 2)
+                {
+                    await _discord.Send(Channel.Spam,
+                        new DiscordMessage {Content = $"Service {service.Key.Name} has missed it's scheduled execution window"});
+                }
+            }
+
+            return report.Count;
+        }
+    }
+}

Wabbajack.Server/Startup.cs

@@ -71,6 +71,8 @@ namespace Wabbajack.Server
             services.AddSingleton<NexusPermissionsUpdater>();
             services.AddSingleton<MirrorUploader>();
             services.AddSingleton<MirrorQueueService>();
+            services.AddSingleton<Watchdog>();
+            services.AddSingleton<DiscordFrontend>();
             
             services.AddMvc();
             services.AddControllers()
@@ -129,6 +131,8 @@ namespace Wabbajack.Server
             app.UseService<NexusPermissionsUpdater>();
             app.UseService<MirrorUploader>();
             app.UseService<MirrorQueueService>();
+            app.UseService<Watchdog>();
+            app.UseService<DiscordFrontend>();
 
             app.Use(next =>
             {

Wabbajack.Server/Wabbajack.Server.csproj

@@ -3,8 +3,8 @@
     <PropertyGroup>
         <OutputType>Exe</OutputType>
         <TargetFramework>netcoreapp3.1</TargetFramework>
-        <AssemblyVersion>2.3.2.0</AssemblyVersion>
-        <FileVersion>2.3.2.0</FileVersion>
+        <AssemblyVersion>2.3.5.1</AssemblyVersion>
+        <FileVersion>2.3.5.1</FileVersion>
         <Copyright>Copyright © 2019-2020</Copyright>
         <Description>Wabbajack Server</Description>
         <RuntimeIdentifier>win-x64</RuntimeIdentifier>
@@ -14,8 +14,8 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="CsvHelper" Version="15.0.8" />
         <PackageReference Include="Dapper" Version="2.0.35" />
+        <PackageReference Include="Discord.Net.WebSocket" Version="2.2.0" />
         <PackageReference Include="FluentFTP" Version="33.0.2" />
         <PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
         <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.9" />

Wabbajack.Server/public/metrics.html

@@ -51,7 +51,7 @@
                         return {
                             label: series.seriesName,
                             fill: false,
-                            data: _.last(series.values, 30)
+                            data: _.last(series.values, 90)
                         }});
                     var ctx = document.getElementById(ele).getContext('2d');
                     var chart = new Chart(ctx, {
@@ -60,7 +60,7 @@
 
                         // The data for our dataset
                         data: {
-                            labels: _.last(labels, 30),
+                            labels: _.last(labels, 90),
                             datasets: datasets},
 
                         // Configuration options go here

Wabbajack.Test/DownloaderTests.cs

@@ -407,13 +407,12 @@ namespace Wabbajack.Test
             Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
         }
         
-        /* Site is down
         [Fact]
         public async Task TESAllDownloader()
         {
             await DownloadDispatcher.GetInstance<TESAllDownloader>().Prepare();
             const string ini = "[General]\n" +
-                               "directURL=https://tesall.ru/files/getdownload/594545-wabbajack-test-file/";
+                               "directURL=https://tesall.ru/files/download/594545";
 
             var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
 
@@ -431,7 +430,6 @@ namespace Wabbajack.Test
 
             Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
         }
-        */
 
         /* WAITING FOR APPROVAL BY MODERATOR
          [Fact]

Wabbajack.Test/Wabbajack.Test.csproj

@@ -28,9 +28,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="CefSharp.Common" Version="85.3.130" />
-    <PackageReference Include="CefSharp.OffScreen" Version="85.3.130" />
-      <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0-preview-20200812-03" />
+    <PackageReference Include="CefSharp.Common" Version="86.0.241" />
+    <PackageReference Include="CefSharp.OffScreen" Version="86.0.241" />
+      <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.3" />
       <PackageReference Include="xunit" Version="2.4.1" />
       <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
       <PackageReference Include="coverlet.collector" Version="1.3.0" />

Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs

@@ -90,7 +90,7 @@ namespace Wabbajack.VirtualFileSystem.Test
                     return await s.xxHashAsync();
                 });
             
-            Assert.Equal(1, results.Count);
+            Assert.Single(results);
             foreach (var (path, hash) in results)
             {
                 Assert.Equal(await temp.Dir.Combine(path).FileHashAsync(), hash);

Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj

@@ -8,7 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-      <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0-preview-20200812-03" />
+      <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.3" />
       <PackageReference Include="xunit" Version="2.4.1" />
       <PackageReference Include="xunit.runner.console" Version="2.4.1" />
       <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />

Wabbajack.VirtualFileSystem/Context.cs

@@ -209,7 +209,7 @@ namespace Wabbajack.VirtualFileSystem
         /// <param name="files"></param>
         /// <param name="callback"></param>
         /// <returns></returns>
-        public async Task Extract(WorkQueue queue, HashSet<VirtualFile> files, Func<VirtualFile, IExtractedFile, ValueTask> callback, AbsolutePath? tempFolder = null)
+        public async Task Extract(WorkQueue queue, HashSet<VirtualFile> files, Func<VirtualFile, IExtractedFile, ValueTask> callback, AbsolutePath? tempFolder = null, StatusUpdateTracker updateTracker = null)
         {
             var top = new VirtualFile();
             var filesByParent = files.SelectMany(f => f.FilesInFullPath)
@@ -238,7 +238,7 @@ namespace Wabbajack.VirtualFileSystem
                             tempFolder: tempFolder,
                             onlyFiles: fileNames.Keys.ToHashSet());
                     }
-                    catch (_7zipReturnError ex)
+                    catch (_7zipReturnError)
                     {
                         await using var stream = await sfn.GetStream();
                         var hash = await stream.xxHashAsync();
@@ -251,7 +251,8 @@ namespace Wabbajack.VirtualFileSystem
                 }
 
             }
-            await filesByParent[top].PMap(queue, async file => await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName) {CanMove = false}));
+            updateTracker ??= new StatusUpdateTracker(1);
+            await filesByParent[top].PMap(queue, updateTracker, async file => await HandleFile(file, new ExtractedNativeFile(file.AbsoluteName) {CanMove = false}));
         }
 
         #region KnownFiles

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -224,11 +224,11 @@ namespace Wabbajack.VirtualFileSystem
 
                 self.Children = list.Values.ToImmutableList();
             }
-            catch (EndOfStreamException ex)
+            catch (EndOfStreamException)
             {
                 return self;
             }
-            catch (Exception ex)
+            catch (Exception)
             {
                 Utils.Log($"Error while examining the contents of {relPath.FileName}");
                 throw;

Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj

@@ -16,9 +16,9 @@
     <ItemGroup>
       <PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
       <PackageReference Include="K4os.Hash.Crc" Version="1.1.4" />
-      <PackageReference Include="OMODFramework" Version="2.0.1" />
+      <PackageReference Include="OMODFramework" Version="2.1.0.2" />
       <PackageReference Include="SharpCompress" Version="0.26.0" />
-      <PackageReference Include="System.Collections.Immutable" Version="5.0.0-preview.6.20305.11" />
+      <PackageReference Include="System.Collections.Immutable" Version="5.0.0" />
     </ItemGroup>
     <ItemGroup>
       <None Update="Extractors\7z.dll">

Wabbajack/Settings.cs

@@ -107,6 +107,7 @@ namespace Wabbajack
         
         private bool _useCompression = false;
         public bool UseCompression { get => _useCompression; set => RaiseAndSetIfChanged(ref _useCompression, value); }
+        public bool ShowUtilityLists { get; set; }
     }
 
     [JsonName("PerformanceSettings")]

Wabbajack/View Models/BackNavigatingVM.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Reactive;
 using System.Reactive.Linq;
+using System.Reactive.Subjects;
 using System.Text;
 using System.Threading.Tasks;
 using System.Windows.Input;
@@ -17,6 +18,9 @@ namespace Wabbajack
     {
         ViewModel NavigateBackTarget { get; set; }
         ReactiveCommand<Unit, Unit> BackCommand { get; }
+        
+        Subject<bool> IsBackEnabledSubject { get; }
+        IObservable<bool> IsBackEnabled { get; }
     }
 
     public class BackNavigatingVM : ViewModel, IBackNavigatingVM
@@ -27,9 +31,13 @@ namespace Wabbajack
 
         private readonly ObservableAsPropertyHelper<bool> _IsActive;
         public bool IsActive => _IsActive.Value;
+        
+        public Subject<bool> IsBackEnabledSubject { get; } = new Subject<bool>();
+        public IObservable<bool> IsBackEnabled { get; }
 
         public BackNavigatingVM(MainWindowVM mainWindowVM)
         {
+            IsBackEnabled = IsBackEnabledSubject.StartWith(true);
             BackCommand = ReactiveCommand.Create(
                 execute: () => Utils.CatchAndLog(() =>
                 {
@@ -53,7 +61,8 @@ namespace Wabbajack
         public static IObservable<bool> ConstructCanNavigateBack(this IBackNavigatingVM vm)
         {
             return vm.WhenAny(x => x.NavigateBackTarget)
-                .Select(x => x != null);
+                .CombineLatest(vm.IsBackEnabled)
+                .Select(x => x.First != null && x.Second);
         }
 
         public static IObservable<bool> ConstructIsActive(this IBackNavigatingVM vm, MainWindowVM mwvm)

Wabbajack/View Models/Compilers/CompilerVM.cs

@@ -10,6 +10,7 @@ using System.Linq;
 using System.Reactive;
 using System.Reactive.Disposables;
 using System.Reactive.Linq;
+using System.Reactive.Subjects;
 using System.Windows.Input;
 using System.Windows.Media.Imaging;
 using Wabbajack.Common;
@@ -18,16 +19,13 @@ using Wabbajack.Lib;
 
 namespace Wabbajack
 {
-    public class CompilerVM : ViewModel, IBackNavigatingVM, ICpuStatusVM
+    public class CompilerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM
     {
         public MainWindowVM MWVM { get; }
 
         private readonly ObservableAsPropertyHelper<BitmapImage> _image;
         public BitmapImage Image => _image.Value;
 
-        [Reactive]
-        public ViewModel NavigateBackTarget { get; set; }
-
         [Reactive]
         public ModManager SelectedCompilerType { get; set; }
 
@@ -69,7 +67,7 @@ namespace Wabbajack
         private readonly ObservableAsPropertyHelper<(int CurrentCPUs, int DesiredCPUs)> _CurrentCpuCount;
         public (int CurrentCPUs, int DesiredCPUs) CurrentCpuCount => _CurrentCpuCount.Value;
 
-        public CompilerVM(MainWindowVM mainWindowVM)
+        public CompilerVM(MainWindowVM mainWindowVM) : base(mainWindowVM)
         {
             MWVM = mainWindowVM;
 
@@ -178,6 +176,7 @@ namespace Wabbajack
                 {
                     try
                     {
+                        IsBackEnabledSubject.OnNext(false);
                         var modList = await this.Compiler.Compile();
                         Completed = ErrorResponse.Create(modList.Succeeded);
                     }
@@ -187,6 +186,10 @@ namespace Wabbajack
                         while (ex.InnerException != null) ex = ex.InnerException;
                         Utils.Error(ex, $"Compiler error");
                     }
+                    finally
+                    {
+                        IsBackEnabledSubject.OnNext(true);
+                    }
                 });
 
             // When sub compiler begins a compile, mark state variable

Wabbajack/View Models/Gallery/ModListGalleryVM.cs

@@ -37,6 +37,9 @@ namespace Wabbajack
 
         [Reactive]
         public bool ShowNSFW { get; set; }
+        
+        [Reactive]
+        public bool ShowUtilityLists { get; set; }
 
         [Reactive]
         public string GameType { get; set; }
@@ -61,6 +64,7 @@ namespace Wabbajack
             {
                 GameType = !string.IsNullOrEmpty(settings.Game) ? settings.Game : ALL_GAME_TYPE;
                 ShowNSFW = settings.ShowNSFW;
+                ShowUtilityLists = settings.ShowUtilityLists;
                 OnlyInstalled = settings.OnlyInstalled;
                 Search = settings.Search;
             }
@@ -77,6 +81,7 @@ namespace Wabbajack
                 {
                     OnlyInstalled = false;
                     ShowNSFW = false;
+                    ShowUtilityLists = false;
                     Search = string.Empty;
                     GameType = ALL_GAME_TYPE;
                 });
@@ -150,6 +155,8 @@ namespace Wabbajack
                         if (!vm.Metadata.NSFW) return true;
                         return vm.Metadata.NSFW && showNSFW;
                     }))
+                .Filter(this.WhenAny(x => x.ShowUtilityLists)
+                    .Select<bool, Func<ModListMetadataVM, bool>>(showUtilityLists => vm => showUtilityLists ? vm.Metadata.UtilityList : !vm.Metadata.UtilityList))
                 // Filter by Game
                 .Filter(this.WhenAny(x => x.GameType)
                     .Debounce(TimeSpan.FromMilliseconds(150), RxApp.MainThreadScheduler)
@@ -163,12 +170,6 @@ namespace Wabbajack
                         return GameType == vm.Metadata.Game.GetDescription<Game>().ToString();
 
                     }))
-                .Filter(this.WhenAny(x => x.ShowNSFW)
-                    .Select<bool, Func<ModListMetadataVM, bool>>(showNSFW => vm =>
-                    {
-                        if (!vm.Metadata.NSFW) return true;
-                        return vm.Metadata.NSFW && showNSFW;
-                    }))
                 // Put broken lists at bottom
                 .Sort(Comparer<ModListMetadataVM>.Create((a, b) => a.IsBroken.CompareTo(b.IsBroken)))
                 .Bind(ModLists)
@@ -204,6 +205,7 @@ namespace Wabbajack
             settings.Game = GameType;
             settings.Search = Search;
             settings.ShowNSFW = ShowNSFW;
+            settings.ShowUtilityLists = ShowUtilityLists;
             settings.OnlyInstalled = OnlyInstalled;
         }
     }

Wabbajack/View Models/Gallery/ModListMetadataVM.cs

@@ -83,7 +83,7 @@ namespace Wabbajack
             });
             DownloadSizeText = "Download size : " + UIUtils.FormatBytes(Metadata.DownloadMetadata.SizeOfArchives);
             InstallSizeText = "Installation size : " + UIUtils.FormatBytes(Metadata.DownloadMetadata.SizeOfInstalledFiles);
-            IsBroken = metadata.ValidationSummary.HasFailures;
+            IsBroken = metadata.ValidationSummary.HasFailures || metadata.ForceDown;
             //https://www.wabbajack.org/#/modlists/info?machineURL=eldersouls
             OpenWebsiteCommand = ReactiveCommand.Create(() => Utils.OpenWebsite(new Uri($"https://www.wabbajack.org/#/modlists/info?machineURL={Metadata.Links.MachineURL}")));
             ExecuteCommand = ReactiveCommand.CreateFromObservable<Unit, Unit>(

Wabbajack/View Models/Installers/InstallerVM.cs

@@ -19,13 +19,14 @@ using DynamicData.Binding;
 using Wabbajack.Common.StatusFeed;
 using System.Reactive;
 using System.Collections.Generic;
+using System.Reactive.Subjects;
 using System.Windows.Input;
 using Microsoft.WindowsAPICodePack.Dialogs;
 using Wabbajack.Common.IO;
 
 namespace Wabbajack
 {
-    public class InstallerVM : ViewModel, IBackNavigatingVM, ICpuStatusVM
+    public class InstallerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM
     {
         public SlideShow Slideshow { get; }
 
@@ -36,9 +37,6 @@ namespace Wabbajack
 
         public FilePickerVM ModListLocation { get; }
 
-        [Reactive]
-        public ViewModel NavigateBackTarget { get; set; }
-
         private readonly ObservableAsPropertyHelper<ISubInstallerVM> _installer;
         public ISubInstallerVM Installer => _installer.Value;
 
@@ -95,12 +93,15 @@ namespace Wabbajack
         public ReactiveCommand<Unit, Unit> OpenReadmeCommand { get; }
         public ReactiveCommand<Unit, Unit> VisitModListWebsiteCommand { get; }
         public ReactiveCommand<Unit, Unit> BackCommand { get; }
+        
+        
         public ReactiveCommand<Unit, Unit> CloseWhenCompleteCommand { get; }
         public ReactiveCommand<Unit, Unit> GoToInstallCommand { get; }
         public ReactiveCommand<Unit, Unit> BeginCommand { get; }
 
-        public InstallerVM(MainWindowVM mainWindowVM)
+        public InstallerVM(MainWindowVM mainWindowVM) : base(mainWindowVM)
         {
+           
             if (Path.GetDirectoryName(Assembly.GetEntryAssembly().Location.ToLower()) == KnownFolders.Downloads.Path.ToLower())
             {
                 Utils.Error(new CriticalFailureIntervention(
@@ -306,11 +307,14 @@ namespace Wabbajack
                         if (err) return "Corrupted Modlist";
                         return name;
                     })
+                .Merge(this.WhenAny(x => x.Installer.ActiveInstallation)
+                    .Where(c => c != null)
+                    .SelectMany(c => c.TextStatus))
                 .ToGuiProperty(this, nameof(ModListName));
 
             ShowManifestCommand = ReactiveCommand.Create(() =>
             {
-                new ManifestWindow(ModList.SourceModList).Show();
+                Utils.OpenWebsite(new Uri("https://www.wabbajack.org/#/modlists/manifest"));
             }, this.WhenAny(x => x.ModList)
                     .Select(x => x?.SourceModList != null)
                 .ObserveOnGuiThread());
@@ -366,6 +370,7 @@ namespace Wabbajack
                     try
                     {
                         Utils.Log($"Starting to install {ModList.Name}");
+                        IsBackEnabledSubject.OnNext(false);
                         var success = await this.Installer.Install();
                         Completed = ErrorResponse.Create(success);
                         try
@@ -378,11 +383,15 @@ namespace Wabbajack
                         }
                     }
                     catch (Exception ex)
-                    { 
+                    {
                         Utils.Error(ex, $"Encountered error, can't continue");
                         while (ex.InnerException != null) ex = ex.InnerException;
                         Completed = ErrorResponse.Fail(ex);
                     }
+                    finally
+                    {
+                        IsBackEnabledSubject.OnNext(true);
+                    }
                 });
 
             // When sub installer begins an install, mark state variable

Wabbajack/View Models/ManifestVM.cs

@@ -1,102 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Reactive;
-using System.Reactive.Linq;
-using ReactiveUI;
-using ReactiveUI.Fody.Helpers;
-using Wabbajack.Common;
-using Wabbajack.Lib;
-
-namespace Wabbajack
-{
-    public enum SortBy { Name, Size }
-
-    public class ManifestVM : ViewModel
-    {
-        public Manifest Manifest { get; set; }
-
-        public string Name => !string.IsNullOrWhiteSpace(Manifest.Name) ? Manifest.Name : "Wabbajack Modlist";
-        public string Author => !string.IsNullOrWhiteSpace(Manifest.Author) ? $"Created by {Manifest.Author}" : "Created by Jyggalag";
-        public string Description => !string.IsNullOrWhiteSpace(Manifest.Description) ? Manifest.Description : "";
-        public string InstallSize => $"Install Size: {Manifest.InstallSize.ToFileSizeString()}";
-        public string DownloadSize => $"Download Size: {Manifest.DownloadSize.ToFileSizeString()}";
-
-        public IEnumerable<Archive> Archives => Manifest.Archives;
-
-        [Reactive]
-        public string SearchTerm { get; set; }
-
-        private readonly ObservableAsPropertyHelper<IEnumerable<Archive>> _searchResults;
-        public IEnumerable<Archive> SearchResults => _searchResults.Value;
-
-        [Reactive]
-        public bool SortAscending { get; set; } = true;
-
-        [Reactive]
-        public SortBy SortEnum { get; set; } = SortBy.Name;
-
-        public ReactiveCommand<Unit, Unit> SortByNameCommand;
-        public ReactiveCommand<Unit, Unit> SortBySizeCommand;
-
-        private IEnumerable<Archive> Order(IEnumerable<Archive> list)
-        {
-            if (SortAscending)
-            {
-                return SortEnum switch
-                {
-                    SortBy.Name => list.OrderBy(x => x.Name),
-                    SortBy.Size => list.OrderBy(x => x.Size),
-                    _ => throw new ArgumentOutOfRangeException()
-                };
-            }
-
-            return SortEnum switch
-            {
-                SortBy.Name => list.OrderByDescending(x => x.Name),
-                SortBy.Size => list.OrderByDescending(x => x.Size),
-                _ => throw new ArgumentOutOfRangeException()
-            };
-        }
-
-        private void Swap(SortBy to)
-        {
-            if (SortEnum != to)
-                SortEnum = to;
-            else
-                SortAscending = !SortAscending;
-        }
-
-        public ManifestVM(Manifest manifest)
-        {
-            Manifest = manifest;
-
-            SortByNameCommand = ReactiveCommand.Create(() => Swap(SortBy.Name));
-
-            SortBySizeCommand = ReactiveCommand.Create(() => Swap(SortBy.Size));
-
-            _searchResults =
-                this.WhenAnyValue(x => x.SearchTerm)
-                    .CombineLatest(
-                        this.WhenAnyValue(x => x.SortAscending),
-                        this.WhenAnyValue(x => x.SortEnum), 
-                        (term, ascending, sort) => term)
-                    .Throttle(TimeSpan.FromMilliseconds(800))
-                    .Select(term => term?.Trim())
-                    //.DistinctUntilChanged()
-                    .Select(term =>
-                    {
-                        if (string.IsNullOrWhiteSpace(term))
-                            return Order(Archives);
-
-                        return Order(Archives.Where(x =>
-                        {
-                            if (term.StartsWith("hash:"))
-                                return x.Hash.ToString().StartsWith(term.Replace("hash:", ""));
-                            return x.Name.StartsWith(term);
-                        }));
-                    })
-                    .ToGuiProperty(this, nameof(SearchResults), Order(Archives));
-        }
-    }
-}

Wabbajack/View Models/WebBrowserVM.cs

@@ -2,6 +2,8 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Reactive;
+using System.Reactive.Linq;
+using System.Reactive.Subjects;
 using System.Text;
 using System.Threading.Tasks;
 using CefSharp;
@@ -28,8 +30,12 @@ namespace Wabbajack
         [Reactive]
         public ReactiveCommand<Unit, Unit> BackCommand { get; set; }
 
+        public Subject<bool> IsBackEnabledSubject { get; } = new Subject<bool>();
+        public IObservable<bool> IsBackEnabled { get; }
+
         private WebBrowserVM(string url = "http://www.wabbajack.org")
         {
+            IsBackEnabled = IsBackEnabledSubject.StartWith(true);
             Instructions = "Wabbajack Web Browser";
         }
 

Wabbajack/Views/ManifestView.xaml

@@ -1,131 +0,0 @@
-<reactiveUi:ReactiveUserControl
-    x:Class="Wabbajack.ManifestView"
-    xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
-    xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-    xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
-    xmlns:local="clr-namespace:Wabbajack"
-    xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
-    xmlns:reactiveUi="http://reactiveui.net"
-    d:DesignHeight="450"
-    d:DesignWidth="800"
-    x:TypeArguments="local:ManifestVM"
-    mc:Ignorable="d">
-    <Grid>
-        <ScrollViewer
-            MouseDown="ScrollViewer_MouseDown"
-            MouseMove="ScrollViewer_MouseMove"
-            MouseUp="ScrollViewer_MouseUp">
-            <ScrollViewer.Resources>
-                <Style x:Key="HeaderStyle" TargetType="{x:Type TextBlock}">
-                    <Setter Property="Foreground" Value="#03DAC6" />
-                </Style>
-                <Style x:Key="HyperlinkStyle" TargetType="{x:Type Hyperlink}">
-                    <Setter Property="Foreground" Value="#BB76FC" />
-                </Style>
-                <Style x:Key="ModTitleStyle" TargetType="{x:Type TextBox}">
-                    <Setter Property="Foreground" Value="#C7FC86" />
-                    <Setter Property="Background" Value="Transparent" />
-                </Style>
-                <SolidColorBrush
-                    x:Key="SearchBarBrush"
-                    Opacity="50"
-                    Color="White" />
-            </ScrollViewer.Resources>
-            <StackPanel x:Name="DynamicStackPanel" Margin="8">
-                <TextBlock
-                    x:Name="Name"
-                    HorizontalAlignment="Center"
-                    FontSize="32"
-                    Style="{StaticResource HeaderStyle}" />
-                <TextBlock
-                    x:Name="Version"
-                    Padding="0,12,0,3"
-                    FontSize="14" />
-                <TextBlock
-                    x:Name="Author"
-                    Padding="0,3,0,3"
-                    FontSize="14" />
-                <TextBlock
-                    x:Name="Description"
-                    FontSize="18"
-                    TextWrapping="Wrap" />
-
-                <TextBlock
-                    x:Name="InstallSize"
-                    Padding="0,24,0,0"
-                    FontSize="24" />
-                <TextBlock
-                    x:Name="DownloadSize"
-                    Padding="0,0,0,12"
-                    FontSize="24" />
-
-                <TextBlock
-                    Padding="6,0,0,0"
-                    FontSize="20"
-                    Text="Search:" />
-                <TextBox
-                    x:Name="SearchBar"
-                    BorderBrush="{StaticResource SearchBarBrush}"
-                    BorderThickness="2"
-                    FontSize="16"
-                    MaxLength="100" />
-
-                <StackPanel Margin="6,6,0,0" Orientation="Horizontal">
-                    <TextBlock Padding="0,1,0,0" FontSize="14">Order by:</TextBlock>
-                    <Button x:Name="OrderByNameButton" Padding="4,0,4,0">
-                        <TextBlock FontSize="14">Name</TextBlock>
-                    </Button>
-                    <Button x:Name="OrderBySizeButton">
-                        <TextBlock FontSize="14">Size</TextBlock>
-                    </Button>
-                </StackPanel>
-
-                <ItemsControl x:Name="ModsList" Padding="0,3,0,6">
-                    <ItemsControl.ItemsPanel>
-                        <ItemsPanelTemplate>
-                            <StackPanel />
-                        </ItemsPanelTemplate>
-                    </ItemsControl.ItemsPanel>
-
-                    <ItemsControl.ItemTemplate>
-                        <DataTemplate>
-                            <Grid Margin="0,6,0,6">
-                                <Grid.ColumnDefinitions>
-                                    <ColumnDefinition Width="Auto" />
-                                    <ColumnDefinition Width="Auto" />
-                                </Grid.ColumnDefinitions>
-                                <Grid.RowDefinitions>
-                                    <RowDefinition Height="Auto" />
-                                    <RowDefinition Height="Auto" />
-                                </Grid.RowDefinitions>
-                                <TextBox Grid.Row="0" Grid.Column="0"
-                                    Padding="6,0,0,0"
-                                    BorderThickness="0"
-                                    FontSize="18"
-                                    IsReadOnly="True"
-                                    Style="{StaticResource ModTitleStyle}"
-                                    Text="{Binding Path=Name}" />
-                                <TextBlock Grid.Row="0" Grid.Column="1"
-                                    Padding="3,0,0,0"
-                                    FontSize="18">
-                                    <Hyperlink
-                                        NavigateUri="{Binding Path=Name}"
-                                        RequestNavigate="Hyperlink_OnRequestNavigate"
-                                        Style="{StaticResource HyperlinkStyle}">
-                                        Link
-                                    </Hyperlink>
-                                </TextBlock>
-                                <TextBox Grid.Row="1" Grid.Column="0"
-                                    Padding="6,0,0,0"
-                                    FontSize="15"
-                                    IsReadOnly="True"
-                                    Text="{Binding Path=Hash}" />
-                            </Grid>
-                        </DataTemplate>
-                    </ItemsControl.ItemTemplate>
-                </ItemsControl>
-            </StackPanel>
-        </ScrollViewer>
-        <Canvas x:Name="TopLayer" IsHitTestVisible="False" />
-    </Grid>
-</reactiveUi:ReactiveUserControl>

Wabbajack/Views/ManifestView.xaml.cs

@@ -1,143 +0,0 @@
-using System;
-using System.Diagnostics;
-using System.Reactive.Disposables;
-using System.Windows;
-using System.Windows.Controls;
-using System.Windows.Documents;
-using System.Windows.Input;
-using System.Windows.Navigation;
-using ReactiveUI;
-using Wabbajack.Lib;
-
-namespace Wabbajack
-{
-    public partial class ManifestView
-    {
-        public ModList Modlist { get; set; }
-
-        public ManifestView(ModList modlist)
-        {
-            Modlist = modlist;
-
-            var manifest = new Manifest(modlist);
-            ViewModel ??= new ManifestVM(manifest);
-
-            InitializeComponent();
-
-            this.WhenActivated(disposable =>
-            {
-                this.OneWayBind(ViewModel, x => x.Name, x => x.Name.Text)
-                    .DisposeWith(disposable);
-                this.OneWayBind(ViewModel, x => x.Manifest.Version, x => x.Version.Text)
-                    .DisposeWith(disposable);
-                this.OneWayBind(ViewModel, x => x.Author, x => x.Author.Text)
-                    .DisposeWith(disposable);
-                this.OneWayBind(ViewModel, x => x.Description, x => x.Description.Text)
-                    .DisposeWith(disposable);
-                this.OneWayBind(ViewModel, x => x.SearchResults, x => x.ModsList.ItemsSource)
-                    .DisposeWith(disposable);
-                this.OneWayBind(ViewModel, x => x.InstallSize, x => x.InstallSize.Text)
-                    .DisposeWith(disposable);
-                this.OneWayBind(ViewModel, x => x.DownloadSize, x => x.DownloadSize.Text)
-                    .DisposeWith(disposable);
-                this.Bind(ViewModel, x => x.SearchTerm, x => x.SearchBar.Text)
-                    .DisposeWith(disposable);
-                this.BindCommand(ViewModel, x => x.SortByNameCommand, x => x.OrderByNameButton)
-                    .DisposeWith(disposable);
-                this.BindCommand(ViewModel, x => x.SortBySizeCommand, x => x.OrderBySizeButton)
-                    .DisposeWith(disposable);
-            });
-        }
-
-        private void Hyperlink_OnRequestNavigate(object sender, RequestNavigateEventArgs e)
-        {
-            if (!(sender is Hyperlink hyperlink)) return;
-            if (!(hyperlink.DataContext is Archive archive)) return;
-
-            var url = archive.State.GetManifestURL(archive);
-            if (string.IsNullOrWhiteSpace(url)) return;
-
-            if (url.StartsWith("https://github.com/"))
-                url = url.Substring(0, url.IndexOf("release", StringComparison.Ordinal));
-
-            //url = url.Replace("&", "^&");
-            url = url.Replace(" ", "%20");
-            Process.Start(new ProcessStartInfo("cmd", $"/c start {url}") {CreateNoWindow = true});
-
-            e.Handled = true;
-        }
-
-        //solution from https://stackoverflow.com/questions/5426232/how-can-i-make-wpf-scrollviewer-middle-click-scroll/5446307#5446307
-
-        private bool _isMoving;                  //False - ignore mouse movements and don't scroll
-        private bool _isDeferredMovingStarted;   //True - Mouse down -> Mouse up without moving -> Move; False - Mouse down -> Move
-        private Point? _startPosition;
-        private const double Slowdown = 10;      //smaller = faster
-
-        private void ScrollViewer_MouseDown(object sender, MouseButtonEventArgs e)
-        {
-            if (_isMoving)
-                CancelScrolling();
-            else if (e.ChangedButton == MouseButton.Middle && e.ButtonState == MouseButtonState.Pressed)
-            {
-                if (_isMoving) return;
-
-                _isMoving = true;
-                _startPosition = e.GetPosition(sender as IInputElement);
-                _isDeferredMovingStarted = true;
-
-                AddScrollSign(e.GetPosition(TopLayer).X, e.GetPosition(TopLayer).Y);
-            }
-        }
-
-        private void ScrollViewer_MouseUp(object sender, MouseButtonEventArgs e)
-        {
-            if(e.ChangedButton == MouseButton.Middle && e.ButtonState == MouseButtonState.Released && _isDeferredMovingStarted != true)
-                CancelScrolling();
-        }
-        
-        private void ScrollViewer_MouseMove(object sender, MouseEventArgs e)
-        {
-            if (!_isMoving || !(sender is ScrollViewer sv))
-                return;
-
-            _isDeferredMovingStarted = false;
-
-            var currentPosition = e.GetPosition(sv);
-            if (_startPosition == null)
-                return;
-
-            var offset = currentPosition - _startPosition.Value;
-            offset.Y /= Slowdown;
-            offset.X /= Slowdown;
-
-            sv.ScrollToVerticalOffset(sv.VerticalOffset + offset.Y);
-            sv.ScrollToHorizontalOffset(sv.HorizontalOffset + offset.X);
-        }
-
-        private void CancelScrolling()
-        {
-            _isMoving = false;
-            _startPosition = null;
-            _isDeferredMovingStarted = false;
-            RemoveScrollSign();
-        }
-
-        private void AddScrollSign(double x, double y)
-        {
-            const double size = 50.0;
-            var img = ResourceLinks.MiddleMouseButton.Value;
-            var icon = new Image {Source = img, Width = size, Height = size};
-            //var icon = new Ellipse { Stroke = Brushes.Red, StrokeThickness = 2.0, Width = 20, Height = 20 };
-
-            TopLayer.Children.Add(icon);
-            Canvas.SetLeft(icon, x - size / 2);
-            Canvas.SetTop(icon, y - size / 2);
-        }
-
-        private void RemoveScrollSign()
-        {
-            TopLayer.Children.Clear();
-        }
-    }
-}

Wabbajack/Views/ManifestWindow.xaml

@@ -1,22 +0,0 @@
-<mah:MetroWindow
-    x:Class="Wabbajack.ManifestWindow"
-    xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
-    xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-    xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
-    xmlns:local="clr-namespace:Wabbajack"
-    xmlns:mah="http://metro.mahapps.com/winfx/xaml/controls"
-    xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
-    Title="Manifest"
-    Width="1280"
-    Height="960"
-    MinWidth="850"
-    MinHeight="650"
-    d:DataContext="{d:DesignInstance local:ManifestWindow}"
-    ResizeMode="CanResize"
-    Style="{StaticResource {x:Type Window}}"
-    TitleBarHeight="25"
-    UseLayoutRounding="True"
-    WindowTitleBrush="{StaticResource MahApps.Brushes.Accent}"
-    mc:Ignorable="d">
-    <Grid x:Name="Grid" />
-</mah:MetroWindow>

Wabbajack/Views/ManifestWindow.xaml.cs

@@ -1,22 +0,0 @@
-using Wabbajack.Lib;
-
-namespace Wabbajack
-{
-    public partial class ManifestWindow
-    {
-        public ModList Modlist { get; set; }
-
-        public ManifestWindow(ModList modlist)
-        {
-            Modlist = modlist;
-
-            InitializeComponent();
-
-            var manifestView = new ManifestView(Modlist);
-
-            Grid.Children.Add(manifestView);
-
-            Title = $"{Modlist.Name} by {Modlist.Author}";
-        }
-    }
-}

Wabbajack/Views/ModListGalleryView.xaml

@@ -116,6 +116,14 @@
                 VerticalAlignment="Center"
                 Content="Show NSFW"
                 Foreground="{StaticResource ForegroundBrush}" />
+            
+            <CheckBox
+                x:Name="ShowUtilityLists"
+                Margin="10,0,10,0"
+                VerticalAlignment="Center"
+                Content="Show Utility Lists"
+                Foreground="{StaticResource ForegroundBrush}" />
+            
             <CheckBox
                 x:Name="OnlyInstalledCheckbox"
                 Margin="10,0,10,0"

Wabbajack/Views/ModListGalleryView.xaml.cs

@@ -63,6 +63,8 @@ namespace Wabbajack
                     .DisposeWith(dispose);
                 this.BindStrict(ViewModel, vm => vm.ShowNSFW, x => x.ShowNSFW.IsChecked)
                     .DisposeWith(dispose);
+                this.BindStrict(ViewModel, vm => vm.ShowUtilityLists, x => x.ShowUtilityLists.IsChecked)
+                    .DisposeWith(dispose);
 
                 this.WhenAny(x => x.ViewModel.ClearFiltersCommand)
                     .BindToStrict(this, x => x.ClearFiltersButton.Command)

Wabbajack/Views/ModListTileView.xaml.cs

@@ -36,12 +36,17 @@ namespace Wabbajack
                     .Select(p => p.Value)
                     .BindToStrict(this, x => x.DownloadProgressBar.Value)
                     .DisposeWith(dispose);
-                this.WhenAny(x => x.ViewModel.Metadata.Title)
+                this.WhenAny(x => x.ViewModel.Metadata)
+                    .Where(x => !x.ImageContainsTitle)
+                    .Select(x => x.Title)
                     .BindToStrict(this, x => x.DescriptionTextShadow.Text)
                     .DisposeWith(dispose);
-                this.WhenAny(x => x.ViewModel.Metadata.Title)
+                this.WhenAny(x => x.ViewModel.Metadata)
+                    .Where(x => !x.ImageContainsTitle)
+                    .Select(x => x.Title)
                     .BindToStrict(this, x => x.ModListTitleShadow.Text)
                     .DisposeWith(dispose);
+
                 this.WhenAny(x => x.ViewModel.IsBroken)
                     .Select(x => x ? Visibility.Visible : Visibility.Collapsed)
                     .BindToStrict(this, x => x.Overlay.Visibility)

Wabbajack/Wabbajack.csproj

@@ -6,8 +6,8 @@
     <UseWPF>true</UseWPF>
     <Platforms>x64</Platforms>
     <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
-      <AssemblyVersion>2.3.2.0</AssemblyVersion>
-      <FileVersion>2.3.2.0</FileVersion>
+      <AssemblyVersion>2.3.5.1</AssemblyVersion>
+      <FileVersion>2.3.5.1</FileVersion>
     <Copyright>Copyright © 2019-2020</Copyright>
     <Description>An automated ModList installer</Description>
     <PublishReadyToRun>true</PublishReadyToRun>
@@ -55,7 +55,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="CefSharp.Wpf" Version="85.3.130" />
+    <PackageReference Include="CefSharp.Wpf" Version="86.0.241" />
     <PackageReference Include="DynamicData" Version="6.17.14" />
     <PackageReference Include="Extended.Wpf.Toolkit" Version="4.0.1" />
     <PackageReference Include="Fody" Version="6.3.0">
@@ -67,7 +67,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="MahApps.Metro" Version="2.3.3" />
+    <PackageReference Include="MahApps.Metro" Version="2.4.3" />
     <PackageReference Include="MahApps.Metro.IconPacks" Version="4.8.0" />
     <PackageReference Include="PInvoke.Gdi32" Version="0.7.78" />
     <PackageReference Include="PInvoke.User32" Version="0.7.78" />