merge master

Wabbajack.CacheServer/DTOs/DownloadState.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Bson.Serialization.Attributes;
+using Wabbajack.Lib.Downloaders;
+
+namespace Wabbajack.CacheServer.DTOs
+{
+    public class DownloadState
+    {
+        [BsonId]
+        public string Key { get; set; }
+        public string Hash { get; set; }
+        
+        public AbstractDownloadState State { get; set; }
+
+        public bool IsValid { get; set; }
+        public DateTime LastValidationTime { get; set; } = DateTime.Now;
+        public DateTime FirstValidationTime { get; set; } = DateTime.Now;
+    }
+}

Wabbajack.CacheServer/DTOs/IndexedFile.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Security.RightsManagement;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Bson.Serialization.Attributes;
+using Wabbajack.VirtualFileSystem;
+
+namespace Wabbajack.CacheServer.DTOs
+{
+    public class IndexedFile
+    {
+        [BsonId]
+        public string Hash { get; set; }
+        public string SHA256 { get; set; }
+        public string SHA1 { get; set; }
+        public string MD5 { get; set; }
+        public string CRC { get; set; }
+        public long Size { get; set; }
+        public bool IsArchive { get; set; }
+        public List<ChildFile> Children { get; set; } = new List<ChildFile>();
+    }
+
+    public class ChildFile
+    {
+        public string Name;
+        public string Extension;
+        public string Hash;
+    }
+}

Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs

@@ -0,0 +1,32 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Bson.Serialization.Attributes;
+using Wabbajack.CacheServer.Jobs;
+
+namespace Wabbajack.CacheServer.DTOs.JobQueue
+{
+    public abstract class AJobPayload
+    {
+        public static List<Type> KnownSubTypes = new List<Type> {typeof(IndexJob)};
+        public static Dictionary<Type, string> TypeToName { get; set; }
+        public static Dictionary<string, Type> NameToType { get; set; }
+
+
+        [BsonIgnore]
+        public abstract string Description { get; }
+
+        public virtual bool UsesNexus { get; } = false;
+
+        public abstract Task<JobResult> Execute();
+
+        static AJobPayload()
+        {
+            NameToType = KnownSubTypes.ToDictionary(t => t.FullName.Substring(t.Namespace.Length + 1), t => t);
+            TypeToName = NameToType.ToDictionary(k => k.Value, k => k.Key);
+        }
+
+    }
+}

Wabbajack.CacheServer/DTOs/JobQueue/Job.cs

@@ -0,0 +1,67 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+using MongoDB.Driver;
+
+namespace Wabbajack.CacheServer.DTOs.JobQueue
+{ 
+    public class Job
+    {
+        public enum JobPriority : int
+        {
+            Low,
+            Normal,
+            High,
+        }
+
+        [BsonId]
+        public Guid Id { get; set; }
+        public DateTime? Started { get; set; }
+        public DateTime? Ended { get; set; }
+        public DateTime Created { get; set; } = DateTime.Now;
+        public JobPriority Priority { get; set; } = JobPriority.Normal;
+
+        public JobResult Result { get; set; }
+        public bool RequiresNexus { get; set; } = true;
+        public AJobPayload Payload { get; set; }
+
+        public static async Task<Guid> Enqueue(Job job)
+        {
+            await Server.Config.JobQueue.Connect().InsertOneAsync(job);
+            return job.Id;
+        }
+
+        public static async Task<Job> GetNext()
+        {
+            var filter = new BsonDocument
+            {
+                {"Started", BsonNull.Value}
+            };
+            var update = new BsonDocument
+            {
+                {"$set", new BsonDocument {{"Started", DateTime.Now}}}
+            };
+            var sort = new {Priority=-1, Created=1}.ToBsonDocument();
+            var job = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update, new FindOneAndUpdateOptions<Job>{Sort = sort});
+            return job;
+        }
+
+        public static async Task<Job> Finish(Job job, JobResult jobResult)
+        {
+            var filter = new BsonDocument
+            {
+                {"query", new BsonDocument {{"Id", job.Id}}},
+            };
+            var update = new BsonDocument
+            {
+                {"$set", new BsonDocument {{"Ended", DateTime.Now}, {"Result", jobResult.ToBsonDocument()}}}
+            };
+            var result = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update);
+            return result;
+        }
+    }
+}

Wabbajack.CacheServer/DTOs/JobQueue/JobResult.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Wabbajack.CacheServer.DTOs.JobQueue
+{
+    public class JobResult
+    {
+        public JobResultType ResultType { get; set; }
+        [BsonIgnoreIfNull]
+        public string Message { get; set; }
+
+        [BsonIgnoreIfNull]
+        public string Stacktrace { get; set; }
+
+        public static JobResult Success()
+        {
+            return new JobResult { ResultType = JobResultType.Success };
+        }
+
+        public static JobResult Error(Exception ex)
+        {
+            return new JobResult {ResultType = JobResultType.Error, Stacktrace = ex.ToString()};
+        }
+
+    }
+
+    public enum JobResultType
+    {
+        Success,
+        Error
+    }
+}

Wabbajack.CacheServer/DTOs/Metric.cs

@@ -0,0 +1,16 @@
+using System;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+
+namespace Wabbajack.CacheServer.DTOs
+{
+    public class Metric
+    {
+        [BsonId]
+        public ObjectId Id;
+        public DateTime Timestamp;
+        public string Action;
+        public string Subject;
+    }
+}

Wabbajack.CacheServer/DTOs/ModListStatus.cs

@@ -0,0 +1,70 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using MongoDB.Bson.Serialization.Attributes;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.Lib;
+using Wabbajack.Lib.ModListRegistry;
+
+namespace Wabbajack.CacheServer.DTOs
+{
+    public class ModListStatus
+    {
+
+        [BsonId]
+        public string Id { get; set; }
+        public ModlistSummary Summary { get; set; }
+
+        public ModlistMetadata Metadata { get; set; }
+        public DetailedStatus DetailedStatus { get; set; }
+
+        public static async Task Update(ModListStatus status)
+        {
+            var id = status.Metadata.Links.MachineURL;
+            await Server.Config.ListValidation.Connect().FindOneAndReplaceAsync<ModListStatus>(s => s.Id == id, status, new FindOneAndReplaceOptions<ModListStatus> {IsUpsert = true});
+        }
+
+        public static IQueryable<ModListStatus> AllSummaries
+        {
+            get
+            {
+                return null;
+            }
+        }
+
+        public static async Task<ModListStatus> ByName(string name)
+        {
+            var result = await Server.Config.ListValidation.Connect()
+                .AsQueryable()
+                .Where(doc => doc.Metadata.Links.MachineURL == name || doc.Metadata.Title == name)
+                .ToListAsync();
+            return result.First();
+        }
+
+        public static IMongoQueryable<ModListStatus> All
+        {
+            get
+            {
+                return Server.Config.ListValidation.Connect().AsQueryable();
+            }
+        }
+    }
+
+    public class DetailedStatus
+    {
+        public string Name { get; set; }
+        public DateTime Checked { get; set; } = DateTime.Now;
+        public List<DetailedStatusItem> Archives { get; set; }
+        public DownloadMetadata DownloadMetaData { get; set; }
+        public bool HasFailures { get; set; }
+        public string MachineName { get; set; }
+    }
+
+    public class DetailedStatusItem
+    {
+        public bool IsFailing { get; set; }
+        public Archive Archive { get; set; }
+    }
+}

Wabbajack.CacheServer/DTOs/MongoDoc.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Bson;
+
+namespace Wabbajack.CacheServer.DTOs
+{
+    public class MongoDoc
+    {
+        public ObjectId _id { get; set; }  = ObjectId.Empty;
+    }
+}

Wabbajack.CacheServer/DTOs/NexusCacheData.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Wabbajack.CacheServer.DTOs
+{
+    public class NexusCacheData<T>
+    {
+        [BsonId]
+        public string Path { get; set; }
+        public T Data { get; set; }
+        public string Game { get; set; }
+        public string ModId { get; set; }
+
+        public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow;
+
+        [BsonIgnoreIfNull]
+        public string FileId { get; set; }
+
+    }
+}

Wabbajack.CacheServer/DTOs/SerializerSettings.cs

@@ -0,0 +1,74 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Bson;
+using MongoDB.Bson.IO;
+using MongoDB.Bson.Serialization;
+using MongoDB.Bson.Serialization.Conventions;
+using Wabbajack.CacheServer.DTOs.JobQueue;
+using Wabbajack.Lib.Downloaders;
+
+namespace Wabbajack.CacheServer.DTOs
+{
+    public static class SerializerSettings
+    {
+        public static void Init()
+        {
+            var dis = new TypeDiscriminator(typeof(AbstractDownloadState), AbstractDownloadState.NameToType,
+                AbstractDownloadState.TypeToName);
+            BsonSerializer.RegisterDiscriminatorConvention(typeof(AbstractDownloadState), dis);
+            BsonClassMap.RegisterClassMap<AbstractDownloadState>(cm => cm.SetIsRootClass(true));
+
+            dis = new TypeDiscriminator(typeof(AJobPayload), AJobPayload.NameToType, AJobPayload.TypeToName);
+            BsonSerializer.RegisterDiscriminatorConvention(typeof(AJobPayload), dis);
+            BsonClassMap.RegisterClassMap<AJobPayload>(cm => cm.SetIsRootClass(true));
+        }
+    }
+
+
+    public class TypeDiscriminator : IDiscriminatorConvention
+    {
+        private readonly Type defaultType;
+        private readonly Dictionary<string, Type> typeMap;
+        private Dictionary<Type, string> revMap;
+
+        public TypeDiscriminator(Type defaultType,
+            Dictionary<string, Type> typeMap, Dictionary<Type, string> revMap)
+        {
+            this.defaultType = defaultType;
+            this.typeMap = typeMap;
+            this.revMap = revMap;
+        }
+
+
+        /// <summary>
+        /// Element Name
+        /// </summary>
+        public string ElementName => "_wjType";
+
+        public Type GetActualType(IBsonReader bsonReader, Type nominalType)
+        {
+            Type type = null;
+            var bookmark = bsonReader.GetBookmark();
+            bsonReader.ReadStartDocument();
+            if (bsonReader.FindElement(ElementName))
+            {
+                var value = bsonReader.ReadString();
+                if (typeMap.ContainsKey(value))
+                    type = typeMap[value];
+            }
+
+            bsonReader.ReturnToBookmark(bookmark);
+            if (type == null)
+                throw new Exception($"Type mis-configuration can't find bson type for ${nominalType}");
+            return type;
+        }
+
+        public BsonValue GetDiscriminator(Type nominalType, Type actualType)
+        {
+            return revMap[actualType];
+        }
+    }
+}

Wabbajack.CacheServer/Extensions.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Expressions;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+
+namespace Wabbajack.CacheServer
+{
+    public static class Extensions
+    {
+        public static async Task<T> FindOneAsync<T>(this IMongoCollection<T> coll, Expression<Func<T, bool>> expr)
+        {
+            return (await coll.AsQueryable().Where(expr).Take(1).ToListAsync()).FirstOrDefault();
+        }
+    }
+}

Wabbajack.CacheServer/JobQueueEndpoints.cs

@@ -0,0 +1,105 @@
+using System;
+using System.Linq;
+using System.Linq.Expressions;
+using System.Security.Policy;
+using System.Threading.Tasks;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Nancy;
+using Nettle;
+using Wabbajack.CacheServer.DTOs.JobQueue;
+using Wabbajack.CacheServer.Jobs;
+
+namespace Wabbajack.CacheServer
+{
+    public class JobQueueEndpoints : NancyModule
+    {
+        public JobQueueEndpoints() : base ("/jobs")
+        {
+            Get("/", HandleListJobs);
+            Get("/enqueue_curated_for_indexing", HandleEnqueueAllCurated);
+        }
+
+        private readonly Func<object, string> HandleListJobsTemplate = NettleEngine.GetCompiler().Compile(@"
+                <html><head/><body>
+
+                <h2>Jobs - {{$.jobs.Count}} Pending</h2>
+                <h3>{{$.time}}</h3>
+                <ol>
+                {{each $.jobs}}
+                    <li>{{$.Description}}</li>
+                {{/each}}
+                </ol>
+
+                <script>
+                setTimeout(function() { location.reload();}, 10000);
+                </script>
+
+                </body></html>");
+
+        private async Task<Response> HandleListJobs(object arg)
+        {
+            var jobs = await Server.Config.JobQueue.Connect()
+                .AsQueryable<Job>()
+                .Where(j => j.Ended == null)
+                .OrderByDescending(j => j.Priority)
+                .ThenBy(j => j.Created)
+                .ToListAsync();
+
+            var response = (Response)HandleListJobsTemplate(new {jobs, time = DateTime.Now});
+            response.ContentType = "text/html";
+            return response;
+        }
+
+
+        private async Task<string> HandleEnqueueAllCurated(object arg)
+        {
+            var states = await Server.Config.ListValidation.Connect()
+                .AsQueryable()
+                .SelectMany(lst => lst.DetailedStatus.Archives)
+                .Select(a => a.Archive)
+                .ToListAsync();
+
+            var jobs = states.Select(state => new IndexJob {Archive = state})
+                .Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus})
+                .ToList();
+
+            if (jobs.Count > 0)
+                await Server.Config.JobQueue.Connect().InsertManyAsync(jobs);
+
+            return $"Enqueued {states.Count} jobs";
+        }
+
+        public static async Task StartJobQueue()
+        {
+            foreach (var task in Enumerable.Range(0, 4))
+            {
+                var tsk = StartJobQueueInner();
+            }
+        }
+
+        private static async Task StartJobQueueInner()
+        {
+            while (true)
+            {
+                try
+                {
+                    var job = await Job.GetNext();
+                    if (job == null)
+                    {
+                        await Task.Delay(5000);
+                        continue;
+                    }
+
+                    var result = await job.Payload.Execute();
+                    await Job.Finish(job, result);
+                }
+                catch (Exception ex)
+                {
+
+                }
+
+            }
+        }
+    }
+}

Wabbajack.CacheServer/Jobs/IndexJob.cs

@@ -0,0 +1,106 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.CacheServer.DTOs;
+using Wabbajack.CacheServer.DTOs.JobQueue;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.VirtualFileSystem;
+
+namespace Wabbajack.CacheServer.Jobs
+{ 
+    public class IndexJob : AJobPayload
+    {
+        public Archive Archive { get; set; }
+        public override string Description { get; } = "Validate and index an archive";
+        public override bool UsesNexus { get => Archive.State is NexusDownloader.State; }
+        public override async Task<JobResult> Execute()
+        {
+
+            var pk = new List<object>();
+            pk.Add(AbstractDownloadState.TypeToName[Archive.State.GetType()]);
+            pk.AddRange(Archive.State.PrimaryKey);
+            var pk_str = string.Join("|",pk.Select(p => p.ToString()));
+
+            var found = await Server.Config.DownloadStates.Connect().AsQueryable().Where(f => f.Key == pk_str).Take(1).ToListAsync();
+            if (found.Count > 0)
+                return JobResult.Success();
+
+            string fileName = Archive.Name;
+            string folder = Guid.NewGuid().ToString();
+            Utils.Log($"Indexer is downloading ${fileName}");
+            var downloadDest = Path.Combine(Server.Config.Indexer.DownloadDir, folder, fileName);
+            await Archive.State.Download(downloadDest);
+
+            using (var queue = new WorkQueue())
+            {
+                var vfs = new Context(queue, true);
+                await vfs.AddRoot(Path.Combine(Server.Config.Indexer.DownloadDir, folder));
+                var archive = vfs.Index.ByRootPath.First();
+                var converted = ConvertArchive(new List<IndexedFile>(), archive.Value);
+                try
+                {
+                    await Server.Config.IndexedFiles.Connect().InsertManyAsync(converted, new InsertManyOptions {IsOrdered = false});
+                }
+                catch (MongoBulkWriteException)
+                {
+                }
+
+                await Server.Config.DownloadStates.Connect().InsertOneAsync(new DownloadState
+                {
+                    Key = pk_str, 
+                    Hash = archive.Value.Hash, 
+                    State = Archive.State, 
+                    IsValid = true
+                });
+
+                var to_path = Path.Combine(Server.Config.Indexer.ArchiveDir,
+                    $"{Path.GetFileName(fileName)}_{archive.Value.Hash.FromBase64().ToHex()}_{Path.GetExtension(fileName)}");
+                if (File.Exists(to_path))
+                    File.Delete(downloadDest);
+                else
+                    File.Move(downloadDest, to_path);
+                Utils.DeleteDirectory(Path.Combine(Server.Config.Indexer.DownloadDir, folder));
+            }
+
+            return JobResult.Success();
+        }
+
+        private List<IndexedFile> ConvertArchive(List<IndexedFile> files, VirtualFile file, bool isTop = true)
+        {
+            var name = isTop ? Path.GetFileName(file.Name) : file.Name;
+            var ifile = new IndexedFile
+            {
+                Hash = file.Hash,
+                SHA256 = file.ExtendedHashes.SHA256,
+                SHA1 = file.ExtendedHashes.SHA1,
+                MD5 = file.ExtendedHashes.MD5,
+                CRC = file.ExtendedHashes.CRC,
+                Size = file.Size,
+                Children = file.Children != null ? file.Children.Select(
+                    f =>
+                    {
+                        ConvertArchive(files, f, false);
+
+                        return new ChildFile
+                        {
+                            Hash = f.Hash,
+                            Name = f.Name.ToLowerInvariant(),
+                            Extension = Path.GetExtension(f.Name.ToLowerInvariant())
+                        };
+                    }).ToList() : new List<ChildFile>()
+            };
+            ifile.IsArchive = ifile.Children.Count > 0;
+            files.Add(ifile);
+            return files;
+        }
+
+
+    }
+}

Wabbajack.CacheServer/ListValidationService.cs

@@ -1,52 +1,41 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
-using System.Collections.Immutable;
 using System.Linq;
 using System.Net.Http;
-using System.Text;
-using System.Threading;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
+using MongoDB.Driver;
 using Nancy;
-using Nancy.Responses;
+using Wabbajack.CacheServer.DTOs;
 using Wabbajack.Common;
 using Wabbajack.Lib;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.ModListRegistry;
+using MongoDB.Driver.Linq;
+using Nettle;
 
 namespace Wabbajack.CacheServer
 {
     public class ListValidationService : NancyModule
     {
-        public class ModListStatus
-        {
-            public string Name;
-            public DateTime Checked = DateTime.Now;
-            public List<(Archive archive, bool)> Archives { get; set; }
-            public DownloadMetadata DownloadMetaData { get; set; }
-            public bool HasFailures { get; set; }
-            public string MachineName { get; set; }
-        }
-
-        public static Dictionary<string, ModListStatus> ModLists { get; set; }
-
         public ListValidationService() : base("/lists")
         {
             Get("/status", HandleGetLists);
+            Get("/force_recheck", HandleForceRecheck);
             Get("/status/{Name}.json", HandleGetListJson);
             Get("/status/{Name}.html", HandleGetListHtml);
+
         }
 
-        private object HandleGetLists(object arg)
+        private async Task<string> HandleForceRecheck(object arg)
         {
-            var summaries = ModLists.Values.Select(m => new ModlistSummary
+            await ValidateLists(false);
-            {
+            return "done";
-                Name = m.Name,
+        }
-                Checked = m.Checked,
+
-                Failed = m.Archives.Count(a => a.Item2),
+        private async Task<string> HandleGetLists(object arg)
-                Passed = m.Archives.Count(a => !a.Item2),
+        {
-            }).ToList();
+            var summaries = await ModListStatus.All.Select(m => m.Summary).ToListAsync();
             return summaries.ToJSON();
         }
 
@@ -63,49 +52,42 @@ namespace Wabbajack.CacheServer
             public List<ArchiveSummary> Passed;
 
         }
-        private object HandleGetListJson(dynamic arg)
+        private async Task<string> HandleGetListJson(dynamic arg)
         {
-            var lst = ModLists[(string)arg.Name];
+            var metric = Metrics.Log("list_validation.get_list_json", (string)arg.Name);
-            var summary = new DetailedSummary
+            var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
-            {
+            return lst.ToJSON();
-                Name = lst.Name,
-                Checked = lst.Checked,
-                Failed = lst.Archives.Where(a => a.Item2)
-                    .Select(a => new ArchiveSummary {Name = a.archive.Name, State = a.archive.State}).ToList(),
-                Passed = lst.Archives.Where(a => !a.Item2)
-                    .Select(a => new ArchiveSummary { Name = a.archive.Name, State = a.archive.State }).ToList(),
-            };
-            return summary.ToJSON();
         }
 
-        private object HandleGetListHtml(dynamic arg)
+        
+        private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
+            <html><body>
+                <h2>{{lst.Name}} - {{lst.Checked}}</h2>
+                <h3>Failed ({{failed.Count}}):</h3>
+                <ul>
+                {{each $.failed }}
+                <li>{{$.Archive.Name}}</li>
+                {{/each}}
+                </ul>
+                <h3>Passed ({{passed.Count}}):</h3>
+                <ul>
+                {{each $.passed }}
+                <li>{{$.Archive.Name}}</li>
+                {{/each}}
+                </ul>
+            </body></html>
+        ");
+
+        private async Task<Response> HandleGetListHtml(dynamic arg)
         {
-            var lst = ModLists[(string)arg.Name];
+            
-            var sb = new StringBuilder();
+            var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
-
+            var response = (Response)HandleGetListTemplate(new
-            sb.Append("<html><body>");
-            sb.Append($"<h2>{lst.Name} - {lst.Checked}</h2>");
-
-            var failed_list = lst.Archives.Where(a => a.Item2).ToList();
-            sb.Append($"<h3>Failed ({failed_list.Count}):</h3>");
-            sb.Append("<ul>");
-            foreach (var archive in failed_list)
             {
-                sb.Append($"<li>{archive.archive.Name}</li>");
+                lst,
-            }
+                failed = lst.Archives.Where(a => a.IsFailing).ToList(),
-            sb.Append("</ul>");
+                passed = lst.Archives.Where(a => !a.IsFailing).ToList()
-
+            });
-            var pased_list = lst.Archives.Where(a => !a.Item2).ToList();
-            sb.Append($"<h3>Passed ({pased_list.Count}):</h3>");
-            sb.Append("<ul>");
-            foreach (var archive in pased_list.OrderBy(f => f.archive.Name))
-            {
-                sb.Append($"<li>{archive.archive.Name}</li>");
-            }
-            sb.Append("</ul>");
-
-            sb.Append("</body></html>");
-            var response = (Response)sb.ToString();
             response.ContentType = "text/html";
             return response;
         }
@@ -130,79 +112,104 @@ namespace Wabbajack.CacheServer
                 }
             }).FireAndForget();
         }
-        public static async Task ValidateLists()
+        public static async Task ValidateLists(bool skipIfNewer = true)
         {
             Utils.Log("Cleaning Nexus Cache");
             var client = new HttpClient();
-            await client.GetAsync("http://build.wabbajack.org/nexus_api_cache/update");
+            //await client.GetAsync("http://build.wabbajack.org/nexus_api_cache/update");
 
             Utils.Log("Starting Modlist Validation");
             var modlists = await ModlistMetadata.LoadFromGithub();
 
-            var statuses = new Dictionary<string, ModListStatus>();
-
             using (var queue = new WorkQueue())
             {
                 foreach (var list in modlists)
                 {
-                    var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ExtensionManager.Extension);
+                    try
-
-                    if (list.NeedsDownload(modlist_path))
                     {
-                        if (File.Exists(modlist_path))
+                        await ValidateList(list, queue, skipIfNewer);
-                            File.Delete(modlist_path);
-
-                        var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
-                        Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
-                        await state.Download(modlist_path);
                     }
-                    else
+                    catch (Exception ex)
                     {
-                        Utils.Log($"No changes detected from downloaded modlist");
                     }
-
+                }
-
-                    Utils.Log($"Loading {modlist_path}");
-
-                    var installer = AInstaller.LoadFromFile(modlist_path);
-
-                    Utils.Log($"{installer.Archives.Count} archives to validate");
- 
-                    DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
-
-                    var validated = (await installer.Archives
-                            .PMap(queue, async archive =>
-                            {
-                                Utils.Log($"Validating: {archive.Name}");
-                                bool is_failed;
-                                try
-                                {
-                                    is_failed = !(await archive.State.Verify());
-                                }
-                                catch (Exception)
-                                {
-                                    is_failed = false;
-                                }
-
-                                return (archive, is_failed);
-                            }))
-                        .ToList();
-
-
-                    var status = new ModListStatus
-                    {
-                        Name = list.Title,
-                        Archives = validated.OrderBy(v => v.archive.Name).ToList(),
-                        DownloadMetaData = list.DownloadMetadata,
-                        HasFailures = validated.Any(v => v.is_failed)
-                    };
-
-                    statuses.Add(status.Name, status);
-                 }
             }
 
-            Utils.Log($"Done validating {statuses.Count} lists");
+            Utils.Log($"Done validating {modlists.Count} lists");
-            ModLists = statuses;
+        }
+
+        private static async Task ValidateList(ModlistMetadata list, WorkQueue queue, bool skipIfNewer = true)
+        {
+            var existing = await Server.Config.ListValidation.Connect().FindOneAsync(l => l.Id == list.Links.MachineURL);
+            if (skipIfNewer && existing != null && DateTime.Now - existing.DetailedStatus.Checked < TimeSpan.FromHours(2))
+                return;
+
+            var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ExtensionManager.Extension);
+
+            if (list.NeedsDownload(modlist_path))
+            {
+                if (File.Exists(modlist_path))
+                    File.Delete(modlist_path);
+
+                var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
+                Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
+                await state.Download(modlist_path);
+            }
+            else
+            {
+                Utils.Log($"No changes detected from downloaded modlist");
+            }
+
+
+            Utils.Log($"Loading {modlist_path}");
+
+            var installer = AInstaller.LoadFromFile(modlist_path);
+
+            Utils.Log($"{installer.Archives.Count} archives to validate");
+
+            DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
+
+            var validated = (await installer.Archives
+                    .PMap(queue, async archive =>
+                    {
+                        Utils.Log($"Validating: {archive.Name}");
+                        bool is_failed;
+                        try
+                        {
+                            is_failed = !(await archive.State.Verify());
+                        }
+                        catch (Exception)
+                        {
+                            is_failed = false;
+                        }
+
+                        return new DetailedStatusItem {IsFailing = is_failed, Archive = archive};
+                    }))
+                .ToList();
+
+
+            var status = new DetailedStatus
+            {
+                Name = list.Title,
+                Archives = validated.OrderBy(v => v.Archive.Name).ToList(),
+                DownloadMetaData = list.DownloadMetadata,
+                HasFailures = validated.Any(v => v.IsFailing)
+            };
+
+            var dto = new ModListStatus
+            {
+                Id = list.Links.MachineURL,
+                Summary = new ModlistSummary
+                {
+                    Name = status.Name,
+                    Checked = status.Checked,
+                    Failed = status.Archives.Count(a => a.IsFailing),
+                    Passed = status.Archives.Count(a => !a.IsFailing),
+                },
+                DetailedStatus = status,
+                Metadata = list
+            };
+            await ModListStatus.Update(dto);
         }
     }
 }

Wabbajack.CacheServer/Metrics.cs

@@ -1,13 +1,13 @@
 using System;
-using System.Collections.Generic;
 using System.Linq;
-using System.Net;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
 using Nancy;
-using ReactiveUI;
+using Wabbajack.CacheServer.DTOs;
 using Wabbajack.Common;
 
 namespace Wabbajack.CacheServer
@@ -19,19 +19,17 @@ namespace Wabbajack.CacheServer
     {
         private static SemaphoreSlim _lockObject = new SemaphoreSlim(1);
 
-        public static async Task Log(params object[] args)
+        public static async Task Log(DateTime timestamp, string action, string subject)
         {
-            var msg = new[] {string.Join("\t", args.Select(a => a.ToString()))};
+            var msg = new[] {string.Join("\t", new[]{timestamp.ToString(), action, subject})};
             Utils.Log(msg.First());
-            await _lockObject.WaitAsync();
+            var db = Server.Config.Metrics.Connect();
-            try
+            await db.InsertOneAsync(new Metric {Timestamp = timestamp, Action = action, Subject = subject});
-            {
+        }
-                File.AppendAllLines("stats.tsv", msg);
+
-            }
+        public static Task Log(string action, string subject)
-            finally
+        {
-            {
+            return Log(DateTime.Now, action, subject);
-                _lockObject.Release();
-            }
         }
 
         public Metrics() : base("/")
@@ -40,6 +38,26 @@ namespace Wabbajack.CacheServer
             Get("/metrics/chart/", HandleChart);
             Get("/metrics/chart/{Action}/", HandleChart);
             Get("/metrics/chart/{Action}/{Value}/", HandleChart);
+            Get("/metrics/ingest/{filename}", HandleBulkIngest);
+        }
+
+        private async Task<string> HandleBulkIngest(dynamic arg)
+        {
+            Log("Bulk Loading " + arg.filename.ToString());
+
+            var lines = File.ReadAllLines(Path.Combine(@"c:\tmp", (string)arg.filename));
+
+            var db = Server.Config.Metrics.Connect();
+            
+            var data = lines.Select(line => line.Split('\t'))
+                .Where(line => line.Length == 3)
+                .Select(line => new Metric{ Timestamp = DateTime.Parse(line[0]), Action = line[1], Subject = line[2] })
+                .ToList();
+
+            foreach (var metric in data)
+                await db.InsertOneAsync(metric);
+
+            return $"Processed {lines.Length} records";
         }
 
         private async Task<string> HandleMetrics(dynamic arg)
@@ -49,36 +67,33 @@ namespace Wabbajack.CacheServer
             return date.ToString();
         }
 
-        private static async Task<string[]> GetData()
-        {
-            await _lockObject.WaitAsync();
-            try
-            {
-                return File.ReadAllLines("stats.tsv");
-            }
-            finally
-            {
-                _lockObject.Release();
-            }
-        }
-
         private async Task<Response> HandleChart(dynamic arg)
         {
-            var data = (await GetData()).Select(line => line.Split('\t'))
+            /*var data = (await GetData()).Select(line => line.Split('\t'))
                 .Where(line => line.Length == 3)
-                .Select(line => new {date = DateTime.Parse(line[0]), Action = line[1], Value = line[2]});
+                .Select(line => new {date = DateTime.Parse(line[0]), Action = line[1], Value = line[2]});*/
+
+            var q = Server.Config.Metrics.Connect().AsQueryable();
 
             // Remove guids / Default, which come from testing
-            data = data.Where(d => !Guid.TryParse(d.Value ?? "", out _) && (d.Value ?? "") != "Default");
 
             if (arg?.Action != null)
-                data = data.Where(d => d.Action == arg.Action);
+            {
+                var action = (string)arg.Action;
+                q = q.Where(d => d.Action == action);
+            }
 
 
             if (arg?.Value != null)
-                data = data.Where(d => d.Value.StartsWith(arg.Value));
+            {
+                var value = (string)arg.Value;
+                q = q.Where(d => d.Subject.StartsWith(value));
+            }
 
-            var grouped_and_counted = data.GroupBy(d => d.date.ToString("yyyy-MM-dd"))
+            var data = (await q.Take(Int32.MaxValue).ToListAsync()).AsEnumerable();
+            data = data.Where(d => !Guid.TryParse(d.Subject ?? "", out Guid v) && (d.Subject ?? "") != "Default");
+
+            var grouped_and_counted = data.GroupBy(d => d.Timestamp.ToString("yyyy-MM-dd"))
                 .OrderBy(d => d.Key)
                 .Select(d => new {Day = d.Key, Count = d.Count()})
                 .ToList();
@@ -116,5 +131,10 @@ namespace Wabbajack.CacheServer
             response.ContentType = "text/html";
             return response;
         }
+
+        public void Log(string l)
+        {
+            Utils.Log("Metrics: " + l);
+        }
     }
 }

Wabbajack.CacheServer/NexusCacheModule.cs

@@ -3,10 +3,13 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
+using System.Reflection;
 using System.Text;
 using System.Threading.Tasks;
+using MongoDB.Driver;
 using Nancy;
-using Nancy.Helpers;
+using Newtonsoft.Json;
+using Wabbajack.CacheServer.DTOs;
 using Wabbajack.Common;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.NexusApi;
@@ -24,12 +27,65 @@ namespace Wabbajack.CacheServer
             Get("/nexus_api_cache/{request}.json", HandleCacheCall);
             Get("/nexus_api_cache", ListCache);
             Get("/nexus_api_cache/update", UpdateCache);
+            Get("/nexus_api_cache/ingest/{Folder}", HandleIngestCache);
+        }
+
+        class UpdatedMod
+        {
+            public long mod_id;
+            public long latest_file_update;
+            public long latest_mod_activity;
         }
 
         public async Task<object> UpdateCache(object arg)
         {
             var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
-            await api.ClearUpdatedModsInCache();
+            
+            var gameTasks = GameRegistry.Games.Values
+                .Where(game => game.NexusName != null)
+                .Select(async game =>
+                {
+                    return (game,
+                        mods: await api.Get<List<UpdatedMod>>(
+                            $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m"));
+                })
+                .Select(async rTask =>
+                {
+                    var (game, mods) = await rTask;
+                    return mods.Select(mod => new { game = game, mod = mod });
+                }).ToList();
+
+            Utils.Log($"Getting update list for {gameTasks.Count} games");
+
+            var purge = (await Task.WhenAll(gameTasks))
+                .SelectMany(i => i)
+                .ToList();
+
+            Utils.Log($"Found {purge.Count} updated mods in the last month");
+            using (var queue = new WorkQueue())
+            {
+                var collected = await purge.Select(d =>
+                {
+                    var a = d.mod.latest_file_update.AsUnixTime();
+                    // Mod activity could hide files
+                    var b = d.mod.latest_mod_activity.AsUnixTime();
+
+                    return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.mod_id.ToString()};
+                }).PMap(queue, async t =>
+                {
+                    var resultA = await Server.Config.NexusModInfos.Connect().DeleteManyAsync(f =>
+                        f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
+                    var resultB = await Server.Config.NexusModFiles.Connect().DeleteManyAsync(f =>
+                        f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
+                    var resultC = await Server.Config.NexusFileInfos.Connect().DeleteManyAsync(f =>
+                        f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
+
+                    return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount;
+                });
+
+                Utils.Log($"Purged {collected.Sum()} cache entries");
+            }
+
             return "Done";
         }
 
@@ -47,30 +103,123 @@ namespace Wabbajack.CacheServer
                 }));
         }
 
-        private async Task<object> HandleModInfo(dynamic arg)
+        private async Task<Response> HandleModInfo(dynamic arg)
         {
             Utils.Log($"{DateTime.Now} - Mod Info - {arg.GameName}/{arg.ModID}/");
-            var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
+            string gameName = arg.GameName;
-            return api.GetModInfo(GameRegistry.GetByNexusName((string)arg.GameName).Game, (string)arg.ModID).ToJSON();
+            string modId = arg.ModId;
+            var result = await Server.Config.NexusModInfos.Connect()
+                .FindOneAsync(info => info.Game == gameName && info.ModId == modId);
+
+            string method = "CACHED";
+            if (result == null)
+            {
+                var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
+                var path = $"/v1/games/{gameName}/mods/{modId}.json";
+                var body = await api.Get<ModInfo>(path);
+                result = new NexusCacheData<ModInfo>
+                {
+                    Data = body, 
+                    Path = path, 
+                    Game = gameName,
+                    ModId = modId
+                };
+                try
+                {
+                    await Server.Config.NexusModInfos.Connect().InsertOneAsync(result);
+                }
+                catch (MongoWriteException)
+                {
+
+                }
+
+                method = "NOT_CACHED";
+            }
+
+            Response response = result.Data.ToJSON();
+            response.Headers.Add("WABBAJACK_CACHE_FROM", method);
+            response.ContentType = "application/json";
+            return response;
         }
 
-        private async Task<object> HandleFileID(dynamic arg)
+        private async Task<Response> HandleFileID(dynamic arg)
         {
             Utils.Log($"{DateTime.Now} - File Info - {arg.GameName}/{arg.ModID}/{arg.FileID}");
-            var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
+            string gameName = arg.GameName;
-            return api.GetFileInfo(new NexusDownloader.State
+            string modId = arg.ModId;
+            string fileId = arg.FileId;
+            var result = await Server.Config.NexusFileInfos.Connect()
+                .FindOneAsync(info => info.Game == gameName && info.ModId == modId && info.FileId == fileId);
+
+            string method = "CACHED";
+            if (result == null)
             {
-                GameName = arg.GameName,
+                var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
-                ModID = arg.ModID,
+                var path = $"/v1/games/{gameName}/mods/{modId}/files/{fileId}.json";
-                FileID = arg.FileID
+                var body = await api.Get<NexusFileInfo>(path);
-            }).ToJSON();
+                result = new NexusCacheData<NexusFileInfo>
+                {
+                    Data = body, 
+                    Path = path, 
+                    Game = gameName, 
+                    ModId = modId,
+                    FileId = fileId
+                };
+                try
+                {
+                    await Server.Config.NexusFileInfos.Connect().InsertOneAsync(result);
+                }
+                catch (MongoWriteException)
+                {
+
+                }
+
+                method = "NOT_CACHED";
+            }
+
+            Response response = result.Data.ToJSON();
+            response.Headers.Add("WABBAJACK_CACHE_FROM", method);
+            response.ContentType = "application/json";
+            return response;
         }
 
-        private async Task<object> HandleGetFiles(dynamic arg)
+        private async Task<Response> HandleGetFiles(dynamic arg)
         {
             Utils.Log($"{DateTime.Now} - Mod Files - {arg.GameName} {arg.ModID}");
-            var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
+            string gameName = arg.GameName;
-            return api.GetModFiles(GameRegistry.GetByNexusName((string)arg.GameName).Game, (int)arg.ModID).ToJSON();
+            string modId = arg.ModId;
+            var result = await Server.Config.NexusModFiles.Connect()
+                .FindOneAsync(info => info.Game == gameName && info.ModId == modId);
+
+            string method = "CACHED";
+            if (result == null)
+            {
+                var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
+                var path = $"/v1/games/{gameName}/mods/{modId}/files.json";
+                var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
+                result = new NexusCacheData<NexusApiClient.GetModFilesResponse>
+                {
+                    Data = body,
+                    Path = path,
+                    Game = gameName,
+                    ModId = modId
+                };
+                try
+                {
+                    await Server.Config.NexusModFiles.Connect().InsertOneAsync(result);
+                }
+                catch (MongoWriteException)
+                {
+
+                }
+
+                method = "NOT_CACHED";
+            }
+
+            Response response = result.Data.ToJSON();
+            response.Headers.Add("WABBAJACK_CACHE_FROM", method);
+            response.ContentType = "application/json";
+            return response;
         }
 
         private async Task<string> HandleCacheCall(dynamic arg)
@@ -79,27 +228,11 @@ namespace Wabbajack.CacheServer
             {
                 string param = (string)arg.request;
                 var url = new Uri(Encoding.UTF8.GetString(param.FromHex()));
-                var path = Path.Combine(NexusApiClient.LocalCacheDir, arg.request + ".json");
 
-                if (!File.Exists(path))
+                var client = new HttpClient();
-                {
+                var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 8080, Scheme = "http"};
-                    Utils.Log($"{DateTime.Now} - Not Cached - {url}");
+                client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]); 
-                    var client = new HttpClient();
+                return await client.GetStringAsync(builder.Uri.ToString());
-                    var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 8080, Scheme = "http"};
-                    client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]);
-                    await client.GetStringAsync(builder.Uri.ToString());
-                    if (!File.Exists(path))
-                    {
-                        Utils.Log($"Still not cached : {path}");
-                        throw new InvalidDataException("Invalid Data");
-                    }
-
-                    Utils.Log($"Is Now Cached : {path}");
-
-                }
-
-                Utils.Log($"{DateTime.Now} - From Cached - {url}");
-                return File.ReadAllText(path);
             }
             catch (Exception ex)
             {
@@ -107,5 +240,105 @@ namespace Wabbajack.CacheServer
                 return "ERROR";
             }
         }
+
+        private async Task<string> HandleIngestCache(dynamic arg)
+        {
+            int count = 0;
+            int failed = 0;
+
+            using (var queue = new WorkQueue())
+            {
+                await Directory.EnumerateFiles(Path.Combine(Server.Config.Settings.TempDir, (string)arg.Folder)).PMap(queue,
+                    async file =>
+                    {
+                        Utils.Log($"Ingesting {file}");
+                        if (!file.EndsWith(".json")) return;
+
+                        var fileInfo = new FileInfo(file);
+                        count++;
+
+                        var url = new Url(
+                            Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(file).FromHex()));
+                        var split = url.Path.Split(new[] {'/'}, StringSplitOptions.RemoveEmptyEntries);
+                        try
+                        {
+                            switch (split.Length)
+                            {
+                                case 5 when split[3] == "mods":
+                                {
+                                    var body = file.FromJSON<ModInfo>();
+
+                                    var payload = new NexusCacheData<ModInfo>();
+                                    payload.Data = body;
+                                    payload.Game = split[2];
+                                    payload.Path = url.Path;
+                                    payload.ModId = body.mod_id;
+                                    payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc;
+
+                                    try
+                                    {
+                                        await Server.Config.NexusModInfos.Connect().InsertOneAsync(payload);
+                                    }
+                                    catch (MongoWriteException ex)
+                                    {
+
+                                    }
+
+                                    break;
+                                }
+                                case 6 when split[5] == "files.json":
+                                {
+                                    var body = file.FromJSON<NexusApiClient.GetModFilesResponse>();
+                                    var payload = new NexusCacheData<NexusApiClient.GetModFilesResponse>();
+                                    payload.Path = url.Path;
+                                    payload.Data = body;
+                                    payload.Game = split[2];
+                                    payload.ModId = split[4];
+                                    payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc;
+
+                                        try
+                                        {
+                                        await Server.Config.NexusModFiles.Connect().InsertOneAsync(payload);
+                                    }
+                                    catch (MongoWriteException ex)
+                                    {
+
+                                    }
+
+                                    break;
+                                }
+                                case 7 when split[5] == "files":
+                                {
+                                    var body = file.FromJSON<NexusFileInfo>();
+                                    var payload = new NexusCacheData<NexusFileInfo>();
+                                    payload.Data = body;
+                                    payload.Path = url.Path;
+                                    payload.Game = split[2];
+                                    payload.FileId = Path.GetFileNameWithoutExtension(split[6]);
+                                    payload.ModId = split[4];
+                                    payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc;
+
+                                        try
+                                        {
+                                        await Server.Config.NexusFileInfos.Connect().InsertOneAsync(payload);
+                                    }
+                                    catch (MongoWriteException ex)
+                                    {
+
+                                    }
+
+                                    break;
+                                }
+                            }
+                        }
+                        catch (Exception ex)
+                        {
+                            failed++;
+                        }
+                    });
+            }
+
+            return $"Inserted {count} caches, {failed} failed";
+        }
     }
 }

Wabbajack.CacheServer/Program.cs

@@ -15,8 +15,13 @@ namespace Wabbajack.CacheServer
             Utils.LogMessages.Subscribe(Console.WriteLine);
             using (var server = new Server("http://localhost:8080"))
             {
-                ListValidationService.Start();
+                Consts.WabbajackCacheHostname = "localhost";
+                Consts.WabbajackCachePort = 8080;
                 server.Start();
+
+                ListValidationService.Start();
+                var tsk = JobQueueEndpoints.StartJobQueue();
+                
                 Console.ReadLine();
             }
         }

Wabbajack.CacheServer/Server.cs

@@ -1,17 +1,13 @@
 using System;
-using System.Collections.Generic;
+using Alphaleonis.Win32.Filesystem;
-using System.Diagnostics;
-using System.Linq;
-using System.Net;
-using System.Net.Sockets;
-using System.Text;
-using System.Threading.Tasks;
 using Nancy;
 using Nancy.Bootstrapper;
 using Nancy.Configuration;
-using Nancy.Diagnostics;
 using Nancy.Hosting.Self;
 using Nancy.TinyIoc;
+using Wabbajack.CacheServer.DTOs;
+using Wabbajack.CacheServer.ServerConfig;
+using Wabbajack.Common;
 
 namespace Wabbajack.CacheServer
 {
@@ -19,15 +15,22 @@ namespace Wabbajack.CacheServer
     {
         private NancyHost _server;
         private HostConfiguration _config;
+        public static BuildServerConfig Config;
+
+        static Server()
+        {
+            SerializerSettings.Init();
+        }
+
 
         public Server(string address)
         {
             Address = address;
-            _config = new HostConfiguration {MaximumConnectionCount = 24, RewriteLocalhost = true};
+            _config = new HostConfiguration {MaximumConnectionCount = 200, RewriteLocalhost = true};
             //_config.UrlReservations.CreateAutomatically = true;
             _server = new NancyHost(_config, new Uri(address));
-            
+
-            
+            Config = File.ReadAllText("config.yaml").FromYaml<BuildServerConfig>();
         }
 
         public string Address { get; }

Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Driver.Core.Configuration;
+using Wabbajack.CacheServer.DTOs;
+using Wabbajack.CacheServer.DTOs.JobQueue;
+using Wabbajack.Lib.NexusApi;
+
+namespace Wabbajack.CacheServer.ServerConfig
+{
+    public class BuildServerConfig
+    {
+        public MongoConfig<Metric> Metrics { get; set; }
+        public MongoConfig<ModListStatus> ListValidation { get; set; }
+
+        public MongoConfig<Job> JobQueue { get; set; }
+
+        public MongoConfig<IndexedFile> IndexedFiles { get; set; }
+        public MongoConfig<DownloadState> DownloadStates { get; set; }
+
+        public MongoConfig<NexusCacheData<ModInfo>> NexusModInfos { get; set; }
+        public MongoConfig<NexusCacheData<NexusApiClient.GetModFilesResponse>> NexusModFiles { get; set; }
+        public MongoConfig<NexusCacheData<NexusFileInfo>> NexusFileInfos { get; set; }
+
+        public IndexerConfig Indexer { get; set; }
+
+        public Settings Settings { get; set; }
+    }
+}

Wabbajack.CacheServer/ServerConfig/IndexerConfig.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Wabbajack.CacheServer.ServerConfig
+{
+    public class IndexerConfig
+    {
+        public string DownloadDir { get; set; }
+        public string TempDir { get; set; }
+
+        public string ArchiveDir { get; set; }
+    }
+}

Wabbajack.CacheServer/ServerConfig/MongoConfig.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using MongoDB.Driver;
+using Wabbajack.CacheServer.DTOs;
+
+namespace Wabbajack.CacheServer.ServerConfig
+{
+    public class MongoConfig<T>
+    {
+        public string Host { get; set; }
+        public string Database { get; set; }
+        public string Collection { get; set; }
+        public string Username { get; set; }
+        public string Password { get; set; }
+
+        private IMongoDatabase Client
+        {
+            get
+            {
+                if (Username != null && Password != null)
+                    return new MongoClient($"mongodb://{Username}:{Password}@{Host}").GetDatabase(Database);
+                return new MongoClient($"mongodb://{Host}").GetDatabase(Database);
+            }
+        }
+
+        public IMongoCollection<T> Connect()
+        {
+            return Client.GetCollection<T>(Collection);
+        }
+    }
+}

Wabbajack.CacheServer/ServerConfig/Settings.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Wabbajack.CacheServer.ServerConfig
+{
+    public class Settings
+    {
+        public string TempDir { get; set; }
+    }
+}

Wabbajack.CacheServer/Wabbajack.CacheServer.csproj

@@ -73,6 +73,19 @@
     <Reference Include="WindowsBase" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="DTOs\DownloadState.cs" />
+    <Compile Include="DTOs\IndexedFile.cs" />
+    <Compile Include="DTOs\JobQueue\AJobPayload.cs" />
+    <Compile Include="DTOs\NexusCacheData.cs" />
+    <Compile Include="Jobs\IndexJob.cs" />
+    <Compile Include="DTOs\JobQueue\Job.cs" />
+    <Compile Include="DTOs\JobQueue\JobResult.cs" />
+    <Compile Include="DTOs\Metric.cs" />
+    <Compile Include="DTOs\ModListStatus.cs" />
+    <Compile Include="DTOs\MongoDoc.cs" />
+    <Compile Include="DTOs\SerializerSettings.cs" />
+    <Compile Include="Extensions.cs" />
+    <Compile Include="JobQueueEndpoints.cs" />
     <Compile Include="ListValidationService.cs" />
     <Compile Include="Metrics.cs" />
     <Compile Include="NexusCacheModule.cs" />
@@ -80,10 +93,17 @@
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Server.cs" />
     <Compile Include="Heartbeat.cs" />
+    <Compile Include="ServerConfig\BuildServerConfig.cs" />
+    <Compile Include="ServerConfig\IndexerConfig.cs" />
+    <Compile Include="ServerConfig\MongoConfig.cs" />
+    <Compile Include="ServerConfig\Settings.cs" />
     <Compile Include="TestingEndpoints.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config" />
+    <None Include="config.yaml">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
@@ -94,11 +114,21 @@
       <Project>{0a820830-a298-497d-85e0-e9a89efef5fe}</Project>
       <Name>Wabbajack.Lib</Name>
     </ProjectReference>
+    <ProjectReference Include="..\Wabbajack.VirtualFileSystem\Wabbajack.VirtualFileSystem.csproj">
+      <Project>{5d6a2eaf-6604-4c51-8ae2-a746b4bc5e3e}</Project>
+      <Name>Wabbajack.VirtualFileSystem</Name>
+    </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <PackageReference Include="MongoDB.Driver">
+      <Version>2.10.0</Version>
+    </PackageReference>
     <PackageReference Include="Nancy.Hosting.Self">
       <Version>2.0.0</Version>
     </PackageReference>
+    <PackageReference Include="Nettle">
+      <Version>1.3.0</Version>
+    </PackageReference>
     <PackageReference Include="Newtonsoft.Json">
       <Version>12.0.3</Version>
     </PackageReference>
@@ -106,8 +136,9 @@
       <Version>11.0.6</Version>
     </PackageReference>
     <PackageReference Include="System.Reactive">
-      <Version>4.3.1</Version>
+      <Version>4.3.2</Version>
     </PackageReference>
   </ItemGroup>
+  <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
 </Project>

Wabbajack.CacheServer/config.yaml

@@ -0,0 +1,39 @@
+---
+Metrics:
+    Host: internal.test.mongodb
+    Database: wabbajack
+    Collection: metrics
+ListValidation:
+    Host: internal.test.mongodb
+    Database: wabbajack
+    Collection: mod_lists
+JobQueue:
+    Host: internal.test.mongodb
+    Database: wabbajack
+    Collection: job_queue
+IndexedFiles:
+    Host: internal.test.mongodb
+    Database: wabbajack
+    Collection: indexed_files
+NexusModInfos:
+    Host: internal.test.mongodb
+    Database: wabbajack
+    Collection: nexus_mod_infos
+NexusModFiles:
+    Host: internal.test.mongodb
+    Database: wabbajack
+    Collection: nexus_mod_files
+NexusFileInfos:
+    Host: internal.test.mongodb
+    Database: wabbajack
+    Collection: nexus_file_infos
+DownloadStates:
+    Host: internal.test.mongodb
+    Database: wabbajack
+    Collection: download_states
+Indexer:
+    DownloadDir: c:\tmp\downloads
+    TempDir: c:\tmp\tmp
+    ArchiveDir: c:\archives
+Settings:
+    TempDir: c:\tmp\tmp

Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>netstandard2.0</TargetFramework>
     </PropertyGroup>
     <ItemGroup>
-      <PackageReference Include="System.Reactive" Version="4.3.1" />
+      <PackageReference Include="System.Reactive" Version="4.3.2" />
       <PackageReference Include="System.Threading.Tasks.Extensions" Version="4.5.3" />
     </ItemGroup>
     <ItemGroup>

Wabbajack.Common/Consts.cs

@@ -92,5 +92,8 @@ namespace Wabbajack.Common
         public static string LocalAppDataPath => Path.Combine(KnownFolders.LocalAppData.Path, "Wabbajack");
 
         public static string WabbajackCacheLocation = "http://build.wabbajack.org/nexus_api_cache/";
+
+        public static string WabbajackCacheHostname = "build.wabbajack.org";
+        public static int WabbajackCachePort = 80;
     }
 }

Wabbajack.Common/SteamHandler.cs

@@ -65,6 +65,8 @@ namespace Wabbajack.Common
         {
             var steamKey = Registry.CurrentUser.OpenSubKey(SteamRegKey);
             SteamPath = steamKey?.GetValue("SteamPath").ToString();
+            if(string.IsNullOrWhiteSpace(SteamPath) || steamKey == null || !Directory.Exists(SteamPath))
+                Utils.ErrorThrow(new Exception("Could not find the Steam folder!"));
             if(!init) return;
             LoadInstallFolders();
             LoadAllSteamGames();
@@ -92,10 +94,15 @@ namespace Wabbajack.Common
                 if (!l.Contains("BaseInstallFolder_")) return;
                 var s = GetVdfValue(l);
                 s = Path.Combine(s, "steamapps");
-                if(Directory.Exists(s))
+                if (!Directory.Exists(s))
-                    paths.Add(s);
+                    return;
+                
+                paths.Add(s);
+                Utils.Log($"Steam Library found at {s}");
             });
 
+            Utils.Log($"Total number of Steam Libraries found: {paths.Count}");
+
             // Default path in the Steam folder isn't in the configs
             if(Directory.Exists(Path.Combine(SteamPath, "steamapps")))
                 paths.Add(Path.Combine(SteamPath, "steamapps"));
@@ -145,9 +152,13 @@ namespace Wabbajack.Common
                             g.RequiredFiles.TrueForAll(s => File.Exists(Path.Combine(steamGame.InstallDir, s)))
                             )?.Game;
                     games.Add(steamGame);
+
+                    Utils.Log($"Found Game: {steamGame.Name} ({steamGame.AppId}) at {steamGame.InstallDir}");
                 });
             });
 
+            Utils.Log($"Total number of Steam Games found: {games.Count}");
+
             Games = games;
         }
 

Wabbajack.Lib/ACompiler.cs

@@ -95,10 +95,8 @@ namespace Wabbajack.Lib
                 ModList.Readme = $"readme{readme.Extension}";
             }
 
-            // DISABLED FOR THIS RELEASE
             //ModList.ReadmeIsWebsite = ReadmeIsWebsite;
 
-            //ModList.ToJSON(Path.Combine(ModListOutputFolder, "modlist.json"));
             ModList.ToCERAS(Path.Combine(ModListOutputFolder, "modlist"), CerasConfig.Config);
 
             if (File.Exists(ModListOutputFile))

Wabbajack.Lib/CerasConfig.cs

@@ -32,7 +32,6 @@ namespace Wabbajack.Lib
 
                 },
             };
-            // DISABLED FOR THIS RELEASE
             //Config.VersionTolerance.Mode = VersionToleranceMode.Standard;
         }
     }

Wabbajack.Lib/Data.cs

@@ -119,7 +119,6 @@ namespace Wabbajack.Lib
         /// <summary>
         ///     Whether readme is a website
         /// </summary>
-        /// DISABLED FOR THIS RELEASE
         //public bool ReadmeIsWebsite;
     }
 
@@ -246,19 +245,19 @@ namespace Wabbajack.Lib
         /// <summary>
         ///     MurMur3 Hash of the archive
         /// </summary>
-        public string Hash;
+        public string Hash { get; set; }
 
         /// <summary>
         /// Meta INI for the downloaded archive
         /// </summary>
-        public string Meta;
+        public string Meta { get; set; }
 
         /// <summary>
         ///     Human friendly name of this archive
         /// </summary>
-        public string Name;
+        public string Name { get; set; }
 
-        public long Size;
+        public long Size { get; set; }
         public AbstractDownloadState State { get; set; }
     }
 

Wabbajack.Lib/Downloaders/AbstractDownloadState.cs

@@ -1,5 +1,9 @@
-using System.Threading.Tasks;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
+using MongoDB.Bson.Serialization.Attributes;
 using Wabbajack.Lib.Validation;
 
 namespace Wabbajack.Lib.Downloaders
@@ -7,8 +11,39 @@ namespace Wabbajack.Lib.Downloaders
     /// <summary>
     /// Base for all abstract downloaders
     /// </summary>
+    [BsonDiscriminator(RootClass = true)]
+    [BsonKnownTypes(typeof(HTTPDownloader.State), typeof(GameFileSourceDownloader.State), typeof(GoogleDriveDownloader.State),
+    typeof(LoversLabDownloader.State), typeof(ManualDownloader.State), typeof(MediaFireDownloader.State), typeof(MegaDownloader.State),
+    typeof(ModDBDownloader.State), typeof(NexusDownloader.State), typeof(SteamWorkshopDownloader.State))]
     public abstract class AbstractDownloadState
     {
+
+        public static List<Type> KnownSubTypes = new List<Type>()
+        {
+            typeof(HTTPDownloader.State),
+            typeof(GameFileSourceDownloader.State),
+            typeof(GoogleDriveDownloader.State),
+            typeof(LoversLabDownloader.State),
+            typeof(ManualDownloader.State),
+            typeof(MediaFireDownloader.State),
+            typeof(MegaDownloader.State),
+            typeof(ModDBDownloader.State),
+            typeof(NexusDownloader.State),
+            typeof(SteamWorkshopDownloader.State)
+        };
+        public static Dictionary<string, Type> NameToType { get; set; }
+        public static Dictionary<Type, string> TypeToName { get; set; }
+
+        static AbstractDownloadState()
+        {
+            NameToType = KnownSubTypes.ToDictionary(t => t.FullName.Substring(t.Namespace.Length + 1), t => t);
+            TypeToName = NameToType.ToDictionary(k => k.Value, k => k.Key);
+        }
+
+        public abstract object[] PrimaryKey { get; }
+
+
+
         /// <summary>
         /// Returns true if this file is allowed to be downloaded via whitelist
         /// </summary>

Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs

@@ -52,6 +52,8 @@ namespace Wabbajack.Lib.Downloaders
 
             internal string SourcePath => Path.Combine(Game.MetaData().GameLocation(), GameFile);
 
+            public override object[] PrimaryKey { get => new object[] {Game, GameFile}; }
+
             public override bool IsWhitelisted(ServerWhitelist whitelist)
             {
                 return true;

Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs

@@ -36,6 +36,8 @@ namespace Wabbajack.Lib.Downloaders
         public class State : AbstractDownloadState
         {
             public string Id { get; set; }
+            public override object[] PrimaryKey { get => new object[] {Id}; }
+
             public override bool IsWhitelisted(ServerWhitelist whitelist)
             {
                 return whitelist.GoogleIDs.Contains(Id);

Wabbajack.Lib/Downloaders/HTTPDownloader.cs

@@ -63,6 +63,8 @@ namespace Wabbajack.Lib.Downloaders
             [Exclude]
             public HttpClient Client { get; set; }
 
+            public override object[] PrimaryKey { get => new object[] {Url};}
+
             public override bool IsWhitelisted(ServerWhitelist whitelist)
             {
                 return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p));

Wabbajack.Lib/Downloaders/LoversLabDownloader.cs

@@ -117,6 +117,8 @@ namespace Wabbajack.Lib.Downloaders
             public string FileID { get; set; }
             public string FileName { get; set; }
 
+            public override object[] PrimaryKey { get => new object[] {FileID, FileName}; }
+
             public override bool IsWhitelisted(ServerWhitelist whitelist)
             {
                 return true;

Wabbajack.Lib/Downloaders/ManualDownloader.cs

@@ -73,6 +73,8 @@ namespace Wabbajack.Lib.Downloaders
         public class State : AbstractDownloadState
         {
             public string Url { get; set; }
+            public override object[] PrimaryKey { get => new object[] {Url}; }
+
             public override bool IsWhitelisted(ServerWhitelist whitelist)
             {
                 return true;

Wabbajack.Lib/Downloaders/MediaFireDownloader.cs

@@ -24,6 +24,8 @@ namespace Wabbajack.Lib.Downloaders
         {
             public string Url { get; set; }
 
+            public override object[] PrimaryKey { get => new object[] {Url};}
+
             public override bool IsWhitelisted(ServerWhitelist whitelist)
             {
                 return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p));

Wabbajack.Lib/Downloaders/ModDBDownloader.cs

@@ -1,4 +1,5 @@
-using System.Net.Http;
+using System;
+using System.Net.Http;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Wabbajack.Common;
@@ -34,6 +35,8 @@ namespace Wabbajack.Lib.Downloaders
         public class State : AbstractDownloadState
         {
             public string Url { get; set; }
+            public override object[] PrimaryKey { get => new object[]{Url}; }
+
             public override bool IsWhitelisted(ServerWhitelist whitelist)
             {
                 // Everything from Moddb is whitelisted

Wabbajack.Lib/Downloaders/NexusDownloader.cs

@@ -112,18 +112,20 @@ namespace Wabbajack.Lib.Downloaders
 
         public class State : AbstractDownloadState
         {
-            public string Author;
+            public string Author { get; set; }
-            public string FileID;
+            public string FileID { get; set; }
-            public string GameName;
+            public string GameName { get; set; }
-            public string ModID;
+            public string ModID { get; set; }
-            public string UploadedBy;
+            public string UploadedBy { get; set; }
-            public string UploaderProfile;
+            public string UploaderProfile { get; set; }
-            public string Version;
+            public string Version { get; set; }
-            public string SlideShowPic;
+            public string SlideShowPic { get; set; }
-            public string ModName;
+            public string ModName { get; set; }
-            public string NexusURL;
+            public string NexusURL { get; set; }
-            public string Summary;
+            public string Summary { get; set; }
-            public bool Adult;
+            public bool Adult { get; set; }
+
+            public override object[] PrimaryKey { get => new object[]{GameName, ModID, FileID};}
 
             public override bool IsWhitelisted(ServerWhitelist whitelist)
             {
@@ -137,7 +139,7 @@ namespace Wabbajack.Lib.Downloaders
                 try
                 {
                     var client = await NexusApiClient.Get();
-                    url = await client.GetNexusDownloadLink(this, false);
+                    url = await client.GetNexusDownloadLink(this);
                 }
                 catch (Exception ex)
                 {

Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs

@@ -41,6 +41,8 @@ namespace Wabbajack.Lib.Downloaders
         public class State : AbstractDownloadState
         {
             public SteamWorkshopItem Item { get; set; }
+            public override object[] PrimaryKey { get => new object[] {Item.Game, Item.ItemID}; }
+
             public override bool IsWhitelisted(ServerWhitelist whitelist)
             {
                 return true;

Wabbajack.Lib/MO2Compiler.cs

@@ -320,7 +320,7 @@ namespace Wabbajack.Lib
 
             Utils.Log(
                 $"Removing {remove.Count} archives from the compilation state, this is probably not an issue but reference this if you have compilation failures");
-            remove.Do(r => Utils.Log($"Resolution failed for: {r.File}"));
+            remove.Do(r => Utils.Log($"Resolution failed for: {r.File.FullPath}"));
             IndexedArchives.RemoveAll(a => remove.Contains(a));
         }
 

Wabbajack.Lib/NexusApi/NexusApi.cs

@@ -179,6 +179,7 @@ namespace Wabbajack.Lib.NexusApi
             {
                 var dailyRemaining = int.Parse(response.Headers.GetValues("x-rl-daily-remaining").First());
                 var hourlyRemaining = int.Parse(response.Headers.GetValues("x-rl-hourly-remaining").First());
+                Utils.Log($"Nexus Requests Remaining: {dailyRemaining} daily - {hourlyRemaining} hourly");
 
                 lock (RemainingLock)
                 {
@@ -200,6 +201,7 @@ namespace Wabbajack.Lib.NexusApi
         {
             ApiKey = apiKey;
 
+            HttpClient.BaseAddress = new Uri("https://api.nexusmods.com");
             // set default headers for all requests to the Nexus API
             var headers = HttpClient.DefaultRequestHeaders;
             headers.Add("User-Agent", Consts.UserAgent);
@@ -218,10 +220,12 @@ namespace Wabbajack.Lib.NexusApi
             return new NexusApiClient(apiKey);
         }
 
-        private async Task<T> Get<T>(string url)
+        public async Task<T> Get<T>(string url)
         {
             var response = await HttpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead);
             UpdateRemaining(response);
+            if (!response.IsSuccessStatusCode)
+                throw new HttpRequestException($"{response.StatusCode} - {response.ReasonPhrase}");
 
             using (var stream = await response.Content.ReadAsStreamAsync())
             {
@@ -231,83 +235,26 @@ namespace Wabbajack.Lib.NexusApi
 
         private async Task<T> GetCached<T>(string url)
         {
-            var code = Encoding.UTF8.GetBytes(url).ToHex() + ".json";
-
-            if (UseLocalCache)
-            {
-                var cache_file = Path.Combine(LocalCacheDir, code);
-
-                lock (_diskLock)
-                {
-                    if (!Directory.Exists(LocalCacheDir))
-                        Directory.CreateDirectory(LocalCacheDir);
-
-
-                    if (File.Exists(cache_file))
-                    {
-                        return cache_file.FromJSON<T>();
-                    }
-                }
-
-                var result = await Get<T>(url);
-
-                if (result == null)
-                    return result;
-
-                lock (_diskLock)
-                {
-                    result.ToJSON(cache_file);
-                }
-
-
-                return result;
-            }
-
             try
             {
-                return await Get<T>(Consts.WabbajackCacheLocation + code);
+                var builder = new UriBuilder(url) { Host = Consts.WabbajackCacheHostname, Port = Consts.WabbajackCachePort, Scheme = "http" };
+                return await Get<T>(builder.ToString());
             }
-            catch (Exception)
+            catch (Exception ex)
             {
                 return await Get<T>(url);
             }
 
         }
 
-        public async Task<string> GetNexusDownloadLink(NexusDownloader.State archive, bool cache = false)
+        public async Task<string> GetNexusDownloadLink(NexusDownloader.State archive)
         {
-            if (cache)
-            {
-                var result = await TryGetCachedLink(archive);
-                if (result.Succeeded)
-                {
-                    return result.Value;
-                }
-            }
-
             ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
 
             var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(archive.GameName)}/mods/{archive.ModID}/files/{archive.FileID}/download_link.json";
             return (await Get<List<DownloadLink>>(url)).First().URI;
         }
 
-        private async Task<GetResponse<string>> TryGetCachedLink(NexusDownloader.State archive)
-        {
-            if (!Directory.Exists(Consts.NexusCacheDirectory))
-                Directory.CreateDirectory(Consts.NexusCacheDirectory);
-
-            var path = Path.Combine(Consts.NexusCacheDirectory, $"link-{archive.GameName}-{archive.ModID}-{archive.FileID}.txt");
-            if (!File.Exists(path) || (DateTime.Now - new FileInfo(path).LastWriteTime).TotalHours > 24)
-            {
-                File.Delete(path);
-                var result = await GetNexusDownloadLink(archive);
-                File.WriteAllText(path, result);
-                return GetResponse<string>.Succeed(result);
-            }
-
-            return GetResponse<string>.Succeed(File.ReadAllText(path));
-        }
-
         public async Task<NexusFileInfo> GetFileInfo(NexusDownloader.State mod)
         {
             var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(mod.GameName)}/mods/{mod.ModID}/files/{mod.FileID}.json";
@@ -358,23 +305,8 @@ namespace Wabbajack.Lib.NexusApi
             public string URI { get; set; }
         }
 
-        private class UpdatedMod
-        {
-            public long mod_id;
-            public long latest_file_update;
-            public long latest_mod_activity;
-        }
-
         private static bool? _useLocalCache;
-        public static bool UseLocalCache
+        public static MethodInfo CacheMethod { get; set; }
-        {
-            get
-            {
-                if (_useLocalCache == null) return LocalCacheDir != null;
-                return _useLocalCache ?? false;
-            }
-            set => _useLocalCache = value;
-        }
 
         private static string _localCacheDir;
         public static string LocalCacheDir
@@ -387,88 +319,5 @@ namespace Wabbajack.Lib.NexusApi
             }
             set => _localCacheDir = value;
         }
-
-
-        public async Task ClearUpdatedModsInCache()
-        {
-            if (!UseLocalCache) return;
-            using (var queue = new WorkQueue())
-            {
-                var invalid_json = (await Directory.EnumerateFiles(LocalCacheDir, "*.json")
-                    .PMap(queue, f =>
-                    {
-                        var s = JsonSerializer.Create();
-                        try
-                        {
-                            using (var tr = File.OpenText(f))
-                                s.Deserialize(new JsonTextReader(tr));
-                            return null;
-                        }
-                        catch (JsonReaderException)
-                        {
-                            return f;
-                        }
-                    })).Where(f => f != null).ToList();
-                Utils.Log($"Found {invalid_json.Count} bad json files");
-                foreach (var file in invalid_json)
-                    File.Delete(file);
-            }
-
-            var gameTasks = GameRegistry.Games.Values
-                .Where(game => game.NexusName != null)
-                .Select(async game =>
-                {
-                    return (game,
-                        mods: await Get<List<UpdatedMod>>(
-                            $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m"));
-                })
-                .Select(async rTask =>
-                {
-                    var (game, mods) = await rTask;
-                    return mods.Select(mod => new { game = game, mod = mod });
-                });
-            var purge = (await Task.WhenAll(gameTasks))
-                .SelectMany(i => i)
-                .ToList();
-
-            Utils.Log($"Found {purge.Count} updated mods in the last month");
-            using (var queue = new WorkQueue())
-            {
-                var to_purge = (await Directory.EnumerateFiles(LocalCacheDir, "*.json")
-                    .PMap(queue, f =>
-                    {
-                        Utils.Status("Cleaning Nexus cache for");
-                        var uri = new Uri(Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(f).FromHex()));
-                        var parts = uri.PathAndQuery.Split('/', '.').ToHashSet();
-                        var found = purge.FirstOrDefault(p =>
-                            parts.Contains(p.game.NexusName) && parts.Contains(p.mod.mod_id.ToString()));
-                        if (found != null)
-                        {
-                            var a = found.mod.latest_file_update.AsUnixTime();
-                            // Mod activity could hide files
-                            var b = found.mod.latest_mod_activity.AsUnixTime();
-                            var should_remove = File.GetLastWriteTimeUtc(f) <= (a > b ? a : b);
-                            return (should_remove, f);
-                        }
-
-                        // ToDo
-                        // Can improve to not read the entire file to see if it starts with null
-                        if (File.ReadAllText(f).StartsWith("null"))
-                            return (true, f);
-
-                        return (false, f);
-                    }))
-                    .Where(p => p.Item1)
-                    .ToList();
-
-                Utils.Log($"Purging {to_purge.Count} cache entries");
-                await to_purge.PMap(queue, f =>
-                {
-                    var uri = new Uri(Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(f.f).FromHex()));
-                    Utils.Log($"Purging {uri}");
-                    File.Delete(f.f);
-                });
-            }
-        }
     }
 }

Wabbajack.Lib/UI/FilePickerVM.cs

@@ -1,4 +1,4 @@
-using DynamicData;
+using DynamicData;
 using Microsoft.WindowsAPICodePack.Dialogs;
 using ReactiveUI;
 using ReactiveUI.Fody.Helpers;
@@ -80,7 +80,7 @@ namespace Wabbajack.Lib
                     this.WhenAny(x => x.TargetPath)
                         // Dont want to debounce the initial value, because we know it's null
                         .Skip(1)
-                        .Debounce(TimeSpan.FromMilliseconds(200))
+                        .Debounce(TimeSpan.FromMilliseconds(200), RxApp.TaskpoolScheduler)
                         .StartWith(default(string)),
                     resultSelector: (existsOption, type, path) => (ExistsOption: existsOption, Type: type, Path: path))
                 .StartWith((ExistsOption: ExistCheckOption, Type: PathType, Path: TargetPath))
@@ -107,7 +107,7 @@ namespace Wabbajack.Lib
                 .Replay(1)
                 .RefCount();
 
-            _exists = Observable.Interval(TimeSpan.FromSeconds(3))
+            _exists = Observable.Interval(TimeSpan.FromSeconds(3), RxApp.TaskpoolScheduler)
                 // Only check exists on timer if desired
                 .FilterSwitch(doExistsCheck)
                 .Unit()
@@ -119,6 +119,7 @@ namespace Wabbajack.Lib
                 .CombineLatest(existsCheckTuple,
                     resultSelector: (_, tuple) => tuple)
                 // Refresh exists
+                .ObserveOn(RxApp.TaskpoolScheduler)
                 .Select(t =>
                 {
                     switch (t.ExistsOption)
@@ -146,7 +147,7 @@ namespace Wabbajack.Lib
                     }
                 })
                 .DistinctUntilChanged()
-                .ObserveOn(RxApp.MainThreadScheduler)
+                .ObserveOnGuiThread()
                 .StartWith(false)
                 .ToProperty(this, nameof(Exists));
 
@@ -217,6 +218,7 @@ namespace Wabbajack.Lib
                         if (filter.Failed) return filter;
                         return ErrorResponse.Convert(err);
                     })
+                .ObserveOnGuiThread()
                 .ToProperty(this, nameof(ErrorState));
 
             _inError = this.WhenAny(x => x.ErrorState)
@@ -242,6 +244,7 @@ namespace Wabbajack.Lib
                         if (!string.IsNullOrWhiteSpace(filters)) return filters;
                         return err?.Reason;
                     })
+                .ObserveOnGuiThread()
                 .ToProperty(this, nameof(ErrorTooltip));
         }
 

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -57,6 +57,9 @@
     <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="MongoDB.Bson">
+      <HintPath>..\..\..\Users\tbald\.nuget\packages\mongodb.bson\2.10.0\lib\net452\MongoDB.Bson.dll</HintPath>
+    </Reference>
     <Reference Include="PresentationCore" />
     <Reference Include="PresentationFramework" />
     <Reference Include="System" />
@@ -214,7 +217,7 @@
       <Version>11.0.6</Version>
     </PackageReference>
     <PackageReference Include="ReactiveUI.Fody">
-      <Version>11.0.1</Version>
+      <Version>11.0.6</Version>
     </PackageReference>
     <PackageReference Include="SharpCompress">
       <Version>0.24.0</Version>
@@ -223,7 +226,7 @@
       <Version>1.2.1</Version>
     </PackageReference>
     <PackageReference Include="System.Reactive">
-      <Version>4.3.1</Version>
+      <Version>4.3.2</Version>
     </PackageReference>
     <PackageReference Include="WebSocketSharpFork">
       <Version>1.0.4</Version>

Wabbajack.Test/DownloaderTests.cs

@@ -222,10 +222,10 @@ namespace Wabbajack.Test
         [TestMethod]
         public async Task NexusDownload()
         {
-            var old_val = NexusApiClient.UseLocalCache;
+            var old_val = NexusApiClient.CacheMethod;
             try
             {
-                NexusApiClient.UseLocalCache = false;
+                NexusApiClient.CacheMethod = null;
                 var ini = @"[General]
                         gameName=SkyrimSE
                         modID = 12604
@@ -251,7 +251,7 @@ namespace Wabbajack.Test
             }
             finally
             {
-                NexusApiClient.UseLocalCache = old_val;
+                NexusApiClient.CacheMethod = old_val;
             }
         }
 

Wabbajack.Test/Wabbajack.Test.csproj

@@ -188,7 +188,7 @@
       <Version>11.0.6</Version>
     </PackageReference>
     <PackageReference Include="System.Reactive">
-      <Version>4.3.1</Version>
+      <Version>4.3.2</Version>
     </PackageReference>
   </ItemGroup>
   <ItemGroup />

Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj

@@ -106,7 +106,7 @@
       <Version>2.0.0</Version>
     </PackageReference>
     <PackageReference Include="System.Reactive">
-      <Version>4.3.1</Version>
+      <Version>4.3.2</Version>
     </PackageReference>
   </ItemGroup>
   <Import Project="$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets" Condition="Exists('$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets')" />

Wabbajack.VirtualFileSystem/Context.cs

@@ -35,12 +35,15 @@ namespace Wabbajack.VirtualFileSystem
         public StatusUpdateTracker UpdateTracker { get; set; } = new StatusUpdateTracker(1);
 
         public WorkQueue  Queue { get; }
+        public bool UseExtendedHashes { get; set; }
 
-        public Context(WorkQueue queue)
+        public Context(WorkQueue queue, bool extendedHashes = false)
         {
             Queue = queue;
+            UseExtendedHashes = extendedHashes;
         }
 
+
         public TemporaryDirectory GetTemporaryFolder()
         {
             return new TemporaryDirectory(Path.Combine(_stagingFolder, Guid.NewGuid().ToString()));

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -3,8 +3,10 @@ using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.IO;
 using System.Linq;
+using System.Security.Cryptography;
 using System.Text;
 using System.Threading.Tasks;
+using K4os.Hash.Crc;
 using Wabbajack.Common;
 using Wabbajack.Common.CSP;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
@@ -40,6 +42,7 @@ namespace Wabbajack.VirtualFileSystem
         }
 
         public string Hash { get; internal set; }
+        public ExtendedHashes ExtendedHashes { get; set; }
         public long Size { get; internal set; }
 
         public long LastModified { get; internal set; }
@@ -143,6 +146,8 @@ namespace Wabbajack.VirtualFileSystem
                 LastAnalyzed = DateTime.Now.Ticks,
                 Hash = abs_path.FileHash()
             };
+            if (context.UseExtendedHashes)
+                self.ExtendedHashes = ExtendedHashes.FromFile(abs_path);
 
             if (FileExtractor.CanExtract(abs_path))
             {
@@ -162,6 +167,7 @@ namespace Wabbajack.VirtualFileSystem
             return self;
         }
 
+
         public void Write(MemoryStream ms)
         {
             using (var bw = new BinaryWriter(ms, Encoding.UTF8, true))
@@ -265,6 +271,42 @@ namespace Wabbajack.VirtualFileSystem
         }
     }
 
+    public class ExtendedHashes
+    {
+        public static ExtendedHashes FromFile(string file)
+        {
+            var hashes = new ExtendedHashes();
+            using (var stream = File.OpenRead(file))
+            {
+                hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex();
+                stream.Position = 0;
+                hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex();
+                stream.Position = 0;
+                hashes.MD5 = System.Security.Cryptography.MD5.Create().ComputeHash(stream).ToHex();
+                stream.Position = 0;
+
+                var bytes = new byte[1024 * 8];
+                var crc = new Crc32();
+                while (true)
+                {
+                    var read = stream.Read(bytes, 0, bytes.Length);
+                    if (read == 0) break;
+                    crc.Update(bytes, 0, read);
+                }
+
+                hashes.CRC = crc.DigestBytes().ToHex();
+            }
+
+            return hashes;
+        }
+
+        public string SHA256 { get; set; }
+        public string SHA1 { get; set; }
+        public string MD5 { get; set; }
+        public string CRC { get; set; }
+    }
+
+
     public class CannotStageNativeFile : Exception
     {
         public CannotStageNativeFile(string cannotStageANativeFile) : base(cannotStageANativeFile)

Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj

@@ -88,6 +88,9 @@
     <PackageReference Include="AlphaFS">
       <Version>2.2.6</Version>
     </PackageReference>
+    <PackageReference Include="K4os.Hash.Crc">
+      <Version>1.1.4</Version>
+    </PackageReference>
     <PackageReference Include="System.Collections.Immutable">
       <Version>1.7.0</Version>
     </PackageReference>

Wabbajack.sln

@@ -34,7 +34,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VirtualFileSystem
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.CacheServer", "Wabbajack.CacheServer\Wabbajack.CacheServer.csproj", "{BDC9A094-D235-47CD-83CA-44199B60AB20}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OMODExtractor", "OMODExtractor\OMODExtractor.csproj", "{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OMODExtractor", "OMODExtractor\OMODExtractor.csproj", "{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -225,8 +225,8 @@ Global
 		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug|x86.Build.0 = Debug|Any CPU
 		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|Any CPU.Build.0 = Release|Any CPU
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x64.ActiveCfg = Release|Any CPU
+		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x64.ActiveCfg = Release|x64
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x64.Build.0 = Release|Any CPU
+		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x64.Build.0 = Release|x64
 		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x86.ActiveCfg = Release|Any CPU
 		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x86.Build.0 = Release|Any CPU
 		{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU

Wabbajack/View Models/Compilers/CompilerVM.cs

@@ -60,6 +60,9 @@ namespace Wabbajack
         [Reactive]
         public ErrorResponse? Completed { get; set; }
 
+        private readonly ObservableAsPropertyHelper<string> _progressTitle;
+        public string ProgressTitle => _progressTitle.Value;
+
         public CompilerVM(MainWindowVM mainWindowVM)
         {
             MWVM = mainWindowVM;
@@ -114,8 +117,9 @@ namespace Wabbajack
 
             _image = this.WhenAny(x => x.CurrentModlistSettings.ImagePath.TargetPath)
                 // Throttle so that it only loads image after any sets of swaps have completed
-                .Throttle(TimeSpan.FromMilliseconds(50), RxApp.MainThreadScheduler)
+                .Throttle(TimeSpan.FromMilliseconds(50), RxApp.TaskpoolScheduler)
                 .DistinctUntilChanged()
+                .ObserveOn(RxApp.MainThreadScheduler)
                 .Select(path =>
                 {
                     if (string.IsNullOrWhiteSpace(path)) return UIUtils.BitmapImageFromResource("Resources/Wabba_Mouth_No_Text.png");
@@ -157,7 +161,7 @@ namespace Wabbajack
                         return ret;
                     })
                 .ToObservableChangeSet(x => x.Status.ID)
-                .Batch(TimeSpan.FromMilliseconds(250), RxApp.TaskpoolScheduler)
+                .Batch(TimeSpan.FromMilliseconds(50), RxApp.TaskpoolScheduler)
                 .EnsureUniqueChanges()
                 .Filter(i => i.Status.IsWorking && i.Status.ID != WorkQueue.UnassignedCpuId)
                 .ObserveOn(RxApp.MainThreadScheduler)
@@ -246,6 +250,22 @@ namespace Wabbajack
                         Process.Start("explorer.exe", OutputLocation.TargetPath);
                     }
                 });
+
+            _progressTitle = Observable.CombineLatest(
+                    this.WhenAny(x => x.Compiling),
+                    this.WhenAny(x => x.StartedCompilation),
+                    resultSelector: (compiling, started) =>
+                    {
+                        if (compiling)
+                        {
+                            return "Compiling";
+                        }
+                        else
+                        {
+                            return started ? "Compiled" : "Configuring";
+                        }
+                    })
+                .ToProperty(this, nameof(ProgressTitle));
         }
     }
 }

Wabbajack/View Models/Installers/InstallerVM.cs

@@ -210,7 +210,7 @@ namespace Wabbajack
             _image = Observable.CombineLatest(
                     this.WhenAny(x => x.ModList.Error),
                     this.WhenAny(x => x.ModList)
-                        .Select(x => x?.ImageObservable ?? Observable.Empty<BitmapImage>())
+                        .Select(x => x?.ImageObservable ?? Observable.Return(WabbajackLogo))
                         .Switch()
                         .StartWith(WabbajackLogo),
                     this.WhenAny(x => x.Slideshow.Image)
@@ -228,21 +228,24 @@ namespace Wabbajack
                 .Select<BitmapImage, ImageSource>(x => x)
                 .ToProperty(this, nameof(Image));
             _titleText = Observable.CombineLatest(
-                    this.WhenAny(x => x.ModList.Name),
+                    this.WhenAny(x => x.ModList)
+                        .Select(modList => modList?.Name ?? string.Empty),
                     this.WhenAny(x => x.Slideshow.TargetMod.ModName)
                         .StartWith(default(string)),
                     this.WhenAny(x => x.Installing),
                     resultSelector: (modList, mod, installing) => installing ? mod : modList)
                 .ToProperty(this, nameof(TitleText));
             _authorText = Observable.CombineLatest(
-                    this.WhenAny(x => x.ModList.Author),
+                    this.WhenAny(x => x.ModList)
+                        .Select(modList => modList?.Author ?? string.Empty),
                     this.WhenAny(x => x.Slideshow.TargetMod.ModAuthor)
                         .StartWith(default(string)),
                     this.WhenAny(x => x.Installing),
                     resultSelector: (modList, mod, installing) => installing ? mod : modList)
                 .ToProperty(this, nameof(AuthorText));
             _description = Observable.CombineLatest(
-                    this.WhenAny(x => x.ModList.Description),
+                    this.WhenAny(x => x.ModList)
+                        .Select(modList => modList?.Description ?? string.Empty),
                     this.WhenAny(x => x.Slideshow.TargetMod.ModDescription)
                         .StartWith(default(string)),
                     this.WhenAny(x => x.Installing),
@@ -278,8 +281,14 @@ namespace Wabbajack
                     this.WhenAny(x => x.StartedInstallation),
                     resultSelector: (installing, started) =>
                     {
-                        if (!installing) return "Configuring";
+                        if (installing)
-                        return started ? "Installing" : "Installed";
+                        {
+                            return "Installing";
+                        }
+                        else
+                        {
+                            return started ? "Installed" : "Configuring";
+                        }
                     })
                 .ToProperty(this, nameof(ProgressTitle));
 
@@ -298,7 +307,7 @@ namespace Wabbajack
                         return ret;
                     })
                 .ToObservableChangeSet(x => x.Status.ID)
-                .Batch(TimeSpan.FromMilliseconds(250), RxApp.TaskpoolScheduler)
+                .Batch(TimeSpan.FromMilliseconds(50), RxApp.TaskpoolScheduler)
                 .EnsureUniqueChanges()
                 .Filter(i => i.Status.IsWorking && i.Status.ID != WorkQueue.UnassignedCpuId)
                 .ObserveOn(RxApp.MainThreadScheduler)

Wabbajack/View Models/Installers/MO2InstallerVM.cs

@@ -59,7 +59,7 @@ namespace Wabbajack
                     this.WhenAny(x => x.Location.TargetPath),
                     this.WhenAny(x => x.DownloadLocation.TargetPath),
                     resultSelector: (target, download) => (target, download))
-                .ObserveOn(RxApp.MainThreadScheduler)
+                .ObserveOn(RxApp.TaskpoolScheduler)
                 .Select(i => MO2Installer.CheckValidInstallPath(i.target, i.download))
                 .ObserveOnGuiThread();
 

Wabbajack/View Models/MainWindowVM.cs

@@ -40,11 +40,8 @@ namespace Wabbajack
         public readonly UserInterventionHandlers UserInterventionHandlers;
         public readonly LoginManagerVM LoginManagerVM;
 
-
         public readonly List<ViewModel> NavigationTrail = new List<ViewModel>();
 
-        public Dispatcher ViewDispatcher { get; set; }
-
         public ICommand CopyVersionCommand { get; }
 
         public ICommand ShowLoginManagerVM { get; }
@@ -54,7 +51,6 @@ namespace Wabbajack
         public MainWindowVM(MainWindow mainWindow, MainSettings settings)
         {
             MainWindow = mainWindow;
-            ViewDispatcher = MainWindow.Dispatcher;
             Settings = settings;
             Installer = new Lazy<InstallerVM>(() => new InstallerVM(this));
             Compiler = new Lazy<CompilerVM>(() => new CompilerVM(this));

Wabbajack/View Models/ModListVM.cs

@@ -87,8 +87,7 @@ namespace Wabbajack
         public void OpenReadmeWindow()
         {
             if (string.IsNullOrEmpty(Readme)) return;
-            // DISABLED FOR THIS RELEASE
+            if (false) //SourceModList.ReadmeIsWebsite)
-            if (false) // SourceModList.ReadmeIsWebsite)
             {
                 Process.Start(Readme);
             }

Wabbajack/Views/Compilers/CompilerView.xaml

@@ -77,7 +77,7 @@
             Grid.ColumnSpan="5"
             OverhangShadow="True"
             ProgressPercent="{Binding PercentCompleted}"
-            StatePrefixTitle="Compiling" />
+            StatePrefixTitle="{Binding ProgressTitle}" />
         <Button
             x:Name="BackButton"
             Grid.Row="0"

Wabbajack/Views/Installers/InstallationView.xaml

@@ -381,6 +381,21 @@
                                 <local:VortexInstallerConfigView />
                             </DataTemplate>
                         </ContentPresenter.Resources>
+                        <ContentPresenter.Style>
+                            <Style TargetType="ContentPresenter">
+                                <Style.Triggers>
+                                    <DataTrigger Binding="{Binding RelativeSource={x:Static RelativeSource.Self}, Path=Content}" Value="{x:Null}">
+                                        <Setter Property="ContentPresenter.ContentTemplate">
+                                            <Setter.Value>
+                                                <DataTemplate>
+                                                    <Rectangle Height="74" />
+                                                </DataTemplate>
+                                            </Setter.Value>
+                                        </Setter>
+                                    </DataTrigger>
+                                </Style.Triggers>
+                            </Style>
+                        </ContentPresenter.Style>
                     </ContentPresenter>
                 </Grid>
                 <local:BeginButton

Wabbajack/Wabbajack.csproj

@@ -486,7 +486,7 @@
       <Version>4.1.0</Version>
     </PackageReference>
     <PackageReference Include="DynamicData">
-      <Version>6.14.1</Version>
+      <Version>6.14.3</Version>
     </PackageReference>
     <PackageReference Include="Fody">
       <Version>6.0.5</Version>
@@ -523,13 +523,13 @@
       <Version>2.4.4</Version>
     </PackageReference>
     <PackageReference Include="ReactiveUI.Events.WPF">
-      <Version>11.0.1</Version>
+      <Version>11.0.6</Version>
     </PackageReference>
     <PackageReference Include="ReactiveUI.Fody">
-      <Version>11.0.1</Version>
+      <Version>11.0.6</Version>
     </PackageReference>
     <PackageReference Include="ReactiveUI.WPF">
-      <Version>11.0.1</Version>
+      <Version>11.0.6</Version>
     </PackageReference>
     <PackageReference Include="SharpCompress">
       <Version>0.24.0</Version>
@@ -541,7 +541,7 @@
       <Version>1.2.1</Version>
     </PackageReference>
     <PackageReference Include="System.Reactive">
-      <Version>4.3.1</Version>
+      <Version>4.3.2</Version>
     </PackageReference>
     <PackageReference Include="WebSocketSharpFork">
       <Version>1.0.4</Version>