Merge pull request #377 from wabbajack-tools/asp-net-core-rewrite

Asp net core rewrite

Compression.BSA/Compression.BSA.csproj

@@ -9,6 +9,7 @@
       <PackageReference Include="Genbox.AlphaFS" Version="2.2.2.1" />
       <PackageReference Include="K4os.Compression.LZ4.Streams" Version="1.1.11" />
       <PackageReference Include="SharpZipLib" Version="1.2.0" />
+      <PackageReference Include="System.Text.Encoding.CodePages" Version="4.7.0" />
     </ItemGroup>
     <ItemGroup>
       <Folder Include="Properties\" />

Compression.BSA/Utils.cs

@@ -8,8 +8,14 @@ namespace Compression.BSA
 {
     internal static class Utils
     {
-        private static readonly Encoding Windows1252 = Encoding.GetEncoding(1252);
+        private static readonly Encoding Windows1252;
 
+        static Utils()
+        {
+            Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
+            Windows1252 = Encoding.GetEncoding(1252);
+        }
+        
         private static Encoding GetEncoding(VersionType version)
         {
             if (version == VersionType.SSE)
@@ -155,4 +161,4 @@ namespace Compression.BSA
             tw.Flush();
         }
     }
-}
+}

Wabbajack.BuildServer/AppSettings.cs

@@ -0,0 +1,15 @@
+using Microsoft.Extensions.Configuration;
+
+namespace Wabbajack.BuildServer
+{
+    public class AppSettings
+    {
+        public AppSettings(IConfiguration config)
+        {
+            config.Bind("WabbajackSettings", this);
+        }
+        
+        public string DownloadDir { get; set; }
+        public string ArchiveDir { get; set; }
+    }
+}

Wabbajack.BuildServer/Controllers/AControllerBase.cs

@@ -0,0 +1,19 @@
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer.Models;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [ApiController]
+    public abstract class AControllerBase<T> : ControllerBase
+    {
+        protected readonly DBContext Db;
+        protected readonly ILogger<T> Logger;
+
+        protected AControllerBase(ILogger<T> logger, DBContext db)
+        {
+            Db = db;
+            Logger = logger;
+        }
+    }
+}

Wabbajack.BuildServer/Controllers/GraphQL.cs

@@ -0,0 +1,41 @@
+using System.Threading.Tasks;
+using GraphQL;
+using GraphQL.Types;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer.GraphQL;
+using Wabbajack.BuildServer.Models;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [Route("graphql")]
+    [ApiController]
+    public class GraphQL : AControllerBase<GraphQL>
+    {
+        public GraphQL(ILogger<GraphQL> logger, DBContext db) : base(logger, db)
+        {
+        }
+
+        [HttpPost]
+        public async Task<IActionResult> Post([FromBody] GraphQLQuery query)
+        {
+            var inputs = query.Variables.ToInputs();
+            var schema = new Schema {Query = new Query(Db), Mutation = new Mutation(Db)};
+
+            var result = await new DocumentExecuter().ExecuteAsync(_ =>
+            {
+                _.Schema = schema;
+                _.Query = query.Query;
+                _.OperationName = query.OperationName;
+                _.Inputs = inputs;
+            });
+
+            if (result.Errors?.Count > 0)
+            {
+                return BadRequest();
+            }
+
+            return Ok(result);
+        }
+    }
+}

Wabbajack.BuildServer/Controllers/Heartbeat.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.Common;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [Route("/heartbeat")]
+    public class Heartbeat : AControllerBase<Heartbeat>
+    {
+        static Heartbeat()
+        {
+            _startTime = DateTime.Now;
+            
+        }
+        private static DateTime _startTime;
+
+        public Heartbeat(ILogger<Heartbeat> logger, DBContext db) : base(logger, db)
+        {
+        }
+        
+        [HttpGet]
+        public async Task<TimeSpan> GetHeartbeat()
+        {
+            return DateTime.Now - _startTime;
+        }
+    }
+}

Wabbajack.BuildServer/Controllers/IndexedFiles.cs

@@ -0,0 +1,123 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using MongoDB.Bson;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.Common;
+using Wabbajack.VirtualFileSystem;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [Route("/indexed_files")]
+    public class IndexedFiles : AControllerBase<IndexedFiles>
+    {
+        public IndexedFiles(ILogger<IndexedFiles> logger, DBContext db) : base(logger, db)
+        {
+        }
+
+        [HttpGet]
+        [Route("{xxHashAsBase64}/meta.ini")]
+        public async Task<IActionResult> GetFileMeta(string xxHashAsBase64)
+        {
+            var id = xxHashAsBase64.FromHex().ToBase64();
+            var state = await Db.DownloadStates.AsQueryable()
+                .Where(d => d.Hash == id && d.IsValid)
+                .OrderByDescending(d => d.LastValidationTime)
+                .Take(1)
+                .ToListAsync();
+
+            if (state.Count == 0)
+                return NotFound();
+            Response.ContentType = "text/plain";
+            return Ok(string.Join("\r\n", state.FirstOrDefault().State.GetMetaIni()));
+        }
+
+        [HttpGet]
+        [Route("{xxHashAsBase64}")]
+        public async Task<IActionResult> GetFile(string xxHashAsBase64)
+        {
+            var id = xxHashAsBase64.FromHex().ToBase64();
+            var query = new[]
+            {
+                new BsonDocument("$match",
+                    new BsonDocument("_id", id)),
+                new BsonDocument("$graphLookup",
+                    new BsonDocument
+                    {
+                        {"from", "indexed_files"},
+                        {"startWith", "$Children.Hash"},
+                        {"connectFromField", "Hash"},
+                        {"connectToField", "_id"},
+                        {"as", "ChildFiles"},
+                        {"maxDepth", 8},
+                        {"restrictSearchWithMatch", new BsonDocument()}
+                    }),
+                new BsonDocument("$project", 
+                new BsonDocument
+                {
+                    // If we return all fields some BSAs will return more that 16MB which is the
+                    // maximum doc size that can can be returned from MongoDB
+                    { "_id", 1 }, 
+                    { "Size", 1 }, 
+                    { "Children.Name", 1 }, 
+                    { "Children.Hash", 1 }, 
+                    { "ChildFiles._id", 1 }, 
+                    { "ChildFiles.Size", 1 }, 
+                    { "ChildFiles.Children.Name", 1 }, 
+                    { "ChildFiles.Children.Hash", 1 }, 
+                    { "ChildFiles.ChildFiles._id", 1 }, 
+                    { "ChildFiles.ChildFiles.Size", 1 }, 
+                    { "ChildFiles.ChildFiles.Children.Name", 1 }, 
+                    { "ChildFiles.ChildFiles.Children.Hash", 1 }, 
+                    { "ChildFiles.ChildFiles.ChildFiles._id", 1 }, 
+                    { "ChildFiles.ChildFiles.ChildFiles.Size", 1 }, 
+                    { "ChildFiles.ChildFiles.ChildFiles.Children.Name", 1 }, 
+                    { "ChildFiles.ChildFiles.ChildFiles.Children.Hash", 1 }, 
+                    { "ChildFiles.ChildFiles.ChildFiles.ChildFiles._id", 1 }, 
+                    { "ChildFiles.ChildFiles.ChildFiles.ChildFiles.Size", 1 }, 
+                    { "ChildFiles.ChildFiles.ChildFiles.ChildFiles.Children.Name", 1 }, 
+                    { "ChildFiles.ChildFiles.ChildFiles.ChildFiles.Children.Hash", 1 }, 
+                    { "ChildFiles.ChildFiles.ChildFiles.ChildFiles.ChildFiles._id", 1 }, 
+                    { "ChildFiles.ChildFiles.ChildFiles.ChildFiles.ChildFiles.Size", 1 }
+                })
+            };
+
+            var result = await Db.IndexedFiles.AggregateAsync<TreeResult>(query);
+
+            IndexedVirtualFile Convert(TreeResult t, string Name = null)
+            {
+                if (t == null)
+                    return null;
+
+                Dictionary<string, TreeResult> indexed_children = new Dictionary<string, TreeResult>();
+                if (t.ChildFiles != null && t.ChildFiles.Count > 0)
+                    indexed_children = t.ChildFiles.ToDictionary(t => t.Hash);
+
+                var file = new IndexedVirtualFile
+                {
+                    Name = Name,
+                    Size = t.Size,
+                    Hash = t.Hash,
+                    Children = t.ChildFiles != null
+                        ? t.Children.Select(child => Convert(indexed_children[child.Hash], child.Name)).ToList()
+                        : new List<IndexedVirtualFile>()
+                };
+                return file;
+            }
+
+            var first = result.FirstOrDefault();
+            if (first == null)
+                return NotFound();
+            return Ok(Convert(first));
+        }
+
+        public class TreeResult : IndexedFile
+        {
+            public List<TreeResult> ChildFiles { get; set; }
+        }
+    }
+}

Wabbajack.BuildServer/Controllers/Jobs.cs

@@ -0,0 +1,30 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.BuildServer.Models.JobQueue;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [ApiController]
+    [Route("/jobs")]
+    public class Jobs : AControllerBase<Jobs>
+    {
+        public Jobs(ILogger<Jobs> logger, DBContext db) : base(logger, db)
+        {
+        }
+
+        [HttpGet]
+        [Route("unfinished")]
+        public async Task<IEnumerable<Job>> GetUnfinished()
+        {
+            return await Db.Jobs.AsQueryable()
+                .Where(j => j.Ended == null)
+                .OrderByDescending(j => j.Priority)
+                .ToListAsync();
+        }
+    }
+}

Wabbajack.BuildServer/Controllers/ListValidation.cs

@@ -0,0 +1,27 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.Lib.ModListRegistry;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [ApiController]
+    [Route("/lists")]
+    public class ListValidation : AControllerBase<ListValidation>
+    {
+        public ListValidation(ILogger<ListValidation> logger, DBContext db) : base(logger, db)
+        {
+        }
+
+        [HttpGet]
+        [Route("status.json")]
+        public async Task<IList<ModlistSummary>> HandleGetLists()
+        {
+            return await Db.ModListStatus.AsQueryable().Select(m => m.Summary).ToListAsync();
+        }
+    }
+}

Wabbajack.BuildServer/Controllers/MetricsController.cs

@@ -0,0 +1,46 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.Common;
+using Wabbajack.Lib.ModListRegistry;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [ApiController]
+    [Route("/metrics")]
+    public class MetricsController : AControllerBase<MetricsController>
+    {
+        public MetricsController(ILogger<MetricsController> logger, DBContext db) : base(logger, db)
+        {
+        }
+
+        [HttpGet]
+        [Route("{Subject}/{Value}")]
+        public async Task<Result> LogMetricAsync(string Subject, string Value)
+        {
+            var date = DateTime.UtcNow;
+            await Log(date, Subject, Value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault());
+            return new Result { Timestamp = date};
+        }
+
+        private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
+        {
+            Logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}");
+            await Db.Metrics.InsertOneAsync(new Metric
+            {
+                Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey
+            });
+        }
+
+        public class Result
+        {
+            public DateTime Timestamp { get; set; }
+        }
+    }
+}

Wabbajack.BuildServer/Controllers/NexusCache.cs

@@ -0,0 +1,123 @@
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.Lib.NexusApi;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    //[Authorize]
+    [ApiController]
+    [Route("/v1/games/")]
+    public class NexusCache : AControllerBase<NexusCache>
+    {
+        public NexusCache(ILogger<NexusCache> logger, DBContext db) : base(logger, db)
+        {
+        }
+
+        /// <summary>
+        ///     Looks up the mod details for a given Gamename/ModId pair. If the entry is not found in the cache it will
+        ///     be requested from the server (using the caller's Nexus API key if provided).
+        /// </summary>
+        /// <param name="db"></param>
+        /// <param name="GameName">The Nexus game name</param>
+        /// <param name="ModId">The Nexus mod id</param>
+        /// <returns>A Mod Info result</returns>
+        [HttpGet]
+        [Route("{GameName}/mods/{ModId}.json")]
+        public async Task<ModInfo> GetModInfo(string GameName, string ModId)
+        {
+            var result = await Db.NexusModInfos.FindOneAsync(info => info.Game == GameName && info.ModId == ModId);
+
+            string method = "CACHED";
+            if (result == null)
+            {
+                var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
+                var path = $"/v1/games/{GameName}/mods/{ModId}.json";
+                var body = await api.Get<ModInfo>(path);
+                result = new NexusCacheData<ModInfo> {Data = body, Path = path, Game = GameName, ModId = ModId};
+                try
+                {
+                    await Db.NexusModInfos.InsertOneAsync(result);
+                }
+                catch (MongoWriteException)
+                {
+                }
+
+                method = "NOT_CACHED";
+            }
+
+            Response.Headers.Add("x-cache-result", method);
+            return result.Data;
+        }
+
+        [HttpGet]
+        [Route("{GameName}/mods/{ModId}/files.json")]
+        public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, string ModId)
+        {
+            var result = await Db.NexusModFiles.FindOneAsync(info => info.Game == GameName && info.ModId == ModId);
+
+            string method = "CACHED";
+            if (result == null)
+            {
+                var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
+                var path = $"/v1/games/{GameName}/mods/{ModId}/files.json";
+                var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
+                result = new NexusCacheData<NexusApiClient.GetModFilesResponse>
+                {
+                    Data = body, Path = path, Game = GameName, ModId = ModId
+                };
+                try
+                {
+                    await Db.NexusModFiles.InsertOneAsync(result);
+                }
+                catch (MongoWriteException)
+                {
+                }
+
+                method = "NOT_CACHED";
+            }
+
+            Response.Headers.Add("x-cache-result", method);
+            return result.Data;
+        }
+
+        [HttpGet]
+        [Route("{GameName}/mods/{ModId}/files/{FileId}.json")]
+        public async Task<object> GetFileInfo(string GameName, string ModId, string FileId)
+        {
+            var result = await Db.NexusFileInfos.FindOneAsync(info =>
+                info.Game == GameName && info.ModId == ModId && info.FileId == FileId);
+
+            string method = "CACHED";
+            if (result == null)
+            {
+                var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
+                var path = $"/v1/games/{GameName}/mods/{ModId}/files/{FileId}.json";
+                var body = await api.Get<NexusFileInfo>(path);
+                result = new NexusCacheData<NexusFileInfo>
+                {
+                    Data = body,
+                    Path = path,
+                    Game = GameName,
+                    ModId = ModId,
+                    FileId = FileId
+                };
+                try
+                {
+                    await Db.NexusFileInfos.InsertOneAsync(result);
+                }
+                catch (MongoWriteException)
+                {
+                }
+
+                method = "NOT_CACHED";
+            }
+
+            Response.Headers.Add("x-cache-method", method);
+            return result.Data;
+        }
+    }
+}

Wabbajack.BuildServer/Extensions.cs

@@ -1,13 +1,14 @@
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using System.Linq.Expressions;
-using System.Text;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.Extensions.Logging;
 using MongoDB.Driver;
 using MongoDB.Driver.Linq;
+using Wabbajack.Common;
 
-namespace Wabbajack.CacheServer
+namespace Wabbajack.BuildServer
 {
     public static class Extensions
     {
@@ -15,5 +16,13 @@ namespace Wabbajack.CacheServer
         {
             return (await coll.AsQueryable().Where(expr).Take(1).ToListAsync()).FirstOrDefault();
         }
+
+        public static void UseJobManager(this IApplicationBuilder b)
+        {
+            var manager = (JobManager)b.ApplicationServices.GetService(typeof(JobManager));
+            var tsk = manager.JobScheduler();
+
+            manager.StartJobRunners();
+        }
     }
 }

Wabbajack.BuildServer/GraphQL/GraphQLQuery.cs

@@ -0,0 +1,15 @@
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace Wabbajack.BuildServer.GraphQL
+{
+    public class GraphQLQuery
+    {
+
+        public string OperationName { get; set; }
+
+        public string NamedQuery { get; set; }
+        public string Query { get; set; }
+        public JObject Variables { get; set; }
+    }
+}

Wabbajack.BuildServer/GraphQL/JobType.cs

@@ -0,0 +1,18 @@
+using GraphQL.Types;
+using Wabbajack.BuildServer.Models.JobQueue;
+
+namespace Wabbajack.BuildServer.GraphQL
+{
+    public class JobType : ObjectGraphType<Job>
+    {
+        public JobType()
+        {
+            Name = "Job";
+            Field(x => x.Id, type: typeof(IdGraphType)).Description("Unique Id of the Job");
+            Field(x => x.Payload.Description).Description("Description of the job's behavior");
+            Field(x => x.Created, type: typeof(DateTimeGraphType)).Description("Creation time of the Job");
+            Field(x => x.Started, type: typeof(DateTimeGraphType)).Description("Started time of the Job");
+            Field(x => x.Ended, type: typeof(DateTimeGraphType)).Description("Ended time of the Job");
+        }
+    }
+}

Wabbajack.BuildServer/GraphQL/MetricType.cs

@@ -0,0 +1,36 @@
+using System.Collections.Generic;
+using GraphQL.Types;
+
+namespace Wabbajack.BuildServer.GraphQL
+{
+    public class MetricEnum : EnumerationGraphType
+    {
+        public MetricEnum()
+        {
+            Name = "MetricType";
+            Description = "The metric grouping";
+            AddValue("BEGIN_INSTALL", "Installation of a modlist started", "begin_install");
+            AddValue("FINISHED_INSTALL", "Installation of a modlist finished", "finish_install");
+            AddValue("BEGIN_DOWNLOAD", "Downloading of a modlist begain started", "downloading");
+        }
+    }
+
+    public class MetricResultType : ObjectGraphType<MetricResult>
+    {
+        public MetricResultType()
+        {
+            Name = "MetricResult";
+            Description =
+                "A single line of data from a metrics graph. For example, the number of unique downloads each day.";
+            Field(x => x.SeriesName).Description("The name of the data series");
+            Field(x => x.Labels).Description("The name for each plot of data (for example the date for each value");
+            Field(x => x.Values).Description("The value for each plot of data");
+        }
+    }
+    public class MetricResult
+    {
+        public string SeriesName { get; set; }
+        public List<string> Labels { get; set; }
+        public List<int> Values { get; set; }
+    }
+}

Wabbajack.BuildServer/GraphQL/ModlistStatusType.cs

@@ -0,0 +1,67 @@
+using System.Collections.Generic;
+using System.Linq;
+using GraphQL.Types;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.Lib.ModListRegistry;
+
+namespace Wabbajack.BuildServer.GraphQL
+{
+    public class ModListStatusType : ObjectGraphType<ModListStatus>
+    {
+        public ModListStatusType()
+        {
+            Name = "ModlistSummary";
+            Description = "Short summary of a modlist status";
+            Field(x => x.Id).Description("Name of the modlist");
+            Field(x => x.Metadata.Title).Description("Human-friendly name of the modlist");
+            Field<ListGraphType<ModListArchiveType>>("Archives",
+                    arguments: new QueryArguments(new QueryArgument<ArchiveEnumFilterType>
+                    {
+                        Name = "filter", Description = "Type of archives to return"
+                    }),
+                    resolve: context =>
+                    {
+                        var arg = context.GetArgument<string>("filter");
+                        var archives = (IEnumerable<DetailedStatusItem>)context.Source.DetailedStatus.Archives;
+                        switch (arg)
+                        {
+                            case "FAILED":
+                                archives = archives.Where(a => a.IsFailing);
+                                break;
+                            case "PASSED":
+                                archives = archives.Where(a => !a.IsFailing);
+                                break;
+                            default:
+                                break;
+                        }
+
+                        return archives;
+                    });
+        }
+       
+    }
+
+    public class ModListArchiveType : ObjectGraphType<DetailedStatusItem>
+    {
+        public ModListArchiveType()
+        {
+            Field(x => x.IsFailing).Description("Is this archive failing validation?");
+            Field(x => x.Archive.Name).Description("Name of the archive");
+            Field(x => x.Archive.Hash).Description("Hash of the archive");
+            Field(x => x.Archive.Size).Description("Size of the archive");
+        }
+    }
+
+    public class ArchiveEnumFilterType : EnumerationGraphType
+    {
+        public ArchiveEnumFilterType()
+        {
+            Name = "ArchiveFilterEnum";
+            Description = "What archives should be returned from a sublist";
+            AddValue("ALL", "All archives are returned", "ALL");
+            AddValue("FAILED", "All archives are returned", "FAILED");
+            AddValue("PASSED", "All archives are returned", "PASSED");
+            
+        }
+    }
+}

Wabbajack.BuildServer/GraphQL/Mutation.cs

@@ -0,0 +1,22 @@
+using GraphQL.Types;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.BuildServer.Models.Jobs;
+
+namespace Wabbajack.BuildServer.GraphQL
+{
+    public class Mutation : ObjectGraphType
+    {
+        public Mutation(DBContext db)
+        {
+            FieldAsync<IdGraphType>("pollNexusForUpdates",
+                resolve: async context =>
+                {
+                    var job = new Job {Payload = new GetNexusUpdatesJob()};
+                    await db.Jobs.InsertOneAsync(job);
+                    return job.Id;
+                });
+        }
+        
+    }
+}

Wabbajack.BuildServer/GraphQL/Query.cs

@@ -0,0 +1,68 @@
+using System;
+using System.Collections.Generic;
+using GraphQL;
+using GraphQL.Types;
+using GraphQLParser.AST;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.Common;
+
+namespace Wabbajack.BuildServer.GraphQL
+{
+    public class Query : ObjectGraphType
+    {
+        public Query(DBContext db)
+        {
+            Field<ListGraphType<JobType>>("unfinishedJobs", resolve: context =>
+            {
+                var data =  db.Jobs.AsQueryable().Where(j => j.Ended == null).ToList();
+                return data;
+            });
+
+            FieldAsync<ListGraphType<ModListStatusType>>("modLists",
+                arguments: new QueryArguments(new QueryArgument<ArchiveEnumFilterType>
+                {
+                    Name = "filter", Description = "Filter lists to those that only have these archive classifications"
+                }),
+                resolve: async context =>
+                {
+                    var arg = context.GetArgument<string>("filter");
+                    var lists = db.ModListStatus.AsQueryable();
+                    switch (arg)
+                    {
+                        case "FAILED":
+                            lists = lists.Where(l => l.DetailedStatus.HasFailures);
+                            break;
+                        case "PASSED":
+                            lists = lists.Where(a => !a.DetailedStatus.HasFailures);
+                            break;
+                        default:
+                            break;
+                    }
+
+                    return await lists.ToListAsync();
+                });
+            
+            FieldAsync<ListGraphType<JobType>>("job",
+                arguments: new QueryArguments(
+                    new QueryArgument<IdGraphType> {Name = "id", Description = "Id of the Job"}),
+                resolve: async context =>
+                {
+                    var id = context.GetArgument<string>("id");
+                    var data = await db.Jobs.AsQueryable().Where(j => j.Id == id).ToListAsync();
+                    return data;
+                });
+
+            FieldAsync<ListGraphType<MetricResultType>>("dailyUniqueMetrics",
+                arguments: new QueryArguments(
+                    new QueryArgument<MetricEnum> {Name = "metric_type", Description = "The grouping of metric data to query"}
+                    ),
+                resolve: async context =>
+                {
+                    var group = context.GetArgument<string>("metric_type");
+                    return await Metric.Report(db, group);
+                });
+        }
+    }
+}

Wabbajack.BuildServer/GraphQL/VirtualFileType.cs

@@ -0,0 +1,32 @@
+using GraphQL.Types;
+using Wabbajack.BuildServer.Models;
+
+namespace Wabbajack.BuildServer.GraphQL
+{
+    public class VirtualFileType : ObjectGraphType<IndexedFileWithChildren>
+    {
+        public VirtualFileType()
+        {
+            Name = "VirtualFile";
+            Field(x => x.Hash, type: typeof(IdGraphType)).Description("xxHash64 of the file, in Base64 encoding");
+            Field(x => x.Size, type: typeof(LongGraphType)).Description("Size of the file");
+            Field(x => x.IsArchive).Description("True if this file is an archive (BSA, zip, 7z, etc.)");
+            Field(x => x.SHA256).Description("SHA256 hash of the file, in hexidecimal encoding");
+            Field(x => x.SHA1).Description("SHA1 hash of the file, in hexidecimal encoding");
+            Field(x => x.MD5).Description("MD5 hash of the file, in hexidecimal encoding");
+            Field(x => x.CRC).Description("CRC32 hash of the file, in hexidecimal encoding");
+            Field(x => x.Children, type: typeof(ChildFileType)).Description("Metadata for the files in this archive (if any)");
+        }
+    }
+
+    public class ChildFileType : ObjectGraphType<ChildFile>
+    {
+        public ChildFileType()
+        {
+            Name = "ChildFile";
+            Field(x => x.Name).Description("The relative path to the file inside the parent archive");
+            Field(x => x.Hash).Description("The hash (xxHash64, Base64 ecoded) of the child file");
+            Field(x => x.Extension).Description("File extension of the child file");
+        }
+    }
+}

Wabbajack.BuildServer/JobManager.cs

@@ -0,0 +1,133 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.BuildServer.Models.Jobs;
+using Wabbajack.Common;
+
+namespace Wabbajack.BuildServer
+{
+    public class JobManager
+    {
+        protected readonly ILogger<JobManager> Logger;
+        protected readonly DBContext Db;
+        protected readonly AppSettings Settings;
+
+        public JobManager(ILogger<JobManager> logger, DBContext db, AppSettings settings)
+        {
+            Db = db;
+            Logger = logger;
+            Settings = settings;
+        }
+
+        public void StartJobRunners()
+        {
+            for (var idx = 0; idx < 2; idx++)
+            {
+                Task.Run(async () =>
+                {
+                    while (true)
+                    {
+                        try
+                        {
+                            var job = await Job.GetNext(Db);
+                            if (job == null)
+                            {
+                                await Task.Delay(5000);
+                                continue;
+                            }
+
+                            Logger.Log(LogLevel.Information, $"Starting Job: {job.Payload.Description}");
+                            JobResult result;
+                            try
+                            {
+                                result = await job.Payload.Execute(Db, Settings);
+                            }
+                            catch (Exception ex)
+                            {
+                                Logger.Log(LogLevel.Error, ex, $"Error while running job: {job.Payload.Description}");
+                                result = JobResult.Error(ex);
+                            }
+
+                            await Job.Finish(Db, job, result);
+                        }
+                        catch (Exception ex)
+                        {
+                            Logger.Log(LogLevel.Error, ex, $"Error getting or updating Job");
+
+                        }
+                    }
+                });
+            }
+        }
+        
+        public async Task JobScheduler()
+        {
+            Utils.LogMessages.Subscribe(msg => Logger.Log(LogLevel.Information, msg.ToString()));
+            while (true)
+            {
+                await KillOrphanedJobs();
+                await ScheduledJob<GetNexusUpdatesJob>(TimeSpan.FromHours(2), Job.JobPriority.High);
+                await ScheduledJob<UpdateModLists>(TimeSpan.FromMinutes(30), Job.JobPriority.High);
+                await ScheduledJob<EnqueueAllArchives>(TimeSpan.FromHours(2), Job.JobPriority.Low);
+                await ScheduledJob<EnqueueAllGameFiles>(TimeSpan.FromHours(24), Job.JobPriority.High);
+                await Task.Delay(10000);
+            }
+        }
+
+        private async Task KillOrphanedJobs()
+        {
+            try
+            {
+                var started = await Db.Jobs.AsQueryable()
+                    .Where(j => j.Started != null && j.Ended == null)
+                    .ToListAsync();
+                foreach (var job in started)
+                {
+                    var runtime = DateTime.Now - job.Started;
+                    if (runtime > TimeSpan.FromMinutes(30))
+                    {
+                        await Job.Finish(Db, job, JobResult.Error(new Exception($"Timeout after {runtime.Value.TotalMinutes}")));
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Logger.Log(LogLevel.Error, ex, "Error in JobScheduler when scheduling KillOrphanedJobs");
+            }
+        }
+        
+        private async Task ScheduledJob<T>(TimeSpan span, Job.JobPriority priority) where T : AJobPayload, new()
+        {
+            try
+            {
+                var jobs = await Db.Jobs.AsQueryable()
+                    .Where(j => j.Payload is T)
+                    .OrderByDescending(j => j.Created)
+                    .Take(10)
+                    .ToListAsync();
+
+                foreach (var job in jobs)
+                {
+                    if (job.Started == null || job.Ended == null) return;
+                    if (DateTime.Now - job.Ended < span) return;
+                }
+                await Db.Jobs.InsertOneAsync(new Job
+                {
+                    Priority = priority,
+                    Payload = new T()
+                });
+            }
+            catch (Exception ex)
+            {
+                
+                Logger.Log(LogLevel.Error, ex, $"Error in JobScheduler when scheduling {typeof(T).Name}");
+            }
+        }
+
+    }
+}

Wabbajack.BuildServer/Models/DBContext.cs

@@ -0,0 +1,42 @@
+using System.Collections.Generic;
+using Microsoft.Extensions.Configuration;
+using MongoDB.Driver;
+using Wabbajack.Lib.NexusApi;
+using Wabbajack.BuildServer.Models.JobQueue;
+
+namespace Wabbajack.BuildServer.Models
+{
+    public class DBContext
+    {
+        private IConfiguration _configuration;
+        private Settings _settings;
+        public DBContext(IConfiguration configuration)
+        {
+            _configuration = configuration;
+
+            _settings = new Settings();
+            _configuration.Bind("MongoDB", _settings);
+        }
+        
+        public IMongoCollection<NexusCacheData<ModInfo>> NexusModInfos => Client.GetCollection<NexusCacheData<ModInfo>>(_settings.Collections["NexusModInfos"]);
+        public IMongoCollection<NexusCacheData<NexusFileInfo>> NexusFileInfos => Client.GetCollection<NexusCacheData<NexusFileInfo>>(_settings.Collections["NexusFileInfos"]);
+        public IMongoCollection<ModListStatus> ModListStatus => Client.GetCollection<ModListStatus>(_settings.Collections["ModListStatus"]);
+        
+        public IMongoCollection<Job> Jobs => Client.GetCollection<Job>(_settings.Collections["JobQueue"]);
+        public IMongoCollection<DownloadState> DownloadStates => Client.GetCollection<DownloadState>(_settings.Collections["DownloadStates"]);
+        public IMongoCollection<Metric> Metrics => Client.GetCollection<Metric>(_settings.Collections["Metrics"]);
+        public IMongoCollection<IndexedFile> IndexedFiles => Client.GetCollection<IndexedFile>(_settings.Collections["IndexedFiles"]);
+
+        public IMongoCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>> NexusModFiles =>
+            Client.GetCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>>(
+                _settings.Collections["NexusModFiles"]);
+
+        private IMongoDatabase Client => new MongoClient($"mongodb://{_settings.Host}").GetDatabase(_settings.Database);
+    }
+    public class Settings
+    {
+        public string Host { get; set; }
+        public string Database { get; set; }
+        public Dictionary<string, string> Collections { get; set; }
+    }
+}

Wabbajack.BuildServer/Models/DownloadState.cs

@@ -6,7 +6,7 @@ using System.Threading.Tasks;
 using MongoDB.Bson.Serialization.Attributes;
 using Wabbajack.Lib.Downloaders;
 
-namespace Wabbajack.CacheServer.DTOs
+namespace Wabbajack.BuildServer.Models
 {
     public class DownloadState
     {

Wabbajack.BuildServer/Models/IndexedFile.cs

@@ -1,13 +1,12 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Security.RightsManagement;
 using System.Text;
 using System.Threading.Tasks;
 using MongoDB.Bson.Serialization.Attributes;
 using Wabbajack.VirtualFileSystem;
 
-namespace Wabbajack.CacheServer.DTOs
+namespace Wabbajack.BuildServer.Models
 {
     public class IndexedFile
     {

Wabbajack.BuildServer/Models/IndexedFileWithChildren.cs

@@ -0,0 +1,9 @@
+using System.Collections;
+using System.Collections.Generic;
+
+namespace Wabbajack.BuildServer.Models
+{
+    public class IndexedFileWithChildren : IndexedFile
+    {
+   }
+}

Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs

@@ -4,13 +4,20 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using MongoDB.Bson.Serialization.Attributes;
-using Wabbajack.CacheServer.Jobs;
+using Wabbajack.BuildServer.Models.Jobs;
 
-namespace Wabbajack.CacheServer.DTOs.JobQueue
+namespace Wabbajack.BuildServer.Models.JobQueue
 {
     public abstract class AJobPayload
     {
-        public static List<Type> KnownSubTypes = new List<Type> {typeof(IndexJob)};
+        public static List<Type> KnownSubTypes = new List<Type>
+        {
+            typeof(IndexJob),
+            typeof(GetNexusUpdatesJob),
+            typeof(UpdateModLists),
+            typeof(EnqueueAllArchives),
+            typeof(EnqueueAllGameFiles)
+        };
         public static Dictionary<Type, string> TypeToName { get; set; }
         public static Dictionary<string, Type> NameToType { get; set; }
 
@@ -20,7 +27,7 @@ namespace Wabbajack.CacheServer.DTOs.JobQueue
 
         public virtual bool UsesNexus { get; } = false;
 
-        public abstract Task<JobResult> Execute();
+        public abstract Task<JobResult> Execute(DBContext db, AppSettings settings);
 
         static AJobPayload()
         {

Wabbajack.BuildServer/Models/JobQueue/Job.cs

@@ -7,7 +7,7 @@ using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
 using MongoDB.Driver;
 
-namespace Wabbajack.CacheServer.DTOs.JobQueue
+namespace Wabbajack.BuildServer.Models.JobQueue
 { 
     public class Job
     {
@@ -18,8 +18,7 @@ namespace Wabbajack.CacheServer.DTOs.JobQueue
             High,
         }
 
-        [BsonId]
-        public Guid Id { get; set; }
+        [BsonId] public String Id { get; set; } = Guid.NewGuid().ToString();
         public DateTime? Started { get; set; }
         public DateTime? Ended { get; set; }
         public DateTime Created { get; set; } = DateTime.Now;
@@ -29,13 +28,13 @@ namespace Wabbajack.CacheServer.DTOs.JobQueue
         public bool RequiresNexus { get; set; } = true;
         public AJobPayload Payload { get; set; }
 
-        public static async Task<Guid> Enqueue(Job job)
+        public static async Task<String> Enqueue(DBContext db, Job job)
         {
-            await Server.Config.JobQueue.Connect().InsertOneAsync(job);
+            await db.Jobs.InsertOneAsync(job);
             return job.Id;
         }
 
-        public static async Task<Job> GetNext()
+        public static async Task<Job> GetNext(DBContext db)
         {
             var filter = new BsonDocument
             {
@@ -46,21 +45,21 @@ namespace Wabbajack.CacheServer.DTOs.JobQueue
                 {"$set", new BsonDocument {{"Started", DateTime.Now}}}
             };
             var sort = new {Priority=-1, Created=1}.ToBsonDocument();
-            var job = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update, new FindOneAndUpdateOptions<Job>{Sort = sort});
+            var job = await db.Jobs.FindOneAndUpdateAsync<Job>(filter, update, new FindOneAndUpdateOptions<Job>{Sort = sort});
             return job;
         }
 
-        public static async Task<Job> Finish(Job job, JobResult jobResult)
+        public static async Task<Job> Finish(DBContext db, Job job, JobResult jobResult)
         {
             var filter = new BsonDocument
             {
-                {"query", new BsonDocument {{"Id", job.Id}}},
+                {"_id", job.Id},
             };
             var update = new BsonDocument
             {
                 {"$set", new BsonDocument {{"Ended", DateTime.Now}, {"Result", jobResult.ToBsonDocument()}}}
             };
-            var result = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync<Job>(filter, update);
+            var result = await db.Jobs.FindOneAndUpdateAsync<Job>(filter, update);
             return result;
         }
     }

Wabbajack.BuildServer/Models/JobQueue/JobResult.cs

@@ -5,7 +5,7 @@ using System.Text;
 using System.Threading.Tasks;
 using MongoDB.Bson.Serialization.Attributes;
 
-namespace Wabbajack.CacheServer.DTOs.JobQueue
+namespace Wabbajack.BuildServer.Models.JobQueue
 {
     public class JobResult
     {

Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs

@@ -0,0 +1,87 @@
+using System;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using System.Linq;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.ModListRegistry;
+
+namespace Wabbajack.BuildServer.Models.Jobs
+{
+    public class EnqueueAllArchives : AJobPayload
+    {
+        public override string Description => "Add missing modlist archives to indexer";
+        public override async Task<JobResult> Execute(DBContext db, AppSettings settings)
+        {
+            Utils.Log("Starting modlist indexing");
+            var modlists = await ModlistMetadata.LoadFromGithub();
+
+            using (var queue = new WorkQueue())
+            {
+                foreach (var list in modlists)
+                {
+                    try
+                    {
+                        await EnqueueFromList(db, list, queue);
+                    }
+                    catch (Exception ex)
+                    {
+                        Utils.Log(ex.ToString());
+                    }
+                }
+            }
+
+            return JobResult.Success();
+        }
+
+        private static async Task EnqueueFromList(DBContext db, ModlistMetadata list, WorkQueue queue)
+        {
+            var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
+
+            var modlist_path = Path.Combine(Consts.ModListDownloadFolder,
+                list.Links.MachineURL + ExtensionManager.Extension);
+
+            if (list.NeedsDownload(modlist_path))
+            {
+                if (File.Exists(modlist_path))
+                    File.Delete(modlist_path);
+
+                var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
+                Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
+                await state.Download(modlist_path);
+            }
+            else
+            {
+                Utils.Log($"No changes detected from downloaded modlist");
+            }
+
+            Utils.Log($"Loading {modlist_path}");
+
+            var installer = AInstaller.LoadFromFile(modlist_path);
+
+            var archives = installer.Archives;
+
+            Utils.Log($"Found {archives.Count} archives in {installer.Name} to index");
+            var searching = archives.Select(a => a.Hash).Distinct().ToArray();
+
+            Utils.Log($"Looking for missing archives");
+            var knownArchives = (await db.IndexedFiles.AsQueryable().Where(a => searching.Contains(a.Hash))
+                .Select(d => d.Hash).ToListAsync()).ToDictionary(a => a);
+
+            Utils.Log($"Found {knownArchives.Count} pre-existing archives");
+            var missing = archives.Where(a => !knownArchives.ContainsKey(a.Hash)).ToList();
+
+            Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs");
+
+            var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}, Priority = Job.JobPriority.Low});
+
+            Utils.Log($"Writing jobs to the DB");
+            await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions {IsOrdered = false});
+            Utils.Log($"Done adding archives for {installer.Name}");
+        }
+    }
+}

Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs

@@ -0,0 +1,74 @@
+using System.Linq;
+using System.Threading.Tasks;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using System.IO;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Directory = Alphaleonis.Win32.Filesystem.Directory;
+using Path = Alphaleonis.Win32.Filesystem.Path;
+
+namespace Wabbajack.BuildServer.Models.Jobs
+{
+    public class EnqueueAllGameFiles : AJobPayload
+    {
+        public override string Description { get => $"Enqueue all game files for indexing"; }
+        public override async Task<JobResult> Execute(DBContext db, AppSettings settings)
+        {
+            using (var queue = new WorkQueue(4))
+            {
+                Utils.Log($"Indexing game files");
+                var states = GameRegistry.Games.Values
+                    .Where(game => game.GameLocation() != null && game.MainExecutable != null)
+                    .SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", SearchOption.AllDirectories)
+                        .Select(file => new GameFileSourceDownloader.State
+                        {
+                            Game = game.Game,
+                            GameVersion = game.InstalledVersion,
+                            GameFile = file.RelativeTo(game.GameLocation()),
+                        }))
+                    .ToList();
+                
+                var pks = states.Select(s => s.PrimaryKeyString).Distinct().ToArray();
+                Utils.Log($"Found {pks.Length} archives to cross-reference with the database");
+
+                var found = (await db.DownloadStates
+                    .AsQueryable().Where(s => pks.Contains(s.Key))
+                    .Select(s => s.Key)
+                    .ToListAsync())
+                    .ToDictionary(s => s);
+
+                states = states.Where(s => !found.ContainsKey(s.PrimaryKeyString)).ToList();
+                Utils.Log($"Found {states.Count} archives to index");
+
+                await states.PMap(queue, async state =>
+                {
+                    var path = Path.Combine(state.Game.MetaData().GameLocation(), state.GameFile);
+                    Utils.Log($"Hashing Game file {path}");
+                    try
+                    {
+                        state.Hash = await path.FileHashAsync();
+                    }
+                    catch (IOException)
+                    {
+                        Utils.Log($"Unable to hash {path}");
+                    }
+                });
+
+                var with_hash = states.Where(state => state.Hash != null).ToList();
+                Utils.Log($"Inserting {with_hash.Count} jobs.");
+                var jobs = states.Select(state => new IndexJob {Archive = new Archive {Name = Path.GetFileName(state.GameFile), State = state}})
+                    .Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus})
+                    .ToList();
+
+                if (jobs.Count > 0)
+                    await db.Jobs.InsertManyAsync(jobs);                
+                
+                return JobResult.Success();
+            }
+            
+        }
+    }
+}

Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs

@@ -0,0 +1,75 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.Common;
+using Wabbajack.Lib.NexusApi;
+using MongoDB.Driver;
+
+
+namespace Wabbajack.BuildServer.Models.Jobs
+{
+    public class GetNexusUpdatesJob : AJobPayload
+    {
+        public override string Description => "Poll the nexus for updated mods, and clean any references to those mods";
+
+        public override async Task<JobResult> Execute(DBContext db, AppSettings settings)
+        {
+            var api = await NexusApiClient.Get();
+            
+            var gameTasks = GameRegistry.Games.Values
+                .Where(game => game.NexusName != null)
+                .Select(async game =>
+                {
+                    return (game,
+                        mods: await api.Get<List<UpdatedMod>>(
+                            $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m"));
+                })
+                .Select(async rTask =>
+                {
+                    var (game, mods) = await rTask;
+                    return mods.Select(mod => new { game = game, mod = mod });
+                }).ToList();
+
+            Utils.Log($"Getting update list for {gameTasks.Count} games");
+
+            var purge = (await Task.WhenAll(gameTasks))
+                .SelectMany(i => i)
+                .ToList();
+
+            Utils.Log($"Found {purge.Count} updated mods in the last month");
+            using (var queue = new WorkQueue())
+            {
+                var collected = await purge.Select(d =>
+                {
+                    var a = d.mod.latest_file_update.AsUnixTime();
+                    // Mod activity could hide files
+                    var b = d.mod.latest_mod_activity.AsUnixTime();
+
+                    return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.mod_id.ToString()};
+                }).PMap(queue, async t =>
+                {
+                    var resultA = await db.NexusModInfos.DeleteManyAsync(f =>
+                        f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
+                    var resultB = await db.NexusModFiles.DeleteManyAsync(f =>
+                        f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
+                    var resultC = await db.NexusFileInfos.DeleteManyAsync(f =>
+                        f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
+
+                    return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount;
+                });
+
+                Utils.Log($"Purged {collected.Sum()} cache entries");
+            }
+
+            return JobResult.Success();
+        }
+        
+        class UpdatedMod
+        {
+            public long mod_id;
+            public long latest_file_update;
+            public long latest_mod_activity;
+        }
+    }
+}

Wabbajack.BuildServer/Models/Jobs/IndexJob.cs

@@ -6,53 +6,53 @@ using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using MongoDB.Driver;
 using MongoDB.Driver.Linq;
-using Wabbajack.CacheServer.DTOs;
-using Wabbajack.CacheServer.DTOs.JobQueue;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.BuildServer.Models.JobQueue;
 using Wabbajack.Common;
 using Wabbajack.Lib;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.VirtualFileSystem;
 
-namespace Wabbajack.CacheServer.Jobs
+namespace Wabbajack.BuildServer.Models.Jobs
 { 
+    
     public class IndexJob : AJobPayload
     {
         public Archive Archive { get; set; }
-        public override string Description { get; } = "Validate and index an archive";
+        public override string Description => $"Index ${Archive.State.PrimaryKeyString} and save the download/file state";
         public override bool UsesNexus { get => Archive.State is NexusDownloader.State; }
-        public override async Task<JobResult> Execute()
+        public override async Task<JobResult> Execute(DBContext db, AppSettings settings)
         {
-
             var pk = new List<object>();
             pk.Add(AbstractDownloadState.TypeToName[Archive.State.GetType()]);
             pk.AddRange(Archive.State.PrimaryKey);
             var pk_str = string.Join("|",pk.Select(p => p.ToString()));
 
-            var found = await Server.Config.DownloadStates.Connect().AsQueryable().Where(f => f.Key == pk_str).Take(1).ToListAsync();
+            var found = await db.DownloadStates.AsQueryable().Where(f => f.Key == pk_str).Take(1).ToListAsync();
             if (found.Count > 0)
                 return JobResult.Success();
 
             string fileName = Archive.Name;
             string folder = Guid.NewGuid().ToString();
             Utils.Log($"Indexer is downloading {fileName}");
-            var downloadDest = Path.Combine(Server.Config.Indexer.DownloadDir, folder, fileName);
+            var downloadDest = Path.Combine(settings.DownloadDir, folder, fileName);
             await Archive.State.Download(downloadDest);
 
             using (var queue = new WorkQueue())
             {
                 var vfs = new Context(queue, true);
-                await vfs.AddRoot(Path.Combine(Server.Config.Indexer.DownloadDir, folder));
+                await vfs.AddRoot(Path.Combine(settings.DownloadDir, folder));
                 var archive = vfs.Index.ByRootPath.First();
                 var converted = ConvertArchive(new List<IndexedFile>(), archive.Value);
                 try
                 {
-                    await Server.Config.IndexedFiles.Connect().InsertManyAsync(converted, new InsertManyOptions {IsOrdered = false});
+                    await db.IndexedFiles.InsertManyAsync(converted, new InsertManyOptions {IsOrdered = false});
                 }
                 catch (MongoBulkWriteException)
                 {
                 }
 
-                await Server.Config.DownloadStates.Connect().InsertOneAsync(new DownloadState
+                await db.DownloadStates.InsertOneAsync(new DownloadState
                 {
                     Key = pk_str, 
                     Hash = archive.Value.Hash, 
@@ -60,13 +60,13 @@ namespace Wabbajack.CacheServer.Jobs
                     IsValid = true
                 });
 
-                var to_path = Path.Combine(Server.Config.Indexer.ArchiveDir,
+                var to_path = Path.Combine(settings.ArchiveDir,
                     $"{Path.GetFileName(fileName)}_{archive.Value.Hash.FromBase64().ToHex()}_{Path.GetExtension(fileName)}");
                 if (File.Exists(to_path))
                     File.Delete(downloadDest);
                 else
                     File.Move(downloadDest, to_path);
-                Utils.DeleteDirectory(Path.Combine(Server.Config.Indexer.DownloadDir, folder));
+                Utils.DeleteDirectory(Path.Combine(settings.DownloadDir, folder));
             }
 
             return JobResult.Success();
@@ -103,4 +103,5 @@ namespace Wabbajack.CacheServer.Jobs
 
 
     }
+    
 }

Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs

@@ -0,0 +1,111 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.ModListRegistry;
+
+namespace Wabbajack.BuildServer.Models.Jobs
+{
+    public class UpdateModLists : AJobPayload
+    {
+        public override string Description => "Validate curated modlists";
+        public override async Task<JobResult> Execute(DBContext db, AppSettings settings)
+        {
+            Utils.Log("Starting Modlist Validation");
+            var modlists = await ModlistMetadata.LoadFromGithub();
+
+            using (var queue = new WorkQueue())
+            {
+                foreach (var list in modlists)
+                {
+                    try
+                    {
+                        await ValidateList(db, list, queue);
+                    }
+                    catch (Exception ex)
+                    {
+                        Utils.Log(ex.ToString());
+                    }
+                }
+            }
+
+            return JobResult.Success();
+        }
+        
+         private static async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue)
+        {
+            var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
+            
+            var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ExtensionManager.Extension);
+
+            if (list.NeedsDownload(modlist_path))
+            {
+                if (File.Exists(modlist_path))
+                    File.Delete(modlist_path);
+
+                var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
+                Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
+                await state.Download(modlist_path);
+            }
+            else
+            {
+                Utils.Log($"No changes detected from downloaded modlist");
+            }
+
+
+            Utils.Log($"Loading {modlist_path}");
+
+            var installer = AInstaller.LoadFromFile(modlist_path);
+
+            Utils.Log($"{installer.Archives.Count} archives to validate");
+
+            DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
+
+            var validated = (await installer.Archives
+                    .PMap(queue, async archive =>
+                    {
+                        Utils.Log($"Validating: {archive.Name}");
+                        bool is_failed;
+                        try
+                        {
+                            is_failed = !(await archive.State.Verify());
+                        }
+                        catch (Exception)
+                        {
+                            is_failed = false;
+                        }
+
+                        return new DetailedStatusItem {IsFailing = is_failed, Archive = archive};
+                    }))
+                .ToList();
+
+
+            var status = new DetailedStatus
+            {
+                Name = list.Title,
+                Archives = validated.OrderBy(v => v.Archive.Name).ToList(),
+                DownloadMetaData = list.DownloadMetadata,
+                HasFailures = validated.Any(v => v.IsFailing)
+            };
+
+            var dto = new ModListStatus
+            {
+                Id = list.Links.MachineURL,
+                Summary = new ModlistSummary
+                {
+                    Name = status.Name,
+                    Checked = status.Checked,
+                    Failed = status.Archives.Count(a => a.IsFailing),
+                    Passed = status.Archives.Count(a => !a.IsFailing),
+                },
+                DetailedStatus = status,
+                Metadata = list
+            };
+            await ModListStatus.Update(db, dto);
+        }
+    }
+}

Wabbajack.BuildServer/Models/Metric.cs

@@ -0,0 +1,71 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.GraphQL;
+using Wabbajack.Common;
+
+
+namespace Wabbajack.BuildServer.Models
+{
+    public class Metric
+    {
+        [BsonId]
+        public ObjectId Id;
+        public DateTime Timestamp;
+        public string Action;
+        public string Subject;
+        public string MetricsKey;
+
+        
+        public static async Task<IEnumerable<MetricResult>> Report(DBContext db, string grouping)
+        {
+            var data = await db.Metrics.AsQueryable()
+                .Where(m => m.MetricsKey != null)
+                .Where(m => m.Action == grouping)
+                .Where(m => m.Subject != "Default")
+                .ToListAsync();
+
+            var minDate = DateTime.Parse(data.Min(d => d.Timestamp.ToString("yyyy-MM-dd")));
+            var maxDate = DateTime.Parse(data.Max(d => d.Timestamp.ToString("yyyy-MM-dd")));
+
+            var dateArray = Enumerable.Range(0, (int)(maxDate - minDate).TotalDays + 1)
+                .Select(idx => minDate + TimeSpan.FromDays(idx))
+                .Select(date => date.ToString("yyyy-MM-dd"))
+                .ToList();
+            
+            var results = data
+                .Where(d => !Guid.TryParse(d.Subject, out var _))
+                .GroupBy(d => d.Subject)
+                .Select(by_series =>
+                {
+                    var by_day = by_series.GroupBy(d => d.Timestamp.ToString("yyyy-MM-dd"))
+                        .Select(d => (d.Key, d.DistinctBy(v => v.MetricsKey ?? "").Count()))
+                        .OrderBy(r => r.Key);
+                    
+                    var by_day_idx = by_day.ToDictionary(d => d.Key);
+
+                    (string Key, int) GetEntry(string date)
+                    {
+                        if (by_day_idx.TryGetValue(date, out var result))
+                            return result;
+                        return (date, 0);
+                    } 
+                    
+                    return new MetricResult
+                    {
+                        SeriesName = by_series.Key,
+                        Labels = dateArray.Select(d => GetEntry(d).Key).ToList(),
+                        Values = dateArray.Select(d => GetEntry(d).Item2).ToList()
+                    };
+                })
+                .OrderBy(f => f.SeriesName);
+
+            return results;
+        }
+    }
+}

Wabbajack.BuildServer/Models/ModListStatus.cs

@@ -8,7 +8,7 @@ using MongoDB.Driver.Linq;
 using Wabbajack.Lib;
 using Wabbajack.Lib.ModListRegistry;
 
-namespace Wabbajack.CacheServer.DTOs
+namespace Wabbajack.BuildServer.Models
 {
     public class ModListStatus
     {
@@ -20,10 +20,10 @@ namespace Wabbajack.CacheServer.DTOs
         public ModlistMetadata Metadata { get; set; }
         public DetailedStatus DetailedStatus { get; set; }
 
-        public static async Task Update(ModListStatus status)
+        public static async Task Update(DBContext db, ModListStatus status)
         {
             var id = status.Metadata.Links.MachineURL;
-            await Server.Config.ListValidation.Connect().FindOneAndReplaceAsync<ModListStatus>(s => s.Id == id, status, new FindOneAndReplaceOptions<ModListStatus> {IsUpsert = true});
+            await db.ModListStatus.FindOneAndReplaceAsync<ModListStatus>(s => s.Id == id, status, new FindOneAndReplaceOptions<ModListStatus> {IsUpsert = true});
         }
 
         public static IQueryable<ModListStatus> AllSummaries
@@ -34,22 +34,14 @@ namespace Wabbajack.CacheServer.DTOs
             }
         }
 
-        public static async Task<ModListStatus> ByName(string name)
+        public static async Task<ModListStatus> ByName(DBContext db, string name)
         {
-            var result = await Server.Config.ListValidation.Connect()
+            var result = await db.ModListStatus
                 .AsQueryable()
                 .Where(doc => doc.Metadata.Links.MachineURL == name || doc.Metadata.Title == name)
                 .ToListAsync();
             return result.First();
         }
-
-        public static IMongoQueryable<ModListStatus> All
-        {
-            get
-            {
-                return Server.Config.ListValidation.Connect().AsQueryable();
-            }
-        }
     }
 
     public class DetailedStatus

Wabbajack.BuildServer/Models/NexusCacheData.cs

@@ -1,11 +1,7 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using MongoDB.Bson.Serialization.Attributes;
 
-namespace Wabbajack.CacheServer.DTOs
+namespace Wabbajack.BuildServer.Models
 {
     public class NexusCacheData<T>
     {

Wabbajack.BuildServer/Program.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace Wabbajack.BuildServer
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            CreateHostBuilder(args).Build().Run();
+        }
+        
+        public static IHostBuilder CreateHostBuilder(string[] args) =>
+            Host.CreateDefaultBuilder(args)
+                .ConfigureWebHostDefaults(webBuilder =>
+                {
+                    webBuilder.UseUrls("http://*:5000");
+                    webBuilder.UseStartup<Startup>();
+                });
+    }
+}

Wabbajack.BuildServer/SerializerSettings.cs

@@ -3,14 +3,16 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc.ModelBinding;
 using MongoDB.Bson;
 using MongoDB.Bson.IO;
 using MongoDB.Bson.Serialization;
 using MongoDB.Bson.Serialization.Conventions;
-using Wabbajack.CacheServer.DTOs.JobQueue;
+using Wabbajack.BuildServer.Models.JobQueue;
 using Wabbajack.Lib.Downloaders;
 
-namespace Wabbajack.CacheServer.DTOs
+
+namespace Wabbajack.BuildServer
 {
     public static class SerializerSettings
     {
@@ -24,10 +26,10 @@ namespace Wabbajack.CacheServer.DTOs
             dis = new TypeDiscriminator(typeof(AJobPayload), AJobPayload.NameToType, AJobPayload.TypeToName);
             BsonSerializer.RegisterDiscriminatorConvention(typeof(AJobPayload), dis);
             BsonClassMap.RegisterClassMap<AJobPayload>(cm => cm.SetIsRootClass(true));
+           
         }
     }
-
-
+    
     public class TypeDiscriminator : IDiscriminatorConvention
     {
         private readonly Type defaultType;

Wabbajack.BuildServer/Startup.cs

@@ -0,0 +1,100 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using GraphiQl;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.HttpsPolicy;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Authentication;
+using Microsoft.AspNetCore.Authentication.AzureAD.UI;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.OpenApi.Models;
+using Newtonsoft.Json;
+using Swashbuckle.AspNetCore.Swagger;
+using Wabbajack.BuildServer.Controllers;
+using Wabbajack.BuildServer.Models;
+using Microsoft.AspNetCore.Mvc.NewtonsoftJson;
+using Wabbajack.BuildServer.Controllers;
+using Microsoft.Extensions.FileProviders;
+using Directory = System.IO.Directory;
+
+
+namespace Wabbajack.BuildServer
+{
+    public class Startup
+    {
+        public Startup(IConfiguration configuration)
+        {
+            Configuration = configuration;
+        }
+
+        public IConfiguration Configuration { get; }
+
+        // This method gets called by the runtime. Use this method to add services to the container.
+        public void ConfigureServices(IServiceCollection services)
+        {
+            services.AddSwaggerGen(c =>
+            {
+                c.SwaggerDoc("v1", new OpenApiInfo {Title = "Wabbajack Build API", Version = "v1"});
+            });
+
+            services.AddSingleton<DBContext>();
+            services.AddSingleton<JobManager>();
+            services.AddSingleton<AppSettings>();
+            services.AddMvc();
+            services.AddControllers()
+                .AddNewtonsoftJson(o =>
+            {
+                
+                o.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore;
+            });
+            
+            
+        }
+
+        // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
+        public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
+        {
+            SerializerSettings.Init();
+            if (env.IsDevelopment())
+            {
+                app.UseDeveloperExceptionPage();
+            }
+
+            app.UseGraphiQl();
+            app.UseDeveloperExceptionPage();
+            app.UseStaticFiles();
+            //app.UseHttpsRedirection();
+
+            app.UseSwagger();
+            app.UseSwaggerUI(c =>
+            {
+                c.SwaggerEndpoint("/swagger/v1/swagger.json", "Wabbajack Build API");
+                c.RoutePrefix = string.Empty;
+            });
+            app.UseRouting();
+
+            app.UseJobManager();
+            app.UseAuthentication();
+            app.UseAuthorization();
+            app.UseFileServer(new FileServerOptions
+            {
+                FileProvider = new PhysicalFileProvider(
+                    Path.Combine(Directory.GetCurrentDirectory(), "public"))
+            });
+
+            app.UseEndpoints(endpoints =>
+            {
+                endpoints.MapControllers();
+            });
+        }
+    }
+}

Wabbajack.BuildServer/Wabbajack.BuildServer.csproj

@@ -0,0 +1,76 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+    <PropertyGroup>
+        <TargetFramework>netcoreapp3.1</TargetFramework>
+        <UserSecretsId>aspnet-Wabbajack.BuildServer-6E798B30-DB04-4436-BE65-F043AF37B314</UserSecretsId>
+        <WebProject_DirectoryAccessLevelKey>0</WebProject_DirectoryAccessLevelKey>
+        <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
+        <Configurations>Debug;Release</Configurations>
+        <Platforms>AnyCPU;x64</Platforms>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="graphiql" Version="1.2.0" />
+        <PackageReference Include="GraphQL" Version="3.0.0-preview-1352" />
+        <PackageReference Include="Microsoft.AspNetCore.Authentication.AzureAD.UI" Version="3.1.0" />
+        <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.0" />
+        <PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
+        <PackageReference Include="Microsoft.OpenApi" Version="1.1.4" />
+        <PackageReference Include="MongoDB.Driver" Version="2.10.0" />
+        <PackageReference Include="MongoDB.Driver.Core" Version="2.10.0" />
+        <PackageReference Include="Swashbuckle.AspNetCore" Version="5.0.0-rc5" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
+      <ProjectReference Include="..\Wabbajack.Lib\Wabbajack.Lib.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <None Remove="chrome_elf.dll" />
+      <None Remove="d3dcompiler_47.dll" />
+      <None Remove="libGLESv2.dll" />
+      <None Remove="CefSharp.dll" />
+      <None Remove="v8_context_snapshot.bin" />
+      <None Remove="CefSharp.Core.dll" />
+      <None Remove="icudtl.dat" />
+      <None Remove="innounp.exe" />
+      <None Remove="CefSharp.Wpf.dll" />
+      <None Remove="snapshot_blob.bin" />
+      <None Remove="libEGL.dll" />
+      <None Remove="libcef.dll" />
+      <None Remove="natives_blob.bin" />
+      <None Remove="CefSharp.OffScreen.dll" />
+      <None Remove="devtools_resources.pak" />
+      <None Remove="CefSharp.BrowserSubprocess.Core.dll" />
+      <None Remove="CefSharp.BrowserSubprocess.exe" />
+      <None Remove="cefsharp.7z" />
+      <None Remove="cef_extensions.pak" />
+      <None Remove="cef_200_percent.pak" />
+      <None Remove="cef_100_percent.pak" />
+      <None Remove="cef.pak" />
+      <None Remove="7z.exe" />
+      <None Remove="7z.dll" />
+      <None Remove="swiftshader\**" />
+      <None Update="public\metrics.html">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
+    </ItemGroup>
+
+    <ItemGroup>
+      <_ContentIncludedByDefault Remove="Views\MetricsDashboard.cshtml" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <Compile Remove="swiftshader\**" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <EmbeddedResource Remove="swiftshader\**" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <Content Remove="swiftshader\**" />
+    </ItemGroup>
+
+</Project>

Wabbajack.BuildServer/appsettings.Development.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft": "Warning",
+      "Microsoft.Hosting.Lifetime": "Information"
+    }
+  }
+}

Wabbajack.BuildServer/appsettings.json

@@ -0,0 +1,34 @@
+{
+  "AzureAd": {
+    "Instance": "https://login.microsoftonline.com/",
+    "Domain": "qualified.domain.name",
+    "TenantId": "22222222-2222-2222-2222-222222222222",
+    "ClientId": "11111111-1111-1111-11111111111111111"
+  },
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft": "Warning",
+      "Microsoft.Hosting.Lifetime": "Information"
+    }
+  },
+  "MongoDB": {
+    "Host": "internal.test.mongodb",
+    "Database": "wabbajack",
+    "Collections": {
+      "NexusModInfos": "nexus_mod_infos",
+      "NexusModFiles": "nexus_mod_files",
+      "NexusFileInfos": "nexus_file_infos",
+      "ModListStatus": "mod_lists",
+      "JobQueue": "job_queue",
+      "DownloadStates": "download_states",
+      "IndexedFiles": "indexed_files",
+      "Metrics": "metrics"
+    }
+  },
+  "WabbajackSettings": {
+    "DownloadDir": "c:\\tmp\\downloads",
+    "ArchiveDir": "c:\\archives"
+  },
+  "AllowedHosts": "*"
+}

Wabbajack.BuildServer/public/metrics.html

@@ -0,0 +1,77 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Wabbajack Metrics</title>
+    <script src="//cdn.jsdelivr.net/npm/graphql.js@0.6.6/graphql.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/chart.js@2.8.0"></script>
+    <script src="https://cdn.jsdelivr.net/npm/chartjs-plugin-colorschemes"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/underscore.js/1.9.1/underscore-min.js"></script>
+</head>
+<body>
+
+<h2>Begin Download</h2>
+<canvas id="begin_download_chart" width="800" height="600"></canvas>
+<hr/>
+
+<h2>Begin Install</h2>
+<canvas id="begin_install_chart" width="800" height="600"></canvas>
+<hr/>
+
+<h2>Finished Install</h2>
+<canvas id="finished_install_chart" width="800" height="600"></canvas>
+<hr/>
+
+
+
+<script>
+    var makeChart = function(ele, group) {
+        var graph = graphql("/graphql",
+                {
+                    method: "POST",
+                    asJSON: true,
+                    headers: {
+                        "Content-Type": "application/json"
+                    }
+                });
+        var metrics = graph.query(`($type: MetricType) {
+  dailyUniqueMetrics(metric_type: $type)
+  {
+    seriesName,
+    labels,
+    values
+  }
+}`);
+
+        var result = metrics({type: group})
+                .then(function (data) {
+                    var data = data.dailyUniqueMetrics;
+                    var labels = _.uniq(_.flatten(_.map(data, series => series.labels))).sort();
+                    var datasets = _.map(data, series => {
+                        return {
+                            label: series.seriesName,
+                            fill: false,
+                            data: series.values
+                        }});
+                    var ctx = document.getElementById(ele).getContext('2d');
+                    var chart = new Chart(ctx, {
+                        // The type of chart we want to create
+                        type: 'line',
+
+                        // The data for our dataset
+                        data: {
+                            labels: labels,
+                            datasets: datasets},
+
+                        // Configuration options go here
+                        options: {}
+                    });
+                });
+    };
+    makeChart("begin_download_chart", "BEGIN_DOWNLOAD");
+    makeChart("begin_install_chart", "BEGIN_INSTALL");
+    makeChart("finished_install_chart", "FINISHED_INSTALL");
+
+</script>
+</body>
+</html>

Wabbajack.CacheServer/App.config

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-    <startup> 
-        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.8"/>
-    </startup>
-</configuration>

Wabbajack.CacheServer/DTOs/Metric.cs

@@ -1,17 +0,0 @@
-using System;
-using MongoDB.Bson;
-using MongoDB.Bson.Serialization.Attributes;
-
-
-namespace Wabbajack.CacheServer.DTOs
-{
-    public class Metric
-    {
-        [BsonId]
-        public ObjectId Id;
-        public DateTime Timestamp;
-        public string Action;
-        public string Subject;
-        public string MetricsKey;
-    }
-}

Wabbajack.CacheServer/DTOs/MongoDoc.cs

@@ -1,14 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using MongoDB.Bson;
-
-namespace Wabbajack.CacheServer.DTOs
-{
-    public class MongoDoc
-    {
-        public ObjectId _id { get; set; }  = ObjectId.Empty;
-    }
-}

Wabbajack.CacheServer/FodyWeavers.xml

@@ -1,3 +0,0 @@
-<Weavers xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="FodyWeavers.xsd">
-  <Costura />
-</Weavers>

Wabbajack.CacheServer/FodyWeavers.xsd

@@ -1,111 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
-  <!-- This file was generated by Fody. Manual changes to this file will be lost when your project is rebuilt. -->
-  <xs:element name="Weavers">
-    <xs:complexType>
-      <xs:all>
-        <xs:element name="Costura" minOccurs="0" maxOccurs="1">
-          <xs:complexType>
-            <xs:all>
-              <xs:element minOccurs="0" maxOccurs="1" name="ExcludeAssemblies" type="xs:string">
-                <xs:annotation>
-                  <xs:documentation>A list of assembly names to exclude from the default action of "embed all Copy Local references", delimited with line breaks</xs:documentation>
-                </xs:annotation>
-              </xs:element>
-              <xs:element minOccurs="0" maxOccurs="1" name="IncludeAssemblies" type="xs:string">
-                <xs:annotation>
-                  <xs:documentation>A list of assembly names to include from the default action of "embed all Copy Local references", delimited with line breaks.</xs:documentation>
-                </xs:annotation>
-              </xs:element>
-              <xs:element minOccurs="0" maxOccurs="1" name="Unmanaged32Assemblies" type="xs:string">
-                <xs:annotation>
-                  <xs:documentation>A list of unmanaged 32 bit assembly names to include, delimited with line breaks.</xs:documentation>
-                </xs:annotation>
-              </xs:element>
-              <xs:element minOccurs="0" maxOccurs="1" name="Unmanaged64Assemblies" type="xs:string">
-                <xs:annotation>
-                  <xs:documentation>A list of unmanaged 64 bit assembly names to include, delimited with line breaks.</xs:documentation>
-                </xs:annotation>
-              </xs:element>
-              <xs:element minOccurs="0" maxOccurs="1" name="PreloadOrder" type="xs:string">
-                <xs:annotation>
-                  <xs:documentation>The order of preloaded assemblies, delimited with line breaks.</xs:documentation>
-                </xs:annotation>
-              </xs:element>
-            </xs:all>
-            <xs:attribute name="CreateTemporaryAssemblies" type="xs:boolean">
-              <xs:annotation>
-                <xs:documentation>This will copy embedded files to disk before loading them into memory. This is helpful for some scenarios that expected an assembly to be loaded from a physical file.</xs:documentation>
-              </xs:annotation>
-            </xs:attribute>
-            <xs:attribute name="IncludeDebugSymbols" type="xs:boolean">
-              <xs:annotation>
-                <xs:documentation>Controls if .pdbs for reference assemblies are also embedded.</xs:documentation>
-              </xs:annotation>
-            </xs:attribute>
-            <xs:attribute name="DisableCompression" type="xs:boolean">
-              <xs:annotation>
-                <xs:documentation>Embedded assemblies are compressed by default, and uncompressed when they are loaded. You can turn compression off with this option.</xs:documentation>
-              </xs:annotation>
-            </xs:attribute>
-            <xs:attribute name="DisableCleanup" type="xs:boolean">
-              <xs:annotation>
-                <xs:documentation>As part of Costura, embedded assemblies are no longer included as part of the build. This cleanup can be turned off.</xs:documentation>
-              </xs:annotation>
-            </xs:attribute>
-            <xs:attribute name="LoadAtModuleInit" type="xs:boolean">
-              <xs:annotation>
-                <xs:documentation>Costura by default will load as part of the module initialization. This flag disables that behavior. Make sure you call CosturaUtility.Initialize() somewhere in your code.</xs:documentation>
-              </xs:annotation>
-            </xs:attribute>
-            <xs:attribute name="IgnoreSatelliteAssemblies" type="xs:boolean">
-              <xs:annotation>
-                <xs:documentation>Costura will by default use assemblies with a name like 'resources.dll' as a satellite resource and prepend the output path. This flag disables that behavior.</xs:documentation>
-              </xs:annotation>
-            </xs:attribute>
-            <xs:attribute name="ExcludeAssemblies" type="xs:string">
-              <xs:annotation>
-                <xs:documentation>A list of assembly names to exclude from the default action of "embed all Copy Local references", delimited with |</xs:documentation>
-              </xs:annotation>
-            </xs:attribute>
-            <xs:attribute name="IncludeAssemblies" type="xs:string">
-              <xs:annotation>
-                <xs:documentation>A list of assembly names to include from the default action of "embed all Copy Local references", delimited with |.</xs:documentation>
-              </xs:annotation>
-            </xs:attribute>
-            <xs:attribute name="Unmanaged32Assemblies" type="xs:string">
-              <xs:annotation>
-                <xs:documentation>A list of unmanaged 32 bit assembly names to include, delimited with |.</xs:documentation>
-              </xs:annotation>
-            </xs:attribute>
-            <xs:attribute name="Unmanaged64Assemblies" type="xs:string">
-              <xs:annotation>
-                <xs:documentation>A list of unmanaged 64 bit assembly names to include, delimited with |.</xs:documentation>
-              </xs:annotation>
-            </xs:attribute>
-            <xs:attribute name="PreloadOrder" type="xs:string">
-              <xs:annotation>
-                <xs:documentation>The order of preloaded assemblies, delimited with |.</xs:documentation>
-              </xs:annotation>
-            </xs:attribute>
-          </xs:complexType>
-        </xs:element>
-      </xs:all>
-      <xs:attribute name="VerifyAssembly" type="xs:boolean">
-        <xs:annotation>
-          <xs:documentation>'true' to run assembly verification (PEVerify) on the target assembly after all weavers have been executed.</xs:documentation>
-        </xs:annotation>
-      </xs:attribute>
-      <xs:attribute name="VerifyIgnoreCodes" type="xs:string">
-        <xs:annotation>
-          <xs:documentation>A comma-separated list of error codes that can be safely ignored in assembly verification.</xs:documentation>
-        </xs:annotation>
-      </xs:attribute>
-      <xs:attribute name="GenerateXsd" type="xs:boolean">
-        <xs:annotation>
-          <xs:documentation>'false' to turn off automatic generation of the XML Schema file.</xs:documentation>
-        </xs:annotation>
-      </xs:attribute>
-    </xs:complexType>
-  </xs:element>
-</xs:schema>

Wabbajack.CacheServer/Heartbeat.cs

@@ -1,24 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using Nancy;
-
-namespace Wabbajack.CacheServer
-{
-    public class Heartbeat : NancyModule
-    {
-        private static DateTime startTime = DateTime.Now;
-
-        public Heartbeat() : base("/")
-        {
-            Get("/heartbeat", HandleHeartbeat);
-        }
-
-        private object HandleHeartbeat(object arg)
-        {
-            return $"Service is live for: {DateTime.Now - startTime}";
-        }
-    }
-}

Wabbajack.CacheServer/JobQueueEndpoints.cs

@@ -1,144 +0,0 @@
-using System;
-using System.IO;
-using System.Linq;
-using System.Linq.Expressions;
-using System.Security.Policy;
-using System.Threading.Tasks;
-using MongoDB.Driver;
-using MongoDB.Driver.Linq;
-using Nancy;
-using Nettle;
-using Wabbajack.CacheServer.DTOs.JobQueue;
-using Wabbajack.CacheServer.Jobs;
-using Wabbajack.Common;
-using Wabbajack.Lib;
-using Wabbajack.Lib.CompilationSteps;
-using Wabbajack.Lib.Downloaders;
-using Directory = Alphaleonis.Win32.Filesystem.Directory;
-using Path = Alphaleonis.Win32.Filesystem.Path;
-
-namespace Wabbajack.CacheServer
-{
-    public class JobQueueEndpoints : NancyModule
-    {
-        public JobQueueEndpoints() : base ("/jobs")
-        {
-            Get("/", HandleListJobs);
-            Get("/enqueue_curated_for_indexing", HandleEnqueueAllCurated);
-            Get("/enqueue_game_files_for_indexing", HandleEnqueueAllGameFiles);
-        }
-
-        private readonly Func<object, string> HandleListJobsTemplate = NettleEngine.GetCompiler().Compile(@"
-                <html><head/><body>
-
-                <h2>Jobs - {{$.jobs.Count}} Pending</h2>
-                <h3>{{$.time}}</h3>
-                <ol>
-                {{each $.jobs}}
-                    <li>{{$.Description}}</li>
-                {{/each}}
-                </ol>
-
-                <script>
-                setTimeout(function() { location.reload();}, 10000);
-                </script>
-
-                </body></html>");
-
-        private async Task<Response> HandleListJobs(object arg)
-        {
-            var jobs = await Server.Config.JobQueue.Connect()
-                .AsQueryable<Job>()
-                .Where(j => j.Ended == null)
-                .OrderByDescending(j => j.Priority)
-                .ThenBy(j => j.Created)
-                .ToListAsync();
-
-            var response = (Response)HandleListJobsTemplate(new {jobs, time = DateTime.Now});
-            response.ContentType = "text/html";
-            return response;
-        }
-
-
-        private async Task<string> HandleEnqueueAllCurated(object arg)
-        {
-            var states = await Server.Config.ListValidation.Connect()
-                .AsQueryable()
-                .SelectMany(lst => lst.DetailedStatus.Archives)
-                .Select(a => a.Archive)
-                .ToListAsync();
-
-            var jobs = states.Select(state => new IndexJob {Archive = state})
-                .Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus})
-                .ToList();
-
-            if (jobs.Count > 0)
-                await Server.Config.JobQueue.Connect().InsertManyAsync(jobs);
-
-            return $"Enqueued {states.Count} jobs";
-        }
-
-        private async Task<string> HandleEnqueueAllGameFiles(object arg)
-        {
-            using (var queue = new WorkQueue(4))
-            {
-                var states = GameRegistry.Games.Values
-                    .Where(game => game.GameLocation() != null && game.MainExecutable != null)
-                    .SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", SearchOption.AllDirectories)
-                        .Select(file => new GameFileSourceDownloader.State
-                        {
-                            Game = game.Game,
-                            GameVersion = game.InstalledVersion,
-                            GameFile = file.RelativeTo(game.GameLocation()),
-                        }))
-                    .ToList();
-
-                await states.PMap(queue, state =>
-                {
-                    state.Hash = Path.Combine(state.Game.MetaData().GameLocation(), state.GameFile).FileHash();
-                });
-                
-                var jobs = states.Select(state => new IndexJob {Archive = new Archive {Name = Path.GetFileName(state.GameFile), State = state}})
-                    .Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus})
-                    .ToList();
-
-                if (jobs.Count > 0)
-                    await Server.Config.JobQueue.Connect().InsertManyAsync(jobs);                
-                
-                return $"Enqueued {states.Count} Jobs";
-            }
-        }
-
-        public static async Task StartJobQueue()
-        {
-            foreach (var task in Enumerable.Range(0, 4))
-            {
-                var tsk = StartJobQueueInner();
-            }
-        }
-
-        private static async Task StartJobQueueInner()
-        {
-            while (true)
-            {
-                try
-                {
-                    var job = await Job.GetNext();
-                    if (job == null)
-                    {
-                        await Task.Delay(5000);
-                        continue;
-                    }
-
-                    var result = await job.Payload.Execute();
-                    await Job.Finish(job, result);
-                }
-                catch (Exception ex)
-                {
-
-                }
-
-            }
-        }
-    }
-}

Wabbajack.CacheServer/ListValidationService.cs

@@ -1,252 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net.Http;
-using System.Threading.Tasks;
-using Alphaleonis.Win32.Filesystem;
-using MongoDB.Driver;
-using Nancy;
-using Wabbajack.CacheServer.DTOs;
-using Wabbajack.Common;
-using Wabbajack.Lib;
-using Wabbajack.Lib.Downloaders;
-using Wabbajack.Lib.ModListRegistry;
-using MongoDB.Driver.Linq;
-using Nettle;
-using Nettle.Functions;
-
-namespace Wabbajack.CacheServer
-{
-    public class ListValidationService : NancyModule
-    {
-        public ListValidationService() : base("/lists")
-        {
-            Get("/status", HandleGetLists);
-            Get("/force_recheck", HandleForceRecheck);
-            Get("/status/{Name}.json", HandleGetListJson);
-            Get("/status/{Name}.html", HandleGetListHtml);
-            Get("/status/{Name}/broken.rss", HandleGetRSSFeed);
-
-        }
-
-        private async Task<string> HandleForceRecheck(object arg)
-        {
-            await ValidateLists(false);
-            return "done";
-        }
-
-        private async Task<string> HandleGetLists(object arg)
-        {
-            var summaries = await ModListStatus.All.Select(m => m.Summary).ToListAsync();
-            return summaries.ToJSON();
-        }
-
-        public class ArchiveSummary
-        {
-            public string Name;
-            public AbstractDownloadState State;
-        }
-        public class DetailedSummary
-        {
-            public string Name;
-            public DateTime Checked;
-            public List<ArchiveSummary> Failed;
-            public List<ArchiveSummary> Passed;
-
-        }
-        private async Task<string> HandleGetListJson(dynamic arg)
-        {
-            var metric = Metrics.Log("list_validation.get_list_json", (string)arg.Name);
-            var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
-            return lst.ToJSON();
-        }
-
-        
-        private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
-            <html><body>
-                <h2>{{lst.Name}} - {{lst.Checked}}</h2>
-                <h3>Failed ({{failed.Count}}):</h3>
-                <ul>
-                {{each $.failed }}
-                <li>{{$.Archive.Name}}</li>
-                {{/each}}
-                </ul>
-                <h3>Passed ({{passed.Count}}):</h3>
-                <ul>
-                {{each $.passed }}
-                <li>{{$.Archive.Name}}</li>
-                {{/each}}
-                </ul>
-            </body></html>
-        ");
-
-        private async Task<Response> HandleGetListHtml(dynamic arg)
-        {
-
-            var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
-            var response = (Response)HandleGetListTemplate(new
-            {
-                lst,
-                failed = lst.Archives.Where(a => a.IsFailing).ToList(),
-                passed = lst.Archives.Where(a => !a.IsFailing).ToList()
-            });
-            response.ContentType = "text/html";
-            return response;
-        }
-
-        private static readonly Func<object, string> HandleGetRSSFeedTemplate = NettleEngine.GetCompiler().Compile(@"
-<?xml version=""1.0""?>
-<rss version=""2.0"">
-  <channel>
-    <title>{{lst.Name}} - Broken Mods</title>
-    <link>http://build.wabbajack.org/status/{{lst.Name}}.html</link>
-    <description>These are mods that are broken and need updating</description>
-    {{ each $.failed }}
-    <item>
-       <title>{{$.Archive.Name}}</title>
-       <link>{{$.Archive.Name}}</link>
-    </item>
-    {{/each}}
-  </channel>
-</rss>
-        ");
-
-        public async Task<Response> HandleGetRSSFeed(dynamic arg)
-        {
-            var metric = Metrics.Log("failed_rss", arg.Name);
-            var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
-            var response = (Response)HandleGetRSSFeedTemplate(new
-            {
-                lst,
-                failed = lst.Archives.Where(a => a.IsFailing).ToList(),
-                passed = lst.Archives.Where(a => !a.IsFailing).ToList()
-            });
-            response.ContentType = "application/rss+xml";
-            await metric;
-            return response;
-
-        }
-
-        public static void Start()
-        {
-            Task.Run(async () =>
-            {
-                while (true)
-                {
-                    try
-                    {
-                        await ValidateLists();
-                    }
-                    catch (Exception ex)
-                    {
-                        Utils.Log(ex.ToString());
-                    }
-
-                    // Sleep for two hours
-                    await Task.Delay(1000 * 60 * 60 * 2);
-                }
-            }).FireAndForget();
-        }
-        public static async Task ValidateLists(bool skipIfNewer = true)
-        {
-            Utils.Log("Cleaning Nexus Cache");
-            var client = new HttpClient();
-            //await client.GetAsync("http://build.wabbajack.org/nexus_api_cache/update");
-
-            Utils.Log("Starting Modlist Validation");
-            var modlists = await ModlistMetadata.LoadFromGithub();
-
-            using (var queue = new WorkQueue())
-            {
-                foreach (var list in modlists)
-                {
-                    try
-                    {
-                        await ValidateList(list, queue, skipIfNewer);
-                    }
-                    catch (Exception ex)
-                    {
-                        Utils.Log(ex.ToString());
-                    }
-                }
-            }
-
-            Utils.Log($"Done validating {modlists.Count} lists");
-        }
-
-        private static async Task ValidateList(ModlistMetadata list, WorkQueue queue, bool skipIfNewer = true)
-        {
-            var existing = await Server.Config.ListValidation.Connect().FindOneAsync(l => l.Id == list.Links.MachineURL);
-            if (skipIfNewer && existing != null && DateTime.UtcNow - existing.DetailedStatus.Checked < TimeSpan.FromHours(2))
-                return;
-
-            var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ExtensionManager.Extension);
-
-            if (list.NeedsDownload(modlist_path))
-            {
-                if (File.Exists(modlist_path))
-                    File.Delete(modlist_path);
-
-                var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
-                Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
-                await state.Download(modlist_path);
-            }
-            else
-            {
-                Utils.Log($"No changes detected from downloaded modlist");
-            }
-
-
-            Utils.Log($"Loading {modlist_path}");
-
-            var installer = AInstaller.LoadFromFile(modlist_path);
-
-            Utils.Log($"{installer.Archives.Count} archives to validate");
-
-            DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
-
-            var validated = (await installer.Archives
-                    .PMap(queue, async archive =>
-                    {
-                        Utils.Log($"Validating: {archive.Name}");
-                        bool is_failed;
-                        try
-                        {
-                            is_failed = !(await archive.State.Verify());
-                        }
-                        catch (Exception)
-                        {
-                            is_failed = false;
-                        }
-
-                        return new DetailedStatusItem {IsFailing = is_failed, Archive = archive};
-                    }))
-                .ToList();
-
-
-            var status = new DetailedStatus
-            {
-                Name = list.Title,
-                Archives = validated.OrderBy(v => v.Archive.Name).ToList(),
-                DownloadMetaData = list.DownloadMetadata,
-                HasFailures = validated.Any(v => v.IsFailing)
-            };
-
-            var dto = new ModListStatus
-            {
-                Id = list.Links.MachineURL,
-                Summary = new ModlistSummary
-                {
-                    Name = status.Name,
-                    Checked = status.Checked,
-                    Failed = status.Archives.Count(a => a.IsFailing),
-                    Passed = status.Archives.Count(a => !a.IsFailing),
-                },
-                DetailedStatus = status,
-                Metadata = list
-            };
-            await ModListStatus.Update(dto);
-        }
-    }
-}
-

Wabbajack.CacheServer/Metrics.cs

@@ -1,140 +0,0 @@
-using System;
-using System.Linq;
-using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
-using Alphaleonis.Win32.Filesystem;
-using MongoDB.Driver;
-using MongoDB.Driver.Linq;
-using Nancy;
-using Wabbajack.CacheServer.DTOs;
-using Wabbajack.Common;
-
-namespace Wabbajack.CacheServer
-{
-    /// <summary>
-    /// Extremely
-    /// </summary>
-    public class Metrics : NancyModule
-    {
-        private static SemaphoreSlim _lockObject = new SemaphoreSlim(1);
-
-        public static async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
-        {
-            var msg = new[] {string.Join("\t", new[]{timestamp.ToString(), metricsKey, action, subject})};
-            Utils.Log(msg.First());
-            var db = Server.Config.Metrics.Connect();
-            await db.InsertOneAsync(new Metric {Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey});
-        }
-
-        public static Task Log(string action, string subject)
-        {
-            return Log(DateTime.Now, action, subject);
-        }
-
-        public Metrics() : base("/")
-        {
-            Get("/metrics/{Action}/{Value}", HandleMetrics);
-            Get("/metrics/chart/", HandleChart);
-            Get("/metrics/chart/{Action}/", HandleChart);
-            Get("/metrics/chart/{Action}/{Value}/", HandleChart);
-            Get("/metrics/ingest/{filename}", HandleBulkIngest);
-        }
-
-        private async Task<string> HandleBulkIngest(dynamic arg)
-        {
-            Log("Bulk Loading " + arg.filename.ToString());
-
-            var lines = File.ReadAllLines(Path.Combine(@"c:\tmp", (string)arg.filename));
-
-            var db = Server.Config.Metrics.Connect();
-            
-            var data = lines.Select(line => line.Split('\t'))
-                .Where(line => line.Length == 3)
-                .Select(line => new Metric{ Timestamp = DateTime.Parse(line[0]), Action = line[1], Subject = line[2] })
-                .ToList();
-
-            foreach (var metric in data)
-                await db.InsertOneAsync(metric);
-
-            return $"Processed {lines.Length} records";
-        }
-
-        private async Task<string> HandleMetrics(dynamic arg)
-        {
-            var date = DateTime.UtcNow;
-            await Log(date, arg.Action, arg.Value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault());
-            return date.ToString();
-        }
-
-        private async Task<Response> HandleChart(dynamic arg)
-        {
-            /*var data = (await GetData()).Select(line => line.Split('\t'))
-                .Where(line => line.Length == 3)
-                .Select(line => new {date = DateTime.Parse(line[0]), Action = line[1], Value = line[2]});*/
-
-            var q = Server.Config.Metrics.Connect().AsQueryable();
-
-            // Remove guids / Default, which come from testing
-
-            if (arg?.Action != null)
-            {
-                var action = (string)arg.Action;
-                q = q.Where(d => d.Action == action);
-            }
-
-
-            if (arg?.Value != null)
-            {
-                var value = (string)arg.Value;
-                q = q.Where(d => d.Subject.StartsWith(value));
-            }
-
-            var data = (await q.Take(Int32.MaxValue).ToListAsync()).AsEnumerable();
-            data = data.Where(d => !Guid.TryParse(d.Subject ?? "", out Guid v) && (d.Subject ?? "") != "Default");
-
-            var grouped_and_counted = data.GroupBy(d => d.Timestamp.ToString("yyyy-MM-dd"))
-                .OrderBy(d => d.Key)
-                .Select(d => new {Day = d.Key, Count = d.Count()})
-                .ToList();
-
-            var sb = new StringBuilder();
-            sb.Append("<html><head><script src=\"https://cdn.jsdelivr.net/npm/chart.js@2.8.0\"></script></head>");
-            sb.Append("<body><canvas id=\"myChart\"></canvas>");
-            sb.Append("<script language='javascript'>");
-            var script = @"var ctx = document.getElementById('myChart').getContext('2d');
-                    var chart = new Chart(ctx, {
-                                    // The type of chart we want to create
-                                        type: 'line',
-
-                        // The data for our dataset
-                        data: {
-                                    labels: [{{LABELS}}],
-                                    datasets: [{
-                                        label: '{{DATASET}}',
-                                        backgroundColor: 'rgb(255, 99, 132)',
-                                        borderColor: 'rgb(255, 99, 132)',
-                                        data: [{{DATA}}]
-                                    }]
-                                },
-
-            // Configuration options go here
-            options: {}
-            });";
-            sb.Append(script.Replace("{{LABELS}}", string.Join(",", grouped_and_counted.Select(e => "'"+e.Day+"'")))
-                .Replace("{{DATA}}", string.Join(",", grouped_and_counted.Select(e => e.Count.ToString())))
-                .Replace("{{DATASET}}", (arg.Action ?? "*") + " - " + (arg.Value ?? "*")));
-
-            sb.Append("</script>");
-            sb.Append("</body></html>");
-            var response = (Response)sb.ToString();
-            response.ContentType = "text/html";
-            return response;
-        }
-
-        public void Log(string l)
-        {
-            Utils.Log("Metrics: " + l);
-        }
-    }
-}

Wabbajack.CacheServer/NexusCacheModule.cs

@@ -1,344 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Net.Http;
-using System.Reflection;
-using System.Text;
-using System.Threading.Tasks;
-using MongoDB.Driver;
-using Nancy;
-using Newtonsoft.Json;
-using Wabbajack.CacheServer.DTOs;
-using Wabbajack.Common;
-using Wabbajack.Lib.Downloaders;
-using Wabbajack.Lib.NexusApi;
-
-namespace Wabbajack.CacheServer
-{
-    public class NexusCacheModule : NancyModule
-    {
-
-        public NexusCacheModule() : base("/")
-        {
-            Get("/v1/games/{GameName}/mods/{ModID}/files/{FileID}.json", HandleFileID);
-            Get("/v1/games/{GameName}/mods/{ModID}/files.json", HandleGetFiles);
-            Get("/v1/games/{GameName}/mods/{ModID}.json", HandleModInfo);
-            Get("/nexus_api_cache/{request}.json", HandleCacheCall);
-            Get("/nexus_api_cache", ListCache);
-            Get("/nexus_api_cache/update", UpdateCache);
-            Get("/nexus_api_cache/ingest/{Folder}", HandleIngestCache);
-        }
-
-        class UpdatedMod
-        {
-            public long mod_id;
-            public long latest_file_update;
-            public long latest_mod_activity;
-        }
-
-        public async Task<object> UpdateCache(object arg)
-        {
-            var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
-            
-            var gameTasks = GameRegistry.Games.Values
-                .Where(game => game.NexusName != null)
-                .Select(async game =>
-                {
-                    return (game,
-                        mods: await api.Get<List<UpdatedMod>>(
-                            $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m"));
-                })
-                .Select(async rTask =>
-                {
-                    var (game, mods) = await rTask;
-                    return mods.Select(mod => new { game = game, mod = mod });
-                }).ToList();
-
-            Utils.Log($"Getting update list for {gameTasks.Count} games");
-
-            var purge = (await Task.WhenAll(gameTasks))
-                .SelectMany(i => i)
-                .ToList();
-
-            Utils.Log($"Found {purge.Count} updated mods in the last month");
-            using (var queue = new WorkQueue())
-            {
-                var collected = await purge.Select(d =>
-                {
-                    var a = d.mod.latest_file_update.AsUnixTime();
-                    // Mod activity could hide files
-                    var b = d.mod.latest_mod_activity.AsUnixTime();
-
-                    return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.mod_id.ToString()};
-                }).PMap(queue, async t =>
-                {
-                    var resultA = await Server.Config.NexusModInfos.Connect().DeleteManyAsync(f =>
-                        f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
-                    var resultB = await Server.Config.NexusModFiles.Connect().DeleteManyAsync(f =>
-                        f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
-                    var resultC = await Server.Config.NexusFileInfos.Connect().DeleteManyAsync(f =>
-                        f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
-
-                    return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount;
-                });
-
-                Utils.Log($"Purged {collected.Sum()} cache entries");
-            }
-
-            return "Done";
-        }
-
-        private string ListCache(object arg)
-        {
-            Utils.Log($"{DateTime.Now} - List Cache");
-            return String.Join("",
-            Directory.EnumerateFiles(NexusApiClient.LocalCacheDir)
-                .Select(f => new FileInfo(f))
-                .OrderByDescending(fi => fi.LastWriteTime)
-                .Select(fi =>
-                {
-                    var decoded = Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(fi.Name).FromHex());
-                    return $"{fi.LastWriteTime} \t {fi.Length.ToFileSizeString()} \t {decoded} \n";
-                }));
-        }
-
-        private async Task<Response> HandleModInfo(dynamic arg)
-        {
-            Utils.Log($"{DateTime.Now} - Mod Info - {arg.GameName}/{arg.ModID}/");
-            string gameName = arg.GameName;
-            string modId = arg.ModId;
-            var result = await Server.Config.NexusModInfos.Connect()
-                .FindOneAsync(info => info.Game == gameName && info.ModId == modId);
-
-            string method = "CACHED";
-            if (result == null)
-            {
-                var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
-                var path = $"/v1/games/{gameName}/mods/{modId}.json";
-                var body = await api.Get<ModInfo>(path);
-                result = new NexusCacheData<ModInfo>
-                {
-                    Data = body, 
-                    Path = path, 
-                    Game = gameName,
-                    ModId = modId
-                };
-                try
-                {
-                    await Server.Config.NexusModInfos.Connect().InsertOneAsync(result);
-                }
-                catch (MongoWriteException)
-                {
-
-                }
-
-                method = "NOT_CACHED";
-            }
-
-            Response response = result.Data.ToJSON();
-            response.Headers.Add("WABBAJACK_CACHE_FROM", method);
-            response.ContentType = "application/json";
-            return response;
-        }
-
-        private async Task<Response> HandleFileID(dynamic arg)
-        {
-            Utils.Log($"{DateTime.Now} - File Info - {arg.GameName}/{arg.ModID}/{arg.FileID}");
-            string gameName = arg.GameName;
-            string modId = arg.ModId;
-            string fileId = arg.FileId;
-            var result = await Server.Config.NexusFileInfos.Connect()
-                .FindOneAsync(info => info.Game == gameName && info.ModId == modId && info.FileId == fileId);
-
-            string method = "CACHED";
-            if (result == null)
-            {
-                var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
-                var path = $"/v1/games/{gameName}/mods/{modId}/files/{fileId}.json";
-                var body = await api.Get<NexusFileInfo>(path);
-                result = new NexusCacheData<NexusFileInfo>
-                {
-                    Data = body, 
-                    Path = path, 
-                    Game = gameName, 
-                    ModId = modId,
-                    FileId = fileId
-                };
-                try
-                {
-                    await Server.Config.NexusFileInfos.Connect().InsertOneAsync(result);
-                }
-                catch (MongoWriteException)
-                {
-
-                }
-
-                method = "NOT_CACHED";
-            }
-
-            Response response = result.Data.ToJSON();
-            response.Headers.Add("WABBAJACK_CACHE_FROM", method);
-            response.ContentType = "application/json";
-            return response;
-        }
-
-        private async Task<Response> HandleGetFiles(dynamic arg)
-        {
-            Utils.Log($"{DateTime.Now} - Mod Files - {arg.GameName} {arg.ModID}");
-            string gameName = arg.GameName;
-            string modId = arg.ModId;
-            var result = await Server.Config.NexusModFiles.Connect()
-                .FindOneAsync(info => info.Game == gameName && info.ModId == modId);
-
-            string method = "CACHED";
-            if (result == null)
-            {
-                var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
-                var path = $"/v1/games/{gameName}/mods/{modId}/files.json";
-                var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
-                result = new NexusCacheData<NexusApiClient.GetModFilesResponse>
-                {
-                    Data = body,
-                    Path = path,
-                    Game = gameName,
-                    ModId = modId
-                };
-                try
-                {
-                    await Server.Config.NexusModFiles.Connect().InsertOneAsync(result);
-                }
-                catch (MongoWriteException)
-                {
-
-                }
-
-                method = "NOT_CACHED";
-            }
-
-            Response response = result.Data.ToJSON();
-            response.Headers.Add("WABBAJACK_CACHE_FROM", method);
-            response.ContentType = "application/json";
-            return response;
-        }
-
-        private async Task<string> HandleCacheCall(dynamic arg)
-        {
-            try
-            {
-                string param = (string)arg.request;
-                var url = new Uri(Encoding.UTF8.GetString(param.FromHex()));
-
-                var client = new HttpClient();
-                var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 8080, Scheme = "http"};
-                client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]); 
-                return await client.GetStringAsync(builder.Uri.ToString());
-            }
-            catch (Exception ex)
-            {
-                Utils.Log(ex.ToString());
-                return "ERROR";
-            }
-        }
-
-        private async Task<string> HandleIngestCache(dynamic arg)
-        {
-            int count = 0;
-            int failed = 0;
-
-            using (var queue = new WorkQueue())
-            {
-                await Directory.EnumerateFiles(Path.Combine(Server.Config.Settings.TempDir, (string)arg.Folder)).PMap(queue,
-                    async file =>
-                    {
-                        Utils.Log($"Ingesting {file}");
-                        if (!file.EndsWith(".json")) return;
-
-                        var fileInfo = new FileInfo(file);
-                        count++;
-
-                        var url = new Url(
-                            Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(file).FromHex()));
-                        var split = url.Path.Split(new[] {'/'}, StringSplitOptions.RemoveEmptyEntries);
-                        try
-                        {
-                            switch (split.Length)
-                            {
-                                case 5 when split[3] == "mods":
-                                {
-                                    var body = file.FromJSON<ModInfo>();
-
-                                    var payload = new NexusCacheData<ModInfo>();
-                                    payload.Data = body;
-                                    payload.Game = split[2];
-                                    payload.Path = url.Path;
-                                    payload.ModId = body.mod_id;
-                                    payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc;
-
-                                    try
-                                    {
-                                        await Server.Config.NexusModInfos.Connect().InsertOneAsync(payload);
-                                    }
-                                    catch (MongoWriteException ex)
-                                    {
-
-                                    }
-
-                                    break;
-                                }
-                                case 6 when split[5] == "files.json":
-                                {
-                                    var body = file.FromJSON<NexusApiClient.GetModFilesResponse>();
-                                    var payload = new NexusCacheData<NexusApiClient.GetModFilesResponse>();
-                                    payload.Path = url.Path;
-                                    payload.Data = body;
-                                    payload.Game = split[2];
-                                    payload.ModId = split[4];
-                                    payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc;
-
-                                        try
-                                        {
-                                        await Server.Config.NexusModFiles.Connect().InsertOneAsync(payload);
-                                    }
-                                    catch (MongoWriteException ex)
-                                    {
-
-                                    }
-
-                                    break;
-                                }
-                                case 7 when split[5] == "files":
-                                {
-                                    var body = file.FromJSON<NexusFileInfo>();
-                                    var payload = new NexusCacheData<NexusFileInfo>();
-                                    payload.Data = body;
-                                    payload.Path = url.Path;
-                                    payload.Game = split[2];
-                                    payload.FileId = Path.GetFileNameWithoutExtension(split[6]);
-                                    payload.ModId = split[4];
-                                    payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc;
-
-                                        try
-                                        {
-                                        await Server.Config.NexusFileInfos.Connect().InsertOneAsync(payload);
-                                    }
-                                    catch (MongoWriteException ex)
-                                    {
-
-                                    }
-
-                                    break;
-                                }
-                            }
-                        }
-                        catch (Exception ex)
-                        {
-                            failed++;
-                        }
-                    });
-            }
-
-            return $"Inserted {count} caches, {failed} failed";
-        }
-    }
-}

Wabbajack.CacheServer/Program.cs

@@ -1,29 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using Nancy.Hosting.Self;
-using Wabbajack.Common;
-
-namespace Wabbajack.CacheServer
-{
-    class Program
-    {
-        static void Main(string[] args)
-        {
-            Utils.LogMessages.Subscribe(Console.WriteLine);
-            using (var server = new Server("http://localhost:8080"))
-            {
-                Consts.WabbajackCacheHostname = "localhost";
-                Consts.WabbajackCachePort = 8080;
-                server.Start();
-
-                ListValidationService.Start();
-                var tsk = JobQueueEndpoints.StartJobQueue();
-                
-                Console.ReadLine();
-            }
-        }
-    }
-}

Wabbajack.CacheServer/Properties/AssemblyInfo.cs

@@ -1,36 +0,0 @@
-using System.Reflection;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-
-// General Information about an assembly is controlled through the following
-// set of attributes. Change these attribute values to modify the information
-// associated with an assembly.
-[assembly: AssemblyTitle("Wabbajack.CacheServer")]
-[assembly: AssemblyDescription("")]
-[assembly: AssemblyConfiguration("")]
-[assembly: AssemblyCompany("")]
-[assembly: AssemblyProduct("Wabbajack.CacheServer")]
-[assembly: AssemblyCopyright("Copyright ©  2019")]
-[assembly: AssemblyTrademark("")]
-[assembly: AssemblyCulture("")]
-
-// Setting ComVisible to false makes the types in this assembly not visible
-// to COM components.  If you need to access a type in this assembly from
-// COM, set the ComVisible attribute to true on that type.
-[assembly: ComVisible(false)]
-
-// The following GUID is for the ID of the typelib if this project is exposed to COM
-[assembly: Guid("bdc9a094-d235-47cd-83ca-44199b60ab20")]
-
-// Version information for an assembly consists of the following four values:
-//
-//      Major Version
-//      Minor Version
-//      Build Number
-//      Revision
-//
-// You can specify all the values or you can default the Build and Revision Numbers
-// by using the '*' as shown below:
-// [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.0.0.0")]
-[assembly: AssemblyFileVersion("1.0.0.0")]

Wabbajack.CacheServer/Readme.md

@@ -1,3 +0,0 @@
-# Wabbajack.CacheServer
-
-CacheServer for caching mod information to reduce the amount of API calls a user has to account for when using Wabbajack to compiler/install a ModList.

Wabbajack.CacheServer/Server.cs

@@ -1,79 +0,0 @@
-using System;
-using Alphaleonis.Win32.Filesystem;
-using Nancy;
-using Nancy.Bootstrapper;
-using Nancy.Configuration;
-using Nancy.Hosting.Self;
-using Nancy.TinyIoc;
-using Wabbajack.CacheServer.DTOs;
-using Wabbajack.CacheServer.ServerConfig;
-using Wabbajack.Common;
-
-namespace Wabbajack.CacheServer
-{
-    public class Server : IDisposable
-    {
-        private NancyHost _server;
-        private HostConfiguration _config;
-        public static BuildServerConfig Config;
-
-        static Server()
-        {
-            SerializerSettings.Init();
-        }
-
-
-        public Server(string address)
-        {
-            Address = address;
-            _config = new HostConfiguration {MaximumConnectionCount = 200, RewriteLocalhost = true};
-            //_config.UrlReservations.CreateAutomatically = true;
-            _server = new NancyHost(_config, new Uri(address));
-
-            Config = File.ReadAllText("config.yaml").FromYaml<BuildServerConfig>();
-        }
-
-        public string Address { get; }
-
-        public void Start()
-        {
-            _server.Start();
-        }
-
-        public void Dispose()
-        {
-            _server?.Dispose();
-        }
-    }
-
-    public class CachingBootstrapper : DefaultNancyBootstrapper
-    {
-        protected override void ApplicationStartup(TinyIoCContainer container, IPipelines pipelines)
-        {
-            pipelines.AfterRequest.AddItemToEndOfPipeline(ctx =>
-            {
-                ctx.Response.WithHeader("Access-Control-Allow-Origin", "*")
-                    .WithHeader("Access-Control-Allow-Methods", "POST, GET")
-                    .WithHeader("Access-Control-Allow-Headers", "Accept, Origin, Content-type")
-                    .WithHeader("Cache-Control","no-store");
-            });
-        }
-
-        public override void Configure(INancyEnvironment environment)
-        {
-            environment.Tracing(
-                enabled: true,
-                displayErrorTraces: true);
-        }
-
-        protected override void ConfigureApplicationContainer(TinyIoCContainer container)
-        {
-            container.Register<Heartbeat>();
-            container.Register<JobQueueEndpoints>();
-            container.Register<ListValidationService>();
-            container.Register<Metrics>();
-            container.Register<NexusCacheModule>();
-            container.Register<TestingEndpoints>();
-        }
-    }
-}

Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs

@@ -1,31 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using MongoDB.Driver.Core.Configuration;
-using Wabbajack.CacheServer.DTOs;
-using Wabbajack.CacheServer.DTOs.JobQueue;
-using Wabbajack.Lib.NexusApi;
-
-namespace Wabbajack.CacheServer.ServerConfig
-{
-    public class BuildServerConfig
-    {
-        public MongoConfig<Metric> Metrics { get; set; }
-        public MongoConfig<ModListStatus> ListValidation { get; set; }
-
-        public MongoConfig<Job> JobQueue { get; set; }
-
-        public MongoConfig<IndexedFile> IndexedFiles { get; set; }
-        public MongoConfig<DownloadState> DownloadStates { get; set; }
-
-        public MongoConfig<NexusCacheData<ModInfo>> NexusModInfos { get; set; }
-        public MongoConfig<NexusCacheData<NexusApiClient.GetModFilesResponse>> NexusModFiles { get; set; }
-        public MongoConfig<NexusCacheData<NexusFileInfo>> NexusFileInfos { get; set; }
-
-        public IndexerConfig Indexer { get; set; }
-
-        public Settings Settings { get; set; }
-    }
-}

Wabbajack.CacheServer/ServerConfig/IndexerConfig.cs

@@ -1,16 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace Wabbajack.CacheServer.ServerConfig
-{
-    public class IndexerConfig
-    {
-        public string DownloadDir { get; set; }
-        public string TempDir { get; set; }
-
-        public string ArchiveDir { get; set; }
-    }
-}

Wabbajack.CacheServer/ServerConfig/MongoConfig.cs

@@ -1,34 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using MongoDB.Driver;
-using Wabbajack.CacheServer.DTOs;
-
-namespace Wabbajack.CacheServer.ServerConfig
-{
-    public class MongoConfig<T>
-    {
-        public string Host { get; set; }
-        public string Database { get; set; }
-        public string Collection { get; set; }
-        public string Username { get; set; }
-        public string Password { get; set; }
-
-        private IMongoDatabase Client
-        {
-            get
-            {
-                if (Username != null && Password != null)
-                    return new MongoClient($"mongodb://{Username}:{Password}@{Host}").GetDatabase(Database);
-                return new MongoClient($"mongodb://{Host}").GetDatabase(Database);
-            }
-        }
-
-        public IMongoCollection<T> Connect()
-        {
-            return Client.GetCollection<T>(Collection);
-        }
-    }
-}

Wabbajack.CacheServer/ServerConfig/Settings.cs

@@ -1,13 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace Wabbajack.CacheServer.ServerConfig
-{
-    public class Settings
-    {
-        public string TempDir { get; set; }
-    }
-}

Wabbajack.CacheServer/TestingEndpoints.cs

@@ -1,28 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using Nancy;
-using Nancy.Responses;
-
-namespace Wabbajack.CacheServer
-{
-    /// <summary>
-    /// These endpoints are used by the testing service to verify that manual and direct
-    /// downloading works as expected.
-    /// </summary>
-    public class TestingEndpoints : NancyModule
-    {
-        public TestingEndpoints() : base("/")
-        {
-            Get("/WABBAJACK_TEST_FILE.txt", _ => "Cheese for Everyone!");
-            Get("/WABBAJACK_TEST_FILE.zip", _ =>
-            {
-                var response = new StreamResponse(() => new MemoryStream(Encoding.UTF8.GetBytes("Cheese for Everyone!")), "application/zip");
-                return response.AsAttachment("WABBAJACK_TEST_FILE.zip");
-            });
-        }
-    }
-}

Wabbajack.CacheServer/Wabbajack.CacheServer.csproj

@@ -1,170 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
-  <PropertyGroup>
-    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
-    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
-    <ProjectGuid>{BDC9A094-D235-47CD-83CA-44199B60AB20}</ProjectGuid>
-    <OutputType>Exe</OutputType>
-    <RootNamespace>Wabbajack.CacheServer</RootNamespace>
-    <AssemblyName>Wabbajack.CacheServer</AssemblyName>
-    <TargetFrameworkVersion>v4.8</TargetFrameworkVersion>
-    <FileAlignment>512</FileAlignment>
-    <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
-    <Deterministic>true</Deterministic>
-    <TargetFrameworkProfile />
-  </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
-    <PlatformTarget>x64</PlatformTarget>
-    <DebugSymbols>true</DebugSymbols>
-    <DebugType>full</DebugType>
-    <Optimize>false</Optimize>
-    <OutputPath>bin\Debug\</OutputPath>
-    <DefineConstants>DEBUG;TRACE</DefineConstants>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <NoWarn>CS1998</NoWarn>
-    <WarningsAsErrors>CS4014</WarningsAsErrors>
-  </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
-    <PlatformTarget>AnyCPU</PlatformTarget>
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
-    <OutputPath>bin\Release\</OutputPath>
-    <DefineConstants>TRACE</DefineConstants>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <NoWarn>CS1998</NoWarn>
-    <WarningsAsErrors>CS4014</WarningsAsErrors>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
-    <DebugSymbols>true</DebugSymbols>
-    <OutputPath>bin\x64\Debug\</OutputPath>
-    <DefineConstants>DEBUG;TRACE</DefineConstants>
-    <DebugType>full</DebugType>
-    <PlatformTarget>x64</PlatformTarget>
-    <LangVersion>7.3</LangVersion>
-    <ErrorReport>prompt</ErrorReport>
-    <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
-    <Prefer32Bit>true</Prefer32Bit>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
-    <OutputPath>bin\x64\Release\</OutputPath>
-    <DefineConstants>TRACE</DefineConstants>
-    <Optimize>true</Optimize>
-    <DebugType>pdbonly</DebugType>
-    <PlatformTarget>x64</PlatformTarget>
-    <LangVersion>7.3</LangVersion>
-    <ErrorReport>prompt</ErrorReport>
-    <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
-    <Prefer32Bit>true</Prefer32Bit>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x86'">
-    <DebugSymbols>true</DebugSymbols>
-    <OutputPath>bin\x86\Debug\</OutputPath>
-    <DefineConstants>DEBUG;TRACE</DefineConstants>
-    <NoWarn>CS1998</NoWarn>
-    <DebugType>full</DebugType>
-    <PlatformTarget>x86</PlatformTarget>
-    <WarningsAsErrors>CS4014</WarningsAsErrors>
-    <LangVersion>7.3</LangVersion>
-    <ErrorReport>prompt</ErrorReport>
-    <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
-    <Prefer32Bit>true</Prefer32Bit>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x86'">
-    <OutputPath>bin\x86\Release\</OutputPath>
-    <DefineConstants>TRACE</DefineConstants>
-    <Optimize>true</Optimize>
-    <NoWarn>CS1998</NoWarn>
-    <DebugType>pdbonly</DebugType>
-    <PlatformTarget>x86</PlatformTarget>
-    <WarningsAsErrors>CS4014</WarningsAsErrors>
-    <LangVersion>7.3</LangVersion>
-    <ErrorReport>prompt</ErrorReport>
-    <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
-    <Prefer32Bit>true</Prefer32Bit>
-  </PropertyGroup>
-  <ItemGroup>
-    <Reference Include="System" />
-    <Reference Include="System.Core" />
-    <Reference Include="System.Windows" />
-    <Reference Include="System.Windows.Forms" />
-    <Reference Include="System.Xml.Linq" />
-    <Reference Include="System.Data.DataSetExtensions" />
-    <Reference Include="Microsoft.CSharp" />
-    <Reference Include="System.Data" />
-    <Reference Include="System.Net.Http" />
-    <Reference Include="System.Xml" />
-    <Reference Include="WindowsBase" />
-  </ItemGroup>
-  <ItemGroup>
-    <Compile Include="DTOs\DownloadState.cs" />
-    <Compile Include="DTOs\IndexedFile.cs" />
-    <Compile Include="DTOs\JobQueue\AJobPayload.cs" />
-    <Compile Include="DTOs\NexusCacheData.cs" />
-    <Compile Include="Jobs\IndexJob.cs" />
-    <Compile Include="DTOs\JobQueue\Job.cs" />
-    <Compile Include="DTOs\JobQueue\JobResult.cs" />
-    <Compile Include="DTOs\Metric.cs" />
-    <Compile Include="DTOs\ModListStatus.cs" />
-    <Compile Include="DTOs\MongoDoc.cs" />
-    <Compile Include="DTOs\SerializerSettings.cs" />
-    <Compile Include="Extensions.cs" />
-    <Compile Include="JobQueueEndpoints.cs" />
-    <Compile Include="ListValidationService.cs" />
-    <Compile Include="Metrics.cs" />
-    <Compile Include="NexusCacheModule.cs" />
-    <Compile Include="Program.cs" />
-    <Compile Include="Properties\AssemblyInfo.cs" />
-    <Compile Include="Server.cs" />
-    <Compile Include="Heartbeat.cs" />
-    <Compile Include="ServerConfig\BuildServerConfig.cs" />
-    <Compile Include="ServerConfig\IndexerConfig.cs" />
-    <Compile Include="ServerConfig\MongoConfig.cs" />
-    <Compile Include="ServerConfig\Settings.cs" />
-    <Compile Include="TestingEndpoints.cs" />
-  </ItemGroup>
-  <ItemGroup>
-    <None Include="App.config" />
-    <None Include="config.yaml">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-  </ItemGroup>
-  <ItemGroup>
-    <ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
-      <Project>{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}</Project>
-      <Name>Wabbajack.Common</Name>
-    </ProjectReference>
-    <ProjectReference Include="..\Wabbajack.Lib\Wabbajack.Lib.csproj">
-      <Project>{0a820830-a298-497d-85e0-e9a89efef5fe}</Project>
-      <Name>Wabbajack.Lib</Name>
-    </ProjectReference>
-    <ProjectReference Include="..\Wabbajack.VirtualFileSystem\Wabbajack.VirtualFileSystem.csproj">
-      <Project>{5d6a2eaf-6604-4c51-8ae2-a746b4bc5e3e}</Project>
-      <Name>Wabbajack.VirtualFileSystem</Name>
-    </ProjectReference>
-  </ItemGroup>
-  <ItemGroup>
-    <PackageReference Include="MongoDB.Driver">
-      <Version>2.10.0</Version>
-    </PackageReference>
-    <PackageReference Include="Nancy.Hosting.Self">
-      <Version>2.0.0</Version>
-    </PackageReference>
-    <PackageReference Include="Nettle">
-      <Version>1.3.0</Version>
-    </PackageReference>
-    <PackageReference Include="Newtonsoft.Json">
-      <Version>12.0.3</Version>
-    </PackageReference>
-    <PackageReference Include="ReactiveUI">
-      <Version>11.1.6</Version>
-    </PackageReference>
-    <PackageReference Include="System.Reactive">
-      <Version>4.3.2</Version>
-    </PackageReference>
-  </ItemGroup>
-  <ItemGroup />
-  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
-</Project>

Wabbajack.CacheServer/config.yaml

@@ -1,39 +0,0 @@
----
-Metrics:
-    Host: internal.test.mongodb
-    Database: wabbajack
-    Collection: metrics
-ListValidation:
-    Host: internal.test.mongodb
-    Database: wabbajack
-    Collection: mod_lists
-JobQueue:
-    Host: internal.test.mongodb
-    Database: wabbajack
-    Collection: job_queue
-IndexedFiles:
-    Host: internal.test.mongodb
-    Database: wabbajack
-    Collection: indexed_files
-NexusModInfos:
-    Host: internal.test.mongodb
-    Database: wabbajack
-    Collection: nexus_mod_infos
-NexusModFiles:
-    Host: internal.test.mongodb
-    Database: wabbajack
-    Collection: nexus_mod_files
-NexusFileInfos:
-    Host: internal.test.mongodb
-    Database: wabbajack
-    Collection: nexus_file_infos
-DownloadStates:
-    Host: internal.test.mongodb
-    Database: wabbajack
-    Collection: download_states
-Indexer:
-    DownloadDir: c:\tmp\downloads
-    TempDir: c:\tmp\tmp
-    ArchiveDir: c:\archives
-Settings:
-    TempDir: c:\tmp\tmp

Wabbajack.Common/FileExtractor.cs

@@ -259,6 +259,7 @@ namespace Wabbajack.Common
                 return p.ExitCode == 0;
             }
 
+
             var testInfo = new ProcessStartInfo
             {
                 FileName = "7z.exe",
@@ -295,5 +296,12 @@ namespace Wabbajack.Common
             testP.WaitForExitAndWarn(TimeSpan.FromSeconds(30), $"Can Extract Check {v}");
             return testP.ExitCode == 0;
         }
+        
+        
+        public static bool MightBeArchive(string path)
+        {
+            var ext = Path.GetExtension(path.ToLower());
+            return ext == ".exe" || Consts.SupportedArchives.Contains(ext) || Consts.SupportedBSAs.Contains(ext);
+        }
     }
 }

Wabbajack.Common/Utils.cs

@@ -202,7 +202,7 @@ namespace Wabbajack.Common
                 throw ex;
             }
         }
-
+        
         public static string FileHashCached(this string file, bool nullOnIOError = false)
         {
             var hashPath = file + Consts.HashFileExtension;
@@ -774,6 +774,7 @@ namespace Wabbajack.Common
             {
                 var key = select(v);
                 if (set.Contains(key)) continue;
+                set.Add(key);
                 yield return v;
             }
         }
@@ -885,7 +886,7 @@ namespace Wabbajack.Common
         }
 
         /// <summary>
-        /// Roundtrips the value throught the JSON routines
+        /// Roundtrips the value through the JSON routines
         /// </summary>
         /// <typeparam name="TV"></typeparam>
         /// <typeparam name="TR"></typeparam>

Wabbajack.Lib/Downloaders/AbstractDownloadState.cs

@@ -35,7 +35,18 @@ namespace Wabbajack.Lib.Downloaders
         }
 
         public abstract object[] PrimaryKey { get; }
-
+        
+        public string PrimaryKeyString
+        {
+            get
+            {
+                var pk = new List<object>();
+                pk.Add(AbstractDownloadState.TypeToName[GetType()]);
+                pk.AddRange(PrimaryKey);
+                var pk_str = string.Join("|",pk.Select(p => p.ToString()));
+                return pk_str;
+            }
+        }
 
 
         /// <summary>
@@ -69,5 +80,6 @@ namespace Wabbajack.Lib.Downloaders
         public abstract IDownloader GetDownloader();
 
         public abstract string GetReportEntry(Archive a);
+        public abstract string[] GetMetaIni();
     }
 }

Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs

@@ -41,7 +41,15 @@ namespace Wabbajack.Lib.Downloaders
             public string FileID { get; set; }
             public string FileName { get; set; }
 
-            public override object[] PrimaryKey { get => new object[] {FileID, FileName}; }
+            public override object[] PrimaryKey
+            {
+                get
+                {
+                    if (FileID == null) 
+                        return new object[] {Downloader.SiteURL, FileName};
+                    return new object[] {Downloader.SiteURL, FileName, FileID};
+                }
+            }
 
             public override bool IsWhitelisted(ServerWhitelist whitelist)
             {
@@ -143,6 +151,25 @@ namespace Wabbajack.Lib.Downloaders
                 var downloader = (INeedsLogin)GetDownloader();
                 return $"* {((INeedsLogin)GetDownloader()).SiteName} - [{a.Name}](https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID})";
             }
+
+            public override string[] GetMetaIni()
+            {
+                var downloader = Downloader;
+                
+                if (FileID != null)
+                    return new[]
+                    {
+                        "[General]",
+                        $"directURL=https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1"
+                    };
+                return new[]
+                {
+                    "[General]",
+                    $"directURL=https://{downloader.SiteURL.Host}/files/file/{FileName}"
+                };
+            }
+
+            private static AbstractNeedsLoginDownloader Downloader => (AbstractNeedsLoginDownloader)(object)DownloadDispatcher.GetInstance<TDownloader>();
         }
 
         protected AbstractIPS4Downloader(Uri loginUri, string encryptedKeyName, string cookieDomain) : 

Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs

@@ -86,6 +86,11 @@ namespace Wabbajack.Lib.Downloaders
             {
                 return $"* Game File {Game} - {GameFile}";
             }
+
+            public override string[] GetMetaIni()
+            {
+                return new[] {"[General]", $"gameName={Game.MetaData().MO2ArchiveName}", $"gameFile={GameFile}"};
+            }
         }
     }
 }

Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs

@@ -79,6 +79,11 @@ namespace Wabbajack.Lib.Downloaders
             {
                 return $"* GoogleDrive - [{a.Name}](https://drive.google.com/uc?id={Id}&export=download)";
             }
+
+            public override string[] GetMetaIni()
+            {
+                return new [] {"[General]",$"directURL=https://drive.google.com/uc?id={Id}&export=download"};
+            }
         }
     }
 }

Wabbajack.Lib/Downloaders/HTTPDownloader.cs

@@ -193,6 +193,17 @@ namespace Wabbajack.Lib.Downloaders
             {
                 return $"* [{a.Name} - {Url}]({Url})";
             }
+
+            public override string[] GetMetaIni()
+            {
+                if (Headers != null)
+                    return new [] {"[General]",
+                          $"directURL={Url}",
+                          $"directURLHeaders={string.Join("|", Headers)}"};
+                else
+                    return new [] {"[General]", $"directURL={Url}"};
+
+            }
         }
     }
 }

Wabbajack.Lib/Downloaders/ManualDownloader.cs

@@ -125,6 +125,15 @@ namespace Wabbajack.Lib.Downloaders
             {
                 return $"* Manual Download - [{a.Name} - {Url}]({Url})";
             }
+
+            public override string[] GetMetaIni()
+            {
+                return new [] {
+                    "[General]",
+                    $"manualURL={Url}"
+                    
+                };
+            }
         }
     }
 }

Wabbajack.Lib/Downloaders/MediaFireDownloader.cs

@@ -69,7 +69,14 @@ namespace Wabbajack.Lib.Downloaders
                 return $"* [{a.Name} - {Url}]({Url})";
             }
 
-
+            public override string[] GetMetaIni()
+            {
+                return new []
+                {
+                    "[General]",
+                    $"directURL={Url}"
+                };
+            }
         }
 
         public async Task Prepare()

Wabbajack.Lib/Downloaders/ModDBDownloader.cs

@@ -107,6 +107,11 @@ namespace Wabbajack.Lib.Downloaders
             {
                 return $"* ModDB - [{a.Name}]({Url})";
             }
+
+            public override string[] GetMetaIni()
+            {
+                return new[] {"[General]", $"directURL={Url}"};
+            }
         }
     }
 }

Wabbajack.Lib/Downloaders/NexusDownloader.cs

@@ -204,6 +204,11 @@ namespace Wabbajack.Lib.Downloaders
                     $"    * Author : [{UploadedBy}]({profile})", 
                     $"    * Version : {Version}");
             }
+
+            public override string[] GetMetaIni()
+            {
+                return new[] {"[General]", $"gameName={GameName}", $"modID={ModID}", $"fileID={FileID}"};
+            }
         }
     }
 }

Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs

@@ -94,6 +94,17 @@ namespace Wabbajack.Lib.Downloaders
             {
                 return $"* Steam - [{Item.ItemID}]";
             }
+
+            public override string[] GetMetaIni()
+            {
+                return new[]
+                {
+                    "[General]", 
+                    $"itemID={Item.ItemID}", 
+                    $"steamID={Item.Game.Game.MetaData().SteamIDs.First()}",
+                    $"itemSize={Item.Size}"
+                };
+            }
         }
     }
 }

Wabbajack.Lib/MO2Compiler.cs

@@ -5,6 +5,7 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
@@ -344,31 +345,20 @@ namespace Wabbajack.Lib
 
             if (to_find.Count == 0) return;
 
-            var games = new[]{CompilingGame}.Concat(GameRegistry.Games.Values.Where(g => g != CompilingGame));
-            var game_files = games
-                .Where(g => g.GameLocation() != null)
-                .SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", DirectoryEnumerationOptions.Recursive).Select(name => (game, name)))
-                .GroupBy(f => (Path.GetFileName(f.name), new FileInfo(f.name).Length))
-                .ToDictionary(f => f.Key);
-
-            await to_find.PMap(Queue, f =>
+            await to_find.PMap(Queue, async f =>
             {
                 var vf = VFS.Index.ByFullPath[f];
-                if (!game_files.TryGetValue((Path.GetFileName(f), vf.Size), out var found))
-                    return;
+                var client = new HttpClient();
+                var response =
+                    await client.GetAsync(
+                        $"http://build.wabbajack.org/indexed_files/{vf.Hash.FromBase64().ToHex()}/meta.ini");
+                
+                if (!response.IsSuccessStatusCode) return;
 
-                var (game, name) = found.FirstOrDefault(ff => ff.name.FileHash() == vf.Hash);
-                if (name == null)
-                    return;
-
-                File.WriteAllLines(f+".meta", new[]
-                {
-                    "[General]",
-                    $"gameName={game.MO2ArchiveName}",
-                    $"gameFile={name.RelativeTo(game.GameLocation()).Replace("\\", "/")}"
-                });
+                var ini_data = await response.Content.ReadAsStringAsync();
+                Utils.Log($"Inferred .meta for {Path.GetFileName(vf.FullPath)}, writing to disk");
+                File.WriteAllText(vf.FullPath + ".meta", ini_data);
             });
-
         }
 
 

Wabbajack.Lib/ModListRegistry/ModListMetadata.cs

@@ -109,12 +109,19 @@ namespace Wabbajack.Lib.ModListRegistry
 
     public class ModlistSummary
     {
-        public string Name;
-        public DateTime Checked;
-        public int Failed;
-        public int Passed;
+        [JsonProperty("name")]
+        public string Name { get; set; }
+        [JsonProperty("checked")]
+        public DateTime Checked { get; set; }
+        [JsonProperty("failed")]
+        public int Failed { get; set; }
+        [JsonProperty("passed")]
+        public int Passed { get; set; }
+        [JsonProperty("link")]
         public string Link => $"/lists/status/{Name}.json";
+        [JsonProperty("report")]
         public string Report => $"/lists/status/{Name}.html";
+        [JsonProperty("has_failures")]
         public bool HasFailures => Failed > 0;
     }
 

Wabbajack.Lib/NexusApi/Dtos.cs

@@ -15,35 +15,35 @@ namespace Wabbajack.Lib.NexusApi
 
     public class NexusFileInfo
     {
-        public ulong category_id;
-        public string category_name;
-        public string changelog_html;
-        public string description;
-        public string external_virus_scan_url;
-        public ulong file_id;
-        public string file_name;
-        public bool is_primary;
-        public string mod_version;
-        public string name;
-        public ulong size;
-        public ulong size_kb;
-        public DateTime uploaded_time;
-        public ulong uploaded_timestamp;
-        public string version;
+        public ulong category_id { get; set; }
+        public string category_name { get; set; }
+        public string changelog_html { get; set; }
+        public string description { get; set; }
+        public string external_virus_scan_url { get; set; }
+        public ulong file_id { get; set; }
+        public string file_name { get; set; }
+        public bool is_primary { get; set; }
+        public string mod_version { get; set; }
+        public string name { get; set; }
+        public ulong size { get; set; }
+        public ulong size_kb { get; set; }
+        public DateTime uploaded_time { get; set; }
+        public ulong uploaded_timestamp { get; set; }
+        public string version { get; set; }
     }
 
     public class ModInfo
     {
-        public uint _internal_version;
-        public string game_name;
-        public string mod_id;
-        public string name;
-        public string summary;
-        public string author;
-        public string uploaded_by;
-        public string uploaded_users_profile_url;
-        public string picture_url;
-        public bool contains_adult_content;
+        public uint _internal_version { get; set; }
+        public string game_name { get; set; }
+        public string mod_id { get; set; }
+        public string name { get; set; }
+        public string summary { get; set; }
+        public string author { get; set; }
+        public string uploaded_by { get; set; }
+        public string uploaded_users_profile_url { get; set; }
+        public string picture_url { get; set; }
+        public bool contains_adult_content { get; set; }
     }
 
     public class MD5Response

Wabbajack.Lib/NexusApi/NexusApi.cs

@@ -272,7 +272,7 @@ namespace Wabbajack.Lib.NexusApi
 
         public class GetModFilesResponse
         {
-            public List<NexusFileInfo> files;
+            public List<NexusFileInfo> files { get; set; }
         }
 
         public async Task<GetModFilesResponse> GetModFiles(Game game, int modid)

Wabbajack.Test/DownloaderTests.cs

@@ -63,7 +63,7 @@ namespace Wabbajack.Test
                 ((MegaDownloader.State)url_state).Url);
 
 
-            var converted = state.ViaJSON();
+            var converted = await state.RoundTripState();
             Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
@@ -93,7 +93,7 @@ namespace Wabbajack.Test
             Assert.AreEqual("https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?dl=1", 
                 ((HTTPDownloader.State)url_state).Url);
 
-            var converted = state.ViaJSON();
+            var converted = await state.RoundTripState();
             Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
@@ -123,7 +123,7 @@ namespace Wabbajack.Test
             Assert.AreEqual("1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_",
                 ((GoogleDriveDownloader.State)url_state).Id);
 
-            var converted = state.ViaJSON();
+            var converted = await state.RoundTripState();
             Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
@@ -152,7 +152,7 @@ namespace Wabbajack.Test
             Assert.AreEqual("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt",
                 ((HTTPDownloader.State)url_state).Url);
 
-            var converted = state.ViaJSON();
+            var converted = await state.RoundTripState();
             Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
@@ -176,7 +176,7 @@ namespace Wabbajack.Test
 
             Assert.IsNotNull(state);
 
-            var converted = state.ViaJSON();
+            var converted = await state.RoundTripState();
             Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
@@ -206,7 +206,7 @@ namespace Wabbajack.Test
             Assert.AreEqual("http://www.mediafire.com/file/agiqzm1xwebczpx/WABBAJACK_TEST_FILE.txt",
                 ((MediaFireDownloader.State) url_state).Url);
 
-            var converted = state.ViaJSON();
+            var converted = await state.RoundTripState();
             Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
@@ -237,7 +237,7 @@ namespace Wabbajack.Test
                 Assert.IsNotNull(state);
 
 
-                var converted = state.ViaJSON();
+                var converted = await state.RoundTripState();
                 Assert.IsTrue(await converted.Verify());
                 // Exercise the cache code
                 Assert.IsTrue(await converted.Verify());
@@ -272,7 +272,7 @@ namespace Wabbajack.Test
             Assert.AreEqual("https://www.moddb.com/downloads/start/124908?referer=https%3A%2F%2Fwww.moddb.com%2Fmods%2Fautopause",
                 ((ModDBDownloader.State)url_state).Url);
 
-            var converted = state.ViaJSON();
+            var converted = await state.RoundTripState();
             Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
@@ -298,7 +298,7 @@ namespace Wabbajack.Test
             Assert.AreEqual("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt",
                 ((HTTPDownloader.State)url_state).Url);
                 */
-            var converted = state.ViaJSON();
+            var converted = await state.RoundTripState();
             Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
@@ -326,7 +326,7 @@ namespace Wabbajack.Test
             Assert.AreEqual("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt",
                 ((HTTPDownloader.State)url_state).Url);
                 */
-            var converted = state.ViaJSON();
+            var converted = await state.RoundTripState();
             Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 
@@ -353,7 +353,7 @@ namespace Wabbajack.Test
 
             Assert.IsNotNull(state);
 
-            var converted = state.ViaJSON();
+            var converted = await state.RoundTripState();
             Assert.IsTrue(await converted.Verify());
             var filename = Guid.NewGuid().ToString();
 

Wabbajack.Test/Extensions.cs

@@ -1,4 +1,7 @@
-using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System.Threading.Tasks;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Wabbajack.Common;
+using Wabbajack.Lib.Downloaders;
 
 namespace Wabbajack.Test
 {
@@ -12,6 +15,16 @@ namespace Wabbajack.Test
         {
             Assert.IsTrue(condition ?? false, string.Empty, (object[])null);
         }
+        
+        public static async Task<T> RoundTripState<T>(this T state) where T : AbstractDownloadState
+        {
+            var ini = string.Join("\r\n", state.GetMetaIni()).LoadIniString();
+            var round = (AbstractDownloadState) await DownloadDispatcher.ResolveArchive(ini);
+            Assert.IsInstanceOfType(round, state.GetType());
+            Assert.AreEqual(state.PrimaryKeyString, round.PrimaryKeyString);
+            CollectionAssert.AreEqual(state.GetMetaIni(), round.GetMetaIni());
+            return (T)round;
+        }
 
     }
 }

Wabbajack.VirtualFileSystem/Context.cs

@@ -19,10 +19,15 @@ namespace Wabbajack.VirtualFileSystem
 {
     public class Context
     {
+        static Context()
+        {
+            Utils.Log("Cleaning VFS, this may take a bit of time");
+            Utils.DeleteDirectory(_stagingFolder);
+        }
         public const ulong FileVersion = 0x02;
         public const string Magic = "WABBAJACK VFS FILE";
 
-        private readonly string _stagingFolder = "vfs_staging";
+        private static readonly string _stagingFolder = "vfs_staging";
         public IndexRoot Index { get; private set; } = IndexRoot.Empty;
 
         /// <summary>
@@ -72,7 +77,7 @@ namespace Wabbajack.VirtualFileSystem
                                         return found;
                                 }
 
-                                return await VirtualFile.Analyze(this, null, f, f);
+                                return await VirtualFile.Analyze(this, null, f, f, true);
                             });
 
             var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());
@@ -109,7 +114,7 @@ namespace Wabbajack.VirtualFileSystem
                             return found;
                     }
 
-                    return await VirtualFile.Analyze(this, null, f, f);
+                    return await VirtualFile.Analyze(this, null, f, f, true);
                 });
 
             var newIndex = await IndexRoot.Empty.Integrate(filtered.Concat(allFiles).ToList());

Wabbajack.VirtualFileSystem/IndexedVirtualFile.cs

@@ -0,0 +1,15 @@
+using System.Collections.Generic;
+
+namespace Wabbajack.VirtualFileSystem
+{
+    /// <summary>
+    /// Response from the Build server for a indexed file
+    /// </summary>
+    public class IndexedVirtualFile
+    {
+        public string Name { get; set; }
+        public string Hash { get; set; }
+        public long Size { get; set; }
+        public List<IndexedVirtualFile> Children { get; set; } = new List<IndexedVirtualFile>();
+    }
+}

Wabbajack.VirtualFileSystem/VirtualFile.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.IO;
 using System.Linq;
+using System.Net.Http;
 using System.Security.Cryptography;
 using System.Text;
 using System.Threading.Tasks;
@@ -133,9 +134,41 @@ namespace Wabbajack.VirtualFileSystem
         }
 
         public static async Task<VirtualFile> Analyze(Context context, VirtualFile parent, string abs_path,
-            string rel_path)
+            string rel_path, bool topLevel)
         {
+            var hash = abs_path.FileHash();
             var fi = new FileInfo(abs_path);
+
+            if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(abs_path))
+            {
+                var result = await TryGetContentsFromServer(hash);
+
+                if (result != null)
+                {
+                    Utils.Log($"Downloaded VFS data for {Path.GetFileName(abs_path)}");
+                    VirtualFile Convert(IndexedVirtualFile file, string path, VirtualFile vparent)
+                    {
+                        var vself = new VirtualFile
+                        {
+                            Context = context,
+                            Name = path,
+                            Parent = vparent,
+                            Size = file.Size,
+                            LastModified = fi.LastWriteTimeUtc.Ticks,
+                            LastAnalyzed = DateTime.Now.Ticks,
+                            Hash = file.Hash,
+
+                        };
+
+                        vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList();
+
+                        return vself;
+                    }
+
+                    return Convert(result, rel_path, parent);
+                }
+            }
+
             var self = new VirtualFile
             {
                 Context = context,
@@ -144,7 +177,7 @@ namespace Wabbajack.VirtualFileSystem
                 Size = fi.Length,
                 LastModified = fi.LastWriteTimeUtc.Ticks,
                 LastAnalyzed = DateTime.Now.Ticks,
-                Hash = abs_path.FileHash()
+                Hash = hash
             };
             if (context.UseExtendedHashes)
                 self.ExtendedHashes = ExtendedHashes.FromFile(abs_path);
@@ -157,7 +190,7 @@ namespace Wabbajack.VirtualFileSystem
                     await FileExtractor.ExtractAll(context.Queue, abs_path, tempFolder.FullName);
 
                     var list = await Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories)
-                                        .PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName)));
+                                        .PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName), false));
 
                     self.Children = list.ToImmutableList();
                 }
@@ -167,6 +200,27 @@ namespace Wabbajack.VirtualFileSystem
             return self;
         }
 
+        private static async Task<IndexedVirtualFile> TryGetContentsFromServer(string hash)
+        {
+            try
+            {
+                var client = new HttpClient();
+                var response = await client.GetAsync($"http://{Consts.WabbajackCacheHostname}/indexed_files/{hash.FromBase64().ToHex()}");
+                if (!response.IsSuccessStatusCode)
+                    return null;
+
+                using (var stream = await response.Content.ReadAsStreamAsync())
+                {
+                    return stream.FromJSON<IndexedVirtualFile>();
+                }
+
+            }
+            catch (Exception ex)
+            {
+                return null;
+            }
+        }
+
 
         public void Write(MemoryStream ms)
         {

Wabbajack.sln

@@ -32,10 +32,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VirtualFileSystem
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.VirtualFileSystem.Test", "Wabbajack.VirtualFileSystem.Test\Wabbajack.VirtualFileSystem.Test.csproj", "{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.CacheServer", "Wabbajack.CacheServer\Wabbajack.CacheServer.csproj", "{BDC9A094-D235-47CD-83CA-44199B60AB20}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OMODExtractor", "OMODExtractor\OMODExtractor.csproj", "{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.BuildServer", "Wabbajack.BuildServer\Wabbajack.BuildServer.csproj", "{DE18D89E-39C5-48FD-8E42-16235E3C4593}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug (no commandargs)|Any CPU = Debug (no commandargs)|Any CPU
@@ -67,12 +67,6 @@ Global
 		{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|x86.Build.0 = Release|Any CPU
 		{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|x64.ActiveCfg = Release|x64
 		{B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5}.Release|x64.Build.0 = Release|x64
-		{33602679-8484-40C7-A10C-774DFF5D8314}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug (no commandargs)|x64
-		{33602679-8484-40C7-A10C-774DFF5D8314}.Debug (no commandargs)|Any CPU.Build.0 = Debug (no commandargs)|x64
-		{33602679-8484-40C7-A10C-774DFF5D8314}.Debug (no commandargs)|x64.ActiveCfg = Debug (no commandargs)|x64
-		{33602679-8484-40C7-A10C-774DFF5D8314}.Debug (no commandargs)|x64.Build.0 = Debug (no commandargs)|x64
-		{33602679-8484-40C7-A10C-774DFF5D8314}.Debug (no commandargs)|x86.ActiveCfg = Debug (no commandargs)|x86
-		{33602679-8484-40C7-A10C-774DFF5D8314}.Debug (no commandargs)|x86.Build.0 = Debug (no commandargs)|x86
 		{33602679-8484-40C7-A10C-774DFF5D8314}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{33602679-8484-40C7-A10C-774DFF5D8314}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{33602679-8484-40C7-A10C-774DFF5D8314}.Debug|x64.ActiveCfg = Debug|x64
@@ -211,24 +205,6 @@ Global
 		{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x86.Build.0 = Release|Any CPU
 		{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x64.ActiveCfg = Release|x64
 		{51CEB604-985A-45B9-AF0D-C5BA8CFA1BF0}.Release|x64.Build.0 = Release|x64
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|x64
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug (no commandargs)|Any CPU.Build.0 = Debug|x64
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug (no commandargs)|x86.ActiveCfg = Debug|Any CPU
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug (no commandargs)|x86.Build.0 = Debug|Any CPU
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug|x64.ActiveCfg = Debug|x64
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug|x64.Build.0 = Debug|x64
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Debug|x86.Build.0 = Debug|Any CPU
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|Any CPU.ActiveCfg = Release|x64
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|Any CPU.Build.0 = Release|x64
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x64.ActiveCfg = Release|x64
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x64.Build.0 = Release|x64
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x86.ActiveCfg = Release|Any CPU
-		{BDC9A094-D235-47CD-83CA-44199B60AB20}.Release|x86.Build.0 = Release|Any CPU
 		{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|x64
 		{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Debug (no commandargs)|Any CPU.Build.0 = Debug|x64
 		{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU
@@ -247,6 +223,24 @@ Global
 		{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x86.Build.0 = Release|Any CPU
 		{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.ActiveCfg = Release|x64
 		{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.Build.0 = Release|x64
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|x86.ActiveCfg = Debug|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|x86.Build.0 = Debug|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.Build.0 = Debug|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x86.Build.0 = Debug|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x86.ActiveCfg = Release|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x86.Build.0 = Release|Any CPU
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.ActiveCfg = Release|x64
+		{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

azure-pipelines.yml

@@ -33,7 +33,7 @@ steps:
 
 - task: VSBuild@1
   inputs:
-    solution: '$(solution)'
+    solution: '**\*.sln'
     platform: '$(buildPlatform)'
     configuration: '$(buildConfiguration)'