From f004595b5fef7225bdc92747d5bb641f734ccddb Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 9 Jan 2020 16:20:03 -0700 Subject: [PATCH] delete old server --- Wabbajack.CacheServer/App.config | 6 - Wabbajack.CacheServer/DTOs/DownloadState.cs | 23 -- Wabbajack.CacheServer/DTOs/IndexedFile.cs | 31 -- .../DTOs/JobQueue/AJobPayload.cs | 32 -- Wabbajack.CacheServer/DTOs/JobQueue/Job.cs | 67 ---- .../DTOs/JobQueue/JobResult.cs | 36 -- Wabbajack.CacheServer/DTOs/Metric.cs | 17 - Wabbajack.CacheServer/DTOs/ModListStatus.cs | 70 ---- Wabbajack.CacheServer/DTOs/MongoDoc.cs | 14 - Wabbajack.CacheServer/DTOs/NexusCacheData.cs | 24 -- .../DTOs/SerializerSettings.cs | 74 ---- Wabbajack.CacheServer/Extensions.cs | 19 - Wabbajack.CacheServer/FodyWeavers.xml | 3 - Wabbajack.CacheServer/FodyWeavers.xsd | 111 ------ Wabbajack.CacheServer/Heartbeat.cs | 24 -- Wabbajack.CacheServer/JobQueueEndpoints.cs | 144 -------- Wabbajack.CacheServer/Jobs/IndexJob.cs | 106 ------ .../ListValidationService.cs | 252 ------------- Wabbajack.CacheServer/Metrics.cs | 140 ------- Wabbajack.CacheServer/NexusCacheModule.cs | 344 ------------------ Wabbajack.CacheServer/Program.cs | 29 -- .../Properties/AssemblyInfo.cs | 36 -- Wabbajack.CacheServer/Readme.md | 3 - Wabbajack.CacheServer/Server.cs | 79 ---- .../ServerConfig/BuildServerConfig.cs | 31 -- .../ServerConfig/IndexerConfig.cs | 16 - .../ServerConfig/MongoConfig.cs | 34 -- .../ServerConfig/Settings.cs | 13 - Wabbajack.CacheServer/TestingEndpoints.cs | 28 -- .../Wabbajack.CacheServer.csproj | 169 --------- Wabbajack.CacheServer/config.yaml | 39 -- 31 files changed, 2014 deletions(-) delete mode 100644 Wabbajack.CacheServer/App.config delete mode 100644 Wabbajack.CacheServer/DTOs/DownloadState.cs delete mode 100644 Wabbajack.CacheServer/DTOs/IndexedFile.cs delete mode 100644 Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs delete mode 100644 Wabbajack.CacheServer/DTOs/JobQueue/Job.cs delete mode 100644 Wabbajack.CacheServer/DTOs/JobQueue/JobResult.cs delete mode 100644 Wabbajack.CacheServer/DTOs/Metric.cs delete mode 100644 Wabbajack.CacheServer/DTOs/ModListStatus.cs delete mode 100644 Wabbajack.CacheServer/DTOs/MongoDoc.cs delete mode 100644 Wabbajack.CacheServer/DTOs/NexusCacheData.cs delete mode 100644 Wabbajack.CacheServer/DTOs/SerializerSettings.cs delete mode 100644 Wabbajack.CacheServer/Extensions.cs delete mode 100644 Wabbajack.CacheServer/FodyWeavers.xml delete mode 100644 Wabbajack.CacheServer/FodyWeavers.xsd delete mode 100644 Wabbajack.CacheServer/Heartbeat.cs delete mode 100644 Wabbajack.CacheServer/JobQueueEndpoints.cs delete mode 100644 Wabbajack.CacheServer/Jobs/IndexJob.cs delete mode 100644 Wabbajack.CacheServer/ListValidationService.cs delete mode 100644 Wabbajack.CacheServer/Metrics.cs delete mode 100644 Wabbajack.CacheServer/NexusCacheModule.cs delete mode 100644 Wabbajack.CacheServer/Program.cs delete mode 100644 Wabbajack.CacheServer/Properties/AssemblyInfo.cs delete mode 100644 Wabbajack.CacheServer/Readme.md delete mode 100644 Wabbajack.CacheServer/Server.cs delete mode 100644 Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs delete mode 100644 Wabbajack.CacheServer/ServerConfig/IndexerConfig.cs delete mode 100644 Wabbajack.CacheServer/ServerConfig/MongoConfig.cs delete mode 100644 Wabbajack.CacheServer/ServerConfig/Settings.cs delete mode 100644 Wabbajack.CacheServer/TestingEndpoints.cs delete mode 100644 Wabbajack.CacheServer/Wabbajack.CacheServer.csproj delete mode 100644 Wabbajack.CacheServer/config.yaml diff --git a/Wabbajack.CacheServer/App.config b/Wabbajack.CacheServer/App.config deleted file mode 100644 index 4bfa0056..00000000 --- a/Wabbajack.CacheServer/App.config +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/Wabbajack.CacheServer/DTOs/DownloadState.cs b/Wabbajack.CacheServer/DTOs/DownloadState.cs deleted file mode 100644 index 9dc2a02c..00000000 --- a/Wabbajack.CacheServer/DTOs/DownloadState.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using MongoDB.Bson.Serialization.Attributes; -using Wabbajack.Lib.Downloaders; - -namespace Wabbajack.CacheServer.DTOs -{ - public class DownloadState - { - [BsonId] - public string Key { get; set; } - public string Hash { get; set; } - - public AbstractDownloadState State { get; set; } - - public bool IsValid { get; set; } - public DateTime LastValidationTime { get; set; } = DateTime.Now; - public DateTime FirstValidationTime { get; set; } = DateTime.Now; - } -} diff --git a/Wabbajack.CacheServer/DTOs/IndexedFile.cs b/Wabbajack.CacheServer/DTOs/IndexedFile.cs deleted file mode 100644 index 5f56926c..00000000 --- a/Wabbajack.CacheServer/DTOs/IndexedFile.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Security.RightsManagement; -using System.Text; -using System.Threading.Tasks; -using MongoDB.Bson.Serialization.Attributes; -using Wabbajack.VirtualFileSystem; - -namespace Wabbajack.CacheServer.DTOs -{ - public class IndexedFile - { - [BsonId] - public string Hash { get; set; } - public string SHA256 { get; set; } - public string SHA1 { get; set; } - public string MD5 { get; set; } - public string CRC { get; set; } - public long Size { get; set; } - public bool IsArchive { get; set; } - public List Children { get; set; } = new List(); - } - - public class ChildFile - { - public string Name; - public string Extension; - public string Hash; - } -} diff --git a/Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs b/Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs deleted file mode 100644 index fac78dfa..00000000 --- a/Wabbajack.CacheServer/DTOs/JobQueue/AJobPayload.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using MongoDB.Bson.Serialization.Attributes; -using Wabbajack.CacheServer.Jobs; - -namespace Wabbajack.CacheServer.DTOs.JobQueue -{ - public abstract class AJobPayload - { - public static List KnownSubTypes = new List {typeof(IndexJob)}; - public static Dictionary TypeToName { get; set; } - public static Dictionary NameToType { get; set; } - - - [BsonIgnore] - public abstract string Description { get; } - - public virtual bool UsesNexus { get; } = false; - - public abstract Task Execute(); - - static AJobPayload() - { - NameToType = KnownSubTypes.ToDictionary(t => t.FullName.Substring(t.Namespace.Length + 1), t => t); - TypeToName = NameToType.ToDictionary(k => k.Value, k => k.Key); - } - - } -} diff --git a/Wabbajack.CacheServer/DTOs/JobQueue/Job.cs b/Wabbajack.CacheServer/DTOs/JobQueue/Job.cs deleted file mode 100644 index a859e80c..00000000 --- a/Wabbajack.CacheServer/DTOs/JobQueue/Job.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using MongoDB.Bson; -using MongoDB.Bson.Serialization.Attributes; -using MongoDB.Driver; - -namespace Wabbajack.CacheServer.DTOs.JobQueue -{ - public class Job - { - public enum JobPriority : int - { - Low, - Normal, - High, - } - - [BsonId] - public Guid Id { get; set; } - public DateTime? Started { get; set; } - public DateTime? Ended { get; set; } - public DateTime Created { get; set; } = DateTime.Now; - public JobPriority Priority { get; set; } = JobPriority.Normal; - - public JobResult Result { get; set; } - public bool RequiresNexus { get; set; } = true; - public AJobPayload Payload { get; set; } - - public static async Task Enqueue(Job job) - { - await Server.Config.JobQueue.Connect().InsertOneAsync(job); - return job.Id; - } - - public static async Task GetNext() - { - var filter = new BsonDocument - { - {"Started", BsonNull.Value} - }; - var update = new BsonDocument - { - {"$set", new BsonDocument {{"Started", DateTime.Now}}} - }; - var sort = new {Priority=-1, Created=1}.ToBsonDocument(); - var job = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync(filter, update, new FindOneAndUpdateOptions{Sort = sort}); - return job; - } - - public static async Task Finish(Job job, JobResult jobResult) - { - var filter = new BsonDocument - { - {"query", new BsonDocument {{"Id", job.Id}}}, - }; - var update = new BsonDocument - { - {"$set", new BsonDocument {{"Ended", DateTime.Now}, {"Result", jobResult.ToBsonDocument()}}} - }; - var result = await Server.Config.JobQueue.Connect().FindOneAndUpdateAsync(filter, update); - return result; - } - } -} diff --git a/Wabbajack.CacheServer/DTOs/JobQueue/JobResult.cs b/Wabbajack.CacheServer/DTOs/JobQueue/JobResult.cs deleted file mode 100644 index 241908c6..00000000 --- a/Wabbajack.CacheServer/DTOs/JobQueue/JobResult.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using MongoDB.Bson.Serialization.Attributes; - -namespace Wabbajack.CacheServer.DTOs.JobQueue -{ - public class JobResult - { - public JobResultType ResultType { get; set; } - [BsonIgnoreIfNull] - public string Message { get; set; } - - [BsonIgnoreIfNull] - public string Stacktrace { get; set; } - - public static JobResult Success() - { - return new JobResult { ResultType = JobResultType.Success }; - } - - public static JobResult Error(Exception ex) - { - return new JobResult {ResultType = JobResultType.Error, Stacktrace = ex.ToString()}; - } - - } - - public enum JobResultType - { - Success, - Error - } -} diff --git a/Wabbajack.CacheServer/DTOs/Metric.cs b/Wabbajack.CacheServer/DTOs/Metric.cs deleted file mode 100644 index bf373d76..00000000 --- a/Wabbajack.CacheServer/DTOs/Metric.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using MongoDB.Bson; -using MongoDB.Bson.Serialization.Attributes; - - -namespace Wabbajack.CacheServer.DTOs -{ - public class Metric - { - [BsonId] - public ObjectId Id; - public DateTime Timestamp; - public string Action; - public string Subject; - public string MetricsKey; - } -} diff --git a/Wabbajack.CacheServer/DTOs/ModListStatus.cs b/Wabbajack.CacheServer/DTOs/ModListStatus.cs deleted file mode 100644 index ddf41f43..00000000 --- a/Wabbajack.CacheServer/DTOs/ModListStatus.cs +++ /dev/null @@ -1,70 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using MongoDB.Bson.Serialization.Attributes; -using MongoDB.Driver; -using MongoDB.Driver.Linq; -using Wabbajack.Lib; -using Wabbajack.Lib.ModListRegistry; - -namespace Wabbajack.CacheServer.DTOs -{ - public class ModListStatus - { - - [BsonId] - public string Id { get; set; } - public ModlistSummary Summary { get; set; } - - public ModlistMetadata Metadata { get; set; } - public DetailedStatus DetailedStatus { get; set; } - - public static async Task Update(ModListStatus status) - { - var id = status.Metadata.Links.MachineURL; - await Server.Config.ListValidation.Connect().FindOneAndReplaceAsync(s => s.Id == id, status, new FindOneAndReplaceOptions {IsUpsert = true}); - } - - public static IQueryable AllSummaries - { - get - { - return null; - } - } - - public static async Task ByName(string name) - { - var result = await Server.Config.ListValidation.Connect() - .AsQueryable() - .Where(doc => doc.Metadata.Links.MachineURL == name || doc.Metadata.Title == name) - .ToListAsync(); - return result.First(); - } - - public static IMongoQueryable All - { - get - { - return Server.Config.ListValidation.Connect().AsQueryable(); - } - } - } - - public class DetailedStatus - { - public string Name { get; set; } - public DateTime Checked { get; set; } = DateTime.Now; - public List Archives { get; set; } - public DownloadMetadata DownloadMetaData { get; set; } - public bool HasFailures { get; set; } - public string MachineName { get; set; } - } - - public class DetailedStatusItem - { - public bool IsFailing { get; set; } - public Archive Archive { get; set; } - } -} diff --git a/Wabbajack.CacheServer/DTOs/MongoDoc.cs b/Wabbajack.CacheServer/DTOs/MongoDoc.cs deleted file mode 100644 index 0d91b231..00000000 --- a/Wabbajack.CacheServer/DTOs/MongoDoc.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using MongoDB.Bson; - -namespace Wabbajack.CacheServer.DTOs -{ - public class MongoDoc - { - public ObjectId _id { get; set; } = ObjectId.Empty; - } -} diff --git a/Wabbajack.CacheServer/DTOs/NexusCacheData.cs b/Wabbajack.CacheServer/DTOs/NexusCacheData.cs deleted file mode 100644 index 68096fd4..00000000 --- a/Wabbajack.CacheServer/DTOs/NexusCacheData.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using MongoDB.Bson.Serialization.Attributes; - -namespace Wabbajack.CacheServer.DTOs -{ - public class NexusCacheData - { - [BsonId] - public string Path { get; set; } - public T Data { get; set; } - public string Game { get; set; } - public string ModId { get; set; } - - public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow; - - [BsonIgnoreIfNull] - public string FileId { get; set; } - - } -} diff --git a/Wabbajack.CacheServer/DTOs/SerializerSettings.cs b/Wabbajack.CacheServer/DTOs/SerializerSettings.cs deleted file mode 100644 index 41ecc746..00000000 --- a/Wabbajack.CacheServer/DTOs/SerializerSettings.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using MongoDB.Bson; -using MongoDB.Bson.IO; -using MongoDB.Bson.Serialization; -using MongoDB.Bson.Serialization.Conventions; -using Wabbajack.CacheServer.DTOs.JobQueue; -using Wabbajack.Lib.Downloaders; - -namespace Wabbajack.CacheServer.DTOs -{ - public static class SerializerSettings - { - public static void Init() - { - var dis = new TypeDiscriminator(typeof(AbstractDownloadState), AbstractDownloadState.NameToType, - AbstractDownloadState.TypeToName); - BsonSerializer.RegisterDiscriminatorConvention(typeof(AbstractDownloadState), dis); - BsonClassMap.RegisterClassMap(cm => cm.SetIsRootClass(true)); - - dis = new TypeDiscriminator(typeof(AJobPayload), AJobPayload.NameToType, AJobPayload.TypeToName); - BsonSerializer.RegisterDiscriminatorConvention(typeof(AJobPayload), dis); - BsonClassMap.RegisterClassMap(cm => cm.SetIsRootClass(true)); - } - } - - - public class TypeDiscriminator : IDiscriminatorConvention - { - private readonly Type defaultType; - private readonly Dictionary typeMap; - private Dictionary revMap; - - public TypeDiscriminator(Type defaultType, - Dictionary typeMap, Dictionary revMap) - { - this.defaultType = defaultType; - this.typeMap = typeMap; - this.revMap = revMap; - } - - - /// - /// Element Name - /// - public string ElementName => "_wjType"; - - public Type GetActualType(IBsonReader bsonReader, Type nominalType) - { - Type type = null; - var bookmark = bsonReader.GetBookmark(); - bsonReader.ReadStartDocument(); - if (bsonReader.FindElement(ElementName)) - { - var value = bsonReader.ReadString(); - if (typeMap.ContainsKey(value)) - type = typeMap[value]; - } - - bsonReader.ReturnToBookmark(bookmark); - if (type == null) - throw new Exception($"Type mis-configuration can't find bson type for ${nominalType}"); - return type; - } - - public BsonValue GetDiscriminator(Type nominalType, Type actualType) - { - return revMap[actualType]; - } - } -} diff --git a/Wabbajack.CacheServer/Extensions.cs b/Wabbajack.CacheServer/Extensions.cs deleted file mode 100644 index c961925f..00000000 --- a/Wabbajack.CacheServer/Extensions.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Linq.Expressions; -using System.Text; -using System.Threading.Tasks; -using MongoDB.Driver; -using MongoDB.Driver.Linq; - -namespace Wabbajack.CacheServer -{ - public static class Extensions - { - public static async Task FindOneAsync(this IMongoCollection coll, Expression> expr) - { - return (await coll.AsQueryable().Where(expr).Take(1).ToListAsync()).FirstOrDefault(); - } - } -} diff --git a/Wabbajack.CacheServer/FodyWeavers.xml b/Wabbajack.CacheServer/FodyWeavers.xml deleted file mode 100644 index 5029e706..00000000 --- a/Wabbajack.CacheServer/FodyWeavers.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file diff --git a/Wabbajack.CacheServer/FodyWeavers.xsd b/Wabbajack.CacheServer/FodyWeavers.xsd deleted file mode 100644 index 44a53744..00000000 --- a/Wabbajack.CacheServer/FodyWeavers.xsd +++ /dev/null @@ -1,111 +0,0 @@ - - - - - - - - - - - - A list of assembly names to exclude from the default action of "embed all Copy Local references", delimited with line breaks - - - - - A list of assembly names to include from the default action of "embed all Copy Local references", delimited with line breaks. - - - - - A list of unmanaged 32 bit assembly names to include, delimited with line breaks. - - - - - A list of unmanaged 64 bit assembly names to include, delimited with line breaks. - - - - - The order of preloaded assemblies, delimited with line breaks. - - - - - - This will copy embedded files to disk before loading them into memory. This is helpful for some scenarios that expected an assembly to be loaded from a physical file. - - - - - Controls if .pdbs for reference assemblies are also embedded. - - - - - Embedded assemblies are compressed by default, and uncompressed when they are loaded. You can turn compression off with this option. - - - - - As part of Costura, embedded assemblies are no longer included as part of the build. This cleanup can be turned off. - - - - - Costura by default will load as part of the module initialization. This flag disables that behavior. Make sure you call CosturaUtility.Initialize() somewhere in your code. - - - - - Costura will by default use assemblies with a name like 'resources.dll' as a satellite resource and prepend the output path. This flag disables that behavior. - - - - - A list of assembly names to exclude from the default action of "embed all Copy Local references", delimited with | - - - - - A list of assembly names to include from the default action of "embed all Copy Local references", delimited with |. - - - - - A list of unmanaged 32 bit assembly names to include, delimited with |. - - - - - A list of unmanaged 64 bit assembly names to include, delimited with |. - - - - - The order of preloaded assemblies, delimited with |. - - - - - - - - 'true' to run assembly verification (PEVerify) on the target assembly after all weavers have been executed. - - - - - A comma-separated list of error codes that can be safely ignored in assembly verification. - - - - - 'false' to turn off automatic generation of the XML Schema file. - - - - - \ No newline at end of file diff --git a/Wabbajack.CacheServer/Heartbeat.cs b/Wabbajack.CacheServer/Heartbeat.cs deleted file mode 100644 index 88f1e05d..00000000 --- a/Wabbajack.CacheServer/Heartbeat.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Nancy; - -namespace Wabbajack.CacheServer -{ - public class Heartbeat : NancyModule - { - private static DateTime startTime = DateTime.Now; - - public Heartbeat() : base("/") - { - Get("/heartbeat", HandleHeartbeat); - } - - private object HandleHeartbeat(object arg) - { - return $"Service is live for: {DateTime.Now - startTime}"; - } - } -} diff --git a/Wabbajack.CacheServer/JobQueueEndpoints.cs b/Wabbajack.CacheServer/JobQueueEndpoints.cs deleted file mode 100644 index f68ce0cb..00000000 --- a/Wabbajack.CacheServer/JobQueueEndpoints.cs +++ /dev/null @@ -1,144 +0,0 @@ -using System; -using System.IO; -using System.Linq; -using System.Linq.Expressions; -using System.Security.Policy; -using System.Threading.Tasks; -using MongoDB.Driver; -using MongoDB.Driver.Linq; -using Nancy; -using Nettle; -using Wabbajack.CacheServer.DTOs.JobQueue; -using Wabbajack.CacheServer.Jobs; -using Wabbajack.Common; -using Wabbajack.Lib; -using Wabbajack.Lib.CompilationSteps; -using Wabbajack.Lib.Downloaders; -using Directory = Alphaleonis.Win32.Filesystem.Directory; -using Path = Alphaleonis.Win32.Filesystem.Path; - -namespace Wabbajack.CacheServer -{ - public class JobQueueEndpoints : NancyModule - { - public JobQueueEndpoints() : base ("/jobs") - { - Get("/", HandleListJobs); - Get("/enqueue_curated_for_indexing", HandleEnqueueAllCurated); - Get("/enqueue_game_files_for_indexing", HandleEnqueueAllGameFiles); - } - - private readonly Func HandleListJobsTemplate = NettleEngine.GetCompiler().Compile(@" - - -

Jobs - {{$.jobs.Count}} Pending

-

{{$.time}}

-
    - {{each $.jobs}} -
  1. {{$.Description}}
    2. - {{/each}} -
- - - - "); - - private async Task HandleListJobs(object arg) - { - var jobs = await Server.Config.JobQueue.Connect() - .AsQueryable() - .Where(j => j.Ended == null) - .OrderByDescending(j => j.Priority) - .ThenBy(j => j.Created) - .ToListAsync(); - - var response = (Response)HandleListJobsTemplate(new {jobs, time = DateTime.Now}); - response.ContentType = "text/html"; - return response; - } - - - private async Task HandleEnqueueAllCurated(object arg) - { - var states = await Server.Config.ListValidation.Connect() - .AsQueryable() - .SelectMany(lst => lst.DetailedStatus.Archives) - .Select(a => a.Archive) - .ToListAsync(); - - var jobs = states.Select(state => new IndexJob {Archive = state}) - .Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus}) - .ToList(); - - if (jobs.Count > 0) - await Server.Config.JobQueue.Connect().InsertManyAsync(jobs); - - return $"Enqueued {states.Count} jobs"; - } - - private async Task HandleEnqueueAllGameFiles(object arg) - { - using (var queue = new WorkQueue(4)) - { - var states = GameRegistry.Games.Values - .Where(game => game.GameLocation() != null && game.MainExecutable != null) - .SelectMany(game => Directory.EnumerateFiles(game.GameLocation(), "*", SearchOption.AllDirectories) - .Select(file => new GameFileSourceDownloader.State - { - Game = game.Game, - GameVersion = game.InstalledVersion, - GameFile = file.RelativeTo(game.GameLocation()), - })) - .ToList(); - - await states.PMap(queue, state => - { - state.Hash = Path.Combine(state.Game.MetaData().GameLocation(), state.GameFile).FileHash(); - }); - - var jobs = states.Select(state => new IndexJob {Archive = new Archive {Name = Path.GetFileName(state.GameFile), State = state}}) - .Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus}) - .ToList(); - - if (jobs.Count > 0) - await Server.Config.JobQueue.Connect().InsertManyAsync(jobs); - - return $"Enqueued {states.Count} Jobs"; - } - } - - public static async Task StartJobQueue() - { - foreach (var task in Enumerable.Range(0, 4)) - { - var tsk = StartJobQueueInner(); - } - } - - private static async Task StartJobQueueInner() - { - while (true) - { - try - { - var job = await Job.GetNext(); - if (job == null) - { - await Task.Delay(5000); - continue; - } - - var result = await job.Payload.Execute(); - await Job.Finish(job, result); - } - catch (Exception ex) - { - - } - - } - } - } -} diff --git a/Wabbajack.CacheServer/Jobs/IndexJob.cs b/Wabbajack.CacheServer/Jobs/IndexJob.cs deleted file mode 100644 index 04bb483e..00000000 --- a/Wabbajack.CacheServer/Jobs/IndexJob.cs +++ /dev/null @@ -1,106 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Alphaleonis.Win32.Filesystem; -using MongoDB.Driver; -using MongoDB.Driver.Linq; -using Wabbajack.CacheServer.DTOs; -using Wabbajack.CacheServer.DTOs.JobQueue; -using Wabbajack.Common; -using Wabbajack.Lib; -using Wabbajack.Lib.Downloaders; -using Wabbajack.VirtualFileSystem; - -namespace Wabbajack.CacheServer.Jobs -{ - public class IndexJob : AJobPayload - { - public Archive Archive { get; set; } - public override string Description { get; } = "Validate and index an archive"; - public override bool UsesNexus { get => Archive.State is NexusDownloader.State; } - public override async Task Execute() - { - - var pk = new List(); - pk.Add(AbstractDownloadState.TypeToName[Archive.State.GetType()]); - pk.AddRange(Archive.State.PrimaryKey); - var pk_str = string.Join("|",pk.Select(p => p.ToString())); - - var found = await Server.Config.DownloadStates.Connect().AsQueryable().Where(f => f.Key == pk_str).Take(1).ToListAsync(); - if (found.Count > 0) - return JobResult.Success(); - - string fileName = Archive.Name; - string folder = Guid.NewGuid().ToString(); - Utils.Log($"Indexer is downloading {fileName}"); - var downloadDest = Path.Combine(Server.Config.Indexer.DownloadDir, folder, fileName); - await Archive.State.Download(downloadDest); - - using (var queue = new WorkQueue()) - { - var vfs = new Context(queue, true); - await vfs.AddRoot(Path.Combine(Server.Config.Indexer.DownloadDir, folder)); - var archive = vfs.Index.ByRootPath.First(); - var converted = ConvertArchive(new List(), archive.Value); - try - { - await Server.Config.IndexedFiles.Connect().InsertManyAsync(converted, new InsertManyOptions {IsOrdered = false}); - } - catch (MongoBulkWriteException) - { - } - - await Server.Config.DownloadStates.Connect().InsertOneAsync(new DownloadState - { - Key = pk_str, - Hash = archive.Value.Hash, - State = Archive.State, - IsValid = true - }); - - var to_path = Path.Combine(Server.Config.Indexer.ArchiveDir, - $"{Path.GetFileName(fileName)}_{archive.Value.Hash.FromBase64().ToHex()}_{Path.GetExtension(fileName)}"); - if (File.Exists(to_path)) - File.Delete(downloadDest); - else - File.Move(downloadDest, to_path); - Utils.DeleteDirectory(Path.Combine(Server.Config.Indexer.DownloadDir, folder)); - } - - return JobResult.Success(); - } - - private List ConvertArchive(List files, VirtualFile file, bool isTop = true) - { - var name = isTop ? Path.GetFileName(file.Name) : file.Name; - var ifile = new IndexedFile - { - Hash = file.Hash, - SHA256 = file.ExtendedHashes.SHA256, - SHA1 = file.ExtendedHashes.SHA1, - MD5 = file.ExtendedHashes.MD5, - CRC = file.ExtendedHashes.CRC, - Size = file.Size, - Children = file.Children != null ? file.Children.Select( - f => - { - ConvertArchive(files, f, false); - - return new ChildFile - { - Hash = f.Hash, - Name = f.Name.ToLowerInvariant(), - Extension = Path.GetExtension(f.Name.ToLowerInvariant()) - }; - }).ToList() : new List() - }; - ifile.IsArchive = ifile.Children.Count > 0; - files.Add(ifile); - return files; - } - - - } -} diff --git a/Wabbajack.CacheServer/ListValidationService.cs b/Wabbajack.CacheServer/ListValidationService.cs deleted file mode 100644 index 0c717b1a..00000000 --- a/Wabbajack.CacheServer/ListValidationService.cs +++ /dev/null @@ -1,252 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Threading.Tasks; -using Alphaleonis.Win32.Filesystem; -using MongoDB.Driver; -using Nancy; -using Wabbajack.CacheServer.DTOs; -using Wabbajack.Common; -using Wabbajack.Lib; -using Wabbajack.Lib.Downloaders; -using Wabbajack.Lib.ModListRegistry; -using MongoDB.Driver.Linq; -using Nettle; -using Nettle.Functions; - -namespace Wabbajack.CacheServer -{ - public class ListValidationService : NancyModule - { - public ListValidationService() : base("/lists") - { - Get("/status", HandleGetLists); - Get("/force_recheck", HandleForceRecheck); - Get("/status/{Name}.json", HandleGetListJson); - Get("/status/{Name}.html", HandleGetListHtml); - Get("/status/{Name}/broken.rss", HandleGetRSSFeed); - - } - - private async Task HandleForceRecheck(object arg) - { - await ValidateLists(false); - return "done"; - } - - private async Task HandleGetLists(object arg) - { - var summaries = await ModListStatus.All.Select(m => m.Summary).ToListAsync(); - return summaries.ToJSON(); - } - - public class ArchiveSummary - { - public string Name; - public AbstractDownloadState State; - } - public class DetailedSummary - { - public string Name; - public DateTime Checked; - public List Failed; - public List Passed; - - } - private async Task HandleGetListJson(dynamic arg) - { - var metric = Metrics.Log("list_validation.get_list_json", (string)arg.Name); - var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus; - return lst.ToJSON(); - } - - - private static readonly Func HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@" - -

{{lst.Name}} - {{lst.Checked}}

-

Failed ({{failed.Count}}):

-
    - {{each $.failed }} -
  • {{$.Archive.Name}}
    • - {{/each}} -
-

Passed ({{passed.Count}}):

-
    - {{each $.passed }} -
  • {{$.Archive.Name}}
    • - {{/each}} -
- - "); - - private async Task HandleGetListHtml(dynamic arg) - { - - var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus; - var response = (Response)HandleGetListTemplate(new - { - lst, - failed = lst.Archives.Where(a => a.IsFailing).ToList(), - passed = lst.Archives.Where(a => !a.IsFailing).ToList() - }); - response.ContentType = "text/html"; - return response; - } - - private static readonly Func HandleGetRSSFeedTemplate = NettleEngine.GetCompiler().Compile(@" - - - - {{lst.Name}} - Broken Mods - http://build.wabbajack.org/status/{{lst.Name}}.html - These are mods that are broken and need updating - {{ each $.failed }} - - {{$.Archive.Name}} - {{$.Archive.Name}} - - {{/each}} - - - "); - - public async Task HandleGetRSSFeed(dynamic arg) - { - var metric = Metrics.Log("failed_rss", arg.Name); - var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus; - var response = (Response)HandleGetRSSFeedTemplate(new - { - lst, - failed = lst.Archives.Where(a => a.IsFailing).ToList(), - passed = lst.Archives.Where(a => !a.IsFailing).ToList() - }); - response.ContentType = "application/rss+xml"; - await metric; - return response; - - } - - public static void Start() - { - Task.Run(async () => - { - while (true) - { - try - { - await ValidateLists(); - } - catch (Exception ex) - { - Utils.Log(ex.ToString()); - } - - // Sleep for two hours - await Task.Delay(1000 * 60 * 60 * 2); - } - }).FireAndForget(); - } - public static async Task ValidateLists(bool skipIfNewer = true) - { - Utils.Log("Cleaning Nexus Cache"); - var client = new HttpClient(); - //await client.GetAsync("http://build.wabbajack.org/nexus_api_cache/update"); - - Utils.Log("Starting Modlist Validation"); - var modlists = await ModlistMetadata.LoadFromGithub(); - - using (var queue = new WorkQueue()) - { - foreach (var list in modlists) - { - try - { - await ValidateList(list, queue, skipIfNewer); - } - catch (Exception ex) - { - Utils.Log(ex.ToString()); - } - } - } - - Utils.Log($"Done validating {modlists.Count} lists"); - } - - private static async Task ValidateList(ModlistMetadata list, WorkQueue queue, bool skipIfNewer = true) - { - var existing = await Server.Config.ListValidation.Connect().FindOneAsync(l => l.Id == list.Links.MachineURL); - if (skipIfNewer && existing != null && DateTime.UtcNow - existing.DetailedStatus.Checked < TimeSpan.FromHours(2)) - return; - - var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ExtensionManager.Extension); - - if (list.NeedsDownload(modlist_path)) - { - if (File.Exists(modlist_path)) - File.Delete(modlist_path); - - var state = DownloadDispatcher.ResolveArchive(list.Links.Download); - Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}"); - await state.Download(modlist_path); - } - else - { - Utils.Log($"No changes detected from downloaded modlist"); - } - - - Utils.Log($"Loading {modlist_path}"); - - var installer = AInstaller.LoadFromFile(modlist_path); - - Utils.Log($"{installer.Archives.Count} archives to validate"); - - DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State)); - - var validated = (await installer.Archives - .PMap(queue, async archive => - { - Utils.Log($"Validating: {archive.Name}"); - bool is_failed; - try - { - is_failed = !(await archive.State.Verify()); - } - catch (Exception) - { - is_failed = false; - } - - return new DetailedStatusItem {IsFailing = is_failed, Archive = archive}; - })) - .ToList(); - - - var status = new DetailedStatus - { - Name = list.Title, - Archives = validated.OrderBy(v => v.Archive.Name).ToList(), - DownloadMetaData = list.DownloadMetadata, - HasFailures = validated.Any(v => v.IsFailing) - }; - - var dto = new ModListStatus - { - Id = list.Links.MachineURL, - Summary = new ModlistSummary - { - Name = status.Name, - Checked = status.Checked, - Failed = status.Archives.Count(a => a.IsFailing), - Passed = status.Archives.Count(a => !a.IsFailing), - }, - DetailedStatus = status, - Metadata = list - }; - await ModListStatus.Update(dto); - } - } -} - diff --git a/Wabbajack.CacheServer/Metrics.cs b/Wabbajack.CacheServer/Metrics.cs deleted file mode 100644 index 9ff873eb..00000000 --- a/Wabbajack.CacheServer/Metrics.cs +++ /dev/null @@ -1,140 +0,0 @@ -using System; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Alphaleonis.Win32.Filesystem; -using MongoDB.Driver; -using MongoDB.Driver.Linq; -using Nancy; -using Wabbajack.CacheServer.DTOs; -using Wabbajack.Common; - -namespace Wabbajack.CacheServer -{ - /// - /// Extremely - /// - public class Metrics : NancyModule - { - private static SemaphoreSlim _lockObject = new SemaphoreSlim(1); - - public static async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null) - { - var msg = new[] {string.Join("\t", new[]{timestamp.ToString(), metricsKey, action, subject})}; - Utils.Log(msg.First()); - var db = Server.Config.Metrics.Connect(); - await db.InsertOneAsync(new Metric {Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey}); - } - - public static Task Log(string action, string subject) - { - return Log(DateTime.Now, action, subject); - } - - public Metrics() : base("/") - { - Get("/metrics/{Action}/{Value}", HandleMetrics); - Get("/metrics/chart/", HandleChart); - Get("/metrics/chart/{Action}/", HandleChart); - Get("/metrics/chart/{Action}/{Value}/", HandleChart); - Get("/metrics/ingest/{filename}", HandleBulkIngest); - } - - private async Task HandleBulkIngest(dynamic arg) - { - Log("Bulk Loading " + arg.filename.ToString()); - - var lines = File.ReadAllLines(Path.Combine(@"c:\tmp", (string)arg.filename)); - - var db = Server.Config.Metrics.Connect(); - - var data = lines.Select(line => line.Split('\t')) - .Where(line => line.Length == 3) - .Select(line => new Metric{ Timestamp = DateTime.Parse(line[0]), Action = line[1], Subject = line[2] }) - .ToList(); - - foreach (var metric in data) - await db.InsertOneAsync(metric); - - return $"Processed {lines.Length} records"; - } - - private async Task HandleMetrics(dynamic arg) - { - var date = DateTime.UtcNow; - await Log(date, arg.Action, arg.Value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault()); - return date.ToString(); - } - - private async Task HandleChart(dynamic arg) - { - /*var data = (await GetData()).Select(line => line.Split('\t')) - .Where(line => line.Length == 3) - .Select(line => new {date = DateTime.Parse(line[0]), Action = line[1], Value = line[2]});*/ - - var q = Server.Config.Metrics.Connect().AsQueryable(); - - // Remove guids / Default, which come from testing - - if (arg?.Action != null) - { - var action = (string)arg.Action; - q = q.Where(d => d.Action == action); - } - - - if (arg?.Value != null) - { - var value = (string)arg.Value; - q = q.Where(d => d.Subject.StartsWith(value)); - } - - var data = (await q.Take(Int32.MaxValue).ToListAsync()).AsEnumerable(); - data = data.Where(d => !Guid.TryParse(d.Subject ?? "", out Guid v) && (d.Subject ?? "") != "Default"); - - var grouped_and_counted = data.GroupBy(d => d.Timestamp.ToString("yyyy-MM-dd")) - .OrderBy(d => d.Key) - .Select(d => new {Day = d.Key, Count = d.Count()}) - .ToList(); - - var sb = new StringBuilder(); - sb.Append(""); - sb.Append(""); - sb.Append(""); - sb.Append(""); - var response = (Response)sb.ToString(); - response.ContentType = "text/html"; - return response; - } - - public void Log(string l) - { - Utils.Log("Metrics: " + l); - } - } -} diff --git a/Wabbajack.CacheServer/NexusCacheModule.cs b/Wabbajack.CacheServer/NexusCacheModule.cs deleted file mode 100644 index a334bb9c..00000000 --- a/Wabbajack.CacheServer/NexusCacheModule.cs +++ /dev/null @@ -1,344 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Reflection; -using System.Text; -using System.Threading.Tasks; -using MongoDB.Driver; -using Nancy; -using Newtonsoft.Json; -using Wabbajack.CacheServer.DTOs; -using Wabbajack.Common; -using Wabbajack.Lib.Downloaders; -using Wabbajack.Lib.NexusApi; - -namespace Wabbajack.CacheServer -{ - public class NexusCacheModule : NancyModule - { - - public NexusCacheModule() : base("/") - { - Get("/v1/games/{GameName}/mods/{ModID}/files/{FileID}.json", HandleFileID); - Get("/v1/games/{GameName}/mods/{ModID}/files.json", HandleGetFiles); - Get("/v1/games/{GameName}/mods/{ModID}.json", HandleModInfo); - Get("/nexus_api_cache/{request}.json", HandleCacheCall); - Get("/nexus_api_cache", ListCache); - Get("/nexus_api_cache/update", UpdateCache); - Get("/nexus_api_cache/ingest/{Folder}", HandleIngestCache); - } - - class UpdatedMod - { - public long mod_id; - public long latest_file_update; - public long latest_mod_activity; - } - - public async Task UpdateCache(object arg) - { - var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - - var gameTasks = GameRegistry.Games.Values - .Where(game => game.NexusName != null) - .Select(async game => - { - return (game, - mods: await api.Get>( - $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m")); - }) - .Select(async rTask => - { - var (game, mods) = await rTask; - return mods.Select(mod => new { game = game, mod = mod }); - }).ToList(); - - Utils.Log($"Getting update list for {gameTasks.Count} games"); - - var purge = (await Task.WhenAll(gameTasks)) - .SelectMany(i => i) - .ToList(); - - Utils.Log($"Found {purge.Count} updated mods in the last month"); - using (var queue = new WorkQueue()) - { - var collected = await purge.Select(d => - { - var a = d.mod.latest_file_update.AsUnixTime(); - // Mod activity could hide files - var b = d.mod.latest_mod_activity.AsUnixTime(); - - return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.mod_id.ToString()}; - }).PMap(queue, async t => - { - var resultA = await Server.Config.NexusModInfos.Connect().DeleteManyAsync(f => - f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); - var resultB = await Server.Config.NexusModFiles.Connect().DeleteManyAsync(f => - f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); - var resultC = await Server.Config.NexusFileInfos.Connect().DeleteManyAsync(f => - f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); - - return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount; - }); - - Utils.Log($"Purged {collected.Sum()} cache entries"); - } - - return "Done"; - } - - private string ListCache(object arg) - { - Utils.Log($"{DateTime.Now} - List Cache"); - return String.Join("", - Directory.EnumerateFiles(NexusApiClient.LocalCacheDir) - .Select(f => new FileInfo(f)) - .OrderByDescending(fi => fi.LastWriteTime) - .Select(fi => - { - var decoded = Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(fi.Name).FromHex()); - return $"{fi.LastWriteTime} \t {fi.Length.ToFileSizeString()} \t {decoded} \n"; - })); - } - - private async Task HandleModInfo(dynamic arg) - { - Utils.Log($"{DateTime.Now} - Mod Info - {arg.GameName}/{arg.ModID}/"); - string gameName = arg.GameName; - string modId = arg.ModId; - var result = await Server.Config.NexusModInfos.Connect() - .FindOneAsync(info => info.Game == gameName && info.ModId == modId); - - string method = "CACHED"; - if (result == null) - { - var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - var path = $"/v1/games/{gameName}/mods/{modId}.json"; - var body = await api.Get(path); - result = new NexusCacheData - { - Data = body, - Path = path, - Game = gameName, - ModId = modId - }; - try - { - await Server.Config.NexusModInfos.Connect().InsertOneAsync(result); - } - catch (MongoWriteException) - { - - } - - method = "NOT_CACHED"; - } - - Response response = result.Data.ToJSON(); - response.Headers.Add("WABBAJACK_CACHE_FROM", method); - response.ContentType = "application/json"; - return response; - } - - private async Task HandleFileID(dynamic arg) - { - Utils.Log($"{DateTime.Now} - File Info - {arg.GameName}/{arg.ModID}/{arg.FileID}"); - string gameName = arg.GameName; - string modId = arg.ModId; - string fileId = arg.FileId; - var result = await Server.Config.NexusFileInfos.Connect() - .FindOneAsync(info => info.Game == gameName && info.ModId == modId && info.FileId == fileId); - - string method = "CACHED"; - if (result == null) - { - var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - var path = $"/v1/games/{gameName}/mods/{modId}/files/{fileId}.json"; - var body = await api.Get(path); - result = new NexusCacheData - { - Data = body, - Path = path, - Game = gameName, - ModId = modId, - FileId = fileId - }; - try - { - await Server.Config.NexusFileInfos.Connect().InsertOneAsync(result); - } - catch (MongoWriteException) - { - - } - - method = "NOT_CACHED"; - } - - Response response = result.Data.ToJSON(); - response.Headers.Add("WABBAJACK_CACHE_FROM", method); - response.ContentType = "application/json"; - return response; - } - - private async Task HandleGetFiles(dynamic arg) - { - Utils.Log($"{DateTime.Now} - Mod Files - {arg.GameName} {arg.ModID}"); - string gameName = arg.GameName; - string modId = arg.ModId; - var result = await Server.Config.NexusModFiles.Connect() - .FindOneAsync(info => info.Game == gameName && info.ModId == modId); - - string method = "CACHED"; - if (result == null) - { - var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - var path = $"/v1/games/{gameName}/mods/{modId}/files.json"; - var body = await api.Get(path); - result = new NexusCacheData - { - Data = body, - Path = path, - Game = gameName, - ModId = modId - }; - try - { - await Server.Config.NexusModFiles.Connect().InsertOneAsync(result); - } - catch (MongoWriteException) - { - - } - - method = "NOT_CACHED"; - } - - Response response = result.Data.ToJSON(); - response.Headers.Add("WABBAJACK_CACHE_FROM", method); - response.ContentType = "application/json"; - return response; - } - - private async Task HandleCacheCall(dynamic arg) - { - try - { - string param = (string)arg.request; - var url = new Uri(Encoding.UTF8.GetString(param.FromHex())); - - var client = new HttpClient(); - var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 8080, Scheme = "http"}; - client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]); - return await client.GetStringAsync(builder.Uri.ToString()); - } - catch (Exception ex) - { - Utils.Log(ex.ToString()); - return "ERROR"; - } - } - - private async Task HandleIngestCache(dynamic arg) - { - int count = 0; - int failed = 0; - - using (var queue = new WorkQueue()) - { - await Directory.EnumerateFiles(Path.Combine(Server.Config.Settings.TempDir, (string)arg.Folder)).PMap(queue, - async file => - { - Utils.Log($"Ingesting {file}"); - if (!file.EndsWith(".json")) return; - - var fileInfo = new FileInfo(file); - count++; - - var url = new Url( - Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(file).FromHex())); - var split = url.Path.Split(new[] {'/'}, StringSplitOptions.RemoveEmptyEntries); - try - { - switch (split.Length) - { - case 5 when split[3] == "mods": - { - var body = file.FromJSON(); - - var payload = new NexusCacheData(); - payload.Data = body; - payload.Game = split[2]; - payload.Path = url.Path; - payload.ModId = body.mod_id; - payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc; - - try - { - await Server.Config.NexusModInfos.Connect().InsertOneAsync(payload); - } - catch (MongoWriteException ex) - { - - } - - break; - } - case 6 when split[5] == "files.json": - { - var body = file.FromJSON(); - var payload = new NexusCacheData(); - payload.Path = url.Path; - payload.Data = body; - payload.Game = split[2]; - payload.ModId = split[4]; - payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc; - - try - { - await Server.Config.NexusModFiles.Connect().InsertOneAsync(payload); - } - catch (MongoWriteException ex) - { - - } - - break; - } - case 7 when split[5] == "files": - { - var body = file.FromJSON(); - var payload = new NexusCacheData(); - payload.Data = body; - payload.Path = url.Path; - payload.Game = split[2]; - payload.FileId = Path.GetFileNameWithoutExtension(split[6]); - payload.ModId = split[4]; - payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc; - - try - { - await Server.Config.NexusFileInfos.Connect().InsertOneAsync(payload); - } - catch (MongoWriteException ex) - { - - } - - break; - } - } - } - catch (Exception ex) - { - failed++; - } - }); - } - - return $"Inserted {count} caches, {failed} failed"; - } - } -} diff --git a/Wabbajack.CacheServer/Program.cs b/Wabbajack.CacheServer/Program.cs deleted file mode 100644 index 049c7ada..00000000 --- a/Wabbajack.CacheServer/Program.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Nancy.Hosting.Self; -using Wabbajack.Common; - -namespace Wabbajack.CacheServer -{ - class Program - { - static void Main(string[] args) - { - Utils.LogMessages.Subscribe(Console.WriteLine); - using (var server = new Server("http://localhost:8080")) - { - Consts.WabbajackCacheHostname = "localhost"; - Consts.WabbajackCachePort = 8080; - server.Start(); - - ListValidationService.Start(); - var tsk = JobQueueEndpoints.StartJobQueue(); - - Console.ReadLine(); - } - } - } -} diff --git a/Wabbajack.CacheServer/Properties/AssemblyInfo.cs b/Wabbajack.CacheServer/Properties/AssemblyInfo.cs deleted file mode 100644 index 4417a3de..00000000 --- a/Wabbajack.CacheServer/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("Wabbajack.CacheServer")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("Wabbajack.CacheServer")] -[assembly: AssemblyCopyright("Copyright © 2019")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("bdc9a094-d235-47cd-83ca-44199b60ab20")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.0")] -[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/Wabbajack.CacheServer/Readme.md b/Wabbajack.CacheServer/Readme.md deleted file mode 100644 index 6953a41a..00000000 --- a/Wabbajack.CacheServer/Readme.md +++ /dev/null @@ -1,3 +0,0 @@ -# Wabbajack.CacheServer - -CacheServer for caching mod information to reduce the amount of API calls a user has to account for when using Wabbajack to compiler/install a ModList. diff --git a/Wabbajack.CacheServer/Server.cs b/Wabbajack.CacheServer/Server.cs deleted file mode 100644 index 404ed688..00000000 --- a/Wabbajack.CacheServer/Server.cs +++ /dev/null @@ -1,79 +0,0 @@ -using System; -using Alphaleonis.Win32.Filesystem; -using Nancy; -using Nancy.Bootstrapper; -using Nancy.Configuration; -using Nancy.Hosting.Self; -using Nancy.TinyIoc; -using Wabbajack.CacheServer.DTOs; -using Wabbajack.CacheServer.ServerConfig; -using Wabbajack.Common; - -namespace Wabbajack.CacheServer -{ - public class Server : IDisposable - { - private NancyHost _server; - private HostConfiguration _config; - public static BuildServerConfig Config; - - static Server() - { - SerializerSettings.Init(); - } - - - public Server(string address) - { - Address = address; - _config = new HostConfiguration {MaximumConnectionCount = 200, RewriteLocalhost = true}; - //_config.UrlReservations.CreateAutomatically = true; - _server = new NancyHost(_config, new Uri(address)); - - Config = File.ReadAllText("config.yaml").FromYaml(); - } - - public string Address { get; } - - public void Start() - { - _server.Start(); - } - - public void Dispose() - { - _server?.Dispose(); - } - } - - public class CachingBootstrapper : DefaultNancyBootstrapper - { - protected override void ApplicationStartup(TinyIoCContainer container, IPipelines pipelines) - { - pipelines.AfterRequest.AddItemToEndOfPipeline(ctx => - { - ctx.Response.WithHeader("Access-Control-Allow-Origin", "*") - .WithHeader("Access-Control-Allow-Methods", "POST, GET") - .WithHeader("Access-Control-Allow-Headers", "Accept, Origin, Content-type") - .WithHeader("Cache-Control","no-store"); - }); - } - - public override void Configure(INancyEnvironment environment) - { - environment.Tracing( - enabled: true, - displayErrorTraces: true); - } - - protected override void ConfigureApplicationContainer(TinyIoCContainer container) - { - container.Register(); - container.Register(); - container.Register(); - container.Register(); - container.Register(); - container.Register(); - } - } -} diff --git a/Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs b/Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs deleted file mode 100644 index 10f6cebd..00000000 --- a/Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using MongoDB.Driver.Core.Configuration; -using Wabbajack.CacheServer.DTOs; -using Wabbajack.CacheServer.DTOs.JobQueue; -using Wabbajack.Lib.NexusApi; - -namespace Wabbajack.CacheServer.ServerConfig -{ - public class BuildServerConfig - { - public MongoConfig Metrics { get; set; } - public MongoConfig ListValidation { get; set; } - - public MongoConfig JobQueue { get; set; } - - public MongoConfig IndexedFiles { get; set; } - public MongoConfig DownloadStates { get; set; } - - public MongoConfig> NexusModInfos { get; set; } - public MongoConfig> NexusModFiles { get; set; } - public MongoConfig> NexusFileInfos { get; set; } - - public IndexerConfig Indexer { get; set; } - - public Settings Settings { get; set; } - } -} diff --git a/Wabbajack.CacheServer/ServerConfig/IndexerConfig.cs b/Wabbajack.CacheServer/ServerConfig/IndexerConfig.cs deleted file mode 100644 index e84208de..00000000 --- a/Wabbajack.CacheServer/ServerConfig/IndexerConfig.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Wabbajack.CacheServer.ServerConfig -{ - public class IndexerConfig - { - public string DownloadDir { get; set; } - public string TempDir { get; set; } - - public string ArchiveDir { get; set; } - } -} diff --git a/Wabbajack.CacheServer/ServerConfig/MongoConfig.cs b/Wabbajack.CacheServer/ServerConfig/MongoConfig.cs deleted file mode 100644 index 519b2684..00000000 --- a/Wabbajack.CacheServer/ServerConfig/MongoConfig.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using MongoDB.Driver; -using Wabbajack.CacheServer.DTOs; - -namespace Wabbajack.CacheServer.ServerConfig -{ - public class MongoConfig - { - public string Host { get; set; } - public string Database { get; set; } - public string Collection { get; set; } - public string Username { get; set; } - public string Password { get; set; } - - private IMongoDatabase Client - { - get - { - if (Username != null && Password != null) - return new MongoClient($"mongodb://{Username}:{Password}@{Host}").GetDatabase(Database); - return new MongoClient($"mongodb://{Host}").GetDatabase(Database); - } - } - - public IMongoCollection Connect() - { - return Client.GetCollection(Collection); - } - } -} diff --git a/Wabbajack.CacheServer/ServerConfig/Settings.cs b/Wabbajack.CacheServer/ServerConfig/Settings.cs deleted file mode 100644 index 26ccfe0b..00000000 --- a/Wabbajack.CacheServer/ServerConfig/Settings.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Wabbajack.CacheServer.ServerConfig -{ - public class Settings - { - public string TempDir { get; set; } - } -} diff --git a/Wabbajack.CacheServer/TestingEndpoints.cs b/Wabbajack.CacheServer/TestingEndpoints.cs deleted file mode 100644 index 181013a5..00000000 --- a/Wabbajack.CacheServer/TestingEndpoints.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Nancy; -using Nancy.Responses; - -namespace Wabbajack.CacheServer -{ - /// - /// These endpoints are used by the testing service to verify that manual and direct - /// downloading works as expected. - /// - public class TestingEndpoints : NancyModule - { - public TestingEndpoints() : base("/") - { - Get("/WABBAJACK_TEST_FILE.txt", _ => "Cheese for Everyone!"); - Get("/WABBAJACK_TEST_FILE.zip", _ => - { - var response = new StreamResponse(() => new MemoryStream(Encoding.UTF8.GetBytes("Cheese for Everyone!")), "application/zip"); - return response.AsAttachment("WABBAJACK_TEST_FILE.zip"); - }); - } - } -} diff --git a/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj b/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj deleted file mode 100644 index 503a3163..00000000 --- a/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj +++ /dev/null @@ -1,169 +0,0 @@ - - - - - Debug - AnyCPU - {BDC9A094-D235-47CD-83CA-44199B60AB20} - Exe - Wabbajack.CacheServer - Wabbajack.CacheServer - v4.8 - 512 - true - true - - - - x64 - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - CS1998 - CS4014 - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - CS1998 - CS4014 - - - true - bin\x64\Debug\ - DEBUG;TRACE - full - x64 - 7.3 - prompt - MinimumRecommendedRules.ruleset - true - - - bin\x64\Release\ - TRACE - true - pdbonly - x64 - 7.3 - prompt - MinimumRecommendedRules.ruleset - true - - - true - bin\x86\Debug\ - DEBUG;TRACE - CS1998 - full - x86 - CS4014 - 7.3 - prompt - MinimumRecommendedRules.ruleset - true - - - bin\x86\Release\ - TRACE - true - CS1998 - pdbonly - x86 - CS4014 - 7.3 - prompt - MinimumRecommendedRules.ruleset - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - PreserveNewest - - - - - {B3F3FB6E-B9EB-4F49-9875-D78578BC7AE5} - Wabbajack.Common - - - {0a820830-a298-497d-85e0-e9a89efef5fe} - Wabbajack.Lib - - - {5d6a2eaf-6604-4c51-8ae2-a746b4bc5e3e} - Wabbajack.VirtualFileSystem - - - - - 2.10.0 - - - 2.0.0 - - - 1.3.0 - - - 12.0.3 - - - 11.1.6 - - - 4.3.2 - - - - - \ No newline at end of file diff --git a/Wabbajack.CacheServer/config.yaml b/Wabbajack.CacheServer/config.yaml deleted file mode 100644 index 95b7811e..00000000 --- a/Wabbajack.CacheServer/config.yaml +++ /dev/null @@ -1,39 +0,0 @@ ---- -Metrics: - Host: internal.test.mongodb - Database: wabbajack - Collection: metrics -ListValidation: - Host: internal.test.mongodb - Database: wabbajack - Collection: mod_lists -JobQueue: - Host: internal.test.mongodb - Database: wabbajack - Collection: job_queue -IndexedFiles: - Host: internal.test.mongodb - Database: wabbajack - Collection: indexed_files -NexusModInfos: - Host: internal.test.mongodb - Database: wabbajack - Collection: nexus_mod_infos -NexusModFiles: - Host: internal.test.mongodb - Database: wabbajack - Collection: nexus_mod_files -NexusFileInfos: - Host: internal.test.mongodb - Database: wabbajack - Collection: nexus_file_infos -DownloadStates: - Host: internal.test.mongodb - Database: wabbajack - Collection: download_states -Indexer: - DownloadDir: c:\tmp\downloads - TempDir: c:\tmp\tmp - ArchiveDir: c:\archives -Settings: - TempDir: c:\tmp\tmp \ No newline at end of file