From 9ed98ace54c54dadf60e9474c2a4678bcb9ccb50 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 7 Jan 2020 21:41:50 -0700 Subject: [PATCH] WIP --- .../Controllers/AControllerBase.cs | 18 +++ Wabbajack.BuildServer/Controllers/Jobs.cs | 31 ++++ .../Controllers/ListValidation.cs | 28 ++++ .../Controllers/NexusCache.cs | 136 ++++++++++++++++++ Wabbajack.BuildServer/Extensions.cs | 17 +++ Wabbajack.BuildServer/Models/DBContext.cs | 41 ++++++ Wabbajack.BuildServer/Models/DownloadState.cs | 23 +++ Wabbajack.BuildServer/Models/IndexedFile.cs | 30 ++++ .../Models/JobQueue/AJobPayload.cs | 36 +++++ Wabbajack.BuildServer/Models/JobQueue/Job.cs | 67 +++++++++ .../Models/JobQueue/JobResult.cs | 36 +++++ Wabbajack.BuildServer/Models/Jobs/IndexJob.cs | 110 ++++++++++++++ Wabbajack.BuildServer/Models/ModListStatus.cs | 62 ++++++++ .../Models/NexusCacheData.cs | 20 +++ Wabbajack.BuildServer/SerializerSettings.cs | 76 ++++++++++ .../Wabbajack.BuildServer.csproj | 49 +++++++ .../Wabbajack.CacheServer.csproj | 1 - .../ModListRegistry/ModListMetadata.cs | 15 +- Wabbajack.Lib/NexusApi/Dtos.cs | 50 +++---- Wabbajack.Lib/NexusApi/NexusApi.cs | 2 +- Wabbajack.sln | 20 +++ 21 files changed, 837 insertions(+), 31 deletions(-) create mode 100644 Wabbajack.BuildServer/Controllers/AControllerBase.cs create mode 100644 Wabbajack.BuildServer/Controllers/Jobs.cs create mode 100644 Wabbajack.BuildServer/Controllers/ListValidation.cs create mode 100644 Wabbajack.BuildServer/Controllers/NexusCache.cs create mode 100644 Wabbajack.BuildServer/Extensions.cs create mode 100644 Wabbajack.BuildServer/Models/DBContext.cs create mode 100644 Wabbajack.BuildServer/Models/DownloadState.cs create mode 100644 Wabbajack.BuildServer/Models/IndexedFile.cs create mode 100644 Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs create mode 100644 Wabbajack.BuildServer/Models/JobQueue/Job.cs create mode 100644 Wabbajack.BuildServer/Models/JobQueue/JobResult.cs create mode 100644 Wabbajack.BuildServer/Models/Jobs/IndexJob.cs create mode 100644 Wabbajack.BuildServer/Models/ModListStatus.cs create mode 100644 Wabbajack.BuildServer/Models/NexusCacheData.cs create mode 100644 Wabbajack.BuildServer/SerializerSettings.cs create mode 100644 Wabbajack.BuildServer/Wabbajack.BuildServer.csproj diff --git a/Wabbajack.BuildServer/Controllers/AControllerBase.cs b/Wabbajack.BuildServer/Controllers/AControllerBase.cs new file mode 100644 index 00000000..a6300cf1 --- /dev/null +++ b/Wabbajack.BuildServer/Controllers/AControllerBase.cs @@ -0,0 +1,18 @@ +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Wabbajack.BuildServer.Models; + +namespace Wabbajack.BuildServer.Controllers +{ + public abstract class AControllerBase : ControllerBase + { + protected readonly ILogger Logger; + protected readonly DBContext Db; + + protected AControllerBase(ILogger logger, DBContext db) + { + Db = db; + Logger = logger; + } + } +} diff --git a/Wabbajack.BuildServer/Controllers/Jobs.cs b/Wabbajack.BuildServer/Controllers/Jobs.cs new file mode 100644 index 00000000..bfddab59 --- /dev/null +++ b/Wabbajack.BuildServer/Controllers/Jobs.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Wabbajack.BuildServer.Models; +using Wabbajack.BuildServer.Models.JobQueue; + +namespace Wabbajack.BuildServer.Controllers +{ + [ApiController] + [Route("/jobs")] + public class Jobs : AControllerBase + { + public Jobs(ILogger logger, DBContext db) : base(logger, db) + { + } + + [HttpGet] + [Route("unfinished")] + public async Task> GetUnfinished() + { + return await Db.Jobs.AsQueryable() + .Where(j => j.Ended == null) + .OrderByDescending(j => j.Priority) + .ToListAsync(); + } + + } +} diff --git a/Wabbajack.BuildServer/Controllers/ListValidation.cs b/Wabbajack.BuildServer/Controllers/ListValidation.cs new file mode 100644 index 00000000..93334a6e --- /dev/null +++ b/Wabbajack.BuildServer/Controllers/ListValidation.cs @@ -0,0 +1,28 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Wabbajack.BuildServer.Models; +using Wabbajack.Lib.ModListRegistry; + +namespace Wabbajack.BuildServer.Controllers +{ + [ApiController] + [Route("/lists")] + + public class ListValidation : AControllerBase + { + public ListValidation(ILogger logger, DBContext db) : base(logger, db) + { + } + + [HttpGet] + [Route("status")] + public async Task> HandleGetLists() + { + return await Db.ModListStatus.AsQueryable().Select(m => m.Summary).ToListAsync(); + } + } +} diff --git a/Wabbajack.BuildServer/Controllers/NexusCache.cs b/Wabbajack.BuildServer/Controllers/NexusCache.cs new file mode 100644 index 00000000..f9bd417d --- /dev/null +++ b/Wabbajack.BuildServer/Controllers/NexusCache.cs @@ -0,0 +1,136 @@ +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using Wabbajack.BuildServer.Models; +using Wabbajack.Lib.NexusApi; + +namespace Wabbajack.BuildServer.Controllers +{ + //[Authorize] + [ApiController] + [Route("/v1/games/")] + public class NexusCache : AControllerBase + { + + public NexusCache(ILogger logger, DBContext db) : base(logger, db) + { + } + + /// + /// Looks up the mod details for a given Gamename/ModId pair. If the entry is not found in the cache it will + /// be requested from the server (using the caller's Nexus API key if provided). + /// + /// + /// The Nexus game name + /// The Nexus mod id + /// A Mod Info result + [HttpGet] + [Route("{GameName}/mods/{ModId}.json")] + public async Task GetModInfo(string GameName, string ModId) + { + var result = await Db.NexusModInfos.FindOneAsync(info => info.Game == GameName && info.ModId == ModId); + + string method = "CACHED"; + if (result == null) + { + var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); + var path = $"/v1/games/{GameName}/mods/{ModId}.json"; + var body = await api.Get(path); + result = new NexusCacheData + { + Data = body, + Path = path, + Game = GameName, + ModId = ModId + }; + try + { + await Db.NexusModInfos.InsertOneAsync(result); + } + catch (MongoWriteException) + { + } + + method = "NOT_CACHED"; + } + + Response.Headers.Add("x-cache-result", method); + return result.Data; + } + + [HttpGet] + [Route("{GameName}/mods/{ModId}/files.json")] + public async Task GetModFiles(string GameName, string ModId) + { + var result = await Db.NexusModFiles.FindOneAsync(info => info.Game == GameName && info.ModId == ModId); + + string method = "CACHED"; + if (result == null) + { + var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); + var path = $"/v1/games/{GameName}/mods/{ModId}/files.json"; + var body = await api.Get(path); + result = new NexusCacheData + { + Data = body, + Path = path, + Game = GameName, + ModId = ModId + }; + try + { + await Db.NexusModFiles.InsertOneAsync(result); + } + catch (MongoWriteException) + { + + } + + method = "NOT_CACHED"; + } + + Response.Headers.Add("x-cache-result", method); + return result.Data; + } + + [HttpGet] + [Route("{GameName}/mods/{ModId}/files/{FileId}.json")] + public async Task GetFileInfo(string GameName, string ModId, string FileId) + { + var result = await Db.NexusFileInfos.FindOneAsync(info => info.Game == GameName && info.ModId == ModId && info.FileId == FileId); + + string method = "CACHED"; + if (result == null) + { + var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); + var path = $"/v1/games/{GameName}/mods/{ModId}/files/{FileId}.json"; + var body = await api.Get(path); + result = new NexusCacheData + { + Data = body, + Path = path, + Game = GameName, + ModId = ModId, + FileId = FileId + }; + try + { + await Db.NexusFileInfos.InsertOneAsync(result); + } + catch (MongoWriteException) + { + + } + + method = "NOT_CACHED"; + } + + Response.Headers.Add("x-cache-method", method); + return result.Data; + } + + + } +} diff --git a/Wabbajack.BuildServer/Extensions.cs b/Wabbajack.BuildServer/Extensions.cs new file mode 100644 index 00000000..bfd4a1a5 --- /dev/null +++ b/Wabbajack.BuildServer/Extensions.cs @@ -0,0 +1,17 @@ +using System; +using System.Linq; +using System.Linq.Expressions; +using System.Threading.Tasks; +using MongoDB.Driver; +using MongoDB.Driver.Linq; + +namespace Wabbajack.BuildServer +{ + public static class Extensions + { + public static async Task FindOneAsync(this IMongoCollection coll, Expression> expr) + { + return (await coll.AsQueryable().Where(expr).Take(1).ToListAsync()).FirstOrDefault(); + } + } +} diff --git a/Wabbajack.BuildServer/Models/DBContext.cs b/Wabbajack.BuildServer/Models/DBContext.cs new file mode 100644 index 00000000..e7e8f6f6 --- /dev/null +++ b/Wabbajack.BuildServer/Models/DBContext.cs @@ -0,0 +1,41 @@ +using System.Collections.Generic; +using Microsoft.Extensions.Configuration; +using MongoDB.Driver; +using Wabbajack.Lib.NexusApi; +using Wabbajack.BuildServer.Models.JobQueue; + +namespace Wabbajack.BuildServer.Models +{ + public class DBContext + { + private IConfiguration _configuration; + private Settings _settings; + public DBContext(IConfiguration configuration) + { + _configuration = configuration; + + _settings = new Settings(); + _configuration.Bind("MongoDB", _settings); + } + + public IMongoCollection> NexusModInfos => Client.GetCollection>(_settings.Collections["NexusModInfos"]); + public IMongoCollection> NexusFileInfos => Client.GetCollection>(_settings.Collections["NexusFileInfos"]); + public IMongoCollection ModListStatus => Client.GetCollection(_settings.Collections["ModListStatus"]); + + public IMongoCollection Jobs => Client.GetCollection(_settings.Collections["JobQueue"]); + public IMongoCollection DownloadStates => Client.GetCollection(_settings.Collections["DownloadStates"]); + public IMongoCollection IndexedFiles => Client.GetCollection(_settings.Collections["IndexedFiles"]); + + public IMongoCollection> NexusModFiles => + Client.GetCollection>( + _settings.Collections["NexusModFiles"]); + + private IMongoDatabase Client => new MongoClient($"mongodb://{_settings.Host}").GetDatabase(_settings.Database); + } + public class Settings + { + public string Host { get; set; } + public string Database { get; set; } + public Dictionary Collections { get; set; } + } +} diff --git a/Wabbajack.BuildServer/Models/DownloadState.cs b/Wabbajack.BuildServer/Models/DownloadState.cs new file mode 100644 index 00000000..b7f6ab24 --- /dev/null +++ b/Wabbajack.BuildServer/Models/DownloadState.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson.Serialization.Attributes; +using Wabbajack.Lib.Downloaders; + +namespace Wabbajack.BuildServer.Models +{ + public class DownloadState + { + [BsonId] + public string Key { get; set; } + public string Hash { get; set; } + + public AbstractDownloadState State { get; set; } + + public bool IsValid { get; set; } + public DateTime LastValidationTime { get; set; } = DateTime.Now; + public DateTime FirstValidationTime { get; set; } = DateTime.Now; + } +} diff --git a/Wabbajack.BuildServer/Models/IndexedFile.cs b/Wabbajack.BuildServer/Models/IndexedFile.cs new file mode 100644 index 00000000..6ef4d224 --- /dev/null +++ b/Wabbajack.BuildServer/Models/IndexedFile.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson.Serialization.Attributes; +using Wabbajack.VirtualFileSystem; + +namespace Wabbajack.BuildServer.Models +{ + public class IndexedFile + { + [BsonId] + public string Hash { get; set; } + public string SHA256 { get; set; } + public string SHA1 { get; set; } + public string MD5 { get; set; } + public string CRC { get; set; } + public long Size { get; set; } + public bool IsArchive { get; set; } + public List Children { get; set; } = new List(); + } + + public class ChildFile + { + public string Name; + public string Extension; + public string Hash; + } +} diff --git a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs new file mode 100644 index 00000000..48b37775 --- /dev/null +++ b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson.Serialization.Attributes; +using Wabbajack.BuildServer.Models.Jobs; + +namespace Wabbajack.BuildServer.Models.JobQueue +{ + public abstract class AJobPayload + { + public static List KnownSubTypes = new List + { + typeof(IndexJob) + + }; + public static Dictionary TypeToName { get; set; } + public static Dictionary NameToType { get; set; } + + + [BsonIgnore] + public abstract string Description { get; } + + public virtual bool UsesNexus { get; } = false; + + public abstract Task Execute(DBContext db); + + static AJobPayload() + { + NameToType = KnownSubTypes.ToDictionary(t => t.FullName.Substring(t.Namespace.Length + 1), t => t); + TypeToName = NameToType.ToDictionary(k => k.Value, k => k.Key); + } + + } +} diff --git a/Wabbajack.BuildServer/Models/JobQueue/Job.cs b/Wabbajack.BuildServer/Models/JobQueue/Job.cs new file mode 100644 index 00000000..2ee713e4 --- /dev/null +++ b/Wabbajack.BuildServer/Models/JobQueue/Job.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver; + +namespace Wabbajack.BuildServer.Models.JobQueue +{ + public class Job + { + public enum JobPriority : int + { + Low, + Normal, + High, + } + + [BsonId] + public Guid Id { get; set; } + public DateTime? Started { get; set; } + public DateTime? Ended { get; set; } + public DateTime Created { get; set; } = DateTime.Now; + public JobPriority Priority { get; set; } = JobPriority.Normal; + + public JobResult Result { get; set; } + public bool RequiresNexus { get; set; } = true; + public AJobPayload Payload { get; set; } + + public static async Task Enqueue(DBContext db, Job job) + { + await db.Jobs.InsertOneAsync(job); + return job.Id; + } + + public static async Task GetNext(DBContext db) + { + var filter = new BsonDocument + { + {"Started", BsonNull.Value} + }; + var update = new BsonDocument + { + {"$set", new BsonDocument {{"Started", DateTime.Now}}} + }; + var sort = new {Priority=-1, Created=1}.ToBsonDocument(); + var job = await db.Jobs.FindOneAndUpdateAsync(filter, update, new FindOneAndUpdateOptions{Sort = sort}); + return job; + } + + public static async Task Finish(DBContext db, Job job, JobResult jobResult) + { + var filter = new BsonDocument + { + {"query", new BsonDocument {{"Id", job.Id}}}, + }; + var update = new BsonDocument + { + {"$set", new BsonDocument {{"Ended", DateTime.Now}, {"Result", jobResult.ToBsonDocument()}}} + }; + var result = await db.Jobs.FindOneAndUpdateAsync(filter, update); + return result; + } + } +} diff --git a/Wabbajack.BuildServer/Models/JobQueue/JobResult.cs b/Wabbajack.BuildServer/Models/JobQueue/JobResult.cs new file mode 100644 index 00000000..6747ef13 --- /dev/null +++ b/Wabbajack.BuildServer/Models/JobQueue/JobResult.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using MongoDB.Bson.Serialization.Attributes; + +namespace Wabbajack.BuildServer.Models.JobQueue +{ + public class JobResult + { + public JobResultType ResultType { get; set; } + [BsonIgnoreIfNull] + public string Message { get; set; } + + [BsonIgnoreIfNull] + public string Stacktrace { get; set; } + + public static JobResult Success() + { + return new JobResult { ResultType = JobResultType.Success }; + } + + public static JobResult Error(Exception ex) + { + return new JobResult {ResultType = JobResultType.Error, Stacktrace = ex.ToString()}; + } + + } + + public enum JobResultType + { + Success, + Error + } +} diff --git a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs new file mode 100644 index 00000000..69c679b1 --- /dev/null +++ b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Wabbajack.BuildServer.Models; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.VirtualFileSystem; + +namespace Wabbajack.BuildServer.Models.Jobs +{ + + public class IndexJob : AJobPayload + { + public Archive Archive { get; set; } + public override string Description { get; } = "Validate and index an archive"; + public override bool UsesNexus { get => Archive.State is NexusDownloader.State; } + public override async Task Execute(DBContext db) + { + + /* + + var pk = new List(); + pk.Add(AbstractDownloadState.TypeToName[Archive.State.GetType()]); + pk.AddRange(Archive.State.PrimaryKey); + var pk_str = string.Join("|",pk.Select(p => p.ToString())); + + var found = await db.DownloadStates.AsQueryable().Where(f => f.Key == pk_str).Take(1).ToListAsync(); + if (found.Count > 0) + return JobResult.Success(); + + string fileName = Archive.Name; + string folder = Guid.NewGuid().ToString(); + Utils.Log($"Indexer is downloading {fileName}"); + var downloadDest = Path.Combine(Server.Config.Indexer.DownloadDir, folder, fileName); + await Archive.State.Download(downloadDest); + + using (var queue = new WorkQueue()) + { + var vfs = new Context(queue, true); + await vfs.AddRoot(Path.Combine(Server.Config.Indexer.DownloadDir, folder)); + var archive = vfs.Index.ByRootPath.First(); + var converted = ConvertArchive(new List(), archive.Value); + try + { + await db.IndexedFiles.InsertManyAsync(converted, new InsertManyOptions {IsOrdered = false}); + } + catch (MongoBulkWriteException) + { + } + + await db.DownloadStates.InsertOneAsync(new DownloadState + { + Key = pk_str, + Hash = archive.Value.Hash, + State = Archive.State, + IsValid = true + }); + + var to_path = Path.Combine(Server.Config.Indexer.ArchiveDir, + $"{Path.GetFileName(fileName)}_{archive.Value.Hash.FromBase64().ToHex()}_{Path.GetExtension(fileName)}"); + if (File.Exists(to_path)) + File.Delete(downloadDest); + else + File.Move(downloadDest, to_path); + Utils.DeleteDirectory(Path.Combine(Server.Config.Indexer.DownloadDir, folder)); + } +*/ + return JobResult.Success(); + } + + private List ConvertArchive(List files, VirtualFile file, bool isTop = true) + { + var name = isTop ? Path.GetFileName(file.Name) : file.Name; + var ifile = new IndexedFile + { + Hash = file.Hash, + SHA256 = file.ExtendedHashes.SHA256, + SHA1 = file.ExtendedHashes.SHA1, + MD5 = file.ExtendedHashes.MD5, + CRC = file.ExtendedHashes.CRC, + Size = file.Size, + Children = file.Children != null ? file.Children.Select( + f => + { + ConvertArchive(files, f, false); + + return new ChildFile + { + Hash = f.Hash, + Name = f.Name.ToLowerInvariant(), + Extension = Path.GetExtension(f.Name.ToLowerInvariant()) + }; + }).ToList() : new List() + }; + ifile.IsArchive = ifile.Children.Count > 0; + files.Add(ifile); + return files; + } + + + } + +} diff --git a/Wabbajack.BuildServer/Models/ModListStatus.cs b/Wabbajack.BuildServer/Models/ModListStatus.cs new file mode 100644 index 00000000..e42b84c7 --- /dev/null +++ b/Wabbajack.BuildServer/Models/ModListStatus.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Wabbajack.Lib; +using Wabbajack.Lib.ModListRegistry; + +namespace Wabbajack.BuildServer.Models +{ + public class ModListStatus + { + + [BsonId] + public string Id { get; set; } + public ModlistSummary Summary { get; set; } + + public ModlistMetadata Metadata { get; set; } + public DetailedStatus DetailedStatus { get; set; } + + public static async Task Update(DBContext db, ModListStatus status) + { + var id = status.Metadata.Links.MachineURL; + await db.ModListStatus.FindOneAndReplaceAsync(s => s.Id == id, status, new FindOneAndReplaceOptions {IsUpsert = true}); + } + + public static IQueryable AllSummaries + { + get + { + return null; + } + } + + public static async Task ByName(DBContext db, string name) + { + var result = await db.ModListStatus + .AsQueryable() + .Where(doc => doc.Metadata.Links.MachineURL == name || doc.Metadata.Title == name) + .ToListAsync(); + return result.First(); + } + } + + public class DetailedStatus + { + public string Name { get; set; } + public DateTime Checked { get; set; } = DateTime.Now; + public List Archives { get; set; } + public DownloadMetadata DownloadMetaData { get; set; } + public bool HasFailures { get; set; } + public string MachineName { get; set; } + } + + public class DetailedStatusItem + { + public bool IsFailing { get; set; } + public Archive Archive { get; set; } + } +} diff --git a/Wabbajack.BuildServer/Models/NexusCacheData.cs b/Wabbajack.BuildServer/Models/NexusCacheData.cs new file mode 100644 index 00000000..4bb0b19a --- /dev/null +++ b/Wabbajack.BuildServer/Models/NexusCacheData.cs @@ -0,0 +1,20 @@ +using System; +using MongoDB.Bson.Serialization.Attributes; + +namespace Wabbajack.BuildServer.Models +{ + public class NexusCacheData + { + [BsonId] + public string Path { get; set; } + public T Data { get; set; } + public string Game { get; set; } + public string ModId { get; set; } + + public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow; + + [BsonIgnoreIfNull] + public string FileId { get; set; } + + } +} diff --git a/Wabbajack.BuildServer/SerializerSettings.cs b/Wabbajack.BuildServer/SerializerSettings.cs new file mode 100644 index 00000000..413f0be9 --- /dev/null +++ b/Wabbajack.BuildServer/SerializerSettings.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc.ModelBinding; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Conventions; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.Lib.Downloaders; +using Newtonsoft.Json. + +namespace Wabbajack.BuildServer +{ + public static class SerializerSettings + { + public static void Init() + { + var dis = new TypeDiscriminator(typeof(AbstractDownloadState), AbstractDownloadState.NameToType, + AbstractDownloadState.TypeToName); + BsonSerializer.RegisterDiscriminatorConvention(typeof(AbstractDownloadState), dis); + BsonClassMap.RegisterClassMap(cm => cm.SetIsRootClass(true)); + + dis = new TypeDiscriminator(typeof(AJobPayload), AJobPayload.NameToType, AJobPayload.TypeToName); + BsonSerializer.RegisterDiscriminatorConvention(typeof(AJobPayload), dis); + BsonClassMap.RegisterClassMap(cm => cm.SetIsRootClass(true)); + + } + } + + public class TypeDiscriminator : IDiscriminatorConvention + { + private readonly Type defaultType; + private readonly Dictionary typeMap; + private Dictionary revMap; + + public TypeDiscriminator(Type defaultType, + Dictionary typeMap, Dictionary revMap) + { + this.defaultType = defaultType; + this.typeMap = typeMap; + this.revMap = revMap; + } + + + /// + /// Element Name + /// + public string ElementName => "_wjType"; + + public Type GetActualType(IBsonReader bsonReader, Type nominalType) + { + Type type = null; + var bookmark = bsonReader.GetBookmark(); + bsonReader.ReadStartDocument(); + if (bsonReader.FindElement(ElementName)) + { + var value = bsonReader.ReadString(); + if (typeMap.ContainsKey(value)) + type = typeMap[value]; + } + + bsonReader.ReturnToBookmark(bookmark); + if (type == null) + throw new Exception($"Type mis-configuration can't find bson type for ${nominalType}"); + return type; + } + + public BsonValue GetDiscriminator(Type nominalType, Type actualType) + { + return revMap[actualType]; + } + } +} diff --git a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj new file mode 100644 index 00000000..5411cfed --- /dev/null +++ b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj @@ -0,0 +1,49 @@ + + + + netcoreapp3.1 + aspnet-Wabbajack.BuildServer-6E798B30-DB04-4436-BE65-F043AF37B314 + 0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj b/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj index 435ad309..c8ab53dc 100644 --- a/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj +++ b/Wabbajack.CacheServer/Wabbajack.CacheServer.csproj @@ -122,7 +122,6 @@ - diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index 038777a8..b024018c 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -109,12 +109,19 @@ namespace Wabbajack.Lib.ModListRegistry public class ModlistSummary { - public string Name; - public DateTime Checked; - public int Failed; - public int Passed; + [JsonProperty("name")] + public string Name { get; set; } + [JsonProperty("checked")] + public DateTime Checked { get; set; } + [JsonProperty("failed")] + public int Failed { get; set; } + [JsonProperty("passed")] + public int Passed { get; set; } + [JsonProperty("link")] public string Link => $"/lists/status/{Name}.json"; + [JsonProperty("report")] public string Report => $"/lists/status/{Name}.html"; + [JsonProperty("has_failures")] public bool HasFailures => Failed > 0; } diff --git a/Wabbajack.Lib/NexusApi/Dtos.cs b/Wabbajack.Lib/NexusApi/Dtos.cs index bd948390..fab90bcd 100644 --- a/Wabbajack.Lib/NexusApi/Dtos.cs +++ b/Wabbajack.Lib/NexusApi/Dtos.cs @@ -15,35 +15,35 @@ namespace Wabbajack.Lib.NexusApi public class NexusFileInfo { - public ulong category_id; - public string category_name; - public string changelog_html; - public string description; - public string external_virus_scan_url; - public ulong file_id; - public string file_name; - public bool is_primary; - public string mod_version; - public string name; - public ulong size; - public ulong size_kb; - public DateTime uploaded_time; - public ulong uploaded_timestamp; - public string version; + public ulong category_id { get; set; } + public string category_name { get; set; } + public string changelog_html { get; set; } + public string description { get; set; } + public string external_virus_scan_url { get; set; } + public ulong file_id { get; set; } + public string file_name { get; set; } + public bool is_primary { get; set; } + public string mod_version { get; set; } + public string name { get; set; } + public ulong size { get; set; } + public ulong size_kb { get; set; } + public DateTime uploaded_time { get; set; } + public ulong uploaded_timestamp { get; set; } + public string version { get; set; } } public class ModInfo { - public uint _internal_version; - public string game_name; - public string mod_id; - public string name; - public string summary; - public string author; - public string uploaded_by; - public string uploaded_users_profile_url; - public string picture_url; - public bool contains_adult_content; + public uint _internal_version { get; set; } + public string game_name { get; set; } + public string mod_id { get; set; } + public string name { get; set; } + public string summary { get; set; } + public string author { get; set; } + public string uploaded_by { get; set; } + public string uploaded_users_profile_url { get; set; } + public string picture_url { get; set; } + public bool contains_adult_content { get; set; } } public class MD5Response diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index c4e417bb..2032ced8 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -272,7 +272,7 @@ namespace Wabbajack.Lib.NexusApi public class GetModFilesResponse { - public List files; + public List files { get; set; } } public async Task GetModFiles(Game game, int modid) diff --git a/Wabbajack.sln b/Wabbajack.sln index 3eaf21ee..05a4fa59 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -36,6 +36,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.CacheServer", "Wa EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OMODExtractor", "OMODExtractor\OMODExtractor.csproj", "{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.BuildServer", "Wabbajack.BuildServer\Wabbajack.BuildServer.csproj", "{DE18D89E-39C5-48FD-8E42-16235E3C4593}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug (no commandargs)|Any CPU = Debug (no commandargs)|Any CPU @@ -247,6 +249,24 @@ Global {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x86.Build.0 = Release|Any CPU {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.ActiveCfg = Release|x64 {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.Build.0 = Release|x64 + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|x86.ActiveCfg = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|x86.Build.0 = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.ActiveCfg = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.Build.0 = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x86.ActiveCfg = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x86.Build.0 = Debug|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|Any CPU.Build.0 = Release|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.ActiveCfg = Release|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.Build.0 = Release|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x86.ActiveCfg = Release|Any CPU + {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE