diff --git a/Wabbajack.BuildServer.Test/ADBTest.cs b/Wabbajack.BuildServer.Test/ADBTest.cs index 627ab4e7..2c422382 100644 --- a/Wabbajack.BuildServer.Test/ADBTest.cs +++ b/Wabbajack.BuildServer.Test/ADBTest.cs @@ -28,7 +28,7 @@ namespace Wabbajack.BuildServer.Test { DBName = "test_db" + Guid.NewGuid().ToString().Replace("-", "_"); User = Guid.NewGuid().ToString().Replace("-", ""); - APIKey = Users.NewAPIKey(); + APIKey = SqlService.NewAPIKey(); } public string APIKey { get; } diff --git a/Wabbajack.BuildServer.Test/LoginTests.cs b/Wabbajack.BuildServer.Test/LoginTests.cs new file mode 100644 index 00000000..e04676f3 --- /dev/null +++ b/Wabbajack.BuildServer.Test/LoginTests.cs @@ -0,0 +1,39 @@ +using System; +using System.Threading.Tasks; +using Wabbajack.Common; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.BuildServer.Test +{ + public class LoginTests : ABuildServerSystemTest + { + public LoginTests(ITestOutputHelper output, SingletonAdaptor fixture) : base(output, fixture) + { + } + + [Fact] + public async Task CanCreateLogins() + { + var newUserName = Guid.NewGuid().ToString(); + + var newKey = await _authedClient.GetStringAsync(MakeURL($"users/add/{newUserName}")); + + Assert.NotEmpty(newKey); + Assert.NotNull(newKey); + Assert.NotEqual(newKey, Fixture.APIKey); + + + var done = await _authedClient.GetStringAsync(MakeURL("users/export")); + Assert.Equal("done", done); + + foreach (var (userName, apiKey) in new[] {(newUserName, newKey), (Fixture.User, Fixture.APIKey)}) + { + var exported = await Fixture.ServerTempFolder.Combine("exported_users", userName, Consts.AuthorAPIKeyFile) + .ReadAllTextAsync(); + Assert.Equal(exported, apiKey); + + } + } + } +} diff --git a/Wabbajack.BuildServer/Controllers/AControllerBase.cs b/Wabbajack.BuildServer/Controllers/AControllerBase.cs index 15c94dbb..3820fdca 100644 --- a/Wabbajack.BuildServer/Controllers/AControllerBase.cs +++ b/Wabbajack.BuildServer/Controllers/AControllerBase.cs @@ -1,13 +1,8 @@ using System; -using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using Alphaleonis.Win32.Filesystem; -using GraphQL; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using Org.BouncyCastle.Ocsp; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; using Wabbajack.Common; @@ -17,13 +12,11 @@ namespace Wabbajack.BuildServer.Controllers [ApiController] public abstract class AControllerBase : ControllerBase { - protected readonly DBContext Db; protected readonly ILogger Logger; protected readonly SqlService SQL; - protected AControllerBase(ILogger logger, DBContext db, SqlService sql) + protected AControllerBase(ILogger logger, SqlService sql) { - Db = db; Logger = logger; SQL = sql; } diff --git a/Wabbajack.BuildServer/Controllers/GraphQL.cs b/Wabbajack.BuildServer/Controllers/GraphQL.cs index 95eb5093..6afa21d1 100644 --- a/Wabbajack.BuildServer/Controllers/GraphQL.cs +++ b/Wabbajack.BuildServer/Controllers/GraphQL.cs @@ -13,18 +13,17 @@ namespace Wabbajack.BuildServer.Controllers [ApiController] public class GraphQL : AControllerBase { - private SqlService _sql; - - public GraphQL(ILogger logger, DBContext db, SqlService sql) : base(logger, db, sql) + public GraphQL(ILogger logger, SqlService sql) : base(logger, sql) { - _sql = sql; } [HttpPost] public async Task Post([FromBody] GraphQLQuery query) { var inputs = query.Variables.ToInputs(); - var schema = new Schema {Query = new Query(Db, _sql), Mutation = new Mutation(Db)}; + var schema = new Schema { + Query = new Query(SQL) + }; var result = await new DocumentExecuter().ExecuteAsync(_ => { diff --git a/Wabbajack.BuildServer/Controllers/Heartbeat.cs b/Wabbajack.BuildServer/Controllers/Heartbeat.cs index 16680d1e..5f696e20 100644 --- a/Wabbajack.BuildServer/Controllers/Heartbeat.cs +++ b/Wabbajack.BuildServer/Controllers/Heartbeat.cs @@ -1,17 +1,10 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Threading.Tasks; -using Alphaleonis.Win32.Filesystem; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; -using MongoDB.Bson; -using MongoDB.Driver; using Wabbajack.BuildServer.Model.Models; -using Wabbajack.BuildServer.Models; -using Wabbajack.Common; using Wabbajack.Common.StatusFeed; namespace Wabbajack.BuildServer.Controllers @@ -26,7 +19,7 @@ namespace Wabbajack.BuildServer.Controllers } private static DateTime _startTime; - public Heartbeat(ILogger logger, DBContext db, SqlService sql) : base(logger, db, sql) + public Heartbeat(ILogger logger, SqlService sql) : base(logger, sql) { } diff --git a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs index 4f3b465a..6db8b742 100644 --- a/Wabbajack.BuildServer/Controllers/IndexedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/IndexedFiles.cs @@ -5,21 +5,15 @@ using System.IO.Compression; using System.Linq; using System.Text; using System.Threading.Tasks; -using DynamicData; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; -using MongoDB.Bson; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.BuildServer.Model.Models; -using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; -using Wabbajack.VirtualFileSystem; using IndexedFile = Wabbajack.BuildServer.Models.IndexedFile; namespace Wabbajack.BuildServer.Controllers @@ -30,7 +24,7 @@ namespace Wabbajack.BuildServer.Controllers private SqlService _sql; private AppSettings _settings; - public IndexedFiles(ILogger logger, DBContext db, SqlService sql, AppSettings settings) : base(logger, db, sql) + public IndexedFiles(ILogger logger, SqlService sql, AppSettings settings) : base(logger, sql) { _settings = settings; _sql = sql; diff --git a/Wabbajack.BuildServer/Controllers/Jobs.cs b/Wabbajack.BuildServer/Controllers/Jobs.cs index 1a8fa84a..210bf318 100644 --- a/Wabbajack.BuildServer/Controllers/Jobs.cs +++ b/Wabbajack.BuildServer/Controllers/Jobs.cs @@ -1,13 +1,9 @@ using System; -using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.BuildServer.Model.Models; -using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; namespace Wabbajack.BuildServer.Controllers @@ -17,27 +13,17 @@ namespace Wabbajack.BuildServer.Controllers [Route("/jobs")] public class Jobs : AControllerBase { - public Jobs(ILogger logger, DBContext db, SqlService sql) : base(logger, db, sql) + public Jobs(ILogger logger, SqlService sql) : base(logger, sql) { } - [HttpGet] - [Route("unfinished")] - public async Task> GetUnfinished() - { - return await Db.Jobs.AsQueryable() - .Where(j => j.Ended == null) - .OrderByDescending(j => j.Priority) - .ToListAsync(); - } - [HttpGet] [Route("enqueue_job/{JobName}")] public async Task EnqueueJob(string JobName) { var jobtype = AJobPayload.NameToType[JobName]; var job = new Job{Priority = Job.JobPriority.High, Payload = (AJobPayload)jobtype.GetConstructor(new Type[0]).Invoke(new object?[0])}; - await Db.Jobs.InsertOneAsync(job); + await SQL.EnqueueJob(job); return job.Id; } } diff --git a/Wabbajack.BuildServer/Controllers/ListValidation.cs b/Wabbajack.BuildServer/Controllers/ListValidation.cs index 0c4c59f3..c57c7215 100644 --- a/Wabbajack.BuildServer/Controllers/ListValidation.cs +++ b/Wabbajack.BuildServer/Controllers/ListValidation.cs @@ -2,16 +2,11 @@ using System.Collections.Generic; using System.Linq; using System.Net; -using System.ServiceModel.Syndication; using System.Threading.Tasks; -using Microsoft.AspNetCore.Http.Extensions; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Nettle; using Wabbajack.BuildServer.Model.Models; -using Wabbajack.BuildServer.Models; using Wabbajack.Common; using Wabbajack.Lib.ModListRegistry; @@ -21,7 +16,7 @@ namespace Wabbajack.BuildServer.Controllers [Route("/lists")] public class ListValidation : AControllerBase { - public ListValidation(ILogger logger, DBContext db, SqlService sql) : base(logger, db, sql) + public ListValidation(ILogger logger, SqlService sql) : base(logger, sql) { } diff --git a/Wabbajack.BuildServer/Controllers/MetricsController.cs b/Wabbajack.BuildServer/Controllers/MetricsController.cs index 4fe8541d..9497649d 100644 --- a/Wabbajack.BuildServer/Controllers/MetricsController.cs +++ b/Wabbajack.BuildServer/Controllers/MetricsController.cs @@ -1,17 +1,11 @@ using System; -using System.Collections.Generic; -using System.Data.SqlTypes; using System.Linq; using System.Threading.Tasks; -using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; using Wabbajack.Common; -using Wabbajack.Lib.ModListRegistry; namespace Wabbajack.BuildServer.Controllers { @@ -19,11 +13,8 @@ namespace Wabbajack.BuildServer.Controllers [Route("/metrics")] public class MetricsController : AControllerBase { - private SqlService _sql; - - public MetricsController(ILogger logger, DBContext db, SqlService sql) : base(logger, db, sql) + public MetricsController(ILogger logger, SqlService sql) : base(logger, sql) { - _sql = sql; } [HttpGet] @@ -35,20 +26,10 @@ namespace Wabbajack.BuildServer.Controllers return new Result { Timestamp = date}; } - [Authorize] - [HttpGet] - [Route("transfer")] - public async Task Transfer() - { - var all_metrics = await Db.Metrics.AsQueryable().ToListAsync(); - await _sql.IngestAllMetrics(all_metrics); - return "done"; - } - private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null) { Logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}"); - await _sql.IngestMetric(new Metric + await SQL.IngestMetric(new Metric { Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey }); diff --git a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs index a695d386..00566e60 100644 --- a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs +++ b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs @@ -8,8 +8,6 @@ using FluentFTP; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; @@ -30,7 +28,7 @@ namespace Wabbajack.BuildServer.Controllers private AppSettings _settings; private SqlService _sql; - public ModlistUpdater(ILogger logger, DBContext db, SqlService sql, AppSettings settings) : base(logger, db, sql) + public ModlistUpdater(ILogger logger, SqlService sql, AppSettings settings) : base(logger, sql) { _settings = settings; _sql = sql; @@ -93,22 +91,21 @@ namespace Wabbajack.BuildServer.Controllers Utils.Log($"Alternative requested for {startingHash}"); await Metric("requested_upgrade", startingHash.ToString()); - var state = await Db.DownloadStates.AsQueryable() + var state = await SQL.GetNexusStateByHash(startingHash); + + /*.DownloadStates.AsQueryable() .Where(s => s.Hash == startingHash) .Where(s => s.State is NexusDownloader.State) - .OrderByDescending(s => s.LastValidationTime).FirstOrDefaultAsync(); + .OrderByDescending(s => s.LastValidationTime).FirstOrDefaultAsync();*/ if (state == null) return NotFound("Original state not found"); var nexusState = state.State as NexusDownloader.State; - var nexusGame = nexusState.Game.MetaData().NexusName; - var mod_files = await Db.NexusModFiles.AsQueryable() - .Where(f => f.Game == nexusGame && f.ModId == nexusState.ModID) - .ToListAsync(); + var nexusGame = nexusState.Game; + var mod_files = (await SQL.GetModFiles(nexusGame, nexusState.ModID)).files; - if (mod_files.SelectMany(f => f.Data.files) - .Any(f => f.category_name != null && f.file_id == nexusState.FileID)) + if (mod_files.Any(f => f.category_name != null && f.file_id == nexusState.FileID)) { await Metric("not_required_upgrade", startingHash.ToString()); return BadRequest("Upgrade Not Required"); @@ -124,7 +121,7 @@ namespace Wabbajack.BuildServer.Controllers Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}"); if (newArchive.Hash == Hash.Empty) { - Db.Jobs.InsertOne(new Job + await SQL.EnqueueJob(new Job { Payload = new IndexJob { @@ -147,7 +144,7 @@ namespace Wabbajack.BuildServer.Controllers if (!PatchArchive.CdnPath(startingHash, newArchive.Hash).Exists) { - Db.Jobs.InsertOne(new Job + await SQL.EnqueueJob(new Job { Priority = Job.JobPriority.High, Payload = new PatchArchive @@ -185,7 +182,7 @@ namespace Wabbajack.BuildServer.Controllers Utils.Log($"Found alternative for {srcHash}"); - var indexed = await Db.DownloadStates.AsQueryable().Where(s => s.Key == archive.State.PrimaryKeyString).FirstOrDefaultAsync(); + var indexed = await SQL.DownloadStateByPrimaryKey(archive.State.PrimaryKeyString); if (indexed == null) { @@ -195,9 +192,6 @@ namespace Wabbajack.BuildServer.Controllers Utils.Log($"Pre-Indexed alternative {indexed.Hash} found for {srcHash}"); archive.Hash = indexed.Hash; return archive; - } - - } } diff --git a/Wabbajack.BuildServer/Controllers/NexusCache.cs b/Wabbajack.BuildServer/Controllers/NexusCache.cs index 20304693..162e874e 100644 --- a/Wabbajack.BuildServer/Controllers/NexusCache.cs +++ b/Wabbajack.BuildServer/Controllers/NexusCache.cs @@ -1,15 +1,11 @@ using System; using System.Collections.Generic; -using System.Globalization; -using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; -using CsvHelper; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; -using MongoDB.Driver; using Newtonsoft.Json; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; @@ -28,7 +24,7 @@ namespace Wabbajack.BuildServer.Controllers private static long CachedCount = 0; private static long ForwardCount = 0; - public NexusCache(ILogger logger, DBContext db, SqlService sql, AppSettings settings) : base(logger, db, sql) + public NexusCache(ILogger logger, SqlService sql, AppSettings settings) : base(logger, sql) { _settings = settings; } @@ -55,13 +51,7 @@ namespace Wabbajack.BuildServer.Controllers var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); var path = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{ModId}.json"; var body = await api.Get(path); - try - { - await SQL.AddNexusModInfo(game, ModId, DateTime.Now, body); - } - catch (MongoWriteException) - { - } + await SQL.AddNexusModInfo(game, ModId, DateTime.Now, body); method = "NOT_CACHED"; Interlocked.Increment(ref ForwardCount); @@ -90,13 +80,8 @@ namespace Wabbajack.BuildServer.Controllers var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files.json"; var body = await api.Get(path); - try - { - await SQL.AddNexusModFiles(game, ModId, DateTime.Now, body); - } - catch (MongoWriteException) - { - } + await SQL.AddNexusModFiles(game, ModId, DateTime.Now, body); + method = "NOT_CACHED"; Interlocked.Increment(ref ForwardCount); diff --git a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs index 57a25a3b..78eb83ad 100644 --- a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs @@ -9,23 +9,16 @@ using System.Text; using System.Threading.Tasks; using FluentFTP; using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Nettle; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using Org.BouncyCastle.Crypto.Engines; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.Common; -using Wabbajack.Lib; -using Wabbajack.Lib.Downloaders; using Path = Alphaleonis.Win32.Filesystem.Path; using AlphaFile = Alphaleonis.Win32.Filesystem.File; @@ -36,7 +29,7 @@ namespace Wabbajack.BuildServer.Controllers private static ConcurrentDictionary _writeLocks = new ConcurrentDictionary(); private AppSettings _settings; - public UploadedFiles(ILogger logger, DBContext db, AppSettings settings, SqlService sql) : base(logger, db, sql) + public UploadedFiles(ILogger logger, AppSettings settings, SqlService sql) : base(logger, sql) { _settings = settings; } @@ -88,7 +81,7 @@ namespace Wabbajack.BuildServer.Controllers [Route("clean_http_uploads")] public async Task CleanUploads() { - var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync(); + var files = await SQL.AllUploadedFiles(); var seen = new HashSet(); var duplicate = new List(); @@ -115,7 +108,7 @@ namespace Wabbajack.BuildServer.Controllers if (await client.FileExistsAsync(dup.MungedName)) await client.DeleteFileAsync(dup.MungedName); - await Db.UploadedFiles.DeleteOneAsync(f => f.Id == dup.Id); + await SQL.DeleteUploadedFile(dup.Id); } } @@ -182,7 +175,7 @@ namespace Wabbajack.BuildServer.Controllers [Route("uploaded_files")] public async Task UploadedFilesGet() { - var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync(); + var files = await SQL.AllUploadedFiles(); var response = HandleGetListTemplate(new { files = files.Select(file => new @@ -221,7 +214,7 @@ namespace Wabbajack.BuildServer.Controllers { var user = User.FindFirstValue(ClaimTypes.Name); Utils.Log($"Delete Uploaded File {user} {name}"); - var files = await Db.UploadedFiles.AsQueryable().Where(f => f.Uploader == user).ToListAsync(); + var files = await SQL.AllUploadedFilesForUser(name); var to_delete = files.First(f => f.MungedName == name); @@ -237,10 +230,8 @@ namespace Wabbajack.BuildServer.Controllers } - var result = await Db.UploadedFiles.DeleteOneAsync(f => f.Id == to_delete.Id); - if (result.DeletedCount == 1) - return Ok($"Deleted {name}"); - return NotFound(name); + await SQL.DeleteUploadedFile(to_delete.Id); + return Ok($"Deleted {to_delete.MungedName}"); } [HttpGet] diff --git a/Wabbajack.BuildServer/Controllers/Users.cs b/Wabbajack.BuildServer/Controllers/Users.cs index aeafaf9d..40aabd6a 100644 --- a/Wabbajack.BuildServer/Controllers/Users.cs +++ b/Wabbajack.BuildServer/Controllers/Users.cs @@ -1,13 +1,8 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Alphaleonis.Win32.Filesystem; +using System.Threading.Tasks; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; -using MongoDB.Driver; using Wabbajack.BuildServer.Model.Models; -using Wabbajack.BuildServer.Models; using Wabbajack.Common; namespace Wabbajack.BuildServer.Controllers @@ -16,47 +11,37 @@ namespace Wabbajack.BuildServer.Controllers [Route("/users")] public class Users : AControllerBase { - public Users(ILogger logger, DBContext db, SqlService sql) : base(logger, db, sql) + private AppSettings _settings; + + public Users(ILogger logger, SqlService sql, AppSettings settings) : base(logger, sql) { + _settings = settings; } [HttpGet] [Route("add/{Name}")] public async Task AddUser(string Name) { - var user = new ApiKey(); - user.Key = NewAPIKey(); - user.Id = Guid.NewGuid().ToString(); - user.Roles = new List(); - user.CanUploadLists = new List(); - - await Db.ApiKeys.InsertOneAsync(user); - - return user.Id; + return await SQL.AddLogin(Name); } [HttpGet] [Route("export")] public async Task Export() { - if (!Directory.Exists("exported_users")) - Directory.CreateDirectory("exported_users"); + var mainFolder = _settings.TempPath.Combine("exported_users"); + mainFolder.CreateDirectory(); - foreach (var user in await Db.ApiKeys.AsQueryable().ToListAsync()) + foreach (var (owner, key) in await SQL.GetAllUserKeys()) { - Directory.CreateDirectory(Path.Combine("exported_users", user.Owner)); - Alphaleonis.Win32.Filesystem.File.WriteAllText(Path.Combine("exported_users", user.Owner, "author-api-key.txt"), user.Key); + var folder = mainFolder.Combine(owner); + folder.CreateDirectory(); + await folder.Combine(Consts.AuthorAPIKeyFile).WriteAllTextAsync(key); } return "done"; } - public static string NewAPIKey() - { - var arr = new byte[128]; - new Random().NextBytes(arr); - return arr.ToHex(); - } } } diff --git a/Wabbajack.BuildServer/Extensions.cs b/Wabbajack.BuildServer/Extensions.cs index 2d9a4529..4a4c8013 100644 --- a/Wabbajack.BuildServer/Extensions.cs +++ b/Wabbajack.BuildServer/Extensions.cs @@ -4,25 +4,15 @@ using System.IO; using System.Linq; using System.Linq.Expressions; using System.Threading.Tasks; -using Alphaleonis.Win32.Filesystem; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Builder; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.Common; -using Directory =Alphaleonis.Win32.Filesystem.Directory; using File = Alphaleonis.Win32.Filesystem.File; -using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.BuildServer { public static class Extensions { - public static async Task FindOneAsync(this IMongoCollection coll, Expression> expr) - { - return (await coll.AsQueryable().Where(expr).Take(1).ToListAsync()).FirstOrDefault(); - } - public static void UseJobManager(this IApplicationBuilder b) { var manager = (JobManager)b.ApplicationServices.GetService(typeof(JobManager)); diff --git a/Wabbajack.BuildServer/GraphQL/Mutation.cs b/Wabbajack.BuildServer/GraphQL/Mutation.cs deleted file mode 100644 index 24da19a7..00000000 --- a/Wabbajack.BuildServer/GraphQL/Mutation.cs +++ /dev/null @@ -1,22 +0,0 @@ -using GraphQL.Types; -using Wabbajack.BuildServer.Models; -using Wabbajack.BuildServer.Models.JobQueue; -using Wabbajack.BuildServer.Models.Jobs; - -namespace Wabbajack.BuildServer.GraphQL -{ - public class Mutation : ObjectGraphType - { - public Mutation(DBContext db) - { - FieldAsync("pollNexusForUpdates", - resolve: async context => - { - var job = new Job {Payload = new GetNexusUpdatesJob()}; - await db.Jobs.InsertOneAsync(job); - return job.Id; - }); - } - - } -} diff --git a/Wabbajack.BuildServer/GraphQL/Query.cs b/Wabbajack.BuildServer/GraphQL/Query.cs index 4d336e4d..0a5b0f41 100644 --- a/Wabbajack.BuildServer/GraphQL/Query.cs +++ b/Wabbajack.BuildServer/GraphQL/Query.cs @@ -1,28 +1,13 @@ -using System; -using System.Linq; -using System.Collections.Generic; -using System.Data.SqlTypes; -using GraphQL; +using System.Linq; using GraphQL.Types; -using GraphQLParser.AST; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.BuildServer.Model.Models; -using Wabbajack.BuildServer.Models; -using Wabbajack.Common; namespace Wabbajack.BuildServer.GraphQL { public class Query : ObjectGraphType { - public Query(DBContext db, SqlService sql) + public Query(SqlService sql) { - Field>("unfinishedJobs", resolve: context => - { - var data = db.Jobs.AsQueryable().Where(j => j.Ended == null).ToList(); - return data; - }); - FieldAsync>("modLists", arguments: new QueryArguments(new QueryArgument { @@ -31,37 +16,17 @@ namespace Wabbajack.BuildServer.GraphQL resolve: async context => { var arg = context.GetArgument("filter"); - var lists = db.ModListStatus.AsQueryable(); + var lists = await sql.GetDetailedModlistStatuses(); switch (arg) { case "FAILED": - lists = lists.Where(l => l.DetailedStatus.HasFailures); - break; + return lists.Where(l => l.HasFailures); case "PASSED": - lists = lists.Where(a => !a.DetailedStatus.HasFailures); - break; + return lists.Where(l => !l.HasFailures); default: - break; + return lists; } - return await lists.ToListAsync(); - }); - - FieldAsync>("job", - arguments: new QueryArguments( - new QueryArgument {Name = "id", Description = "Id of the Job"}), - resolve: async context => - { - var id = context.GetArgument("id"); - var data = await db.Jobs.AsQueryable().Where(j => j.Id == id).ToListAsync(); - return data; - }); - - FieldAsync>("uploadedFiles", - resolve: async context => - { - var data = await db.UploadedFiles.AsQueryable().ToListAsync(); - return data; }); FieldAsync>("dailyUniqueMetrics", diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index 9f5bc9a9..d4a6ddf3 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -3,8 +3,6 @@ using System.Linq; using System.Reactive.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Nettle; using Wabbajack.BuildServer.Controllers; using Wabbajack.BuildServer.Model.Models; @@ -18,13 +16,11 @@ namespace Wabbajack.BuildServer public class JobManager { protected readonly ILogger Logger; - protected readonly DBContext Db; protected readonly AppSettings Settings; protected SqlService Sql; - public JobManager(ILogger logger, DBContext db, SqlService sql, AppSettings settings) + public JobManager(ILogger logger, SqlService sql, AppSettings settings) { - Db = db; Logger = logger; Settings = settings; Sql = sql; @@ -42,7 +38,7 @@ namespace Wabbajack.BuildServer { try { - var job = await Job.GetNext(Db); + var job = await Sql.GetJob(); if (job == null) { await Task.Delay(5000); @@ -50,18 +46,17 @@ namespace Wabbajack.BuildServer } Logger.Log(LogLevel.Information, $"Starting job: {job.Payload.Description}"); - JobResult result; try { - result = await job.Payload.Execute(Db, Sql, Settings); + job.Result = await job.Payload.Execute(Sql, Settings); } catch (Exception ex) { Logger.Log(LogLevel.Error, ex, $"Error while running job: {job.Payload.Description}"); - result = JobResult.Error(ex); + job.Result = JobResult.Error(ex); } - await Job.Finish(Db, job, result); + await Sql.FinishJob(job); } catch (Exception ex) { @@ -95,16 +90,15 @@ namespace Wabbajack.BuildServer { try { - var started = await Db.Jobs.AsQueryable() - .Where(j => j.Started != null && j.Ended == null) - .ToListAsync(); + var started = await Sql.GetRunningJobs(); foreach (var job in started) { var runtime = DateTime.Now - job.Started; - if (runtime > TimeSpan.FromMinutes(30)) - { - await Job.Finish(Db, job, JobResult.Error(new Exception($"Timeout after {runtime.Value.TotalMinutes}"))); - } + + if (!(runtime > TimeSpan.FromMinutes(30))) continue; + + job.Result = JobResult.Error(new Exception($"Timeout after {runtime.Value.TotalMinutes}")); + await Sql.FinishJob(job); } } catch (Exception ex) @@ -119,18 +113,17 @@ namespace Wabbajack.BuildServer if (!Settings.RunFrontEndJobs && typeof(T).ImplementsInterface(typeof(IFrontEndJob))) return; try { - var jobs = await Db.Jobs.AsQueryable() + var jobs = (await Sql.GetUnfinishedJobs()) .Where(j => j.Payload is T) .OrderByDescending(j => j.Created) - .Take(10) - .ToListAsync(); + .Take(10); foreach (var job in jobs) { if (job.Started == null || job.Ended == null) return; if (DateTime.Now - job.Ended < span) return; } - await Db.Jobs.InsertOneAsync(new Job + await Sql.EnqueueJob(new Job { Priority = priority, Payload = new T() diff --git a/Wabbajack.BuildServer/Models/ApiKey.cs b/Wabbajack.BuildServer/Models/ApiKey.cs index 2fe63b7a..b07d1999 100644 --- a/Wabbajack.BuildServer/Models/ApiKey.cs +++ b/Wabbajack.BuildServer/Models/ApiKey.cs @@ -1,7 +1,5 @@ using System.Collections.Generic; using System.Threading.Tasks; -using MongoDB.Driver; -using MongoDB.Driver.Linq; namespace Wabbajack.BuildServer.Models { @@ -14,10 +12,5 @@ namespace Wabbajack.BuildServer.Models public List CanUploadLists { get; set; } public List Roles { get; set; } - - public static async Task Get(DBContext db, string key) - { - return await db.ApiKeys.AsQueryable().Where(k => k.Key == key).FirstOrDefaultAsync(); - } } } diff --git a/Wabbajack.BuildServer/Models/DBContext.cs b/Wabbajack.BuildServer/Models/DBContext.cs deleted file mode 100644 index ed6ce178..00000000 --- a/Wabbajack.BuildServer/Models/DBContext.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System.Collections.Generic; -using Microsoft.Extensions.Configuration; -using MongoDB.Driver; -using Wabbajack.Lib.NexusApi; -using Wabbajack.BuildServer.Models.JobQueue; - -namespace Wabbajack.BuildServer.Models -{ - public class DBContext - { - private IConfiguration _configuration; - private Settings _settings; - public DBContext(IConfiguration configuration) - { - _configuration = configuration; - - _settings = new Settings(); - _configuration.Bind("MongoDB", _settings); - } - - public IMongoCollection> NexusModInfos => Client.GetCollection>(_settings.Collections["NexusModInfos"]); - public IMongoCollection> NexusFileInfos => Client.GetCollection>(_settings.Collections["NexusFileInfos"]); - public IMongoCollection ModListStatus => Client.GetCollection(_settings.Collections["ModListStatus"]); - - public IMongoCollection Jobs => Client.GetCollection(_settings.Collections["JobQueue"]); - public IMongoCollection DownloadStates => Client.GetCollection(_settings.Collections["DownloadStates"]); - public IMongoCollection Metrics => Client.GetCollection(_settings.Collections["Metrics"]); - public IMongoCollection IndexedFiles => Client.GetCollection(_settings.Collections["IndexedFiles"]); - public IMongoCollection>> NexusUpdates => Client.GetCollection>>(_settings.Collections["NexusUpdates"]); - - public IMongoCollection ApiKeys => Client.GetCollection(_settings.Collections["ApiKeys"]); - public IMongoCollection UploadedFiles => Client.GetCollection(_settings.Collections["UploadedFiles"]); - - public IMongoCollection> NexusModFiles => - Client.GetCollection>( - _settings.Collections["NexusModFiles"]); - private IMongoDatabase Client => new MongoClient($"mongodb://{_settings.Host}").GetDatabase(_settings.Database); - } - public class Settings - { - public string Host { get; set; } - public string Database { get; set; } - public Dictionary Collections { get; set; } - public string SqlConnection { get; set; } - } -} diff --git a/Wabbajack.BuildServer/Models/DownloadState.cs b/Wabbajack.BuildServer/Models/DownloadState.cs index bfd5e183..77455c1d 100644 --- a/Wabbajack.BuildServer/Models/DownloadState.cs +++ b/Wabbajack.BuildServer/Models/DownloadState.cs @@ -1,5 +1,4 @@ using System; -using MongoDB.Bson.Serialization.Attributes; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; @@ -7,7 +6,6 @@ namespace Wabbajack.BuildServer.Models { public class DownloadState { - [BsonId] public string Key { get; set; } public Hash Hash { get; set; } diff --git a/Wabbajack.BuildServer/Models/IndexedFile.cs b/Wabbajack.BuildServer/Models/IndexedFile.cs index 241b619c..3ce3d9df 100644 --- a/Wabbajack.BuildServer/Models/IndexedFile.cs +++ b/Wabbajack.BuildServer/Models/IndexedFile.cs @@ -1,17 +1,10 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using MongoDB.Bson.Serialization.Attributes; +using System.Collections.Generic; using Wabbajack.Common; -using Wabbajack.VirtualFileSystem; namespace Wabbajack.BuildServer.Models { public class IndexedFile { - [BsonId] public Hash Hash { get; set; } public string SHA256 { get; set; } public string SHA1 { get; set; } diff --git a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs index f56b50da..a28f2048 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs @@ -1,9 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading.Tasks; -using MongoDB.Bson.Serialization.Attributes; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models.Jobs; @@ -18,7 +16,6 @@ namespace Wabbajack.BuildServer.Models.JobQueue typeof(UpdateModLists), typeof(EnqueueAllArchives), typeof(EnqueueAllGameFiles), - typeof(EnqueueRecentFiles), typeof(UploadToCDN), typeof(IndexDynDOLOD), typeof(ReindexArchives), @@ -28,12 +25,11 @@ namespace Wabbajack.BuildServer.Models.JobQueue public static Dictionary NameToType { get; set; } - [BsonIgnore] public abstract string Description { get; } public virtual bool UsesNexus { get; } = false; - public abstract Task Execute(DBContext db, SqlService sql,AppSettings settings); + public abstract Task Execute(SqlService sql,AppSettings settings); static AJobPayload() { diff --git a/Wabbajack.BuildServer/Models/JobQueue/Job.cs b/Wabbajack.BuildServer/Models/JobQueue/Job.cs index 4c013d2d..f0353222 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/Job.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/Job.cs @@ -1,12 +1,5 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Threading.Tasks; -using MongoDB.Bson; -using MongoDB.Bson.Serialization.Attributes; -using MongoDB.Driver; -using Wabbajack.Lib.NexusApi; namespace Wabbajack.BuildServer.Models.JobQueue { @@ -29,39 +22,5 @@ namespace Wabbajack.BuildServer.Models.JobQueue public AJobPayload Payload { get; set; } public Job OnSuccess { get; set; } - - public static async Task GetNext(DBContext db) - { - var filter = new BsonDocument - { - {"Started", BsonNull.Value} - }; - var update = new BsonDocument - { - {"$set", new BsonDocument {{"Started", DateTime.Now}}} - }; - var sort = new {Priority=-1, Created=1}.ToBsonDocument(); - var job = await db.Jobs.FindOneAndUpdateAsync(filter, update, new FindOneAndUpdateOptions{Sort = sort}); - return job; - } - - public static async Task Finish(DBContext db, Job job, JobResult jobResult) - { - if (jobResult.ResultType == JobResultType.Success && job.OnSuccess != null) - { - await db.Jobs.InsertOneAsync(job.OnSuccess); - } - - var filter = new BsonDocument - { - {"_id", job.Id}, - }; - var update = new BsonDocument - { - {"$set", new BsonDocument {{"Ended", DateTime.Now}, {"Result", jobResult.ToBsonDocument()}}} - }; - var result = await db.Jobs.FindOneAndUpdateAsync(filter, update); - return result; - } } } diff --git a/Wabbajack.BuildServer/Models/JobQueue/JobResult.cs b/Wabbajack.BuildServer/Models/JobQueue/JobResult.cs index 6747ef13..8f3ca2a6 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/JobResult.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/JobResult.cs @@ -1,19 +1,13 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using MongoDB.Bson.Serialization.Attributes; namespace Wabbajack.BuildServer.Models.JobQueue { public class JobResult { public JobResultType ResultType { get; set; } - [BsonIgnoreIfNull] + public string Message { get; set; } - [BsonIgnoreIfNull] public string Stacktrace { get; set; } public static JobResult Success() diff --git a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs index c00625ae..36e70c80 100644 --- a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs +++ b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs @@ -1,10 +1,6 @@ using System; using System.Threading.Tasks; -using Alphaleonis.Win32.Filesystem; using System.Linq; -using FluentFTP; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; @@ -17,7 +13,7 @@ namespace Wabbajack.BuildServer.Models.Jobs public class EnqueueAllArchives : AJobPayload, IBackEndJob { public override string Description => "Add missing modlist archives to indexer"; - public override async Task Execute(DBContext db, SqlService sql, AppSettings settings) + public override async Task Execute(SqlService sql, AppSettings settings) { Utils.Log("Starting ModList indexing"); var modlists = await ModlistMetadata.LoadFromGithub(); @@ -28,7 +24,7 @@ namespace Wabbajack.BuildServer.Models.Jobs { try { - await EnqueueFromList(db, list, queue); + await EnqueueFromList(sql, list, queue); } catch (Exception ex) { @@ -40,10 +36,8 @@ namespace Wabbajack.BuildServer.Models.Jobs return JobResult.Success(); } - private static async Task EnqueueFromList(DBContext db, ModlistMetadata list, WorkQueue queue) + private static async Task EnqueueFromList(SqlService sql, ModlistMetadata list, WorkQueue queue) { - var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL); - var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension); if (list.NeedsDownload(modlistPath)) @@ -66,21 +60,23 @@ namespace Wabbajack.BuildServer.Models.Jobs var archives = installer.Archives; Utils.Log($"Found {archives.Count} archives in {installer.Name} to index"); - var searching = archives.Select(a => a.Hash).Distinct().ToArray(); + var searching = archives.Select(a => a.Hash).ToHashSet(); Utils.Log($"Looking for missing archives"); - var knownArchives = (await db.IndexedFiles.AsQueryable().Where(a => searching.Contains(a.Hash)) - .Select(d => d.Hash).ToListAsync()).ToDictionary(a => a); + var knownArchives = await sql.FilterByExistingIndexedArchives(searching); Utils.Log($"Found {knownArchives.Count} pre-existing archives"); - var missing = archives.Where(a => !knownArchives.ContainsKey(a.Hash)).ToList(); + var missing = archives.Where(a => !knownArchives.Contains(a.Hash)).ToList(); Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs"); var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}, Priority = Job.JobPriority.Low}); Utils.Log($"Writing jobs to the database"); - await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions {IsOrdered = false}); + + foreach (var job in jobs) + await sql.EnqueueJob(job); + Utils.Log($"Done adding archives for {installer.Name}"); } } diff --git a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs index e0aa8aa1..d0596580 100644 --- a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs +++ b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs @@ -5,18 +5,14 @@ using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using System.IO; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.BuildServer.Model.Models; -using Directory = Alphaleonis.Win32.Filesystem.Directory; -using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.BuildServer.Models.Jobs { public class EnqueueAllGameFiles : AJobPayload, IBackEndJob { public override string Description { get => $"Enqueue all game files for indexing"; } - public override async Task Execute(DBContext db, SqlService sql, AppSettings settings) + public override async Task Execute(SqlService sql, AppSettings settings) { using (var queue = new WorkQueue(4)) { @@ -32,16 +28,12 @@ namespace Wabbajack.BuildServer.Models.Jobs })) .ToList(); - var pks = states.Select(s => s.PrimaryKeyString).Distinct().ToArray(); - Utils.Log($"Found {pks.Length} archives to cross-reference with the database"); + var pks = states.Select(s => s.PrimaryKeyString).ToHashSet(); + Utils.Log($"Found {pks.Count} archives to cross-reference with the database"); - var found = (await db.DownloadStates - .AsQueryable().Where(s => pks.Contains(s.Key)) - .Select(s => s.Key) - .ToListAsync()) - .ToDictionary(s => s); - - states = states.Where(s => !found.ContainsKey(s.PrimaryKeyString)).ToList(); + var found = await sql.FilterByExistingPrimaryKeys(pks); + + states = states.Where(s => !found.Contains(s.PrimaryKeyString)).ToList(); Utils.Log($"Found {states.Count} archives to index"); await states.PMap(queue, async state => @@ -58,14 +50,14 @@ namespace Wabbajack.BuildServer.Models.Jobs } }); - var with_hash = states.Where(state => state.Hash != null).ToList(); + var with_hash = states.Where(state => state.Hash != default).ToList(); Utils.Log($"Inserting {with_hash.Count} jobs."); var jobs = states.Select(state => new IndexJob {Archive = new Archive {Name = state.GameFile.FileName.ToString(), State = state}}) .Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus}) .ToList(); - if (jobs.Count > 0) - await db.Jobs.InsertManyAsync(jobs); + foreach (var job in jobs) + await sql.EnqueueJob(job); return JobResult.Success(); } diff --git a/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs b/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs deleted file mode 100644 index 8e92f0a8..00000000 --- a/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs +++ /dev/null @@ -1,85 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using MongoDB.Driver; -using MongoDB.Driver.Core.Authentication; -using MongoDB.Driver.Linq; -using Wabbajack.BuildServer.Model.Models; -using Wabbajack.BuildServer.Models.JobQueue; -using Wabbajack.Common; -using Wabbajack.Lib; -using Wabbajack.Lib.Downloaders; -using Wabbajack.Lib.NexusApi; - -namespace Wabbajack.BuildServer.Models.Jobs -{ - public class EnqueueRecentFiles : AJobPayload, IFrontEndJob - { - public override string Description => "Enqueue the past days worth of mods for indexing"; - - private static HashSet GamesToScan = new HashSet - { - Game.Fallout3, Game.Fallout4, Game.Skyrim, Game.SkyrimSpecialEdition, Game.SkyrimVR, Game.FalloutNewVegas, Game.Oblivion - }; - public override async Task Execute(DBContext db, SqlService sql, AppSettings settings) - { - using (var queue = new WorkQueue()) - { - var updates = await db.NexusUpdates.AsQueryable().ToListAsync(); - var mods = updates - .Where(list => GamesToScan.Contains(GameRegistry.GetByNexusName(list.Game).Game)) - .SelectMany(list => - list.Data.Where(mod => DateTime.UtcNow - mod.LatestFileUpdate.AsUnixTime() < TimeSpan.FromDays(1)) - .Select(mod => (list.Game, mod.ModId))); - var mod_files = (await mods.PMap(queue, async mod => - { - var client = await NexusApiClient.Get(); - try - { - var files = await client.GetModFiles(GameRegistry.GetByNexusName(mod.Game).Game, - (int)mod.ModId); - return (Game: GameRegistry.GetByFuzzyName(mod.Game).Game, mod.ModId, files.files); - } - catch (Exception) - { - return default; - } - })).Where(t => t.Game != default).ToList(); - - var archives = - mod_files.SelectMany(mod => mod.files.Select(file => (mod.Game, mod.ModId, File:file)).Where(f => !string.IsNullOrEmpty(f.File.category_name) )) - .Select(tuple => - { - var state = new NexusDownloader.State - { - Game = tuple.Game, ModID = tuple.ModId, FileID = tuple.File.file_id - }; - return new Archive {State = state, Name = tuple.File.file_name}; - }).ToList(); - - Utils.Log($"Found {archives.Count} archives from recent Nexus updates to index"); - var searching = archives.Select(a => a.State.PrimaryKeyString).Distinct().ToArray(); - - Utils.Log($"Looking for missing states"); - var knownArchives = (await db.DownloadStates.AsQueryable().Where(s => searching.Contains(s.Key)) - .Select(d => d.Key).ToListAsync()).ToDictionary(a => a); - - Utils.Log($"Found {knownArchives.Count} pre-existing archives"); - var missing = archives.Where(a => !knownArchives.ContainsKey(a.State.PrimaryKeyString)) - .DistinctBy(d => d.State.PrimaryKeyString) - .ToList(); - - Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs"); - - var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}, Priority = Job.JobPriority.Low}); - - Utils.Log($"Writing jobs to the DB"); - await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions {IsOrdered = false}); - Utils.Log($"Done adding archives for Nexus Updates"); - - return JobResult.Success(); - } - } - } -} diff --git a/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs b/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs index 4598d82b..22b2893e 100644 --- a/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs @@ -4,8 +4,6 @@ using System.Threading.Tasks; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; using Wabbajack.Lib.NexusApi; -using MongoDB.Driver; -using Newtonsoft.Json; using Wabbajack.BuildServer.Model.Models; @@ -15,7 +13,7 @@ namespace Wabbajack.BuildServer.Models.Jobs { public override string Description => "Poll the Nexus for updated mods, and clean any references to those mods"; - public override async Task Execute(DBContext db, SqlService sql, AppSettings settings) + public override async Task Execute(SqlService sql, AppSettings settings) { var api = await NexusApiClient.Get(); @@ -31,8 +29,6 @@ namespace Wabbajack.BuildServer.Models.Jobs entry.Path = $"/v1/games/{game.NexusName}/mods/updated.json?period=1m"; entry.Data = mods; - await entry.Upsert(db.NexusUpdates); - return (game, mods); }) .Select(async rTask => @@ -56,19 +52,14 @@ namespace Wabbajack.BuildServer.Models.Jobs // Mod activity could hide files var b = d.mod.LastestModActivity.AsUnixTime(); - return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.ModId}; + return new {Game = d.game.Game, Date = (a > b ? a : b), ModId = d.mod.ModId}; }); var purged = await collected.PMap(queue, async t => { - var resultA = await db.NexusModInfos.DeleteManyAsync(f => - f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); - var resultB = await db.NexusModFiles.DeleteManyAsync(f => - f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); - var resultC = await db.NexusFileInfos.DeleteManyAsync(f => - f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); - - return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount; + var resultA = await sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date); + var resultB = await sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date); + return resultA + resultB; }); Utils.Log($"Purged {purged.Sum()} cache entries"); diff --git a/Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs b/Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs index 06a559a6..7430ca74 100644 --- a/Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs +++ b/Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs @@ -3,8 +3,6 @@ using System.Linq; using System.Net.Http; using System.Threading.Tasks; using HtmlAgilityPack; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; @@ -20,7 +18,7 @@ namespace Wabbajack.BuildServer.Models.Jobs public class IndexDynDOLOD : AJobPayload { public override string Description => "Queue MEGA URLs from the DynDOLOD Post"; - public override async Task Execute(DBContext db, SqlService sql, AppSettings settings) + public override async Task Execute(SqlService sql, AppSettings settings) { var doc = new HtmlDocument(); var body = await new HttpClient().GetStringAsync(new Uri( @@ -54,11 +52,11 @@ namespace Wabbajack.BuildServer.Models.Jobs foreach (var job in matches) { var key = ((MegaDownloader.State)((IndexJob)job.Payload).Archive.State).PrimaryKeyString; - var found = await db.DownloadStates.AsQueryable().Where(s => s.Key == key).FirstOrDefaultAsync(); + var found = await sql.DownloadStateByPrimaryKey(key); if (found != null) continue; Utils.Log($"Queuing {key} for indexing"); - await db.Jobs.InsertOneAsync(job); + await sql.EnqueueJob(job); } return JobResult.Success(); diff --git a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs index 4e849f02..e5c6cd30 100644 --- a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs @@ -1,14 +1,9 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; -using FluentFTP; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.BuildServer.Model.Models; -using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; using Wabbajack.Lib; @@ -23,7 +18,7 @@ namespace Wabbajack.BuildServer.Models.Jobs public Archive Archive { get; set; } public override string Description => $"Index ${Archive.State.PrimaryKeyString} and save the download/file state"; public override bool UsesNexus { get => Archive.State is NexusDownloader.State; } - public override async Task Execute(DBContext db, SqlService sql, AppSettings settings) + public override async Task Execute(SqlService sql, AppSettings settings) { if (Archive.State is ManualDownloader.State) return JobResult.Success(); @@ -33,8 +28,8 @@ namespace Wabbajack.BuildServer.Models.Jobs pk.AddRange(Archive.State.PrimaryKey); var pk_str = string.Join("|",pk.Select(p => p.ToString())); - var found = await db.DownloadStates.AsQueryable().Where(f => f.Key == pk_str).Take(1).ToListAsync(); - if (found.Count > 0) + var found = await sql.DownloadStateByPrimaryKey(pk_str); + if (found == null) return JobResult.Success(); string fileName = Archive.Name; @@ -51,10 +46,7 @@ namespace Wabbajack.BuildServer.Models.Jobs await sql.MergeVirtualFile(archive); - await db.DownloadStates.InsertOneAsync(new DownloadState - { - Key = pk_str, Hash = archive.Hash, State = Archive.State, IsValid = true - }); + await sql.AddDownloadState(archive.Hash, Archive.State); var to_path = settings.ArchiveDir.Combine( $"{Path.GetFileName(fileName)}_{archive.Hash.ToHex()}_{Path.GetExtension(fileName)}"); @@ -66,43 +58,9 @@ namespace Wabbajack.BuildServer.Models.Jobs await settings.DownloadDir.Combine(folder).DeleteDirectory(); } - - - return JobResult.Success(); } - /* - private List ConvertArchive(List files, VirtualFile file, bool isTop = true) - { - var name = isTop ? file.Name.FileName : file.Name; - var ifile = new IndexedFile - { - Hash = file.Hash, - SHA256 = file.ExtendedHashes.SHA256, - SHA1 = file.ExtendedHashes.SHA1, - MD5 = file.ExtendedHashes.MD5, - CRC = file.ExtendedHashes.CRC, - Size = file.Size, - Children = file.Children != null ? file.Children.Select( - f => - { - ConvertArchive(files, f, false); - - return new ChildFile - { - Hash = f.Hash, - Name = f.Name, - Extension = Path.GetExtension(f.Name.ToLowerInvariant()) - }; - }).ToList() : new List() - }; - ifile.IsArchive = ifile.Children.Count > 0; - files.Add(ifile); - return files; - }*/ - - } } diff --git a/Wabbajack.BuildServer/Models/Jobs/ReindexArchives.cs b/Wabbajack.BuildServer/Models/Jobs/ReindexArchives.cs index ac5c1e8e..12d3e93e 100644 --- a/Wabbajack.BuildServer/Models/Jobs/ReindexArchives.cs +++ b/Wabbajack.BuildServer/Models/Jobs/ReindexArchives.cs @@ -14,7 +14,7 @@ namespace Wabbajack.BuildServer.Models.Jobs public class ReindexArchives : AJobPayload { public override string Description => "Reindex all files in the mod archive folder"; - public override async Task Execute(DBContext db, SqlService sql, AppSettings settings) + public override async Task Execute(SqlService sql, AppSettings settings) { using (var queue = new WorkQueue()) { diff --git a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs index 84d1662b..57a5dbf6 100644 --- a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs +++ b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs @@ -1,9 +1,6 @@ using System; using System.Linq; using System.Threading.Tasks; -using Alphaleonis.Win32.Filesystem; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; @@ -12,14 +9,13 @@ using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.Validation; -using File = Alphaleonis.Win32.Filesystem.File; namespace Wabbajack.BuildServer.Models.Jobs { public class UpdateModLists : AJobPayload, IFrontEndJob { public override string Description => "Validate curated modlists"; - public override async Task Execute(DBContext db, SqlService sql, AppSettings settings) + public override async Task Execute(SqlService sql, AppSettings settings) { Utils.Log("Starting Modlist Validation"); var modlists = await ModlistMetadata.LoadFromGithub(); @@ -34,7 +30,7 @@ namespace Wabbajack.BuildServer.Models.Jobs { try { - await ValidateList(db, list, queue, whitelists); + await ValidateList(sql, list, queue, whitelists); } catch (Exception ex) { @@ -46,7 +42,7 @@ namespace Wabbajack.BuildServer.Models.Jobs return JobResult.Success(); } - private async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists) + private async Task ValidateList(SqlService sql, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists) { var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension); @@ -76,7 +72,7 @@ namespace Wabbajack.BuildServer.Models.Jobs var validated = (await installer.Archives .PMap(queue, async archive => { - var isValid = await IsValid(db, whitelists, archive); + var isValid = await IsValid(sql, whitelists, archive); return new DetailedStatusItem {IsFailing = !isValid, Archive = archive}; })) @@ -107,13 +103,13 @@ namespace Wabbajack.BuildServer.Models.Jobs }; Utils.Log( $"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed"); - await ModListStatus.Update(db, dto); + await sql.UpdateModListStatus(dto); Utils.Log( $"Done updating {dto.Summary.Name}"); } - private async Task IsValid(DBContext db, ValidateModlist whitelists, Archive archive) + private async Task IsValid(SqlService sql, ValidateModlist whitelists, Archive archive) { try { @@ -123,7 +119,7 @@ namespace Wabbajack.BuildServer.Models.Jobs { if (archive.State is NexusDownloader.State state) { - if (await ValidateNexusFast(db, state)) return true; + if (await ValidateNexusFast(sql, state)) return true; } else if (archive.State is GoogleDriveDownloader.State) @@ -166,13 +162,11 @@ namespace Wabbajack.BuildServer.Models.Jobs } } - private async Task ValidateNexusFast(DBContext db, NexusDownloader.State state) + private async Task ValidateNexusFast(SqlService sql, NexusDownloader.State state) { try { - var gameMeta = state.Game.MetaData(); - - var modFiles = (await db.NexusModFiles.AsQueryable().Where(g => g.Game == gameMeta.NexusName && g.ModId == state.ModID).FirstOrDefaultAsync())?.Data; + var modFiles = await sql.GetModFiles(state.Game, state.ModID); if (modFiles == null) { diff --git a/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs b/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs index a54ef2d4..1f955257 100644 --- a/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs +++ b/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs @@ -2,11 +2,7 @@ using System.Net; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; -using BunnyCDN.Net.Storage; -using CG.Web.MegaApiClient; using FluentFTP; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; @@ -22,11 +18,11 @@ namespace Wabbajack.BuildServer.Models.Jobs public Guid FileId { get; set; } - public override async Task Execute(DBContext db, SqlService sql, AppSettings settings) + public override async Task Execute(SqlService sql, AppSettings settings) { int retries = 0; TOP: - var file = await db.UploadedFiles.AsQueryable().Where(f => f.Id == FileId).FirstOrDefaultAsync(); + var file = await sql.UploadedFileById(FileId); if (settings.BunnyCDN_User == "TEST" && settings.BunnyCDN_Password == "TEST") { @@ -53,7 +49,7 @@ namespace Wabbajack.BuildServer.Models.Jobs } } - await db.Jobs.InsertOneAsync(new Job + await sql.EnqueueJob(new Job { Priority = Job.JobPriority.High, Payload = new IndexJob diff --git a/Wabbajack.BuildServer/Models/Metric.cs b/Wabbajack.BuildServer/Models/Metric.cs index b95c6261..82f09e3e 100644 --- a/Wabbajack.BuildServer/Models/Metric.cs +++ b/Wabbajack.BuildServer/Models/Metric.cs @@ -3,71 +3,17 @@ using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; -using MongoDB.Bson; -using MongoDB.Bson.Serialization.Attributes; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.BuildServer.GraphQL; -using Wabbajack.Common; +using Wabbajack.BuildServer.Model.Models; namespace Wabbajack.BuildServer.Models { public class Metric { - [BsonId] - public ObjectId Id { get; set; } public DateTime Timestamp { get; set; } public string Action { get; set; } public string Subject { get; set; } public string MetricsKey { get; set; } - - - public static async Task> Report(DBContext db, string grouping) - { - var regex = new Regex("\\d+\\."); - var data = await db.Metrics.AsQueryable() - .Where(m => m.MetricsKey != null) - .Where(m => m.Action == grouping) - .Where(m => m.Subject != "Default") - .ToListAsync(); - - var minDate = DateTime.Parse(data.Min(d => d.Timestamp.ToString("yyyy-MM-dd"))); - var maxDate = DateTime.Parse(data.Max(d => d.Timestamp.ToString("yyyy-MM-dd"))); - - var dateArray = Enumerable.Range(0, (int)(maxDate - minDate).TotalDays + 1) - .Select(idx => minDate + TimeSpan.FromDays(idx)) - .Select(date => date.ToString("yyyy-MM-dd")) - .ToList(); - - var results = data - .Where(d => !Guid.TryParse(d.Subject, out var _)) - .GroupBy(d => regex.Split(d.Subject).First()) - .Select(by_series => - { - var by_day = by_series.GroupBy(d => d.Timestamp.ToString("yyyy-MM-dd")) - .Select(d => (d.Key, d.DistinctBy(v => v.MetricsKey ?? "").Count())) - .OrderBy(r => r.Key); - - var by_day_idx = by_day.ToDictionary(d => d.Key); - - (string Key, int) GetEntry(string date) - { - if (by_day_idx.TryGetValue(date, out var result)) - return result; - return (date, 0); - } - - return new MetricResult - { - SeriesName = by_series.Key, - Labels = dateArray.Select(d => GetEntry(d).Key).ToList(), - Values = dateArray.Select(d => GetEntry(d).Item2).ToList() - }; - }) - .OrderBy(f => f.SeriesName); - - return results; - } } } diff --git a/Wabbajack.BuildServer/Models/ModListStatus.cs b/Wabbajack.BuildServer/Models/ModListStatus.cs index 04b9c8e7..1b1d90ff 100644 --- a/Wabbajack.BuildServer/Models/ModListStatus.cs +++ b/Wabbajack.BuildServer/Models/ModListStatus.cs @@ -2,9 +2,6 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using MongoDB.Bson.Serialization.Attributes; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.Lib; using Wabbajack.Lib.ModListRegistry; @@ -12,20 +9,12 @@ namespace Wabbajack.BuildServer.Models { public class ModListStatus { - - [BsonId] public string Id { get; set; } public ModlistSummary Summary { get; set; } public ModlistMetadata Metadata { get; set; } public DetailedStatus DetailedStatus { get; set; } - public static async Task Update(DBContext db, ModListStatus status) - { - var id = status.Metadata.Links.MachineURL; - await db.ModListStatus.FindOneAndReplaceAsync(s => s.Id == id, status, new FindOneAndReplaceOptions {IsUpsert = true}); - } - public static IQueryable AllSummaries { get @@ -33,15 +22,6 @@ namespace Wabbajack.BuildServer.Models return null; } } - - public static async Task ByName(DBContext db, string name) - { - var result = await db.ModListStatus - .AsQueryable() - .Where(doc => doc.Metadata.Links.MachineURL == name || doc.Metadata.Title == name) - .ToListAsync(); - return result.First(); - } } public class DetailedStatus diff --git a/Wabbajack.BuildServer/Models/NexusCacheData.cs b/Wabbajack.BuildServer/Models/NexusCacheData.cs index 9a25e6bd..1a641643 100644 --- a/Wabbajack.BuildServer/Models/NexusCacheData.cs +++ b/Wabbajack.BuildServer/Models/NexusCacheData.cs @@ -1,28 +1,19 @@ using System; using System.Threading.Tasks; -using MongoDB.Bson.Serialization.Attributes; -using MongoDB.Driver; namespace Wabbajack.BuildServer.Models { public class NexusCacheData { - [BsonId] public string Path { get; set; } public T Data { get; set; } public string Game { get; set; } - [BsonIgnoreIfNull] public long ModId { get; set; } public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow; - [BsonIgnoreIfNull] public string FileId { get; set; } - public async Task Upsert(IMongoCollection> coll) - { - await coll.FindOneAndReplaceAsync>(s => s.Path == Path, this, new FindOneAndReplaceOptions> {IsUpsert = true}); - } } } diff --git a/Wabbajack.BuildServer/Models/NexusUpdateEntry.cs b/Wabbajack.BuildServer/Models/NexusUpdateEntry.cs index a1a78944..23c29325 100644 --- a/Wabbajack.BuildServer/Models/NexusUpdateEntry.cs +++ b/Wabbajack.BuildServer/Models/NexusUpdateEntry.cs @@ -1,6 +1,4 @@ -using System; -using MongoDB.Bson.Serialization.Attributes; -using Newtonsoft.Json; +using Newtonsoft.Json; namespace Wabbajack.BuildServer.Models { diff --git a/Wabbajack.BuildServer/Models/PatchArchive.cs b/Wabbajack.BuildServer/Models/PatchArchive.cs index 2d18005b..5a8d4b7c 100644 --- a/Wabbajack.BuildServer/Models/PatchArchive.cs +++ b/Wabbajack.BuildServer/Models/PatchArchive.cs @@ -1,17 +1,11 @@ using System; -using System.IO; using System.Net; using System.Threading.Tasks; using FluentFTP; -using MongoDB.Driver; -using MongoDB.Driver.Linq; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.Common; -using Wabbajack.Lib; -using Wabbajack.Lib.Downloaders; -using File = Alphaleonis.Win32.Filesystem.File; namespace Wabbajack.BuildServer.Models { @@ -20,10 +14,10 @@ namespace Wabbajack.BuildServer.Models public override string Description => "Create a archive update patch"; public Hash Src { get; set; } public string DestPK { get; set; } - public override async Task Execute(DBContext db, SqlService sql, AppSettings settings) + public override async Task Execute(SqlService sql, AppSettings settings) { var srcPath = settings.PathForArchive(Src); - var destHash = (await db.DownloadStates.AsQueryable().Where(s => s.Key == DestPK).FirstOrDefaultAsync()).Hash; + var destHash = (await sql.DownloadStateByPrimaryKey(DestPK)).Hash; var destPath = settings.PathForArchive(destHash); if (Src == destHash) diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index 7f24bd4c..afb22f7e 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Data; using System.Data.SqlClient; @@ -11,6 +12,7 @@ using Wabbajack.BuildServer.Model.Models.Results; using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.Common; +using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.NexusApi; @@ -168,18 +170,6 @@ namespace Wabbajack.BuildServer.Model.Models .ToList(); } - #region UserRoutines - - public async Task LoginByAPIKey(string key) - { - await using var conn = await Open(); - var result = await conn.QueryAsync(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @Key", - new {Key = key}); - return result.FirstOrDefault(); - } - - #endregion - #region JobRoutines /// @@ -233,6 +223,24 @@ namespace Wabbajack.BuildServer.Model.Models new {RunBy = Guid.NewGuid().ToString()}); return result.FirstOrDefault(); } + + + public async Task> GetRunningJobs() + { + await using var conn = await Open(); + var results = + await conn.QueryAsync("SELECT * from dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL "); + return results; + } + + + public async Task> GetUnfinishedJobs() + { + await using var conn = await Open(); + var results = + await conn.QueryAsync("SELECT * from dbo.Jobs WHERE Ended IS NULL "); + return results; + } #endregion @@ -292,6 +300,15 @@ namespace Wabbajack.BuildServer.Model.Models uf.CDNName }); } + + + public async Task UploadedFileById(Guid fileId) + { + await using var conn = await Open(); + return await conn.QueryFirstAsync("SELECT * FROM dbo.UploadedFiles WHERE Id = @Id", + new {Id = fileId.ToString()}); + + } public async Task> AllUploadedFilesForUser(string user) { @@ -299,6 +316,24 @@ namespace Wabbajack.BuildServer.Model.Models return await conn.QueryAsync("SELECT * FROM dbo.UploadedFiles WHERE UploadedBy = @uploadedBy", new {UploadedBy = user}); } + + + public async Task> AllUploadedFiles() + { + await using var conn = await Open(); + return await conn.QueryAsync("SELECT * FROM dbo.UploadedFiles ORDER BY UploadDate DESC"); + } + + public async Task DeleteUploadedFile(Guid dupId) + { + await using var conn = await Open(); + await conn.ExecuteAsync("SELECT * FROM dbo.UploadedFiles WHERE Id = @id", + new + { + Id = dupId.ToString() + }); + } + public async Task AddDownloadState(Hash hash, AbstractDownloadState state) { @@ -429,7 +464,6 @@ namespace Wabbajack.BuildServer.Model.Models }); return result.FromJSONString(); } - public async Task> GetDetailedModlistStatuses() { await using var conn = await Open(); @@ -440,5 +474,141 @@ namespace Wabbajack.BuildServer.Model.Models #endregion + + #region Logins + public async Task AddLogin(string name) + { + var key = NewAPIKey(); + await using var conn = await Open(); + + + await conn.ExecuteAsync("INSERT INTO dbo.ApiKeys (Owner, ApiKey) VALUES (@Owner, @ApiKey)", + new {Owner = name, ApiKey = key}); + return key; + } + + public static string NewAPIKey() + { + var arr = new byte[128]; + new Random().NextBytes(arr); + return arr.ToHex(); + } + + public async Task LoginByAPIKey(string key) + { + await using var conn = await Open(); + var result = await conn.QueryAsync(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @ApiKey", + new {ApiKey = key}); + return result.FirstOrDefault(); + } + + public async Task> GetAllUserKeys() + { + await using var conn = await Open(); + var result = await conn.QueryAsync<(string Owner, string Key)>("SELECT Owner, ApiKey FROM dbo.ApiKeys"); + return result; + } + + + #endregion + + #region Auto-healing routines + + public async Task GetNexusStateByHash(Hash startingHash) + { + await using var conn = await Open(); + var result = await conn.QueryFirstOrDefaultAsync(@"SELECT JsonState FROM dbo.DownloadStates + WHERE Hash = @hash AND PrimaryKey like 'NexusDownloader+State|%' + ORDER BY LastValidated DESC", + new {Hash = startingHash}); + return result == null ? null : new Archive + { + State = result.FromJSONString(), + Hash = startingHash + }; + } + + public async Task DownloadStateByPrimaryKey(string primaryKey) + { + await using var conn = await Open(); + var result = await conn.QueryFirstOrDefaultAsync<(long Hash, string State)>(@"SELECT Hash, JsonState FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey", + new {PrimaryKey = primaryKey}); + return result == default ? null : new Archive + { + State = result.State.FromJSONString(), + Hash = Hash.FromLong(result.Hash) + }; + } + + + #endregion + + + /// + /// Returns a hashset the only contains hashes from the input that do not exist in IndexedArchives + /// + /// + /// + /// + public async Task> FilterByExistingIndexedArchives(HashSet searching) + { + await using var conn = await Open(); + var found = await conn.QueryAsync("SELECT Hash from dbo.IndexedFile WHERE Hash in @Hashes", + new {Hashes = searching.Select(h => (long)h)}); + return searching.Except(found.Select(h => Hash.FromLong(h)).ToHashSet()).ToHashSet(); + } + + + /// + /// Returns a hashset the only contains primary keys from the input that do not exist in IndexedArchives + /// + /// + /// + /// + public async Task> FilterByExistingPrimaryKeys(HashSet pks) + { + await using var conn = await Open(); + var found = await conn.QueryAsync("SELECT Hash from dbo.IndexedFile WHERE PrimaryKey in @PrimaryKeys", + new {PrimaryKeys = pks.ToList()}); + return pks.Except(found.ToHashSet()).ToHashSet(); + } + + public async Task DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date) + { + await using var conn = await Open(); + var deleted = await conn.ExecuteScalarAsync( + @"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date + SELECT @@ROWCOUNT AS Deleted", + new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date}); + return deleted; + } + + public async Task DeleteNexusModFilesUpdatedBeforeDate(Game game, long modId, DateTime date) + { + await using var conn = await Open(); + var deleted = await conn.ExecuteScalarAsync( + @"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date + SELECT @@ROWCOUNT AS Deleted", + new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date}); + return deleted; + } + + public async Task UpdateModListStatus(ModListStatus dto) + { + await using var conn = await Open(); + await conn.ExecuteAsync(@"MERGE dbo.ModLists AS Target + USING (SELECT @MachineUrl MachineUrl, @Metadata Metadata, @Summary Summary, @DetailedStatus DetailedStatus) AS Source + ON Target.MachineUrl = Source.MachineUrl + WHEN MATCHED THEN UPDATE SET Target.Summary = Source.Summary, Target.Metadata = Source.Metadata, Target.DetailedStatus = Source.DetailedStats + WHEN NOT MATCHED THEN INSERT (MachineUrl, Summary, Metadata, DetailedStatus) VALUES (@MachineUrl, @Summary, @Metadata, @DetailedStatus)", + new + { + MachineUrl = dto.Metadata.Links.MachineURL, + Metadata = dto.Metadata.ToJSON(), + Summary = dto.Summary.ToJSON(), + DetailedStatus = dto.DetailedStatus.ToJSON() + }); + } + } } diff --git a/Wabbajack.BuildServer/Models/UploadedFile.cs b/Wabbajack.BuildServer/Models/UploadedFile.cs index 09f4d2fa..47d6b414 100644 --- a/Wabbajack.BuildServer/Models/UploadedFile.cs +++ b/Wabbajack.BuildServer/Models/UploadedFile.cs @@ -1,8 +1,4 @@ using System; -using System.IO; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Http; -using MongoDB.Bson.Serialization.Attributes; using Wabbajack.Common; using Path = Alphaleonis.Win32.Filesystem.Path; @@ -19,10 +15,8 @@ namespace Wabbajack.BuildServer.Models public string CDNName { get; set; } - [BsonIgnore] public string MungedName => $"{Path.GetFileNameWithoutExtension(Name)}-{Id}{Path.GetExtension(Name)}"; - [BsonIgnore] public string Uri => CDNName == null ? $"https://wabbajack.b-cdn.net/{MungedName}" : $"https://{CDNName}.b-cdn.net/{MungedName}"; } } diff --git a/Wabbajack.BuildServer/SerializerSettings.cs b/Wabbajack.BuildServer/SerializerSettings.cs deleted file mode 100644 index 213e2b3f..00000000 --- a/Wabbajack.BuildServer/SerializerSettings.cs +++ /dev/null @@ -1,76 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Mvc.ModelBinding; -using MongoDB.Bson; -using MongoDB.Bson.IO; -using MongoDB.Bson.Serialization; -using MongoDB.Bson.Serialization.Conventions; -using Wabbajack.BuildServer.Models.JobQueue; -using Wabbajack.Lib.Downloaders; - - -namespace Wabbajack.BuildServer -{ - public static class SerializerSettings - { - public static void Init() - { - var dis = new TypeDiscriminator(typeof(AbstractDownloadState), AbstractDownloadState.NameToType, - AbstractDownloadState.TypeToName); - BsonSerializer.RegisterDiscriminatorConvention(typeof(AbstractDownloadState), dis); - BsonClassMap.RegisterClassMap(cm => cm.SetIsRootClass(true)); - - dis = new TypeDiscriminator(typeof(AJobPayload), AJobPayload.NameToType, AJobPayload.TypeToName); - BsonSerializer.RegisterDiscriminatorConvention(typeof(AJobPayload), dis); - BsonClassMap.RegisterClassMap(cm => cm.SetIsRootClass(true)); - - } - } - - public class TypeDiscriminator : IDiscriminatorConvention - { - private readonly Type defaultType; - private readonly Dictionary typeMap; - private Dictionary revMap; - - public TypeDiscriminator(Type defaultType, - Dictionary typeMap, Dictionary revMap) - { - this.defaultType = defaultType; - this.typeMap = typeMap; - this.revMap = revMap; - } - - - /// - /// Element Name - /// - public string ElementName => "_wjType"; - - public Type GetActualType(IBsonReader bsonReader, Type nominalType) - { - Type type = null; - var bookmark = bsonReader.GetBookmark(); - bsonReader.ReadStartDocument(); - if (bsonReader.FindElement(ElementName)) - { - var value = bsonReader.ReadString(); - if (typeMap.ContainsKey(value)) - type = typeMap[value]; - } - - bsonReader.ReturnToBookmark(bookmark); - if (type == null) - throw new Exception($"Type mis-configuration can't find bson type for ${nominalType}"); - return type; - } - - public BsonValue GetDiscriminator(Type nominalType, Type actualType) - { - return revMap[actualType]; - } - } -} diff --git a/Wabbajack.BuildServer/Startup.cs b/Wabbajack.BuildServer/Startup.cs index a5bfbd84..2e4b5962 100644 --- a/Wabbajack.BuildServer/Startup.cs +++ b/Wabbajack.BuildServer/Startup.cs @@ -71,7 +71,6 @@ namespace Wabbajack.BuildServer x.MultipartBodyLengthLimit = int.MaxValue; }); - services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -89,7 +88,6 @@ namespace Wabbajack.BuildServer // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { - SerializerSettings.Init(); if (env.IsDevelopment()) { app.UseDeveloperExceptionPage(); diff --git a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj index aea23a8d..ac530c15 100644 --- a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj +++ b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj @@ -16,7 +16,6 @@ - @@ -28,8 +27,6 @@ - - diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index d4303e33..2346dca0 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -137,5 +137,6 @@ namespace Wabbajack.Common public static RelativePath ModListTxt = (RelativePath)"modlist.txt"; public static RelativePath ModOrganizer2Exe = (RelativePath)"ModOrganizer.exe"; public static RelativePath ModOrganizer2Ini = (RelativePath)"ModOrganizer.ini"; + public static string AuthorAPIKeyFile = "author-api-key.txt"; } } diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index f310aacf..ef30fe37 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -4,12 +4,10 @@ using System.Reactive; using System.Reactive.Linq; using System.Threading.Tasks; using MessagePack; -using MongoDB.Bson.Serialization.Attributes; using Newtonsoft.Json; using ReactiveUI; using Wabbajack.Common; using Wabbajack.Common.StatusFeed.Errors; -using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.Validation; using Game = Wabbajack.Common.Game; @@ -130,7 +128,6 @@ namespace Wabbajack.Lib.Downloaders } } - [BsonIgnoreExtraElements] [MessagePackObject] public class State : AbstractDownloadState, IMetaState { diff --git a/Wabbajack.Lib/FileUploader/AuthorAPI.cs b/Wabbajack.Lib/FileUploader/AuthorAPI.cs index e6fc80f5..425fce97 100644 --- a/Wabbajack.Lib/FileUploader/AuthorAPI.cs +++ b/Wabbajack.Lib/FileUploader/AuthorAPI.cs @@ -19,11 +19,11 @@ namespace Wabbajack.Lib.FileUploader { public class AuthorAPI { - public static IObservable HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable("author-api-key.txt"); + public static IObservable HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable(Consts.AuthorAPIKeyFile); public static async Task GetAPIKey(string apiKey = null) { - return apiKey ?? (await Consts.LocalAppDataPath.Combine("author-api-key.txt").ReadAllTextAsync()).Trim(); + return apiKey ?? (await Consts.LocalAppDataPath.Combine(Consts.AuthorAPIKeyFile).ReadAllTextAsync()).Trim(); } public static Uri UploadURL => new Uri($"{Consts.WabbajackBuildServerUri}upload_file"); diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index a89efda4..6a3c250b 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -1,13 +1,9 @@ using System; using System.Collections.Generic; -using System.Drawing; using System.Linq; -using System.Net.Http; using System.Threading.Tasks; -using MongoDB.Bson.Serialization.Attributes; using Newtonsoft.Json; using Wabbajack.Common; -using File = System.IO.File; using Game = Wabbajack.Common.Game; namespace Wabbajack.Lib.ModListRegistry @@ -40,7 +36,6 @@ namespace Wabbajack.Lib.ModListRegistry [JsonIgnore] public ModlistSummary ValidationSummary { get; set; } = new ModlistSummary(); - [BsonIgnoreExtraElements] public class LinksObject { [JsonProperty("image")] diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index d5e598a0..73d330ba 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -36,12 +36,6 @@ 2.1.0 - - 2.10.1 - - - 2.10.1 - 11.2.3