Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Purge all remaining references to MongoDB and BJSON

Browse Source
This commit is contained in:
Timothy Baldridge 2020-04-05 15:15:01 -06:00
parent 653c840323
commit 3d16b1e838
48 changed files with 343 additions and 776 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.BuildServer.Test/ADBTest.cs
View File

@ -28,7 +28,7 @@ namespace Wabbajack.BuildServer.Test
{
DBName = "test_db" + Guid.NewGuid().ToString().Replace("-", "_");
User = Guid.NewGuid().ToString().Replace("-", "");
APIKey = Users.NewAPIKey();
APIKey = SqlService.NewAPIKey();
}
public string APIKey { get; }

39
Wabbajack.BuildServer.Test/LoginTests.cs Normal file
View File

@ -0,0 +1,39 @@
using System;
using System.Threading.Tasks;
using Wabbajack.Common;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class LoginTests : ABuildServerSystemTest
{
public LoginTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanCreateLogins()
{
var newUserName = Guid.NewGuid().ToString();
var newKey = await _authedClient.GetStringAsync(MakeURL($"users/add/{newUserName}"));
Assert.NotEmpty(newKey);
Assert.NotNull(newKey);
Assert.NotEqual(newKey, Fixture.APIKey);
var done = await _authedClient.GetStringAsync(MakeURL("users/export"));
Assert.Equal("done", done);
foreach (var (userName, apiKey) in new[] {(newUserName, newKey), (Fixture.User, Fixture.APIKey)})
{
var exported = await Fixture.ServerTempFolder.Combine("exported_users", userName, Consts.AuthorAPIKeyFile)
.ReadAllTextAsync();
Assert.Equal(exported, apiKey);
}
}
}
}

9
Wabbajack.BuildServer/Controllers/AControllerBase.cs
View File

@ -1,13 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using GraphQL;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using Org.BouncyCastle.Ocsp;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
@ -17,13 +12,11 @@ namespace Wabbajack.BuildServer.Controllers
[ApiController]
public abstract class AControllerBase<T> : ControllerBase
{
protected readonly DBContext Db;
protected readonly ILogger<T> Logger;
protected readonly SqlService SQL;
protected AControllerBase(ILogger<T> logger, DBContext db, SqlService sql)
protected AControllerBase(ILogger<T> logger, SqlService sql)
{
Db = db;
Logger = logger;
SQL = sql;
}

9
Wabbajack.BuildServer/Controllers/GraphQL.cs
View File

@ -13,18 +13,17 @@ namespace Wabbajack.BuildServer.Controllers
[ApiController]
public class GraphQL : AControllerBase<GraphQL>
{
private SqlService _sql;
public GraphQL(ILogger<GraphQL> logger, DBContext db, SqlService sql) : base(logger, db, sql)
public GraphQL(ILogger<GraphQL> logger, SqlService sql) : base(logger, sql)
{
_sql = sql;
}
[HttpPost]
public async Task<IActionResult> Post([FromBody] GraphQLQuery query)
{
var inputs = query.Variables.ToInputs();
var schema = new Schema {Query = new Query(Db, _sql), Mutation = new Mutation(Db)};
var schema = new Schema {
Query = new Query(SQL)
};
var result = await new DocumentExecuter().ExecuteAsync(_ =>
{

9
Wabbajack.BuildServer/Controllers/Heartbeat.cs
View File

@ -1,17 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Bson;
using MongoDB.Driver;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
using Wabbajack.Common.StatusFeed;
namespace Wabbajack.BuildServer.Controllers
@ -26,7 +19,7 @@ namespace Wabbajack.BuildServer.Controllers
}
private static DateTime _startTime;
public Heartbeat(ILogger<Heartbeat> logger, DBContext db, SqlService sql) : base(logger, db, sql)
public Heartbeat(ILogger<Heartbeat> logger, SqlService sql) : base(logger, sql)
{
}

8
Wabbajack.BuildServer/Controllers/IndexedFiles.cs
View File

@ -5,21 +5,15 @@ using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using DynamicData;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.VirtualFileSystem;
using IndexedFile = Wabbajack.BuildServer.Models.IndexedFile;
namespace Wabbajack.BuildServer.Controllers
@ -30,7 +24,7 @@ namespace Wabbajack.BuildServer.Controllers
private SqlService _sql;
private AppSettings _settings;
public IndexedFiles(ILogger<IndexedFiles> logger, DBContext db, SqlService sql, AppSettings settings) : base(logger, db, sql)
public IndexedFiles(ILogger<IndexedFiles> logger, SqlService sql, AppSettings settings) : base(logger, sql)
{
_settings = settings;
_sql = sql;

18
Wabbajack.BuildServer/Controllers/Jobs.cs
View File

@ -1,13 +1,9 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
namespace Wabbajack.BuildServer.Controllers
@ -17,27 +13,17 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/jobs")]
public class Jobs : AControllerBase<Jobs>
{
public Jobs(ILogger<Jobs> logger, DBContext db, SqlService sql) : base(logger, db, sql)
public Jobs(ILogger<Jobs> logger, SqlService sql) : base(logger, sql)
{
}
[HttpGet]
[Route("unfinished")]
public async Task<IEnumerable<Job>> GetUnfinished()
{
return await Db.Jobs.AsQueryable()
.Where(j => j.Ended == null)
.OrderByDescending(j => j.Priority)
.ToListAsync();
}
[HttpGet]
[Route("enqueue_job/{JobName}")]
public async Task<long> EnqueueJob(string JobName)
{
var jobtype = AJobPayload.NameToType[JobName];
var job = new Job{Priority = Job.JobPriority.High, Payload = (AJobPayload)jobtype.GetConstructor(new Type[0]).Invoke(new object?[0])};
await Db.Jobs.InsertOneAsync(job);
await SQL.EnqueueJob(job);
return job.Id;
}
}

7
Wabbajack.BuildServer/Controllers/ListValidation.cs
View File

@ -2,16 +2,11 @@
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.ServiceModel.Syndication;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http.Extensions;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nettle;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
using Wabbajack.Lib.ModListRegistry;
@ -21,7 +16,7 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/lists")]
public class ListValidation : AControllerBase<ListValidation>
{
public ListValidation(ILogger<ListValidation> logger, DBContext db, SqlService sql) : base(logger, db, sql)
public ListValidation(ILogger<ListValidation> logger, SqlService sql) : base(logger, sql)
{
}

23
Wabbajack.BuildServer/Controllers/MetricsController.cs
View File

@ -1,17 +1,11 @@
using System;
using System.Collections.Generic;
using System.Data.SqlTypes;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
using Wabbajack.Lib.ModListRegistry;
namespace Wabbajack.BuildServer.Controllers
{
@ -19,11 +13,8 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/metrics")]
public class MetricsController : AControllerBase<MetricsController>
{
private SqlService _sql;
public MetricsController(ILogger<MetricsController> logger, DBContext db, SqlService sql) : base(logger, db, sql)
public MetricsController(ILogger<MetricsController> logger, SqlService sql) : base(logger, sql)
{
_sql = sql;
}
[HttpGet]
@ -35,20 +26,10 @@ namespace Wabbajack.BuildServer.Controllers
return new Result { Timestamp = date};
}
[Authorize]
[HttpGet]
[Route("transfer")]
public async Task<string> Transfer()
{
var all_metrics = await Db.Metrics.AsQueryable().ToListAsync();
await _sql.IngestAllMetrics(all_metrics);
return "done";
}
private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
{
Logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}");
await _sql.IngestMetric(new Metric
await SQL.IngestMetric(new Metric
{
Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey
});

28
Wabbajack.BuildServer/Controllers/ModlistUpdater.cs
View File

@ -8,8 +8,6 @@ using FluentFTP;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
@ -30,7 +28,7 @@ namespace Wabbajack.BuildServer.Controllers
private AppSettings _settings;
private SqlService _sql;
public ModlistUpdater(ILogger<ModlistUpdater> logger, DBContext db, SqlService sql, AppSettings settings) : base(logger, db, sql)
public ModlistUpdater(ILogger<ModlistUpdater> logger, SqlService sql, AppSettings settings) : base(logger, sql)
{
_settings = settings;
_sql = sql;
@ -93,22 +91,21 @@ namespace Wabbajack.BuildServer.Controllers
Utils.Log($"Alternative requested for {startingHash}");
await Metric("requested_upgrade", startingHash.ToString());
var state = await Db.DownloadStates.AsQueryable()
var state = await SQL.GetNexusStateByHash(startingHash);
/*.DownloadStates.AsQueryable()
.Where(s => s.Hash == startingHash)
.Where(s => s.State is NexusDownloader.State)
.OrderByDescending(s => s.LastValidationTime).FirstOrDefaultAsync();
.OrderByDescending(s => s.LastValidationTime).FirstOrDefaultAsync();*/
if (state == null)
return NotFound("Original state not found");
var nexusState = state.State as NexusDownloader.State;
var nexusGame = nexusState.Game.MetaData().NexusName;
var mod_files = await Db.NexusModFiles.AsQueryable()
.Where(f => f.Game == nexusGame && f.ModId == nexusState.ModID)
.ToListAsync();
var nexusGame = nexusState.Game;
var mod_files = (await SQL.GetModFiles(nexusGame, nexusState.ModID)).files;
if (mod_files.SelectMany(f => f.Data.files)
.Any(f => f.category_name != null && f.file_id == nexusState.FileID))
if (mod_files.Any(f => f.category_name != null && f.file_id == nexusState.FileID))
{
await Metric("not_required_upgrade", startingHash.ToString());
return BadRequest("Upgrade Not Required");
@ -124,7 +121,7 @@ namespace Wabbajack.BuildServer.Controllers
Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}");
if (newArchive.Hash == Hash.Empty)
{
Db.Jobs.InsertOne(new Job
await SQL.EnqueueJob(new Job
{
Payload = new IndexJob
{
@ -147,7 +144,7 @@ namespace Wabbajack.BuildServer.Controllers
if (!PatchArchive.CdnPath(startingHash, newArchive.Hash).Exists)
{
Db.Jobs.InsertOne(new Job
await SQL.EnqueueJob(new Job
{
Priority = Job.JobPriority.High,
Payload = new PatchArchive
@ -185,7 +182,7 @@ namespace Wabbajack.BuildServer.Controllers
Utils.Log($"Found alternative for {srcHash}");
var indexed = await Db.DownloadStates.AsQueryable().Where(s => s.Key == archive.State.PrimaryKeyString).FirstOrDefaultAsync();
var indexed = await SQL.DownloadStateByPrimaryKey(archive.State.PrimaryKeyString);
if (indexed == null)
{
@ -195,9 +192,6 @@ namespace Wabbajack.BuildServer.Controllers
Utils.Log($"Pre-Indexed alternative {indexed.Hash} found for {srcHash}");
archive.Hash = indexed.Hash;
return archive;
}
}
}

23
Wabbajack.BuildServer/Controllers/NexusCache.cs
View File

@ -1,15 +1,11 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using CsvHelper;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using Newtonsoft.Json;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
@ -28,7 +24,7 @@ namespace Wabbajack.BuildServer.Controllers
private static long CachedCount = 0;
private static long ForwardCount = 0;
public NexusCache(ILogger<NexusCache> logger, DBContext db, SqlService sql, AppSettings settings) : base(logger, db, sql)
public NexusCache(ILogger<NexusCache> logger, SqlService sql, AppSettings settings) : base(logger, sql)
{
_settings = settings;
}
@ -55,13 +51,7 @@ namespace Wabbajack.BuildServer.Controllers
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{ModId}.json";
var body = await api.Get<ModInfo>(path);
try
{
await SQL.AddNexusModInfo(game, ModId, DateTime.Now, body);
}
catch (MongoWriteException)
{
}
await SQL.AddNexusModInfo(game, ModId, DateTime.Now, body);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);
@ -90,13 +80,8 @@ namespace Wabbajack.BuildServer.Controllers
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files.json";
var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
try
{
await SQL.AddNexusModFiles(game, ModId, DateTime.Now, body);
}
catch (MongoWriteException)
{
}
await SQL.AddNexusModFiles(game, ModId, DateTime.Now, body);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);

23
Wabbajack.BuildServer/Controllers/UploadedFiles.cs
View File

@ -9,23 +9,16 @@ using System.Text;
using System.Threading.Tasks;
using FluentFTP;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Identity;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nettle;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Org.BouncyCastle.Crypto.Engines;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Path = Alphaleonis.Win32.Filesystem.Path;
using AlphaFile = Alphaleonis.Win32.Filesystem.File;
@ -36,7 +29,7 @@ namespace Wabbajack.BuildServer.Controllers
private static ConcurrentDictionary<string, AsyncLock> _writeLocks = new ConcurrentDictionary<string, AsyncLock>();
private AppSettings _settings;
public UploadedFiles(ILogger<UploadedFiles> logger, DBContext db, AppSettings settings, SqlService sql) : base(logger, db, sql)
public UploadedFiles(ILogger<UploadedFiles> logger, AppSettings settings, SqlService sql) : base(logger, sql)
{
_settings = settings;
}
@ -88,7 +81,7 @@ namespace Wabbajack.BuildServer.Controllers
[Route("clean_http_uploads")]
public async Task<IActionResult> CleanUploads()
{
var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync();
var files = await SQL.AllUploadedFiles();
var seen = new HashSet<string>();
var duplicate = new List<UploadedFile>();
@ -115,7 +108,7 @@ namespace Wabbajack.BuildServer.Controllers
if (await client.FileExistsAsync(dup.MungedName))
await client.DeleteFileAsync(dup.MungedName);
await Db.UploadedFiles.DeleteOneAsync(f => f.Id == dup.Id);
await SQL.DeleteUploadedFile(dup.Id);
}
}
@ -182,7 +175,7 @@ namespace Wabbajack.BuildServer.Controllers
[Route("uploaded_files")]
public async Task<ContentResult> UploadedFilesGet()
{
var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync();
var files = await SQL.AllUploadedFiles();
var response = HandleGetListTemplate(new
{
files = files.Select(file => new
@ -221,7 +214,7 @@ namespace Wabbajack.BuildServer.Controllers
{
var user = User.FindFirstValue(ClaimTypes.Name);
Utils.Log($"Delete Uploaded File {user} {name}");
var files = await Db.UploadedFiles.AsQueryable().Where(f => f.Uploader == user).ToListAsync();
var files = await SQL.AllUploadedFilesForUser(name);
var to_delete = files.First(f => f.MungedName == name);
@ -237,10 +230,8 @@ namespace Wabbajack.BuildServer.Controllers
}
var result = await Db.UploadedFiles.DeleteOneAsync(f => f.Id == to_delete.Id);
if (result.DeletedCount == 1)
return Ok($"Deleted {name}");
return NotFound(name);
await SQL.DeleteUploadedFile(to_delete.Id);
return Ok($"Deleted {to_delete.MungedName}");
}
[HttpGet]

39
Wabbajack.BuildServer/Controllers/Users.cs
View File

@ -1,13 +1,8 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
namespace Wabbajack.BuildServer.Controllers
@ -16,47 +11,37 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/users")]
public class Users : AControllerBase<Users>
{
public Users(ILogger<Users> logger, DBContext db, SqlService sql) : base(logger, db, sql)
private AppSettings _settings;
public Users(ILogger<Users> logger, SqlService sql, AppSettings settings) : base(logger, sql)
{
_settings = settings;
}
[HttpGet]
[Route("add/{Name}")]
public async Task<string> AddUser(string Name)
{
var user = new ApiKey();
user.Key = NewAPIKey();
user.Id = Guid.NewGuid().ToString();
user.Roles = new List<string>();
user.CanUploadLists = new List<string>();
await Db.ApiKeys.InsertOneAsync(user);
return user.Id;
return await SQL.AddLogin(Name);
}
[HttpGet]
[Route("export")]
public async Task<string> Export()
{
if (!Directory.Exists("exported_users"))
Directory.CreateDirectory("exported_users");
var mainFolder = _settings.TempPath.Combine("exported_users");
mainFolder.CreateDirectory();
foreach (var user in await Db.ApiKeys.AsQueryable().ToListAsync())
foreach (var (owner, key) in await SQL.GetAllUserKeys())
{
Directory.CreateDirectory(Path.Combine("exported_users", user.Owner));
Alphaleonis.Win32.Filesystem.File.WriteAllText(Path.Combine("exported_users", user.Owner, "author-api-key.txt"), user.Key);
var folder = mainFolder.Combine(owner);
folder.CreateDirectory();
await folder.Combine(Consts.AuthorAPIKeyFile).WriteAllTextAsync(key);
}
return "done";
}
public static string NewAPIKey()
{
var arr = new byte[128];
new Random().NextBytes(arr);
return arr.ToHex();
}
}
}

10
Wabbajack.BuildServer/Extensions.cs
View File

@ -4,25 +4,15 @@ using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Builder;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.Common;
using Directory =Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.BuildServer
{
public static class Extensions
{
public static async Task<T> FindOneAsync<T>(this IMongoCollection<T> coll, Expression<Func<T, bool>> expr)
{
return (await coll.AsQueryable().Where(expr).Take(1).ToListAsync()).FirstOrDefault();
}
public static void UseJobManager(this IApplicationBuilder b)
{
var manager = (JobManager)b.ApplicationServices.GetService(typeof(JobManager));

22
Wabbajack.BuildServer/GraphQL/Mutation.cs
View File

@ -1,22 +0,0 @@
using GraphQL.Types;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
namespace Wabbajack.BuildServer.GraphQL
{
public class Mutation : ObjectGraphType
{
public Mutation(DBContext db)
{
FieldAsync<IdGraphType>("pollNexusForUpdates",
resolve: async context =>
{
var job = new Job {Payload = new GetNexusUpdatesJob()};
await db.Jobs.InsertOneAsync(job);
return job.Id;
});
}
}
}

47
Wabbajack.BuildServer/GraphQL/Query.cs
View File

@ -1,28 +1,13 @@
using System;
using System.Linq;
using System.Collections.Generic;
using System.Data.SqlTypes;
using GraphQL;
using System.Linq;
using GraphQL.Types;
using GraphQLParser.AST;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
namespace Wabbajack.BuildServer.GraphQL
{
public class Query : ObjectGraphType
{
public Query(DBContext db, SqlService sql)
public Query(SqlService sql)
{
Field<ListGraphType<JobType>>("unfinishedJobs", resolve: context =>
{
var data = db.Jobs.AsQueryable().Where(j => j.Ended == null).ToList();
return data;
});
FieldAsync<ListGraphType<ModListStatusType>>("modLists",
arguments: new QueryArguments(new QueryArgument<ArchiveEnumFilterType>
{
@ -31,37 +16,17 @@ namespace Wabbajack.BuildServer.GraphQL
resolve: async context =>
{
var arg = context.GetArgument<string>("filter");
var lists = db.ModListStatus.AsQueryable();
var lists = await sql.GetDetailedModlistStatuses();
switch (arg)
{
case "FAILED":
lists = lists.Where(l => l.DetailedStatus.HasFailures);
break;
return lists.Where(l => l.HasFailures);
case "PASSED":
lists = lists.Where(a => !a.DetailedStatus.HasFailures);
break;
return lists.Where(l => !l.HasFailures);
default:
break;
return lists;
}
return await lists.ToListAsync();
});
FieldAsync<ListGraphType<JobType>>("job",
arguments: new QueryArguments(
new QueryArgument<IntGraphType> {Name = "id", Description = "Id of the Job"}),
resolve: async context =>
{
var id = context.GetArgument<long>("id");
var data = await db.Jobs.AsQueryable().Where(j => j.Id == id).ToListAsync();
return data;
});
FieldAsync<ListGraphType<UploadedFileType>>("uploadedFiles",
resolve: async context =>
{
var data = await db.UploadedFiles.AsQueryable().ToListAsync();
return data;
});
FieldAsync<ListGraphType<MetricResultType>>("dailyUniqueMetrics",

35
Wabbajack.BuildServer/JobManager.cs
View File

@ -3,8 +3,6 @@ using System.Linq;
using System.Reactive.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nettle;
using Wabbajack.BuildServer.Controllers;
using Wabbajack.BuildServer.Model.Models;
@ -18,13 +16,11 @@ namespace Wabbajack.BuildServer
public class JobManager
{
protected readonly ILogger<JobManager> Logger;
protected readonly DBContext Db;
protected readonly AppSettings Settings;
protected SqlService Sql;
public JobManager(ILogger<JobManager> logger, DBContext db, SqlService sql, AppSettings settings)
public JobManager(ILogger<JobManager> logger, SqlService sql, AppSettings settings)
{
Db = db;
Logger = logger;
Settings = settings;
Sql = sql;
@ -42,7 +38,7 @@ namespace Wabbajack.BuildServer
{
try
{
var job = await Job.GetNext(Db);
var job = await Sql.GetJob();
if (job == null)
{
await Task.Delay(5000);
@ -50,18 +46,17 @@ namespace Wabbajack.BuildServer
}
Logger.Log(LogLevel.Information, $"Starting job: {job.Payload.Description}");
JobResult result;
try
{
result = await job.Payload.Execute(Db, Sql, Settings);
job.Result = await job.Payload.Execute(Sql, Settings);
}
catch (Exception ex)
{
Logger.Log(LogLevel.Error, ex, $"Error while running job: {job.Payload.Description}");
result = JobResult.Error(ex);
job.Result = JobResult.Error(ex);
}
await Job.Finish(Db, job, result);
await Sql.FinishJob(job);
}
catch (Exception ex)
{
@ -95,16 +90,15 @@ namespace Wabbajack.BuildServer
{
try
{
var started = await Db.Jobs.AsQueryable()
.Where(j => j.Started != null && j.Ended == null)
.ToListAsync();
var started = await Sql.GetRunningJobs();
foreach (var job in started)
{
var runtime = DateTime.Now - job.Started;
if (runtime > TimeSpan.FromMinutes(30))
{
await Job.Finish(Db, job, JobResult.Error(new Exception($"Timeout after {runtime.Value.TotalMinutes}")));
}
if (!(runtime > TimeSpan.FromMinutes(30))) continue;
job.Result = JobResult.Error(new Exception($"Timeout after {runtime.Value.TotalMinutes}"));
await Sql.FinishJob(job);
}
}
catch (Exception ex)
@ -119,18 +113,17 @@ namespace Wabbajack.BuildServer
if (!Settings.RunFrontEndJobs && typeof(T).ImplementsInterface(typeof(IFrontEndJob))) return;
try
{
var jobs = await Db.Jobs.AsQueryable()
var jobs = (await Sql.GetUnfinishedJobs())
.Where(j => j.Payload is T)
.OrderByDescending(j => j.Created)
.Take(10)
.ToListAsync();
.Take(10);
foreach (var job in jobs)
{
if (job.Started == null || job.Ended == null) return;
if (DateTime.Now - job.Ended < span) return;
}
await Db.Jobs.InsertOneAsync(new Job
await Sql.EnqueueJob(new Job
{
Priority = priority,
Payload = new T()

7
Wabbajack.BuildServer/Models/ApiKey.cs
View File

@ -1,7 +1,5 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
namespace Wabbajack.BuildServer.Models
{
@ -14,10 +12,5 @@ namespace Wabbajack.BuildServer.Models
public List<string> CanUploadLists { get; set; }
public List<string> Roles { get; set; }
public static async Task<ApiKey> Get(DBContext db, string key)
{
return await db.ApiKeys.AsQueryable().Where(k => k.Key == key).FirstOrDefaultAsync();
}
}
}

46
Wabbajack.BuildServer/Models/DBContext.cs
View File

@ -1,46 +0,0 @@
using System.Collections.Generic;
using Microsoft.Extensions.Configuration;
using MongoDB.Driver;
using Wabbajack.Lib.NexusApi;
using Wabbajack.BuildServer.Models.JobQueue;
namespace Wabbajack.BuildServer.Models
{
public class DBContext
{
private IConfiguration _configuration;
private Settings _settings;
public DBContext(IConfiguration configuration)
{
_configuration = configuration;
_settings = new Settings();
_configuration.Bind("MongoDB", _settings);
}
public IMongoCollection<NexusCacheData<ModInfo>> NexusModInfos => Client.GetCollection<NexusCacheData<ModInfo>>(_settings.Collections["NexusModInfos"]);
public IMongoCollection<NexusCacheData<NexusFileInfo>> NexusFileInfos => Client.GetCollection<NexusCacheData<NexusFileInfo>>(_settings.Collections["NexusFileInfos"]);
public IMongoCollection<ModListStatus> ModListStatus => Client.GetCollection<ModListStatus>(_settings.Collections["ModListStatus"]);
public IMongoCollection<Job> Jobs => Client.GetCollection<Job>(_settings.Collections["JobQueue"]);
public IMongoCollection<DownloadState> DownloadStates => Client.GetCollection<DownloadState>(_settings.Collections["DownloadStates"]);
public IMongoCollection<Metric> Metrics => Client.GetCollection<Metric>(_settings.Collections["Metrics"]);
public IMongoCollection<IndexedFile> IndexedFiles => Client.GetCollection<IndexedFile>(_settings.Collections["IndexedFiles"]);
public IMongoCollection<NexusCacheData<List<NexusUpdateEntry>>> NexusUpdates => Client.GetCollection<NexusCacheData<List<NexusUpdateEntry>>>(_settings.Collections["NexusUpdates"]);
public IMongoCollection<ApiKey> ApiKeys => Client.GetCollection<ApiKey>(_settings.Collections["ApiKeys"]);
public IMongoCollection<UploadedFile> UploadedFiles => Client.GetCollection<UploadedFile>(_settings.Collections["UploadedFiles"]);
public IMongoCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>> NexusModFiles =>
Client.GetCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>>(
_settings.Collections["NexusModFiles"]);
private IMongoDatabase Client => new MongoClient($"mongodb://{_settings.Host}").GetDatabase(_settings.Database);
}
public class Settings
{
public string Host { get; set; }
public string Database { get; set; }
public Dictionary<string, string> Collections { get; set; }
public string SqlConnection { get; set; }
}
}

2
Wabbajack.BuildServer/Models/DownloadState.cs
View File

@ -1,5 +1,4 @@
using System;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
@ -7,7 +6,6 @@ namespace Wabbajack.BuildServer.Models
{
public class DownloadState
{
[BsonId]
public string Key { get; set; }
public Hash Hash { get; set; }

9
Wabbajack.BuildServer/Models/IndexedFile.cs
View File

@ -1,17 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using System.Collections.Generic;
using Wabbajack.Common;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.BuildServer.Models
{
public class IndexedFile
{
[BsonId]
public Hash Hash { get; set; }
public string SHA256 { get; set; }
public string SHA1 { get; set; }

6
Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs
View File

@ -1,9 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.Jobs;
@ -18,7 +16,6 @@ namespace Wabbajack.BuildServer.Models.JobQueue
typeof(UpdateModLists),
typeof(EnqueueAllArchives),
typeof(EnqueueAllGameFiles),
typeof(EnqueueRecentFiles),
typeof(UploadToCDN),
typeof(IndexDynDOLOD),
typeof(ReindexArchives),
@ -28,12 +25,11 @@ namespace Wabbajack.BuildServer.Models.JobQueue
public static Dictionary<string, Type> NameToType { get; set; }
[BsonIgnore]
public abstract string Description { get; }
public virtual bool UsesNexus { get; } = false;
public abstract Task<JobResult> Execute(DBContext db, SqlService sql,AppSettings settings);
public abstract Task<JobResult> Execute(SqlService sql,AppSettings settings);
static AJobPayload()
{

41
Wabbajack.BuildServer/Models/JobQueue/Job.cs
View File

@ -1,12 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.BuildServer.Models.JobQueue
{
@ -29,39 +22,5 @@ namespace Wabbajack.BuildServer.Models.JobQueue
public AJobPayload Payload { get; set; }
public Job OnSuccess { get; set; }
public static async Task<Job> GetNext(DBContext db)
{
var filter = new BsonDocument
{
{"Started", BsonNull.Value}
};
var update = new BsonDocument
{
{"$set", new BsonDocument {{"Started", DateTime.Now}}}
};
var sort = new {Priority=-1, Created=1}.ToBsonDocument();
var job = await db.Jobs.FindOneAndUpdateAsync<Job>(filter, update, new FindOneAndUpdateOptions<Job>{Sort = sort});
return job;
}
public static async Task<Job> Finish(DBContext db, Job job, JobResult jobResult)
{
if (jobResult.ResultType == JobResultType.Success && job.OnSuccess != null)
{
await db.Jobs.InsertOneAsync(job.OnSuccess);
}
var filter = new BsonDocument
{
{"_id", job.Id},
};
var update = new BsonDocument
{
{"$set", new BsonDocument {{"Ended", DateTime.Now}, {"Result", jobResult.ToBsonDocument()}}}
};
var result = await db.Jobs.FindOneAndUpdateAsync<Job>(filter, update);
return result;
}
}
}

8
Wabbajack.BuildServer/Models/JobQueue/JobResult.cs
View File

@ -1,19 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
namespace Wabbajack.BuildServer.Models.JobQueue
{
public class JobResult
{
public JobResultType ResultType { get; set; }
[BsonIgnoreIfNull]
public string Message { get; set; }
[BsonIgnoreIfNull]
public string Stacktrace { get; set; }
public static JobResult Success()

24
Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs
View File

@ -1,10 +1,6 @@
using System;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using System.Linq;
using FluentFTP;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
@ -17,7 +13,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
public class EnqueueAllArchives : AJobPayload, IBackEndJob
{
public override string Description => "Add missing modlist archives to indexer";
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{
Utils.Log("Starting ModList indexing");
var modlists = await ModlistMetadata.LoadFromGithub();
@ -28,7 +24,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
{
try
{
await EnqueueFromList(db, list, queue);
await EnqueueFromList(sql, list, queue);
}
catch (Exception ex)
{
@ -40,10 +36,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
private static async Task EnqueueFromList(DBContext db, ModlistMetadata list, WorkQueue queue)
private static async Task EnqueueFromList(SqlService sql, ModlistMetadata list, WorkQueue queue)
{
var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
if (list.NeedsDownload(modlistPath))
@ -66,21 +60,23 @@ namespace Wabbajack.BuildServer.Models.Jobs
var archives = installer.Archives;
Utils.Log($"Found {archives.Count} archives in {installer.Name} to index");
var searching = archives.Select(a => a.Hash).Distinct().ToArray();
var searching = archives.Select(a => a.Hash).ToHashSet();
Utils.Log($"Looking for missing archives");
var knownArchives = (await db.IndexedFiles.AsQueryable().Where(a => searching.Contains(a.Hash))
.Select(d => d.Hash).ToListAsync()).ToDictionary(a => a);
var knownArchives = await sql.FilterByExistingIndexedArchives(searching);
Utils.Log($"Found {knownArchives.Count} pre-existing archives");
var missing = archives.Where(a => !knownArchives.ContainsKey(a.Hash)).ToList();
var missing = archives.Where(a => !knownArchives.Contains(a.Hash)).ToList();
Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs");
var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}, Priority = Job.JobPriority.Low});
Utils.Log($"Writing jobs to the database");
await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions {IsOrdered = false});
foreach (var job in jobs)
await sql.EnqueueJob(job);
Utils.Log($"Done adding archives for {installer.Name}");
}
}

26
Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs
View File

@ -5,18 +5,14 @@ using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using System.IO;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class EnqueueAllGameFiles : AJobPayload, IBackEndJob
{
public override string Description { get => $"Enqueue all game files for indexing"; }
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{
using (var queue = new WorkQueue(4))
{
@ -32,16 +28,12 @@ namespace Wabbajack.BuildServer.Models.Jobs
}))
.ToList();
var pks = states.Select(s => s.PrimaryKeyString).Distinct().ToArray();
Utils.Log($"Found {pks.Length} archives to cross-reference with the database");
var pks = states.Select(s => s.PrimaryKeyString).ToHashSet();
Utils.Log($"Found {pks.Count} archives to cross-reference with the database");
var found = (await db.DownloadStates
.AsQueryable().Where(s => pks.Contains(s.Key))
.Select(s => s.Key)
.ToListAsync())
.ToDictionary(s => s);
states = states.Where(s => !found.ContainsKey(s.PrimaryKeyString)).ToList();
var found = await sql.FilterByExistingPrimaryKeys(pks);
states = states.Where(s => !found.Contains(s.PrimaryKeyString)).ToList();
Utils.Log($"Found {states.Count} archives to index");
await states.PMap(queue, async state =>
@ -58,14 +50,14 @@ namespace Wabbajack.BuildServer.Models.Jobs
}
});
var with_hash = states.Where(state => state.Hash != null).ToList();
var with_hash = states.Where(state => state.Hash != default).ToList();
Utils.Log($"Inserting {with_hash.Count} jobs.");
var jobs = states.Select(state => new IndexJob {Archive = new Archive {Name = state.GameFile.FileName.ToString(), State = state}})
.Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus})
.ToList();
if (jobs.Count > 0)
await db.Jobs.InsertManyAsync(jobs);
foreach (var job in jobs)
await sql.EnqueueJob(job);
return JobResult.Success();
}

85
Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs
View File

@ -1,85 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using MongoDB.Driver;
using MongoDB.Driver.Core.Authentication;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class EnqueueRecentFiles : AJobPayload, IFrontEndJob
{
public override string Description => "Enqueue the past days worth of mods for indexing";
private static HashSet<Game> GamesToScan = new HashSet<Game>
{
Game.Fallout3, Game.Fallout4, Game.Skyrim, Game.SkyrimSpecialEdition, Game.SkyrimVR, Game.FalloutNewVegas, Game.Oblivion
};
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
{
using (var queue = new WorkQueue())
{
var updates = await db.NexusUpdates.AsQueryable().ToListAsync();
var mods = updates
.Where(list => GamesToScan.Contains(GameRegistry.GetByNexusName(list.Game).Game))
.SelectMany(list =>
list.Data.Where(mod => DateTime.UtcNow - mod.LatestFileUpdate.AsUnixTime() < TimeSpan.FromDays(1))
.Select(mod => (list.Game, mod.ModId)));
var mod_files = (await mods.PMap(queue, async mod =>
{
var client = await NexusApiClient.Get();
try
{
var files = await client.GetModFiles(GameRegistry.GetByNexusName(mod.Game).Game,
(int)mod.ModId);
return (Game: GameRegistry.GetByFuzzyName(mod.Game).Game, mod.ModId, files.files);
}
catch (Exception)
{
return default;
}
})).Where(t => t.Game != default).ToList();
var archives =
mod_files.SelectMany(mod => mod.files.Select(file => (mod.Game, mod.ModId, File:file)).Where(f => !string.IsNullOrEmpty(f.File.category_name) ))
.Select(tuple =>
{
var state = new NexusDownloader.State
{
Game = tuple.Game, ModID = tuple.ModId, FileID = tuple.File.file_id
};
return new Archive {State = state, Name = tuple.File.file_name};
}).ToList();
Utils.Log($"Found {archives.Count} archives from recent Nexus updates to index");
var searching = archives.Select(a => a.State.PrimaryKeyString).Distinct().ToArray();
Utils.Log($"Looking for missing states");
var knownArchives = (await db.DownloadStates.AsQueryable().Where(s => searching.Contains(s.Key))
.Select(d => d.Key).ToListAsync()).ToDictionary(a => a);
Utils.Log($"Found {knownArchives.Count} pre-existing archives");
var missing = archives.Where(a => !knownArchives.ContainsKey(a.State.PrimaryKeyString))
.DistinctBy(d => d.State.PrimaryKeyString)
.ToList();
Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs");
var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}, Priority = Job.JobPriority.Low});
Utils.Log($"Writing jobs to the DB");
await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions {IsOrdered = false});
Utils.Log($"Done adding archives for Nexus Updates");
return JobResult.Success();
}
}
}
}

19
Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs
View File

@ -4,8 +4,6 @@ using System.Threading.Tasks;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
using MongoDB.Driver;
using Newtonsoft.Json;
using Wabbajack.BuildServer.Model.Models;
@ -15,7 +13,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
{
public override string Description => "Poll the Nexus for updated mods, and clean any references to those mods";
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{
var api = await NexusApiClient.Get();
@ -31,8 +29,6 @@ namespace Wabbajack.BuildServer.Models.Jobs
entry.Path = $"/v1/games/{game.NexusName}/mods/updated.json?period=1m";
entry.Data = mods;
await entry.Upsert(db.NexusUpdates);
return (game, mods);
})
.Select(async rTask =>
@ -56,19 +52,14 @@ namespace Wabbajack.BuildServer.Models.Jobs
// Mod activity could hide files
var b = d.mod.LastestModActivity.AsUnixTime();
return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.ModId};
return new {Game = d.game.Game, Date = (a > b ? a : b), ModId = d.mod.ModId};
});
var purged = await collected.PMap(queue, async t =>
{
var resultA = await db.NexusModInfos.DeleteManyAsync(f =>
f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
var resultB = await db.NexusModFiles.DeleteManyAsync(f =>
f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
var resultC = await db.NexusFileInfos.DeleteManyAsync(f =>
f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount;
var resultA = await sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date);
var resultB = await sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date);
return resultA + resultB;
});
Utils.Log($"Purged {purged.Sum()} cache entries");

8
Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs
View File

@ -3,8 +3,6 @@ using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using HtmlAgilityPack;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
@ -20,7 +18,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
public class IndexDynDOLOD : AJobPayload
{
public override string Description => "Queue MEGA URLs from the DynDOLOD Post";
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{
var doc = new HtmlDocument();
var body = await new HttpClient().GetStringAsync(new Uri(
@ -54,11 +52,11 @@ namespace Wabbajack.BuildServer.Models.Jobs
foreach (var job in matches)
{
var key = ((MegaDownloader.State)((IndexJob)job.Payload).Archive.State).PrimaryKeyString;
var found = await db.DownloadStates.AsQueryable().Where(s => s.Key == key).FirstOrDefaultAsync();
var found = await sql.DownloadStateByPrimaryKey(key);
if (found != null) continue;
Utils.Log($"Queuing {key} for indexing");
await db.Jobs.InsertOneAsync(job);
await sql.EnqueueJob(job);
}
return JobResult.Success();

50
Wabbajack.BuildServer/Models/Jobs/IndexJob.cs
View File

@ -1,14 +1,9 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using FluentFTP;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib;
@ -23,7 +18,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
public Archive Archive { get; set; }
public override string Description => $"Index ${Archive.State.PrimaryKeyString} and save the download/file state";
public override bool UsesNexus { get => Archive.State is NexusDownloader.State; }
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{
if (Archive.State is ManualDownloader.State)
return JobResult.Success();
@ -33,8 +28,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
pk.AddRange(Archive.State.PrimaryKey);
var pk_str = string.Join("|",pk.Select(p => p.ToString()));
var found = await db.DownloadStates.AsQueryable().Where(f => f.Key == pk_str).Take(1).ToListAsync();
if (found.Count > 0)
var found = await sql.DownloadStateByPrimaryKey(pk_str);
if (found == null)
return JobResult.Success();
string fileName = Archive.Name;
@ -51,10 +46,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
await sql.MergeVirtualFile(archive);
await db.DownloadStates.InsertOneAsync(new DownloadState
{
Key = pk_str, Hash = archive.Hash, State = Archive.State, IsValid = true
});
await sql.AddDownloadState(archive.Hash, Archive.State);
var to_path = settings.ArchiveDir.Combine(
$"{Path.GetFileName(fileName)}_{archive.Hash.ToHex()}_{Path.GetExtension(fileName)}");
@ -66,43 +58,9 @@ namespace Wabbajack.BuildServer.Models.Jobs
await settings.DownloadDir.Combine(folder).DeleteDirectory();
}
return JobResult.Success();
}
/*
private List<IndexedFile> ConvertArchive(List<IndexedFile> files, VirtualFile file, bool isTop = true)
{
var name = isTop ? file.Name.FileName : file.Name;
var ifile = new IndexedFile
{
Hash = file.Hash,
SHA256 = file.ExtendedHashes.SHA256,
SHA1 = file.ExtendedHashes.SHA1,
MD5 = file.ExtendedHashes.MD5,
CRC = file.ExtendedHashes.CRC,
Size = file.Size,
Children = file.Children != null ? file.Children.Select(
f =>
{
ConvertArchive(files, f, false);
return new ChildFile
{
Hash = f.Hash,
Name = f.Name,
Extension = Path.GetExtension(f.Name.ToLowerInvariant())
};
}).ToList() : new List<ChildFile>()
};
ifile.IsArchive = ifile.Children.Count > 0;
files.Add(ifile);
return files;
}*/
}
}

2
Wabbajack.BuildServer/Models/Jobs/ReindexArchives.cs
View File

@ -14,7 +14,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
public class ReindexArchives : AJobPayload
{
public override string Description => "Reindex all files in the mod archive folder";
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{
using (var queue = new WorkQueue())
{

24
Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs
View File

@ -1,9 +1,6 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
@ -12,14 +9,13 @@ using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using File = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class UpdateModLists : AJobPayload, IFrontEndJob
{
public override string Description => "Validate curated modlists";
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{
Utils.Log("Starting Modlist Validation");
var modlists = await ModlistMetadata.LoadFromGithub();
@ -34,7 +30,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
{
try
{
await ValidateList(db, list, queue, whitelists);
await ValidateList(sql, list, queue, whitelists);
}
catch (Exception ex)
{
@ -46,7 +42,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
private async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
private async Task ValidateList(SqlService sql, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
{
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
@ -76,7 +72,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
var validated = (await installer.Archives
.PMap(queue, async archive =>
{
var isValid = await IsValid(db, whitelists, archive);
var isValid = await IsValid(sql, whitelists, archive);
return new DetailedStatusItem {IsFailing = !isValid, Archive = archive};
}))
@ -107,13 +103,13 @@ namespace Wabbajack.BuildServer.Models.Jobs
};
Utils.Log(
$"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed");
await ModListStatus.Update(db, dto);
await sql.UpdateModListStatus(dto);
Utils.Log(
$"Done updating {dto.Summary.Name}");
}
private async Task<bool> IsValid(DBContext db, ValidateModlist whitelists, Archive archive)
private async Task<bool> IsValid(SqlService sql, ValidateModlist whitelists, Archive archive)
{
try
{
@ -123,7 +119,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
{
if (archive.State is NexusDownloader.State state)
{
if (await ValidateNexusFast(db, state)) return true;
if (await ValidateNexusFast(sql, state)) return true;
}
else if (archive.State is GoogleDriveDownloader.State)
@ -166,13 +162,11 @@ namespace Wabbajack.BuildServer.Models.Jobs
}
}
private async Task<bool> ValidateNexusFast(DBContext db, NexusDownloader.State state)
private async Task<bool> ValidateNexusFast(SqlService sql, NexusDownloader.State state)
{
try
{
var gameMeta = state.Game.MetaData();
var modFiles = (await db.NexusModFiles.AsQueryable().Where(g => g.Game == gameMeta.NexusName && g.ModId == state.ModID).FirstOrDefaultAsync())?.Data;
var modFiles = await sql.GetModFiles(state.Game, state.ModID);
if (modFiles == null)
{

10
Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs
View File

@ -2,11 +2,7 @@
using System.Net;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using BunnyCDN.Net.Storage;
using CG.Web.MegaApiClient;
using FluentFTP;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
@ -22,11 +18,11 @@ namespace Wabbajack.BuildServer.Models.Jobs
public Guid FileId { get; set; }
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{
int retries = 0;
TOP:
var file = await db.UploadedFiles.AsQueryable().Where(f => f.Id == FileId).FirstOrDefaultAsync();
var file = await sql.UploadedFileById(FileId);
if (settings.BunnyCDN_User == "TEST" && settings.BunnyCDN_Password == "TEST")
{
@ -53,7 +49,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
}
}
await db.Jobs.InsertOneAsync(new Job
await sql.EnqueueJob(new Job
{
Priority = Job.JobPriority.High,
Payload = new IndexJob

56
Wabbajack.BuildServer/Models/Metric.cs
View File

@ -3,71 +3,17 @@ using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.GraphQL;
using Wabbajack.Common;
using Wabbajack.BuildServer.Model.Models;
namespace Wabbajack.BuildServer.Models
{
public class Metric
{
[BsonId]
public ObjectId Id { get; set; }
public DateTime Timestamp { get; set; }
public string Action { get; set; }
public string Subject { get; set; }
public string MetricsKey { get; set; }
public static async Task<IEnumerable<MetricResult>> Report(DBContext db, string grouping)
{
var regex = new Regex("\\d+\\.");
var data = await db.Metrics.AsQueryable()
.Where(m => m.MetricsKey != null)
.Where(m => m.Action == grouping)
.Where(m => m.Subject != "Default")
.ToListAsync();
var minDate = DateTime.Parse(data.Min(d => d.Timestamp.ToString("yyyy-MM-dd")));
var maxDate = DateTime.Parse(data.Max(d => d.Timestamp.ToString("yyyy-MM-dd")));
var dateArray = Enumerable.Range(0, (int)(maxDate - minDate).TotalDays + 1)
.Select(idx => minDate + TimeSpan.FromDays(idx))
.Select(date => date.ToString("yyyy-MM-dd"))
.ToList();
var results = data
.Where(d => !Guid.TryParse(d.Subject, out var _))
.GroupBy(d => regex.Split(d.Subject).First())
.Select(by_series =>
{
var by_day = by_series.GroupBy(d => d.Timestamp.ToString("yyyy-MM-dd"))
.Select(d => (d.Key, d.DistinctBy(v => v.MetricsKey ?? "").Count()))
.OrderBy(r => r.Key);
var by_day_idx = by_day.ToDictionary(d => d.Key);
(string Key, int) GetEntry(string date)
{
if (by_day_idx.TryGetValue(date, out var result))
return result;
return (date, 0);
}
return new MetricResult
{
SeriesName = by_series.Key,
Labels = dateArray.Select(d => GetEntry(d).Key).ToList(),
Values = dateArray.Select(d => GetEntry(d).Item2).ToList()
};
})
.OrderBy(f => f.SeriesName);
return results;
}
}
}

20
Wabbajack.BuildServer/Models/ModListStatus.cs
View File

@ -2,9 +2,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.Lib;
using Wabbajack.Lib.ModListRegistry;
@ -12,20 +9,12 @@ namespace Wabbajack.BuildServer.Models
{
public class ModListStatus
{
[BsonId]
public string Id { get; set; }
public ModlistSummary Summary { get; set; }
public ModlistMetadata Metadata { get; set; }
public DetailedStatus DetailedStatus { get; set; }
public static async Task Update(DBContext db, ModListStatus status)
{
var id = status.Metadata.Links.MachineURL;
await db.ModListStatus.FindOneAndReplaceAsync<ModListStatus>(s => s.Id == id, status, new FindOneAndReplaceOptions<ModListStatus> {IsUpsert = true});
}
public static IQueryable<ModListStatus> AllSummaries
{
get
@ -33,15 +22,6 @@ namespace Wabbajack.BuildServer.Models
return null;
}
}
public static async Task<ModListStatus> ByName(DBContext db, string name)
{
var result = await db.ModListStatus
.AsQueryable()
.Where(doc => doc.Metadata.Links.MachineURL == name || doc.Metadata.Title == name)
.ToListAsync();
return result.First();
}
}
public class DetailedStatus

9
Wabbajack.BuildServer/Models/NexusCacheData.cs
View File

@ -1,28 +1,19 @@
using System;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
namespace Wabbajack.BuildServer.Models
{
public class NexusCacheData<T>
{
[BsonId]
public string Path { get; set; }
public T Data { get; set; }
public string Game { get; set; }
[BsonIgnoreIfNull]
public long ModId { get; set; }
public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow;
[BsonIgnoreIfNull]
public string FileId { get; set; }
public async Task Upsert(IMongoCollection<NexusCacheData<T>> coll)
{
await coll.FindOneAndReplaceAsync<NexusCacheData<T>>(s => s.Path == Path, this, new FindOneAndReplaceOptions<NexusCacheData<T>> {IsUpsert = true});
}
}
}

4
Wabbajack.BuildServer/Models/NexusUpdateEntry.cs
View File

@ -1,6 +1,4 @@
using System;
using MongoDB.Bson.Serialization.Attributes;
using Newtonsoft.Json;
using Newtonsoft.Json;
namespace Wabbajack.BuildServer.Models
{

10
Wabbajack.BuildServer/Models/PatchArchive.cs
View File

@ -1,17 +1,11 @@
using System;
using System.IO;
using System.Net;
using System.Threading.Tasks;
using FluentFTP;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using File = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.BuildServer.Models
{
@ -20,10 +14,10 @@ namespace Wabbajack.BuildServer.Models
public override string Description => "Create a archive update patch";
public Hash Src { get; set; }
public string DestPK { get; set; }
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{
var srcPath = settings.PathForArchive(Src);
var destHash = (await db.DownloadStates.AsQueryable().Where(s => s.Key == DestPK).FirstOrDefaultAsync()).Hash;
var destHash = (await sql.DownloadStateByPrimaryKey(DestPK)).Hash;
var destPath = settings.PathForArchive(destHash);
if (Src == destHash)

196
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Data;
using System.Data.SqlClient;
@ -11,6 +12,7 @@ using Wabbajack.BuildServer.Model.Models.Results;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
@ -168,18 +170,6 @@ namespace Wabbajack.BuildServer.Model.Models
.ToList();
}
#region UserRoutines
public async Task<string> LoginByAPIKey(string key)
{
await using var conn = await Open();
var result = await conn.QueryAsync<string>(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @Key",
new {Key = key});
return result.FirstOrDefault();
}
#endregion
#region JobRoutines
/// <summary>
@ -233,6 +223,24 @@ namespace Wabbajack.BuildServer.Model.Models
new {RunBy = Guid.NewGuid().ToString()});
return result.FirstOrDefault();
}
public async Task<IEnumerable<Job>> GetRunningJobs()
{
await using var conn = await Open();
var results =
await conn.QueryAsync<Job>("SELECT * from dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL ");
return results;
}
public async Task<IEnumerable<Job>> GetUnfinishedJobs()
{
await using var conn = await Open();
var results =
await conn.QueryAsync<Job>("SELECT * from dbo.Jobs WHERE Ended IS NULL ");
return results;
}
#endregion
@ -292,6 +300,15 @@ namespace Wabbajack.BuildServer.Model.Models
uf.CDNName
});
}
public async Task<UploadedFile> UploadedFileById(Guid fileId)
{
await using var conn = await Open();
return await conn.QueryFirstAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles WHERE Id = @Id",
new {Id = fileId.ToString()});
}
public async Task<IEnumerable<UploadedFile>> AllUploadedFilesForUser(string user)
{
@ -299,6 +316,24 @@ namespace Wabbajack.BuildServer.Model.Models
return await conn.QueryAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles WHERE UploadedBy = @uploadedBy",
new {UploadedBy = user});
}
public async Task<IEnumerable<UploadedFile>> AllUploadedFiles()
{
await using var conn = await Open();
return await conn.QueryAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles ORDER BY UploadDate DESC");
}
public async Task DeleteUploadedFile(Guid dupId)
{
await using var conn = await Open();
await conn.ExecuteAsync("SELECT * FROM dbo.UploadedFiles WHERE Id = @id",
new
{
Id = dupId.ToString()
});
}
public async Task AddDownloadState(Hash hash, AbstractDownloadState state)
{
@ -429,7 +464,6 @@ namespace Wabbajack.BuildServer.Model.Models
});
return result.FromJSONString<DetailedStatus>();
}
public async Task<List<DetailedStatus>> GetDetailedModlistStatuses()
{
await using var conn = await Open();
@ -440,5 +474,141 @@ namespace Wabbajack.BuildServer.Model.Models
#endregion
#region Logins
public async Task<string> AddLogin(string name)
{
var key = NewAPIKey();
await using var conn = await Open();
await conn.ExecuteAsync("INSERT INTO dbo.ApiKeys (Owner, ApiKey) VALUES (@Owner, @ApiKey)",
new {Owner = name, ApiKey = key});
return key;
}
public static string NewAPIKey()
{
var arr = new byte[128];
new Random().NextBytes(arr);
return arr.ToHex();
}
public async Task<string> LoginByAPIKey(string key)
{
await using var conn = await Open();
var result = await conn.QueryAsync<string>(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @ApiKey",
new {ApiKey = key});
return result.FirstOrDefault();
}
public async Task<IEnumerable<(string Owner, string Key)>> GetAllUserKeys()
{
await using var conn = await Open();
var result = await conn.QueryAsync<(string Owner, string Key)>("SELECT Owner, ApiKey FROM dbo.ApiKeys");
return result;
}
#endregion
#region Auto-healing routines
public async Task<Archive> GetNexusStateByHash(Hash startingHash)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(@"SELECT JsonState FROM dbo.DownloadStates
WHERE Hash = @hash AND PrimaryKey like 'NexusDownloader+State|%'
ORDER BY LastValidated DESC",
new {Hash = startingHash});
return result == null ? null : new Archive
{
State = result.FromJSONString<AbstractDownloadState>(),
Hash = startingHash
};
}
public async Task<Archive> DownloadStateByPrimaryKey(string primaryKey)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(long Hash, string State)>(@"SELECT Hash, JsonState FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey",
new {PrimaryKey = primaryKey});
return result == default ? null : new Archive
{
State = result.State.FromJSONString<AbstractDownloadState>(),
Hash = Hash.FromLong(result.Hash)
};
}
#endregion
/// <summary>
/// Returns a hashset the only contains hashes from the input that do not exist in IndexedArchives
/// </summary>
/// <param name="searching"></param>
/// <returns></returns>
/// <exception cref="NotImplementedException"></exception>
public async Task<HashSet<Hash>> FilterByExistingIndexedArchives(HashSet<Hash> searching)
{
await using var conn = await Open();
var found = await conn.QueryAsync<long>("SELECT Hash from dbo.IndexedFile WHERE Hash in @Hashes",
new {Hashes = searching.Select(h => (long)h)});
return searching.Except(found.Select(h => Hash.FromLong(h)).ToHashSet()).ToHashSet();
}
/// <summary>
/// Returns a hashset the only contains primary keys from the input that do not exist in IndexedArchives
/// </summary>
/// <param name="searching"></param>
/// <returns></returns>
/// <exception cref="NotImplementedException"></exception>
public async Task<HashSet<string>> FilterByExistingPrimaryKeys(HashSet<string> pks)
{
await using var conn = await Open();
var found = await conn.QueryAsync<string>("SELECT Hash from dbo.IndexedFile WHERE PrimaryKey in @PrimaryKeys",
new {PrimaryKeys = pks.ToList()});
return pks.Except(found.ToHashSet()).ToHashSet();
}
public async Task<long> DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date)
{
await using var conn = await Open();
var deleted = await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
return deleted;
}
public async Task<long> DeleteNexusModFilesUpdatedBeforeDate(Game game, long modId, DateTime date)
{
await using var conn = await Open();
var deleted = await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
return deleted;
}
public async Task UpdateModListStatus(ModListStatus dto)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"MERGE dbo.ModLists AS Target
USING (SELECT @MachineUrl MachineUrl, @Metadata Metadata, @Summary Summary, @DetailedStatus DetailedStatus) AS Source
ON Target.MachineUrl = Source.MachineUrl
WHEN MATCHED THEN UPDATE SET Target.Summary = Source.Summary, Target.Metadata = Source.Metadata, Target.DetailedStatus = Source.DetailedStats
WHEN NOT MATCHED THEN INSERT (MachineUrl, Summary, Metadata, DetailedStatus) VALUES (@MachineUrl, @Summary, @Metadata, @DetailedStatus)",
new
{
MachineUrl = dto.Metadata.Links.MachineURL,
Metadata = dto.Metadata.ToJSON(),
Summary = dto.Summary.ToJSON(),
DetailedStatus = dto.DetailedStatus.ToJSON()
});
}
}
}

6
Wabbajack.BuildServer/Models/UploadedFile.cs
View File

@ -1,8 +1,4 @@
using System;
using System.IO;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.Common;
using Path = Alphaleonis.Win32.Filesystem.Path;
@ -19,10 +15,8 @@ namespace Wabbajack.BuildServer.Models
public string CDNName { get; set; }
[BsonIgnore]
public string MungedName => $"{Path.GetFileNameWithoutExtension(Name)}-{Id}{Path.GetExtension(Name)}";
[BsonIgnore]
public string Uri => CDNName == null ? $"https://wabbajack.b-cdn.net/{MungedName}" : $"https://{CDNName}.b-cdn.net/{MungedName}";
}
}

76
Wabbajack.BuildServer/SerializerSettings.cs
View File

@ -1,76 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc.ModelBinding;
using MongoDB.Bson;
using MongoDB.Bson.IO;
using MongoDB.Bson.Serialization;
using MongoDB.Bson.Serialization.Conventions;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.BuildServer
{
public static class SerializerSettings
{
public static void Init()
{
var dis = new TypeDiscriminator(typeof(AbstractDownloadState), AbstractDownloadState.NameToType,
AbstractDownloadState.TypeToName);
BsonSerializer.RegisterDiscriminatorConvention(typeof(AbstractDownloadState), dis);
BsonClassMap.RegisterClassMap<AbstractDownloadState>(cm => cm.SetIsRootClass(true));
dis = new TypeDiscriminator(typeof(AJobPayload), AJobPayload.NameToType, AJobPayload.TypeToName);
BsonSerializer.RegisterDiscriminatorConvention(typeof(AJobPayload), dis);
BsonClassMap.RegisterClassMap<AJobPayload>(cm => cm.SetIsRootClass(true));
}
}
public class TypeDiscriminator : IDiscriminatorConvention
{
private readonly Type defaultType;
private readonly Dictionary<string, Type> typeMap;
private Dictionary<Type, string> revMap;
public TypeDiscriminator(Type defaultType,
Dictionary<string, Type> typeMap, Dictionary<Type, string> revMap)
{
this.defaultType = defaultType;
this.typeMap = typeMap;
this.revMap = revMap;
}
/// <summary>
/// Element Name
/// </summary>
public string ElementName => "_wjType";
public Type GetActualType(IBsonReader bsonReader, Type nominalType)
{
Type type = null;
var bookmark = bsonReader.GetBookmark();
bsonReader.ReadStartDocument();
if (bsonReader.FindElement(ElementName))
{
var value = bsonReader.ReadString();
if (typeMap.ContainsKey(value))
type = typeMap[value];
}
bsonReader.ReturnToBookmark(bookmark);
if (type == null)
throw new Exception($"Type mis-configuration can't find bson type for ${nominalType}");
return type;
}
public BsonValue GetDiscriminator(Type nominalType, Type actualType)
{
return revMap[actualType];
}
}
}

2
Wabbajack.BuildServer/Startup.cs
View File

@ -71,7 +71,6 @@ namespace Wabbajack.BuildServer
x.MultipartBodyLengthLimit = int.MaxValue;
});
services.AddSingleton<DBContext>();
services.AddSingleton<JobManager>();
services.AddSingleton<AppSettings>();
services.AddSingleton<SqlService>();
@ -89,7 +88,6 @@ namespace Wabbajack.BuildServer
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
SerializerSettings.Init();
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();

3
Wabbajack.BuildServer/Wabbajack.BuildServer.csproj
View File

@ -16,7 +16,6 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="BunnyCDN.Net.Storage" Version="1.0.2" />
<PackageReference Include="CsvHelper" Version="15.0.3" />
<PackageReference Include="Dapper" Version="2.0.30" />
<PackageReference Include="FluentFTP" Version="32.2.2" />
@ -28,8 +27,6 @@
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.0" />
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Microsoft.OpenApi" Version="1.1.4" />
<PackageReference Include="MongoDB.Driver" Version="2.10.2" />
<PackageReference Include="MongoDB.Driver.Core" Version="2.10.2" />
<PackageReference Include="Nettle" Version="1.3.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.1.0" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.1" />

1
Wabbajack.Common/Consts.cs
View File

@ -137,5 +137,6 @@ namespace Wabbajack.Common
public static RelativePath ModListTxt = (RelativePath)"modlist.txt";
public static RelativePath ModOrganizer2Exe = (RelativePath)"ModOrganizer.exe";
public static RelativePath ModOrganizer2Ini = (RelativePath)"ModOrganizer.ini";
public static string AuthorAPIKeyFile = "author-api-key.txt";
}
}

3
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -4,12 +4,10 @@ using System.Reactive;
using System.Reactive.Linq;
using System.Threading.Tasks;
using MessagePack;
using MongoDB.Bson.Serialization.Attributes;
using Newtonsoft.Json;
using ReactiveUI;
using Wabbajack.Common;
using Wabbajack.Common.StatusFeed.Errors;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using Game = Wabbajack.Common.Game;
@ -130,7 +128,6 @@ namespace Wabbajack.Lib.Downloaders
}
}
[BsonIgnoreExtraElements]
[MessagePackObject]
public class State : AbstractDownloadState, IMetaState
{

4
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -19,11 +19,11 @@ namespace Wabbajack.Lib.FileUploader
{
public class AuthorAPI
{
public static IObservable<bool> HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable("author-api-key.txt");
public static IObservable<bool> HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable(Consts.AuthorAPIKeyFile);
public static async Task<string> GetAPIKey(string apiKey = null)
{
return apiKey ?? (await Consts.LocalAppDataPath.Combine("author-api-key.txt").ReadAllTextAsync()).Trim();
return apiKey ?? (await Consts.LocalAppDataPath.Combine(Consts.AuthorAPIKeyFile).ReadAllTextAsync()).Trim();
}
public static Uri UploadURL => new Uri($"{Consts.WabbajackBuildServerUri}upload_file");

5
Wabbajack.Lib/ModListRegistry/ModListMetadata.cs
View File

@ -1,13 +1,9 @@
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using Newtonsoft.Json;
using Wabbajack.Common;
using File = System.IO.File;
using Game = Wabbajack.Common.Game;
namespace Wabbajack.Lib.ModListRegistry
@ -40,7 +36,6 @@ namespace Wabbajack.Lib.ModListRegistry
[JsonIgnore]
public ModlistSummary ValidationSummary { get; set; } = new ModlistSummary();
[BsonIgnoreExtraElements]
public class LinksObject
{
[JsonProperty("image")]

6
Wabbajack.Lib/Wabbajack.Lib.csproj
View File

@ -36,12 +36,6 @@
<PackageReference Include="ModuleInit.Fody">
<Version>2.1.0</Version>
</PackageReference>
<PackageReference Include="MongoDB.Bson">
<Version>2.10.1</Version>
</PackageReference>
<PackageReference Include="MongoDB.Bson">
<Version>2.10.1</Version>
</PackageReference>
<PackageReference Include="ReactiveUI">
<Version>11.2.3</Version>
</PackageReference>