Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Purge all remaining references to MongoDB and BJSON

Browse Source
This commit is contained in:
Timothy Baldridge 2020-04-05 15:15:01 -06:00
parent 653c840323
commit 3d16b1e838
48 changed files with 343 additions and 776 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.BuildServer.Test/ADBTest.cs
View File

@ -28,7 +28,7 @@ namespace Wabbajack.BuildServer.Test
{ {
DBName = "test_db" + Guid.NewGuid().ToString().Replace("-", "_"); DBName = "test_db" + Guid.NewGuid().ToString().Replace("-", "_");
User = Guid.NewGuid().ToString().Replace("-", ""); User = Guid.NewGuid().ToString().Replace("-", "");
APIKey = Users.NewAPIKey(); APIKey = SqlService.NewAPIKey();
} }
public string APIKey { get; } public string APIKey { get; }

39
Wabbajack.BuildServer.Test/LoginTests.cs Normal file
View File

@ -0,0 +1,39 @@
using System;
using System.Threading.Tasks;
using Wabbajack.Common;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class LoginTests : ABuildServerSystemTest
{
public LoginTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanCreateLogins()
{
var newUserName = Guid.NewGuid().ToString();
var newKey = await _authedClient.GetStringAsync(MakeURL($"users/add/{newUserName}"));
Assert.NotEmpty(newKey);
Assert.NotNull(newKey);
Assert.NotEqual(newKey, Fixture.APIKey);
var done = await _authedClient.GetStringAsync(MakeURL("users/export"));
Assert.Equal("done", done);
foreach (var (userName, apiKey) in new[] {(newUserName, newKey), (Fixture.User, Fixture.APIKey)})
{
var exported = await Fixture.ServerTempFolder.Combine("exported_users", userName, Consts.AuthorAPIKeyFile)
.ReadAllTextAsync();
Assert.Equal(exported, apiKey);
}
}
}
}

9
Wabbajack.BuildServer/Controllers/AControllerBase.cs
View File

@ -1,13 +1,8 @@
using System; using System;
using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using GraphQL;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using Org.BouncyCastle.Ocsp;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models;
using Wabbajack.Common; using Wabbajack.Common;
@ -17,13 +12,11 @@ namespace Wabbajack.BuildServer.Controllers
[ApiController] [ApiController]
public abstract class AControllerBase<T> : ControllerBase public abstract class AControllerBase<T> : ControllerBase
{ {
protected readonly DBContext Db;
protected readonly ILogger<T> Logger; protected readonly ILogger<T> Logger;
protected readonly SqlService SQL; protected readonly SqlService SQL;
protected AControllerBase(ILogger<T> logger, DBContext db, SqlService sql) protected AControllerBase(ILogger<T> logger, SqlService sql)
{ {
Db = db;
Logger = logger; Logger = logger;
SQL = sql; SQL = sql;
} }

9
Wabbajack.BuildServer/Controllers/GraphQL.cs
View File

@ -13,18 +13,17 @@ namespace Wabbajack.BuildServer.Controllers
[ApiController] [ApiController]
public class GraphQL : AControllerBase<GraphQL> public class GraphQL : AControllerBase<GraphQL>
{ {
private SqlService _sql; public GraphQL(ILogger<GraphQL> logger, SqlService sql) : base(logger, sql)
public GraphQL(ILogger<GraphQL> logger, DBContext db, SqlService sql) : base(logger, db, sql)
{ {
_sql = sql;
} }
[HttpPost] [HttpPost]
public async Task<IActionResult> Post([FromBody] GraphQLQuery query) public async Task<IActionResult> Post([FromBody] GraphQLQuery query)
{ {
var inputs = query.Variables.ToInputs(); var inputs = query.Variables.ToInputs();
var schema = new Schema {Query = new Query(Db, _sql), Mutation = new Mutation(Db)}; var schema = new Schema {
Query = new Query(SQL)
};
var result = await new DocumentExecuter().ExecuteAsync(_ => var result = await new DocumentExecuter().ExecuteAsync(_ =>
{ {

9
Wabbajack.BuildServer/Controllers/Heartbeat.cs
View File

@ -1,17 +1,10 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MongoDB.Bson;
using MongoDB.Driver;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
using Wabbajack.Common.StatusFeed; using Wabbajack.Common.StatusFeed;
namespace Wabbajack.BuildServer.Controllers namespace Wabbajack.BuildServer.Controllers
@ -26,7 +19,7 @@ namespace Wabbajack.BuildServer.Controllers
} }
private static DateTime _startTime; private static DateTime _startTime;
public Heartbeat(ILogger<Heartbeat> logger, DBContext db, SqlService sql) : base(logger, db, sql) public Heartbeat(ILogger<Heartbeat> logger, SqlService sql) : base(logger, sql)
{ {
} }

8
Wabbajack.BuildServer/Controllers/IndexedFiles.cs
View File

@ -5,21 +5,15 @@ using System.IO.Compression;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using DynamicData;
using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.Lib; using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.Downloaders;
using Wabbajack.VirtualFileSystem;
using IndexedFile = Wabbajack.BuildServer.Models.IndexedFile; using IndexedFile = Wabbajack.BuildServer.Models.IndexedFile;
namespace Wabbajack.BuildServer.Controllers namespace Wabbajack.BuildServer.Controllers
@ -30,7 +24,7 @@ namespace Wabbajack.BuildServer.Controllers
private SqlService _sql; private SqlService _sql;
private AppSettings _settings; private AppSettings _settings;
public IndexedFiles(ILogger<IndexedFiles> logger, DBContext db, SqlService sql, AppSettings settings) : base(logger, db, sql) public IndexedFiles(ILogger<IndexedFiles> logger, SqlService sql, AppSettings settings) : base(logger, sql)
{ {
_settings = settings; _settings = settings;
_sql = sql; _sql = sql;

18
Wabbajack.BuildServer/Controllers/Jobs.cs
View File

@ -1,13 +1,9 @@
using System; using System;
using System.Collections.Generic;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.JobQueue;
namespace Wabbajack.BuildServer.Controllers namespace Wabbajack.BuildServer.Controllers
@ -17,27 +13,17 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/jobs")] [Route("/jobs")]
public class Jobs : AControllerBase<Jobs> public class Jobs : AControllerBase<Jobs>
{ {
public Jobs(ILogger<Jobs> logger, DBContext db, SqlService sql) : base(logger, db, sql) public Jobs(ILogger<Jobs> logger, SqlService sql) : base(logger, sql)
{ {
} }
[HttpGet]
[Route("unfinished")]
public async Task<IEnumerable<Job>> GetUnfinished()
{
return await Db.Jobs.AsQueryable()
.Where(j => j.Ended == null)
.OrderByDescending(j => j.Priority)
.ToListAsync();
}
[HttpGet] [HttpGet]
[Route("enqueue_job/{JobName}")] [Route("enqueue_job/{JobName}")]
public async Task<long> EnqueueJob(string JobName) public async Task<long> EnqueueJob(string JobName)
{ {
var jobtype = AJobPayload.NameToType[JobName]; var jobtype = AJobPayload.NameToType[JobName];
var job = new Job{Priority = Job.JobPriority.High, Payload = (AJobPayload)jobtype.GetConstructor(new Type[0]).Invoke(new object?[0])}; var job = new Job{Priority = Job.JobPriority.High, Payload = (AJobPayload)jobtype.GetConstructor(new Type[0]).Invoke(new object?[0])};
await Db.Jobs.InsertOneAsync(job); await SQL.EnqueueJob(job);
return job.Id; return job.Id;
} }
} }

7
Wabbajack.BuildServer/Controllers/ListValidation.cs
View File

@ -2,16 +2,11 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Net; using System.Net;
using System.ServiceModel.Syndication;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.AspNetCore.Http.Extensions;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nettle; using Nettle;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.ModListRegistry;
@ -21,7 +16,7 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/lists")] [Route("/lists")]
public class ListValidation : AControllerBase<ListValidation> public class ListValidation : AControllerBase<ListValidation>
{ {
public ListValidation(ILogger<ListValidation> logger, DBContext db, SqlService sql) : base(logger, db, sql) public ListValidation(ILogger<ListValidation> logger, SqlService sql) : base(logger, sql)
{ {
} }

23
Wabbajack.BuildServer/Controllers/MetricsController.cs
View File

@ -1,17 +1,11 @@
using System; using System;
using System.Collections.Generic;
using System.Data.SqlTypes;
using System.Linq; using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.Lib.ModListRegistry;
namespace Wabbajack.BuildServer.Controllers namespace Wabbajack.BuildServer.Controllers
{ {
@ -19,11 +13,8 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/metrics")] [Route("/metrics")]
public class MetricsController : AControllerBase<MetricsController> public class MetricsController : AControllerBase<MetricsController>
{ {
private SqlService _sql; public MetricsController(ILogger<MetricsController> logger, SqlService sql) : base(logger, sql)
public MetricsController(ILogger<MetricsController> logger, DBContext db, SqlService sql) : base(logger, db, sql)
{ {
_sql = sql;
} }
[HttpGet] [HttpGet]
@ -35,20 +26,10 @@ namespace Wabbajack.BuildServer.Controllers
return new Result { Timestamp = date}; return new Result { Timestamp = date};
} }
[Authorize]
[HttpGet]
[Route("transfer")]
public async Task<string> Transfer()
{
var all_metrics = await Db.Metrics.AsQueryable().ToListAsync();
await _sql.IngestAllMetrics(all_metrics);
return "done";
}
private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null) private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
{ {
Logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}"); Logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}");
await _sql.IngestMetric(new Metric await SQL.IngestMetric(new Metric
{ {
Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey
}); });

28
Wabbajack.BuildServer/Controllers/ModlistUpdater.cs
View File

@ -8,8 +8,6 @@ using FluentFTP;
using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.JobQueue;
@ -30,7 +28,7 @@ namespace Wabbajack.BuildServer.Controllers
private AppSettings _settings; private AppSettings _settings;
private SqlService _sql; private SqlService _sql;
public ModlistUpdater(ILogger<ModlistUpdater> logger, DBContext db, SqlService sql, AppSettings settings) : base(logger, db, sql) public ModlistUpdater(ILogger<ModlistUpdater> logger, SqlService sql, AppSettings settings) : base(logger, sql)
{ {
_settings = settings; _settings = settings;
_sql = sql; _sql = sql;
@ -93,22 +91,21 @@ namespace Wabbajack.BuildServer.Controllers
Utils.Log($"Alternative requested for {startingHash}"); Utils.Log($"Alternative requested for {startingHash}");
await Metric("requested_upgrade", startingHash.ToString()); await Metric("requested_upgrade", startingHash.ToString());
var state = await Db.DownloadStates.AsQueryable() var state = await SQL.GetNexusStateByHash(startingHash);
/*.DownloadStates.AsQueryable()
.Where(s => s.Hash == startingHash) .Where(s => s.Hash == startingHash)
.Where(s => s.State is NexusDownloader.State) .Where(s => s.State is NexusDownloader.State)
.OrderByDescending(s => s.LastValidationTime).FirstOrDefaultAsync(); .OrderByDescending(s => s.LastValidationTime).FirstOrDefaultAsync();*/
if (state == null) if (state == null)
return NotFound("Original state not found"); return NotFound("Original state not found");
var nexusState = state.State as NexusDownloader.State; var nexusState = state.State as NexusDownloader.State;
var nexusGame = nexusState.Game.MetaData().NexusName; var nexusGame = nexusState.Game;
var mod_files = await Db.NexusModFiles.AsQueryable() var mod_files = (await SQL.GetModFiles(nexusGame, nexusState.ModID)).files;
.Where(f => f.Game == nexusGame && f.ModId == nexusState.ModID)
.ToListAsync();
if (mod_files.SelectMany(f => f.Data.files) if (mod_files.Any(f => f.category_name != null && f.file_id == nexusState.FileID))
.Any(f => f.category_name != null && f.file_id == nexusState.FileID))
{ {
await Metric("not_required_upgrade", startingHash.ToString()); await Metric("not_required_upgrade", startingHash.ToString());
return BadRequest("Upgrade Not Required"); return BadRequest("Upgrade Not Required");
@ -124,7 +121,7 @@ namespace Wabbajack.BuildServer.Controllers
Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}"); Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}");
if (newArchive.Hash == Hash.Empty) if (newArchive.Hash == Hash.Empty)
{ {
Db.Jobs.InsertOne(new Job await SQL.EnqueueJob(new Job
{ {
Payload = new IndexJob Payload = new IndexJob
{ {
@ -147,7 +144,7 @@ namespace Wabbajack.BuildServer.Controllers
if (!PatchArchive.CdnPath(startingHash, newArchive.Hash).Exists) if (!PatchArchive.CdnPath(startingHash, newArchive.Hash).Exists)
{ {
Db.Jobs.InsertOne(new Job await SQL.EnqueueJob(new Job
{ {
Priority = Job.JobPriority.High, Priority = Job.JobPriority.High,
Payload = new PatchArchive Payload = new PatchArchive
@ -185,7 +182,7 @@ namespace Wabbajack.BuildServer.Controllers
Utils.Log($"Found alternative for {srcHash}"); Utils.Log($"Found alternative for {srcHash}");
var indexed = await Db.DownloadStates.AsQueryable().Where(s => s.Key == archive.State.PrimaryKeyString).FirstOrDefaultAsync(); var indexed = await SQL.DownloadStateByPrimaryKey(archive.State.PrimaryKeyString);
if (indexed == null) if (indexed == null)
{ {
@ -195,9 +192,6 @@ namespace Wabbajack.BuildServer.Controllers
Utils.Log($"Pre-Indexed alternative {indexed.Hash} found for {srcHash}"); Utils.Log($"Pre-Indexed alternative {indexed.Hash} found for {srcHash}");
archive.Hash = indexed.Hash; archive.Hash = indexed.Hash;
return archive; return archive;
} }
} }
} }

23
Wabbajack.BuildServer/Controllers/NexusCache.cs
View File

@ -1,15 +1,11 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq; using System.Linq;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using CsvHelper;
using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using Newtonsoft.Json; using Newtonsoft.Json;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models;
@ -28,7 +24,7 @@ namespace Wabbajack.BuildServer.Controllers
private static long CachedCount = 0; private static long CachedCount = 0;
private static long ForwardCount = 0; private static long ForwardCount = 0;
public NexusCache(ILogger<NexusCache> logger, DBContext db, SqlService sql, AppSettings settings) : base(logger, db, sql) public NexusCache(ILogger<NexusCache> logger, SqlService sql, AppSettings settings) : base(logger, sql)
{ {
_settings = settings; _settings = settings;
} }
@ -55,13 +51,7 @@ namespace Wabbajack.BuildServer.Controllers
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{ModId}.json"; var path = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{ModId}.json";
var body = await api.Get<ModInfo>(path); var body = await api.Get<ModInfo>(path);
try await SQL.AddNexusModInfo(game, ModId, DateTime.Now, body);
{
await SQL.AddNexusModInfo(game, ModId, DateTime.Now, body);
}
catch (MongoWriteException)
{
}
method = "NOT_CACHED"; method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount); Interlocked.Increment(ref ForwardCount);
@ -90,13 +80,8 @@ namespace Wabbajack.BuildServer.Controllers
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files.json"; var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files.json";
var body = await api.Get<NexusApiClient.GetModFilesResponse>(path); var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
try await SQL.AddNexusModFiles(game, ModId, DateTime.Now, body);
{
await SQL.AddNexusModFiles(game, ModId, DateTime.Now, body);
}
catch (MongoWriteException)
{
}
method = "NOT_CACHED"; method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount); Interlocked.Increment(ref ForwardCount);

23
Wabbajack.BuildServer/Controllers/UploadedFiles.cs
View File

@ -9,23 +9,16 @@ using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using FluentFTP; using FluentFTP;
using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Identity;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nettle; using Nettle;
using Newtonsoft.Json; using Newtonsoft.Json;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Org.BouncyCastle.Crypto.Engines;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Path = Alphaleonis.Win32.Filesystem.Path; using Path = Alphaleonis.Win32.Filesystem.Path;
using AlphaFile = Alphaleonis.Win32.Filesystem.File; using AlphaFile = Alphaleonis.Win32.Filesystem.File;
@ -36,7 +29,7 @@ namespace Wabbajack.BuildServer.Controllers
private static ConcurrentDictionary<string, AsyncLock> _writeLocks = new ConcurrentDictionary<string, AsyncLock>(); private static ConcurrentDictionary<string, AsyncLock> _writeLocks = new ConcurrentDictionary<string, AsyncLock>();
private AppSettings _settings; private AppSettings _settings;
public UploadedFiles(ILogger<UploadedFiles> logger, DBContext db, AppSettings settings, SqlService sql) : base(logger, db, sql) public UploadedFiles(ILogger<UploadedFiles> logger, AppSettings settings, SqlService sql) : base(logger, sql)
{ {
_settings = settings; _settings = settings;
} }
@ -88,7 +81,7 @@ namespace Wabbajack.BuildServer.Controllers
[Route("clean_http_uploads")] [Route("clean_http_uploads")]
public async Task<IActionResult> CleanUploads() public async Task<IActionResult> CleanUploads()
{ {
var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync(); var files = await SQL.AllUploadedFiles();
var seen = new HashSet<string>(); var seen = new HashSet<string>();
var duplicate = new List<UploadedFile>(); var duplicate = new List<UploadedFile>();
@ -115,7 +108,7 @@ namespace Wabbajack.BuildServer.Controllers
if (await client.FileExistsAsync(dup.MungedName)) if (await client.FileExistsAsync(dup.MungedName))
await client.DeleteFileAsync(dup.MungedName); await client.DeleteFileAsync(dup.MungedName);
await Db.UploadedFiles.DeleteOneAsync(f => f.Id == dup.Id); await SQL.DeleteUploadedFile(dup.Id);
} }
} }
@ -182,7 +175,7 @@ namespace Wabbajack.BuildServer.Controllers
[Route("uploaded_files")] [Route("uploaded_files")]
public async Task<ContentResult> UploadedFilesGet() public async Task<ContentResult> UploadedFilesGet()
{ {
var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync(); var files = await SQL.AllUploadedFiles();
var response = HandleGetListTemplate(new var response = HandleGetListTemplate(new
{ {
files = files.Select(file => new files = files.Select(file => new
@ -221,7 +214,7 @@ namespace Wabbajack.BuildServer.Controllers
{ {
var user = User.FindFirstValue(ClaimTypes.Name); var user = User.FindFirstValue(ClaimTypes.Name);
Utils.Log($"Delete Uploaded File {user} {name}"); Utils.Log($"Delete Uploaded File {user} {name}");
var files = await Db.UploadedFiles.AsQueryable().Where(f => f.Uploader == user).ToListAsync(); var files = await SQL.AllUploadedFilesForUser(name);
var to_delete = files.First(f => f.MungedName == name); var to_delete = files.First(f => f.MungedName == name);
@ -237,10 +230,8 @@ namespace Wabbajack.BuildServer.Controllers
} }
var result = await Db.UploadedFiles.DeleteOneAsync(f => f.Id == to_delete.Id); await SQL.DeleteUploadedFile(to_delete.Id);
if (result.DeletedCount == 1) return Ok($"Deleted {to_delete.MungedName}");
return Ok($"Deleted {name}");
return NotFound(name);
} }
[HttpGet] [HttpGet]

39
Wabbajack.BuildServer/Controllers/Users.cs
View File

@ -1,13 +1,8 @@
using System; using System.Threading.Tasks;
using System.Collections.Generic;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common; using Wabbajack.Common;
namespace Wabbajack.BuildServer.Controllers namespace Wabbajack.BuildServer.Controllers
@ -16,47 +11,37 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/users")] [Route("/users")]
public class Users : AControllerBase<Users> public class Users : AControllerBase<Users>
{ {
public Users(ILogger<Users> logger, DBContext db, SqlService sql) : base(logger, db, sql) private AppSettings _settings;
public Users(ILogger<Users> logger, SqlService sql, AppSettings settings) : base(logger, sql)
{ {
_settings = settings;
} }
[HttpGet] [HttpGet]
[Route("add/{Name}")] [Route("add/{Name}")]
public async Task<string> AddUser(string Name) public async Task<string> AddUser(string Name)
{ {
var user = new ApiKey(); return await SQL.AddLogin(Name);
user.Key = NewAPIKey();
user.Id = Guid.NewGuid().ToString();
user.Roles = new List<string>();
user.CanUploadLists = new List<string>();
await Db.ApiKeys.InsertOneAsync(user);
return user.Id;
} }
[HttpGet] [HttpGet]
[Route("export")] [Route("export")]
public async Task<string> Export() public async Task<string> Export()
{ {
if (!Directory.Exists("exported_users")) var mainFolder = _settings.TempPath.Combine("exported_users");
Directory.CreateDirectory("exported_users"); mainFolder.CreateDirectory();
foreach (var user in await Db.ApiKeys.AsQueryable().ToListAsync()) foreach (var (owner, key) in await SQL.GetAllUserKeys())
{ {
Directory.CreateDirectory(Path.Combine("exported_users", user.Owner)); var folder = mainFolder.Combine(owner);
Alphaleonis.Win32.Filesystem.File.WriteAllText(Path.Combine("exported_users", user.Owner, "author-api-key.txt"), user.Key); folder.CreateDirectory();
await folder.Combine(Consts.AuthorAPIKeyFile).WriteAllTextAsync(key);
} }
return "done"; return "done";
} }
public static string NewAPIKey()
{
var arr = new byte[128];
new Random().NextBytes(arr);
return arr.ToHex();
}
} }
} }

10
Wabbajack.BuildServer/Extensions.cs
View File

@ -4,25 +4,15 @@ using System.IO;
using System.Linq; using System.Linq;
using System.Linq.Expressions; using System.Linq.Expressions;
using System.Threading.Tasks; using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Builder;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.Common; using Wabbajack.Common;
using Directory =Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File; using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.BuildServer namespace Wabbajack.BuildServer
{ {
public static class Extensions public static class Extensions
{ {
public static async Task<T> FindOneAsync<T>(this IMongoCollection<T> coll, Expression<Func<T, bool>> expr)
{
return (await coll.AsQueryable().Where(expr).Take(1).ToListAsync()).FirstOrDefault();
}
public static void UseJobManager(this IApplicationBuilder b) public static void UseJobManager(this IApplicationBuilder b)
{ {
var manager = (JobManager)b.ApplicationServices.GetService(typeof(JobManager)); var manager = (JobManager)b.ApplicationServices.GetService(typeof(JobManager));

22
Wabbajack.BuildServer/GraphQL/Mutation.cs
View File

@ -1,22 +0,0 @@
using GraphQL.Types;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
namespace Wabbajack.BuildServer.GraphQL
{
public class Mutation : ObjectGraphType
{
public Mutation(DBContext db)
{
FieldAsync<IdGraphType>("pollNexusForUpdates",
resolve: async context =>
{
var job = new Job {Payload = new GetNexusUpdatesJob()};
await db.Jobs.InsertOneAsync(job);
return job.Id;
});
}
}
}

47
Wabbajack.BuildServer/GraphQL/Query.cs
View File

@ -1,28 +1,13 @@
using System; using System.Linq;
using System.Linq;
using System.Collections.Generic;
using System.Data.SqlTypes;
using GraphQL;
using GraphQL.Types; using GraphQL.Types;
using GraphQLParser.AST;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
namespace Wabbajack.BuildServer.GraphQL namespace Wabbajack.BuildServer.GraphQL
{ {
public class Query : ObjectGraphType public class Query : ObjectGraphType
{ {
public Query(DBContext db, SqlService sql) public Query(SqlService sql)
{ {
Field<ListGraphType<JobType>>("unfinishedJobs", resolve: context =>
{
var data = db.Jobs.AsQueryable().Where(j => j.Ended == null).ToList();
return data;
});
FieldAsync<ListGraphType<ModListStatusType>>("modLists", FieldAsync<ListGraphType<ModListStatusType>>("modLists",
arguments: new QueryArguments(new QueryArgument<ArchiveEnumFilterType> arguments: new QueryArguments(new QueryArgument<ArchiveEnumFilterType>
{ {
@ -31,37 +16,17 @@ namespace Wabbajack.BuildServer.GraphQL
resolve: async context => resolve: async context =>
{ {
var arg = context.GetArgument<string>("filter"); var arg = context.GetArgument<string>("filter");
var lists = db.ModListStatus.AsQueryable(); var lists = await sql.GetDetailedModlistStatuses();
switch (arg) switch (arg)
{ {
case "FAILED": case "FAILED":
lists = lists.Where(l => l.DetailedStatus.HasFailures); return lists.Where(l => l.HasFailures);
break;
case "PASSED": case "PASSED":
lists = lists.Where(a => !a.DetailedStatus.HasFailures); return lists.Where(l => !l.HasFailures);
break;
default: default:
break; return lists;
} }
return await lists.ToListAsync();
});
FieldAsync<ListGraphType<JobType>>("job",
arguments: new QueryArguments(
new QueryArgument<IntGraphType> {Name = "id", Description = "Id of the Job"}),
resolve: async context =>
{
var id = context.GetArgument<long>("id");
var data = await db.Jobs.AsQueryable().Where(j => j.Id == id).ToListAsync();
return data;
});
FieldAsync<ListGraphType<UploadedFileType>>("uploadedFiles",
resolve: async context =>
{
var data = await db.UploadedFiles.AsQueryable().ToListAsync();
return data;
}); });
FieldAsync<ListGraphType<MetricResultType>>("dailyUniqueMetrics", FieldAsync<ListGraphType<MetricResultType>>("dailyUniqueMetrics",

35
Wabbajack.BuildServer/JobManager.cs
View File

@ -3,8 +3,6 @@ using System.Linq;
using System.Reactive.Linq; using System.Reactive.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nettle; using Nettle;
using Wabbajack.BuildServer.Controllers; using Wabbajack.BuildServer.Controllers;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
@ -18,13 +16,11 @@ namespace Wabbajack.BuildServer
public class JobManager public class JobManager
{ {
protected readonly ILogger<JobManager> Logger; protected readonly ILogger<JobManager> Logger;
protected readonly DBContext Db;
protected readonly AppSettings Settings; protected readonly AppSettings Settings;
protected SqlService Sql; protected SqlService Sql;
public JobManager(ILogger<JobManager> logger, DBContext db, SqlService sql, AppSettings settings) public JobManager(ILogger<JobManager> logger, SqlService sql, AppSettings settings)
{ {
Db = db;
Logger = logger; Logger = logger;
Settings = settings; Settings = settings;
Sql = sql; Sql = sql;
@ -42,7 +38,7 @@ namespace Wabbajack.BuildServer
{ {
try try
{ {
var job = await Job.GetNext(Db); var job = await Sql.GetJob();
if (job == null) if (job == null)
{ {
await Task.Delay(5000); await Task.Delay(5000);
@ -50,18 +46,17 @@ namespace Wabbajack.BuildServer
} }
Logger.Log(LogLevel.Information, $"Starting job: {job.Payload.Description}"); Logger.Log(LogLevel.Information, $"Starting job: {job.Payload.Description}");
JobResult result;
try try
{ {
result = await job.Payload.Execute(Db, Sql, Settings); job.Result = await job.Payload.Execute(Sql, Settings);
} }
catch (Exception ex) catch (Exception ex)
{ {
Logger.Log(LogLevel.Error, ex, $"Error while running job: {job.Payload.Description}"); Logger.Log(LogLevel.Error, ex, $"Error while running job: {job.Payload.Description}");
result = JobResult.Error(ex); job.Result = JobResult.Error(ex);
} }
await Job.Finish(Db, job, result); await Sql.FinishJob(job);
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -95,16 +90,15 @@ namespace Wabbajack.BuildServer
{ {
try try
{ {
var started = await Db.Jobs.AsQueryable() var started = await Sql.GetRunningJobs();
.Where(j => j.Started != null && j.Ended == null)
.ToListAsync();
foreach (var job in started) foreach (var job in started)
{ {
var runtime = DateTime.Now - job.Started; var runtime = DateTime.Now - job.Started;
if (runtime > TimeSpan.FromMinutes(30))
{ if (!(runtime > TimeSpan.FromMinutes(30))) continue;
await Job.Finish(Db, job, JobResult.Error(new Exception($"Timeout after {runtime.Value.TotalMinutes}")));
} job.Result = JobResult.Error(new Exception($"Timeout after {runtime.Value.TotalMinutes}"));
await Sql.FinishJob(job);
} }
} }
catch (Exception ex) catch (Exception ex)
@ -119,18 +113,17 @@ namespace Wabbajack.BuildServer
if (!Settings.RunFrontEndJobs && typeof(T).ImplementsInterface(typeof(IFrontEndJob))) return; if (!Settings.RunFrontEndJobs && typeof(T).ImplementsInterface(typeof(IFrontEndJob))) return;
try try
{ {
var jobs = await Db.Jobs.AsQueryable() var jobs = (await Sql.GetUnfinishedJobs())
.Where(j => j.Payload is T) .Where(j => j.Payload is T)
.OrderByDescending(j => j.Created) .OrderByDescending(j => j.Created)
.Take(10) .Take(10);
.ToListAsync();
foreach (var job in jobs) foreach (var job in jobs)
{ {
if (job.Started == null || job.Ended == null) return; if (job.Started == null || job.Ended == null) return;
if (DateTime.Now - job.Ended < span) return; if (DateTime.Now - job.Ended < span) return;
} }
await Db.Jobs.InsertOneAsync(new Job await Sql.EnqueueJob(new Job
{ {
Priority = priority, Priority = priority,
Payload = new T() Payload = new T()

7
Wabbajack.BuildServer/Models/ApiKey.cs
View File

@ -1,7 +1,5 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
namespace Wabbajack.BuildServer.Models namespace Wabbajack.BuildServer.Models
{ {
@ -14,10 +12,5 @@ namespace Wabbajack.BuildServer.Models
public List<string> CanUploadLists { get; set; } public List<string> CanUploadLists { get; set; }
public List<string> Roles { get; set; } public List<string> Roles { get; set; }
public static async Task<ApiKey> Get(DBContext db, string key)
{
return await db.ApiKeys.AsQueryable().Where(k => k.Key == key).FirstOrDefaultAsync();
}
} }
} }

46
Wabbajack.BuildServer/Models/DBContext.cs
View File

@ -1,46 +0,0 @@
using System.Collections.Generic;
using Microsoft.Extensions.Configuration;
using MongoDB.Driver;
using Wabbajack.Lib.NexusApi;
using Wabbajack.BuildServer.Models.JobQueue;
namespace Wabbajack.BuildServer.Models
{
public class DBContext
{
private IConfiguration _configuration;
private Settings _settings;
public DBContext(IConfiguration configuration)
{
_configuration = configuration;
_settings = new Settings();
_configuration.Bind("MongoDB", _settings);
}
public IMongoCollection<NexusCacheData<ModInfo>> NexusModInfos => Client.GetCollection<NexusCacheData<ModInfo>>(_settings.Collections["NexusModInfos"]);
public IMongoCollection<NexusCacheData<NexusFileInfo>> NexusFileInfos => Client.GetCollection<NexusCacheData<NexusFileInfo>>(_settings.Collections["NexusFileInfos"]);
public IMongoCollection<ModListStatus> ModListStatus => Client.GetCollection<ModListStatus>(_settings.Collections["ModListStatus"]);
public IMongoCollection<Job> Jobs => Client.GetCollection<Job>(_settings.Collections["JobQueue"]);
public IMongoCollection<DownloadState> DownloadStates => Client.GetCollection<DownloadState>(_settings.Collections["DownloadStates"]);
public IMongoCollection<Metric> Metrics => Client.GetCollection<Metric>(_settings.Collections["Metrics"]);
public IMongoCollection<IndexedFile> IndexedFiles => Client.GetCollection<IndexedFile>(_settings.Collections["IndexedFiles"]);
public IMongoCollection<NexusCacheData<List<NexusUpdateEntry>>> NexusUpdates => Client.GetCollection<NexusCacheData<List<NexusUpdateEntry>>>(_settings.Collections["NexusUpdates"]);
public IMongoCollection<ApiKey> ApiKeys => Client.GetCollection<ApiKey>(_settings.Collections["ApiKeys"]);
public IMongoCollection<UploadedFile> UploadedFiles => Client.GetCollection<UploadedFile>(_settings.Collections["UploadedFiles"]);
public IMongoCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>> NexusModFiles =>
Client.GetCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>>(
_settings.Collections["NexusModFiles"]);
private IMongoDatabase Client => new MongoClient($"mongodb://{_settings.Host}").GetDatabase(_settings.Database);
}
public class Settings
{
public string Host { get; set; }
public string Database { get; set; }
public Dictionary<string, string> Collections { get; set; }
public string SqlConnection { get; set; }
}
}

2
Wabbajack.BuildServer/Models/DownloadState.cs
View File

@ -1,5 +1,4 @@
using System; using System;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.Downloaders;
@ -7,7 +6,6 @@ namespace Wabbajack.BuildServer.Models
{ {
public class DownloadState public class DownloadState
{ {
[BsonId]
public string Key { get; set; } public string Key { get; set; }
public Hash Hash { get; set; } public Hash Hash { get; set; }

9
Wabbajack.BuildServer/Models/IndexedFile.cs
View File

@ -1,17 +1,10 @@
using System; using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.VirtualFileSystem;
namespace Wabbajack.BuildServer.Models namespace Wabbajack.BuildServer.Models
{ {
public class IndexedFile public class IndexedFile
{ {
[BsonId]
public Hash Hash { get; set; } public Hash Hash { get; set; }
public string SHA256 { get; set; } public string SHA256 { get; set; }
public string SHA1 { get; set; } public string SHA1 { get; set; }

6
Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs
View File

@ -1,9 +1,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.BuildServer.Models.Jobs;
@ -18,7 +16,6 @@ namespace Wabbajack.BuildServer.Models.JobQueue
typeof(UpdateModLists), typeof(UpdateModLists),
typeof(EnqueueAllArchives), typeof(EnqueueAllArchives),
typeof(EnqueueAllGameFiles), typeof(EnqueueAllGameFiles),
typeof(EnqueueRecentFiles),
typeof(UploadToCDN), typeof(UploadToCDN),
typeof(IndexDynDOLOD), typeof(IndexDynDOLOD),
typeof(ReindexArchives), typeof(ReindexArchives),
@ -28,12 +25,11 @@ namespace Wabbajack.BuildServer.Models.JobQueue
public static Dictionary<string, Type> NameToType { get; set; } public static Dictionary<string, Type> NameToType { get; set; }
[BsonIgnore]
public abstract string Description { get; } public abstract string Description { get; }
public virtual bool UsesNexus { get; } = false; public virtual bool UsesNexus { get; } = false;
public abstract Task<JobResult> Execute(DBContext db, SqlService sql,AppSettings settings); public abstract Task<JobResult> Execute(SqlService sql,AppSettings settings);
static AJobPayload() static AJobPayload()
{ {

41
Wabbajack.BuildServer/Models/JobQueue/Job.cs
View File

@ -1,12 +1,5 @@
using System; using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.BuildServer.Models.JobQueue namespace Wabbajack.BuildServer.Models.JobQueue
{ {
@ -29,39 +22,5 @@ namespace Wabbajack.BuildServer.Models.JobQueue
public AJobPayload Payload { get; set; } public AJobPayload Payload { get; set; }
public Job OnSuccess { get; set; } public Job OnSuccess { get; set; }
public static async Task<Job> GetNext(DBContext db)
{
var filter = new BsonDocument
{
{"Started", BsonNull.Value}
};
var update = new BsonDocument
{
{"$set", new BsonDocument {{"Started", DateTime.Now}}}
};
var sort = new {Priority=-1, Created=1}.ToBsonDocument();
var job = await db.Jobs.FindOneAndUpdateAsync<Job>(filter, update, new FindOneAndUpdateOptions<Job>{Sort = sort});
return job;
}
public static async Task<Job> Finish(DBContext db, Job job, JobResult jobResult)
{
if (jobResult.ResultType == JobResultType.Success && job.OnSuccess != null)
{
await db.Jobs.InsertOneAsync(job.OnSuccess);
}
var filter = new BsonDocument
{
{"_id", job.Id},
};
var update = new BsonDocument
{
{"$set", new BsonDocument {{"Ended", DateTime.Now}, {"Result", jobResult.ToBsonDocument()}}}
};
var result = await db.Jobs.FindOneAndUpdateAsync<Job>(filter, update);
return result;
}
} }
} }

8
Wabbajack.BuildServer/Models/JobQueue/JobResult.cs
View File

@ -1,19 +1,13 @@
using System; using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
namespace Wabbajack.BuildServer.Models.JobQueue namespace Wabbajack.BuildServer.Models.JobQueue
{ {
public class JobResult public class JobResult
{ {
public JobResultType ResultType { get; set; } public JobResultType ResultType { get; set; }
[BsonIgnoreIfNull]
public string Message { get; set; } public string Message { get; set; }
[BsonIgnoreIfNull]
public string Stacktrace { get; set; } public string Stacktrace { get; set; }
public static JobResult Success() public static JobResult Success()

24
Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs
View File

@ -1,10 +1,6 @@
using System; using System;
using System.Threading.Tasks; using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using System.Linq; using System.Linq;
using FluentFTP;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common; using Wabbajack.Common;
@ -17,7 +13,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
public class EnqueueAllArchives : AJobPayload, IBackEndJob public class EnqueueAllArchives : AJobPayload, IBackEndJob
{ {
public override string Description => "Add missing modlist archives to indexer"; public override string Description => "Add missing modlist archives to indexer";
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings) public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{ {
Utils.Log("Starting ModList indexing"); Utils.Log("Starting ModList indexing");
var modlists = await ModlistMetadata.LoadFromGithub(); var modlists = await ModlistMetadata.LoadFromGithub();
@ -28,7 +24,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
{ {
try try
{ {
await EnqueueFromList(db, list, queue); await EnqueueFromList(sql, list, queue);
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -40,10 +36,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success(); return JobResult.Success();
} }
private static async Task EnqueueFromList(DBContext db, ModlistMetadata list, WorkQueue queue) private static async Task EnqueueFromList(SqlService sql, ModlistMetadata list, WorkQueue queue)
{ {
var existing = await db.ModListStatus.FindOneAsync(l => l.Id == list.Links.MachineURL);
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension); var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
if (list.NeedsDownload(modlistPath)) if (list.NeedsDownload(modlistPath))
@ -66,21 +60,23 @@ namespace Wabbajack.BuildServer.Models.Jobs
var archives = installer.Archives; var archives = installer.Archives;
Utils.Log($"Found {archives.Count} archives in {installer.Name} to index"); Utils.Log($"Found {archives.Count} archives in {installer.Name} to index");
var searching = archives.Select(a => a.Hash).Distinct().ToArray(); var searching = archives.Select(a => a.Hash).ToHashSet();
Utils.Log($"Looking for missing archives"); Utils.Log($"Looking for missing archives");
var knownArchives = (await db.IndexedFiles.AsQueryable().Where(a => searching.Contains(a.Hash)) var knownArchives = await sql.FilterByExistingIndexedArchives(searching);
.Select(d => d.Hash).ToListAsync()).ToDictionary(a => a);
Utils.Log($"Found {knownArchives.Count} pre-existing archives"); Utils.Log($"Found {knownArchives.Count} pre-existing archives");
var missing = archives.Where(a => !knownArchives.ContainsKey(a.Hash)).ToList(); var missing = archives.Where(a => !knownArchives.Contains(a.Hash)).ToList();
Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs"); Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs");
var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}, Priority = Job.JobPriority.Low}); var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}, Priority = Job.JobPriority.Low});
Utils.Log($"Writing jobs to the database"); Utils.Log($"Writing jobs to the database");
await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions {IsOrdered = false});
foreach (var job in jobs)
await sql.EnqueueJob(job);
Utils.Log($"Done adding archives for {installer.Name}"); Utils.Log($"Done adding archives for {installer.Name}");
} }
} }

26
Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs
View File

@ -5,18 +5,14 @@ using Wabbajack.Common;
using Wabbajack.Lib; using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.Downloaders;
using System.IO; using System.IO;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.BuildServer.Models.Jobs namespace Wabbajack.BuildServer.Models.Jobs
{ {
public class EnqueueAllGameFiles : AJobPayload, IBackEndJob public class EnqueueAllGameFiles : AJobPayload, IBackEndJob
{ {
public override string Description { get => $"Enqueue all game files for indexing"; } public override string Description { get => $"Enqueue all game files for indexing"; }
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings) public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{ {
using (var queue = new WorkQueue(4)) using (var queue = new WorkQueue(4))
{ {
@ -32,16 +28,12 @@ namespace Wabbajack.BuildServer.Models.Jobs
})) }))
.ToList(); .ToList();
var pks = states.Select(s => s.PrimaryKeyString).Distinct().ToArray(); var pks = states.Select(s => s.PrimaryKeyString).ToHashSet();
Utils.Log($"Found {pks.Length} archives to cross-reference with the database"); Utils.Log($"Found {pks.Count} archives to cross-reference with the database");
var found = (await db.DownloadStates var found = await sql.FilterByExistingPrimaryKeys(pks);
.AsQueryable().Where(s => pks.Contains(s.Key))
.Select(s => s.Key) states = states.Where(s => !found.Contains(s.PrimaryKeyString)).ToList();
.ToListAsync())
.ToDictionary(s => s);
states = states.Where(s => !found.ContainsKey(s.PrimaryKeyString)).ToList();
Utils.Log($"Found {states.Count} archives to index"); Utils.Log($"Found {states.Count} archives to index");
await states.PMap(queue, async state => await states.PMap(queue, async state =>
@ -58,14 +50,14 @@ namespace Wabbajack.BuildServer.Models.Jobs
} }
}); });
var with_hash = states.Where(state => state.Hash != null).ToList(); var with_hash = states.Where(state => state.Hash != default).ToList();
Utils.Log($"Inserting {with_hash.Count} jobs."); Utils.Log($"Inserting {with_hash.Count} jobs.");
var jobs = states.Select(state => new IndexJob {Archive = new Archive {Name = state.GameFile.FileName.ToString(), State = state}}) var jobs = states.Select(state => new IndexJob {Archive = new Archive {Name = state.GameFile.FileName.ToString(), State = state}})
.Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus}) .Select(j => new Job {Payload = j, RequiresNexus = j.UsesNexus})
.ToList(); .ToList();
if (jobs.Count > 0) foreach (var job in jobs)
await db.Jobs.InsertManyAsync(jobs); await sql.EnqueueJob(job);
return JobResult.Success(); return JobResult.Success();
} }

85
Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs
View File

@ -1,85 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using MongoDB.Driver;
using MongoDB.Driver.Core.Authentication;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class EnqueueRecentFiles : AJobPayload, IFrontEndJob
{
public override string Description => "Enqueue the past days worth of mods for indexing";
private static HashSet<Game> GamesToScan = new HashSet<Game>
{
Game.Fallout3, Game.Fallout4, Game.Skyrim, Game.SkyrimSpecialEdition, Game.SkyrimVR, Game.FalloutNewVegas, Game.Oblivion
};
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
{
using (var queue = new WorkQueue())
{
var updates = await db.NexusUpdates.AsQueryable().ToListAsync();
var mods = updates
.Where(list => GamesToScan.Contains(GameRegistry.GetByNexusName(list.Game).Game))
.SelectMany(list =>
list.Data.Where(mod => DateTime.UtcNow - mod.LatestFileUpdate.AsUnixTime() < TimeSpan.FromDays(1))
.Select(mod => (list.Game, mod.ModId)));
var mod_files = (await mods.PMap(queue, async mod =>
{
var client = await NexusApiClient.Get();
try
{
var files = await client.GetModFiles(GameRegistry.GetByNexusName(mod.Game).Game,
(int)mod.ModId);
return (Game: GameRegistry.GetByFuzzyName(mod.Game).Game, mod.ModId, files.files);
}
catch (Exception)
{
return default;
}
})).Where(t => t.Game != default).ToList();
var archives =
mod_files.SelectMany(mod => mod.files.Select(file => (mod.Game, mod.ModId, File:file)).Where(f => !string.IsNullOrEmpty(f.File.category_name) ))
.Select(tuple =>
{
var state = new NexusDownloader.State
{
Game = tuple.Game, ModID = tuple.ModId, FileID = tuple.File.file_id
};
return new Archive {State = state, Name = tuple.File.file_name};
}).ToList();
Utils.Log($"Found {archives.Count} archives from recent Nexus updates to index");
var searching = archives.Select(a => a.State.PrimaryKeyString).Distinct().ToArray();
Utils.Log($"Looking for missing states");
var knownArchives = (await db.DownloadStates.AsQueryable().Where(s => searching.Contains(s.Key))
.Select(d => d.Key).ToListAsync()).ToDictionary(a => a);
Utils.Log($"Found {knownArchives.Count} pre-existing archives");
var missing = archives.Where(a => !knownArchives.ContainsKey(a.State.PrimaryKeyString))
.DistinctBy(d => d.State.PrimaryKeyString)
.ToList();
Utils.Log($"Found {missing.Count} missing archives, enqueing indexing jobs");
var jobs = missing.Select(a => new Job {Payload = new IndexJob {Archive = a}, Priority = Job.JobPriority.Low});
Utils.Log($"Writing jobs to the DB");
await db.Jobs.InsertManyAsync(jobs, new InsertManyOptions {IsOrdered = false});
Utils.Log($"Done adding archives for Nexus Updates");
return JobResult.Success();
}
}
}
}

19
Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs
View File

@ -4,8 +4,6 @@ using System.Threading.Tasks;
using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.NexusApi;
using MongoDB.Driver;
using Newtonsoft.Json;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
@ -15,7 +13,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
{ {
public override string Description => "Poll the Nexus for updated mods, and clean any references to those mods"; public override string Description => "Poll the Nexus for updated mods, and clean any references to those mods";
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings) public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{ {
var api = await NexusApiClient.Get(); var api = await NexusApiClient.Get();
@ -31,8 +29,6 @@ namespace Wabbajack.BuildServer.Models.Jobs
entry.Path = $"/v1/games/{game.NexusName}/mods/updated.json?period=1m"; entry.Path = $"/v1/games/{game.NexusName}/mods/updated.json?period=1m";
entry.Data = mods; entry.Data = mods;
await entry.Upsert(db.NexusUpdates);
return (game, mods); return (game, mods);
}) })
.Select(async rTask => .Select(async rTask =>
@ -56,19 +52,14 @@ namespace Wabbajack.BuildServer.Models.Jobs
// Mod activity could hide files // Mod activity could hide files
var b = d.mod.LastestModActivity.AsUnixTime(); var b = d.mod.LastestModActivity.AsUnixTime();
return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.ModId}; return new {Game = d.game.Game, Date = (a > b ? a : b), ModId = d.mod.ModId};
}); });
var purged = await collected.PMap(queue, async t => var purged = await collected.PMap(queue, async t =>
{ {
var resultA = await db.NexusModInfos.DeleteManyAsync(f => var resultA = await sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date);
f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); var resultB = await sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date);
var resultB = await db.NexusModFiles.DeleteManyAsync(f => return resultA + resultB;
f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
var resultC = await db.NexusFileInfos.DeleteManyAsync(f =>
f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount;
}); });
Utils.Log($"Purged {purged.Sum()} cache entries"); Utils.Log($"Purged {purged.Sum()} cache entries");

8
Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs
View File

@ -3,8 +3,6 @@ using System.Linq;
using System.Net.Http; using System.Net.Http;
using System.Threading.Tasks; using System.Threading.Tasks;
using HtmlAgilityPack; using HtmlAgilityPack;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common; using Wabbajack.Common;
@ -20,7 +18,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
public class IndexDynDOLOD : AJobPayload public class IndexDynDOLOD : AJobPayload
{ {
public override string Description => "Queue MEGA URLs from the DynDOLOD Post"; public override string Description => "Queue MEGA URLs from the DynDOLOD Post";
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings) public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{ {
var doc = new HtmlDocument(); var doc = new HtmlDocument();
var body = await new HttpClient().GetStringAsync(new Uri( var body = await new HttpClient().GetStringAsync(new Uri(
@ -54,11 +52,11 @@ namespace Wabbajack.BuildServer.Models.Jobs
foreach (var job in matches) foreach (var job in matches)
{ {
var key = ((MegaDownloader.State)((IndexJob)job.Payload).Archive.State).PrimaryKeyString; var key = ((MegaDownloader.State)((IndexJob)job.Payload).Archive.State).PrimaryKeyString;
var found = await db.DownloadStates.AsQueryable().Where(s => s.Key == key).FirstOrDefaultAsync(); var found = await sql.DownloadStateByPrimaryKey(key);
if (found != null) continue; if (found != null) continue;
Utils.Log($"Queuing {key} for indexing"); Utils.Log($"Queuing {key} for indexing");
await db.Jobs.InsertOneAsync(job); await sql.EnqueueJob(job);
} }
return JobResult.Success(); return JobResult.Success();

50
Wabbajack.BuildServer/Models/Jobs/IndexJob.cs
View File

@ -1,14 +1,9 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem; using Alphaleonis.Win32.Filesystem;
using FluentFTP;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.Lib; using Wabbajack.Lib;
@ -23,7 +18,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
public Archive Archive { get; set; } public Archive Archive { get; set; }
public override string Description => $"Index ${Archive.State.PrimaryKeyString} and save the download/file state"; public override string Description => $"Index ${Archive.State.PrimaryKeyString} and save the download/file state";
public override bool UsesNexus { get => Archive.State is NexusDownloader.State; } public override bool UsesNexus { get => Archive.State is NexusDownloader.State; }
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings) public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{ {
if (Archive.State is ManualDownloader.State) if (Archive.State is ManualDownloader.State)
return JobResult.Success(); return JobResult.Success();
@ -33,8 +28,8 @@ namespace Wabbajack.BuildServer.Models.Jobs
pk.AddRange(Archive.State.PrimaryKey); pk.AddRange(Archive.State.PrimaryKey);
var pk_str = string.Join("|",pk.Select(p => p.ToString())); var pk_str = string.Join("|",pk.Select(p => p.ToString()));
var found = await db.DownloadStates.AsQueryable().Where(f => f.Key == pk_str).Take(1).ToListAsync(); var found = await sql.DownloadStateByPrimaryKey(pk_str);
if (found.Count > 0) if (found == null)
return JobResult.Success(); return JobResult.Success();
string fileName = Archive.Name; string fileName = Archive.Name;
@ -51,10 +46,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
await sql.MergeVirtualFile(archive); await sql.MergeVirtualFile(archive);
await db.DownloadStates.InsertOneAsync(new DownloadState await sql.AddDownloadState(archive.Hash, Archive.State);
{
Key = pk_str, Hash = archive.Hash, State = Archive.State, IsValid = true
});
var to_path = settings.ArchiveDir.Combine( var to_path = settings.ArchiveDir.Combine(
$"{Path.GetFileName(fileName)}_{archive.Hash.ToHex()}_{Path.GetExtension(fileName)}"); $"{Path.GetFileName(fileName)}_{archive.Hash.ToHex()}_{Path.GetExtension(fileName)}");
@ -66,43 +58,9 @@ namespace Wabbajack.BuildServer.Models.Jobs
await settings.DownloadDir.Combine(folder).DeleteDirectory(); await settings.DownloadDir.Combine(folder).DeleteDirectory();
} }
return JobResult.Success(); return JobResult.Success();
} }
/*
private List<IndexedFile> ConvertArchive(List<IndexedFile> files, VirtualFile file, bool isTop = true)
{
var name = isTop ? file.Name.FileName : file.Name;
var ifile = new IndexedFile
{
Hash = file.Hash,
SHA256 = file.ExtendedHashes.SHA256,
SHA1 = file.ExtendedHashes.SHA1,
MD5 = file.ExtendedHashes.MD5,
CRC = file.ExtendedHashes.CRC,
Size = file.Size,
Children = file.Children != null ? file.Children.Select(
f =>
{
ConvertArchive(files, f, false);
return new ChildFile
{
Hash = f.Hash,
Name = f.Name,
Extension = Path.GetExtension(f.Name.ToLowerInvariant())
};
}).ToList() : new List<ChildFile>()
};
ifile.IsArchive = ifile.Children.Count > 0;
files.Add(ifile);
return files;
}*/
} }
} }

2
Wabbajack.BuildServer/Models/Jobs/ReindexArchives.cs
View File

@ -14,7 +14,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
public class ReindexArchives : AJobPayload public class ReindexArchives : AJobPayload
{ {
public override string Description => "Reindex all files in the mod archive folder"; public override string Description => "Reindex all files in the mod archive folder";
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings) public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{ {
using (var queue = new WorkQueue()) using (var queue = new WorkQueue())
{ {

24
Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs
View File

@ -1,9 +1,6 @@
using System; using System;
using System.Linq; using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common; using Wabbajack.Common;
@ -12,14 +9,13 @@ using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation; using Wabbajack.Lib.Validation;
using File = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.BuildServer.Models.Jobs namespace Wabbajack.BuildServer.Models.Jobs
{ {
public class UpdateModLists : AJobPayload, IFrontEndJob public class UpdateModLists : AJobPayload, IFrontEndJob
{ {
public override string Description => "Validate curated modlists"; public override string Description => "Validate curated modlists";
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings) public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{ {
Utils.Log("Starting Modlist Validation"); Utils.Log("Starting Modlist Validation");
var modlists = await ModlistMetadata.LoadFromGithub(); var modlists = await ModlistMetadata.LoadFromGithub();
@ -34,7 +30,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
{ {
try try
{ {
await ValidateList(db, list, queue, whitelists); await ValidateList(sql, list, queue, whitelists);
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -46,7 +42,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success(); return JobResult.Success();
} }
private async Task ValidateList(DBContext db, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists) private async Task ValidateList(SqlService sql, ModlistMetadata list, WorkQueue queue, ValidateModlist whitelists)
{ {
var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension); var modlistPath = Consts.ModListDownloadFolder.Combine(list.Links.MachineURL + Consts.ModListExtension);
@ -76,7 +72,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
var validated = (await installer.Archives var validated = (await installer.Archives
.PMap(queue, async archive => .PMap(queue, async archive =>
{ {
var isValid = await IsValid(db, whitelists, archive); var isValid = await IsValid(sql, whitelists, archive);
return new DetailedStatusItem {IsFailing = !isValid, Archive = archive}; return new DetailedStatusItem {IsFailing = !isValid, Archive = archive};
})) }))
@ -107,13 +103,13 @@ namespace Wabbajack.BuildServer.Models.Jobs
}; };
Utils.Log( Utils.Log(
$"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed"); $"Writing Update for {dto.Summary.Name} - {dto.Summary.Failed} failed - {dto.Summary.Passed} passed");
await ModListStatus.Update(db, dto); await sql.UpdateModListStatus(dto);
Utils.Log( Utils.Log(
$"Done updating {dto.Summary.Name}"); $"Done updating {dto.Summary.Name}");
} }
private async Task<bool> IsValid(DBContext db, ValidateModlist whitelists, Archive archive) private async Task<bool> IsValid(SqlService sql, ValidateModlist whitelists, Archive archive)
{ {
try try
{ {
@ -123,7 +119,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
{ {
if (archive.State is NexusDownloader.State state) if (archive.State is NexusDownloader.State state)
{ {
if (await ValidateNexusFast(db, state)) return true; if (await ValidateNexusFast(sql, state)) return true;
} }
else if (archive.State is GoogleDriveDownloader.State) else if (archive.State is GoogleDriveDownloader.State)
@ -166,13 +162,11 @@ namespace Wabbajack.BuildServer.Models.Jobs
} }
} }
private async Task<bool> ValidateNexusFast(DBContext db, NexusDownloader.State state) private async Task<bool> ValidateNexusFast(SqlService sql, NexusDownloader.State state)
{ {
try try
{ {
var gameMeta = state.Game.MetaData(); var modFiles = await sql.GetModFiles(state.Game, state.ModID);
var modFiles = (await db.NexusModFiles.AsQueryable().Where(g => g.Game == gameMeta.NexusName && g.ModId == state.ModID).FirstOrDefaultAsync())?.Data;
if (modFiles == null) if (modFiles == null)
{ {

10
Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs
View File

@ -2,11 +2,7 @@
using System.Net; using System.Net;
using System.Threading.Tasks; using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem; using Alphaleonis.Win32.Filesystem;
using BunnyCDN.Net.Storage;
using CG.Web.MegaApiClient;
using FluentFTP; using FluentFTP;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common; using Wabbajack.Common;
@ -22,11 +18,11 @@ namespace Wabbajack.BuildServer.Models.Jobs
public Guid FileId { get; set; } public Guid FileId { get; set; }
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings) public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{ {
int retries = 0; int retries = 0;
TOP: TOP:
var file = await db.UploadedFiles.AsQueryable().Where(f => f.Id == FileId).FirstOrDefaultAsync(); var file = await sql.UploadedFileById(FileId);
if (settings.BunnyCDN_User == "TEST" && settings.BunnyCDN_Password == "TEST") if (settings.BunnyCDN_User == "TEST" && settings.BunnyCDN_Password == "TEST")
{ {
@ -53,7 +49,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
} }
} }
await db.Jobs.InsertOneAsync(new Job await sql.EnqueueJob(new Job
{ {
Priority = Job.JobPriority.High, Priority = Job.JobPriority.High,
Payload = new IndexJob Payload = new IndexJob

56
Wabbajack.BuildServer/Models/Metric.cs
View File

@ -3,71 +3,17 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.GraphQL; using Wabbajack.BuildServer.GraphQL;
using Wabbajack.Common; using Wabbajack.BuildServer.Model.Models;
namespace Wabbajack.BuildServer.Models namespace Wabbajack.BuildServer.Models
{ {
public class Metric public class Metric
{ {
[BsonId]
public ObjectId Id { get; set; }
public DateTime Timestamp { get; set; } public DateTime Timestamp { get; set; }
public string Action { get; set; } public string Action { get; set; }
public string Subject { get; set; } public string Subject { get; set; }
public string MetricsKey { get; set; } public string MetricsKey { get; set; }
public static async Task<IEnumerable<MetricResult>> Report(DBContext db, string grouping)
{
var regex = new Regex("\\d+\\.");
var data = await db.Metrics.AsQueryable()
.Where(m => m.MetricsKey != null)
.Where(m => m.Action == grouping)
.Where(m => m.Subject != "Default")
.ToListAsync();
var minDate = DateTime.Parse(data.Min(d => d.Timestamp.ToString("yyyy-MM-dd")));
var maxDate = DateTime.Parse(data.Max(d => d.Timestamp.ToString("yyyy-MM-dd")));
var dateArray = Enumerable.Range(0, (int)(maxDate - minDate).TotalDays + 1)
.Select(idx => minDate + TimeSpan.FromDays(idx))
.Select(date => date.ToString("yyyy-MM-dd"))
.ToList();
var results = data
.Where(d => !Guid.TryParse(d.Subject, out var _))
.GroupBy(d => regex.Split(d.Subject).First())
.Select(by_series =>
{
var by_day = by_series.GroupBy(d => d.Timestamp.ToString("yyyy-MM-dd"))
.Select(d => (d.Key, d.DistinctBy(v => v.MetricsKey ?? "").Count()))
.OrderBy(r => r.Key);
var by_day_idx = by_day.ToDictionary(d => d.Key);
(string Key, int) GetEntry(string date)
{
if (by_day_idx.TryGetValue(date, out var result))
return result;
return (date, 0);
}
return new MetricResult
{
SeriesName = by_series.Key,
Labels = dateArray.Select(d => GetEntry(d).Key).ToList(),
Values = dateArray.Select(d => GetEntry(d).Item2).ToList()
};
})
.OrderBy(f => f.SeriesName);
return results;
}
} }
} }

20
Wabbajack.BuildServer/Models/ModListStatus.cs
View File

@ -2,9 +2,6 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.Lib; using Wabbajack.Lib;
using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.ModListRegistry;
@ -12,20 +9,12 @@ namespace Wabbajack.BuildServer.Models
{ {
public class ModListStatus public class ModListStatus
{ {
[BsonId]
public string Id { get; set; } public string Id { get; set; }
public ModlistSummary Summary { get; set; } public ModlistSummary Summary { get; set; }
public ModlistMetadata Metadata { get; set; } public ModlistMetadata Metadata { get; set; }
public DetailedStatus DetailedStatus { get; set; } public DetailedStatus DetailedStatus { get; set; }
public static async Task Update(DBContext db, ModListStatus status)
{
var id = status.Metadata.Links.MachineURL;
await db.ModListStatus.FindOneAndReplaceAsync<ModListStatus>(s => s.Id == id, status, new FindOneAndReplaceOptions<ModListStatus> {IsUpsert = true});
}
public static IQueryable<ModListStatus> AllSummaries public static IQueryable<ModListStatus> AllSummaries
{ {
get get
@ -33,15 +22,6 @@ namespace Wabbajack.BuildServer.Models
return null; return null;
} }
} }
public static async Task<ModListStatus> ByName(DBContext db, string name)
{
var result = await db.ModListStatus
.AsQueryable()
.Where(doc => doc.Metadata.Links.MachineURL == name || doc.Metadata.Title == name)
.ToListAsync();
return result.First();
}
} }
public class DetailedStatus public class DetailedStatus

9
Wabbajack.BuildServer/Models/NexusCacheData.cs
View File

@ -1,28 +1,19 @@
using System; using System;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
namespace Wabbajack.BuildServer.Models namespace Wabbajack.BuildServer.Models
{ {
public class NexusCacheData<T> public class NexusCacheData<T>
{ {
[BsonId]
public string Path { get; set; } public string Path { get; set; }
public T Data { get; set; } public T Data { get; set; }
public string Game { get; set; } public string Game { get; set; }
[BsonIgnoreIfNull]
public long ModId { get; set; } public long ModId { get; set; }
public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow; public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow;
[BsonIgnoreIfNull]
public string FileId { get; set; } public string FileId { get; set; }
public async Task Upsert(IMongoCollection<NexusCacheData<T>> coll)
{
await coll.FindOneAndReplaceAsync<NexusCacheData<T>>(s => s.Path == Path, this, new FindOneAndReplaceOptions<NexusCacheData<T>> {IsUpsert = true});
}
} }
} }

4
Wabbajack.BuildServer/Models/NexusUpdateEntry.cs
View File

@ -1,6 +1,4 @@
using System; using Newtonsoft.Json;
using MongoDB.Bson.Serialization.Attributes;
using Newtonsoft.Json;
namespace Wabbajack.BuildServer.Models namespace Wabbajack.BuildServer.Models
{ {

10
Wabbajack.BuildServer/Models/PatchArchive.cs
View File

@ -1,17 +1,11 @@
using System; using System;
using System.IO;
using System.Net; using System.Net;
using System.Threading.Tasks; using System.Threading.Tasks;
using FluentFTP; using FluentFTP;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using File = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.BuildServer.Models namespace Wabbajack.BuildServer.Models
{ {
@ -20,10 +14,10 @@ namespace Wabbajack.BuildServer.Models
public override string Description => "Create a archive update patch"; public override string Description => "Create a archive update patch";
public Hash Src { get; set; } public Hash Src { get; set; }
public string DestPK { get; set; } public string DestPK { get; set; }
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings) public override async Task<JobResult> Execute(SqlService sql, AppSettings settings)
{ {
var srcPath = settings.PathForArchive(Src); var srcPath = settings.PathForArchive(Src);
var destHash = (await db.DownloadStates.AsQueryable().Where(s => s.Key == DestPK).FirstOrDefaultAsync()).Hash; var destHash = (await sql.DownloadStateByPrimaryKey(DestPK)).Hash;
var destPath = settings.PathForArchive(destHash); var destPath = settings.PathForArchive(destHash);
if (Src == destHash) if (Src == destHash)

196
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -1,4 +1,5 @@
using System; using System;
using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using System.Data; using System.Data;
using System.Data.SqlClient; using System.Data.SqlClient;
@ -11,6 +12,7 @@ using Wabbajack.BuildServer.Model.Models.Results;
using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.NexusApi;
@ -168,18 +170,6 @@ namespace Wabbajack.BuildServer.Model.Models
.ToList(); .ToList();
} }
#region UserRoutines
public async Task<string> LoginByAPIKey(string key)
{
await using var conn = await Open();
var result = await conn.QueryAsync<string>(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @Key",
new {Key = key});
return result.FirstOrDefault();
}
#endregion
#region JobRoutines #region JobRoutines
/// <summary> /// <summary>
@ -233,6 +223,24 @@ namespace Wabbajack.BuildServer.Model.Models
new {RunBy = Guid.NewGuid().ToString()}); new {RunBy = Guid.NewGuid().ToString()});
return result.FirstOrDefault(); return result.FirstOrDefault();
} }
public async Task<IEnumerable<Job>> GetRunningJobs()
{
await using var conn = await Open();
var results =
await conn.QueryAsync<Job>("SELECT * from dbo.Jobs WHERE Started IS NOT NULL AND Ended IS NULL ");
return results;
}
public async Task<IEnumerable<Job>> GetUnfinishedJobs()
{
await using var conn = await Open();
var results =
await conn.QueryAsync<Job>("SELECT * from dbo.Jobs WHERE Ended IS NULL ");
return results;
}
#endregion #endregion
@ -292,6 +300,15 @@ namespace Wabbajack.BuildServer.Model.Models
uf.CDNName uf.CDNName
}); });
} }
public async Task<UploadedFile> UploadedFileById(Guid fileId)
{
await using var conn = await Open();
return await conn.QueryFirstAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles WHERE Id = @Id",
new {Id = fileId.ToString()});
}
public async Task<IEnumerable<UploadedFile>> AllUploadedFilesForUser(string user) public async Task<IEnumerable<UploadedFile>> AllUploadedFilesForUser(string user)
{ {
@ -299,6 +316,24 @@ namespace Wabbajack.BuildServer.Model.Models
return await conn.QueryAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles WHERE UploadedBy = @uploadedBy", return await conn.QueryAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles WHERE UploadedBy = @uploadedBy",
new {UploadedBy = user}); new {UploadedBy = user});
} }
public async Task<IEnumerable<UploadedFile>> AllUploadedFiles()
{
await using var conn = await Open();
return await conn.QueryAsync<UploadedFile>("SELECT * FROM dbo.UploadedFiles ORDER BY UploadDate DESC");
}
public async Task DeleteUploadedFile(Guid dupId)
{
await using var conn = await Open();
await conn.ExecuteAsync("SELECT * FROM dbo.UploadedFiles WHERE Id = @id",
new
{
Id = dupId.ToString()
});
}
public async Task AddDownloadState(Hash hash, AbstractDownloadState state) public async Task AddDownloadState(Hash hash, AbstractDownloadState state)
{ {
@ -429,7 +464,6 @@ namespace Wabbajack.BuildServer.Model.Models
}); });
return result.FromJSONString<DetailedStatus>(); return result.FromJSONString<DetailedStatus>();
} }
public async Task<List<DetailedStatus>> GetDetailedModlistStatuses() public async Task<List<DetailedStatus>> GetDetailedModlistStatuses()
{ {
await using var conn = await Open(); await using var conn = await Open();
@ -440,5 +474,141 @@ namespace Wabbajack.BuildServer.Model.Models
#endregion #endregion
#region Logins
public async Task<string> AddLogin(string name)
{
var key = NewAPIKey();
await using var conn = await Open();
await conn.ExecuteAsync("INSERT INTO dbo.ApiKeys (Owner, ApiKey) VALUES (@Owner, @ApiKey)",
new {Owner = name, ApiKey = key});
return key;
}
public static string NewAPIKey()
{
var arr = new byte[128];
new Random().NextBytes(arr);
return arr.ToHex();
}
public async Task<string> LoginByAPIKey(string key)
{
await using var conn = await Open();
var result = await conn.QueryAsync<string>(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @ApiKey",
new {ApiKey = key});
return result.FirstOrDefault();
}
public async Task<IEnumerable<(string Owner, string Key)>> GetAllUserKeys()
{
await using var conn = await Open();
var result = await conn.QueryAsync<(string Owner, string Key)>("SELECT Owner, ApiKey FROM dbo.ApiKeys");
return result;
}
#endregion
#region Auto-healing routines
public async Task<Archive> GetNexusStateByHash(Hash startingHash)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(@"SELECT JsonState FROM dbo.DownloadStates
WHERE Hash = @hash AND PrimaryKey like 'NexusDownloader+State|%'
ORDER BY LastValidated DESC",
new {Hash = startingHash});
return result == null ? null : new Archive
{
State = result.FromJSONString<AbstractDownloadState>(),
Hash = startingHash
};
}
public async Task<Archive> DownloadStateByPrimaryKey(string primaryKey)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<(long Hash, string State)>(@"SELECT Hash, JsonState FROM dbo.DownloadStates WHERE PrimaryKey = @PrimaryKey",
new {PrimaryKey = primaryKey});
return result == default ? null : new Archive
{
State = result.State.FromJSONString<AbstractDownloadState>(),
Hash = Hash.FromLong(result.Hash)
};
}
#endregion
/// <summary>
/// Returns a hashset the only contains hashes from the input that do not exist in IndexedArchives
/// </summary>
/// <param name="searching"></param>
/// <returns></returns>
/// <exception cref="NotImplementedException"></exception>
public async Task<HashSet<Hash>> FilterByExistingIndexedArchives(HashSet<Hash> searching)
{
await using var conn = await Open();
var found = await conn.QueryAsync<long>("SELECT Hash from dbo.IndexedFile WHERE Hash in @Hashes",
new {Hashes = searching.Select(h => (long)h)});
return searching.Except(found.Select(h => Hash.FromLong(h)).ToHashSet()).ToHashSet();
}
/// <summary>
/// Returns a hashset the only contains primary keys from the input that do not exist in IndexedArchives
/// </summary>
/// <param name="searching"></param>
/// <returns></returns>
/// <exception cref="NotImplementedException"></exception>
public async Task<HashSet<string>> FilterByExistingPrimaryKeys(HashSet<string> pks)
{
await using var conn = await Open();
var found = await conn.QueryAsync<string>("SELECT Hash from dbo.IndexedFile WHERE PrimaryKey in @PrimaryKeys",
new {PrimaryKeys = pks.ToList()});
return pks.Except(found.ToHashSet()).ToHashSet();
}
public async Task<long> DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date)
{
await using var conn = await Open();
var deleted = await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
return deleted;
}
public async Task<long> DeleteNexusModFilesUpdatedBeforeDate(Game game, long modId, DateTime date)
{
await using var conn = await Open();
var deleted = await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
return deleted;
}
public async Task UpdateModListStatus(ModListStatus dto)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"MERGE dbo.ModLists AS Target
USING (SELECT @MachineUrl MachineUrl, @Metadata Metadata, @Summary Summary, @DetailedStatus DetailedStatus) AS Source
ON Target.MachineUrl = Source.MachineUrl
WHEN MATCHED THEN UPDATE SET Target.Summary = Source.Summary, Target.Metadata = Source.Metadata, Target.DetailedStatus = Source.DetailedStats
WHEN NOT MATCHED THEN INSERT (MachineUrl, Summary, Metadata, DetailedStatus) VALUES (@MachineUrl, @Summary, @Metadata, @DetailedStatus)",
new
{
MachineUrl = dto.Metadata.Links.MachineURL,
Metadata = dto.Metadata.ToJSON(),
Summary = dto.Summary.ToJSON(),
DetailedStatus = dto.DetailedStatus.ToJSON()
});
}
} }
} }

6
Wabbajack.BuildServer/Models/UploadedFile.cs
View File

@ -1,8 +1,4 @@
using System; using System;
using System.IO;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.Common; using Wabbajack.Common;
using Path = Alphaleonis.Win32.Filesystem.Path; using Path = Alphaleonis.Win32.Filesystem.Path;
@ -19,10 +15,8 @@ namespace Wabbajack.BuildServer.Models
public string CDNName { get; set; } public string CDNName { get; set; }
[BsonIgnore]
public string MungedName => $"{Path.GetFileNameWithoutExtension(Name)}-{Id}{Path.GetExtension(Name)}"; public string MungedName => $"{Path.GetFileNameWithoutExtension(Name)}-{Id}{Path.GetExtension(Name)}";
[BsonIgnore]
public string Uri => CDNName == null ? $"https://wabbajack.b-cdn.net/{MungedName}" : $"https://{CDNName}.b-cdn.net/{MungedName}"; public string Uri => CDNName == null ? $"https://wabbajack.b-cdn.net/{MungedName}" : $"https://{CDNName}.b-cdn.net/{MungedName}";
} }
} }

76
Wabbajack.BuildServer/SerializerSettings.cs
View File

@ -1,76 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc.ModelBinding;
using MongoDB.Bson;
using MongoDB.Bson.IO;
using MongoDB.Bson.Serialization;
using MongoDB.Bson.Serialization.Conventions;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.BuildServer
{
public static class SerializerSettings
{
public static void Init()
{
var dis = new TypeDiscriminator(typeof(AbstractDownloadState), AbstractDownloadState.NameToType,
AbstractDownloadState.TypeToName);
BsonSerializer.RegisterDiscriminatorConvention(typeof(AbstractDownloadState), dis);
BsonClassMap.RegisterClassMap<AbstractDownloadState>(cm => cm.SetIsRootClass(true));
dis = new TypeDiscriminator(typeof(AJobPayload), AJobPayload.NameToType, AJobPayload.TypeToName);
BsonSerializer.RegisterDiscriminatorConvention(typeof(AJobPayload), dis);
BsonClassMap.RegisterClassMap<AJobPayload>(cm => cm.SetIsRootClass(true));
}
}
public class TypeDiscriminator : IDiscriminatorConvention
{
private readonly Type defaultType;
private readonly Dictionary<string, Type> typeMap;
private Dictionary<Type, string> revMap;
public TypeDiscriminator(Type defaultType,
Dictionary<string, Type> typeMap, Dictionary<Type, string> revMap)
{
this.defaultType = defaultType;
this.typeMap = typeMap;
this.revMap = revMap;
}
/// <summary>
/// Element Name
/// </summary>
public string ElementName => "_wjType";
public Type GetActualType(IBsonReader bsonReader, Type nominalType)
{
Type type = null;
var bookmark = bsonReader.GetBookmark();
bsonReader.ReadStartDocument();
if (bsonReader.FindElement(ElementName))
{
var value = bsonReader.ReadString();
if (typeMap.ContainsKey(value))
type = typeMap[value];
}
bsonReader.ReturnToBookmark(bookmark);
if (type == null)
throw new Exception($"Type mis-configuration can't find bson type for ${nominalType}");
return type;
}
public BsonValue GetDiscriminator(Type nominalType, Type actualType)
{
return revMap[actualType];
}
}
}

2
Wabbajack.BuildServer/Startup.cs
View File

@ -71,7 +71,6 @@ namespace Wabbajack.BuildServer
x.MultipartBodyLengthLimit = int.MaxValue; x.MultipartBodyLengthLimit = int.MaxValue;
}); });
services.AddSingleton<DBContext>();
services.AddSingleton<JobManager>(); services.AddSingleton<JobManager>();
services.AddSingleton<AppSettings>(); services.AddSingleton<AppSettings>();
services.AddSingleton<SqlService>(); services.AddSingleton<SqlService>();
@ -89,7 +88,6 @@ namespace Wabbajack.BuildServer
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline. // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env) public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{ {
SerializerSettings.Init();
if (env.IsDevelopment()) if (env.IsDevelopment())
{ {
app.UseDeveloperExceptionPage(); app.UseDeveloperExceptionPage();

3
Wabbajack.BuildServer/Wabbajack.BuildServer.csproj
View File

@ -16,7 +16,6 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="BunnyCDN.Net.Storage" Version="1.0.2" />
<PackageReference Include="CsvHelper" Version="15.0.3" /> <PackageReference Include="CsvHelper" Version="15.0.3" />
<PackageReference Include="Dapper" Version="2.0.30" /> <PackageReference Include="Dapper" Version="2.0.30" />
<PackageReference Include="FluentFTP" Version="32.2.2" /> <PackageReference Include="FluentFTP" Version="32.2.2" />
@ -28,8 +27,6 @@
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.0" /> <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.0" />
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" /> <PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Microsoft.OpenApi" Version="1.1.4" /> <PackageReference Include="Microsoft.OpenApi" Version="1.1.4" />
<PackageReference Include="MongoDB.Driver" Version="2.10.2" />
<PackageReference Include="MongoDB.Driver.Core" Version="2.10.2" />
<PackageReference Include="Nettle" Version="1.3.0" /> <PackageReference Include="Nettle" Version="1.3.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.1.0" /> <PackageReference Include="Swashbuckle.AspNetCore" Version="5.1.0" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.1" /> <PackageReference Include="System.Data.SqlClient" Version="4.8.1" />

1
Wabbajack.Common/Consts.cs
View File

@ -137,5 +137,6 @@ namespace Wabbajack.Common
public static RelativePath ModListTxt = (RelativePath)"modlist.txt"; public static RelativePath ModListTxt = (RelativePath)"modlist.txt";
public static RelativePath ModOrganizer2Exe = (RelativePath)"ModOrganizer.exe"; public static RelativePath ModOrganizer2Exe = (RelativePath)"ModOrganizer.exe";
public static RelativePath ModOrganizer2Ini = (RelativePath)"ModOrganizer.ini"; public static RelativePath ModOrganizer2Ini = (RelativePath)"ModOrganizer.ini";
public static string AuthorAPIKeyFile = "author-api-key.txt";
} }
} }

3
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -4,12 +4,10 @@ using System.Reactive;
using System.Reactive.Linq; using System.Reactive.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using MessagePack; using MessagePack;
using MongoDB.Bson.Serialization.Attributes;
using Newtonsoft.Json; using Newtonsoft.Json;
using ReactiveUI; using ReactiveUI;
using Wabbajack.Common; using Wabbajack.Common;
using Wabbajack.Common.StatusFeed.Errors; using Wabbajack.Common.StatusFeed.Errors;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation; using Wabbajack.Lib.Validation;
using Game = Wabbajack.Common.Game; using Game = Wabbajack.Common.Game;
@ -130,7 +128,6 @@ namespace Wabbajack.Lib.Downloaders
} }
} }
[BsonIgnoreExtraElements]
[MessagePackObject] [MessagePackObject]
public class State : AbstractDownloadState, IMetaState public class State : AbstractDownloadState, IMetaState
{ {

4
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -19,11 +19,11 @@ namespace Wabbajack.Lib.FileUploader
{ {
public class AuthorAPI public class AuthorAPI
{ {
public static IObservable<bool> HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable("author-api-key.txt"); public static IObservable<bool> HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable(Consts.AuthorAPIKeyFile);
public static async Task<string> GetAPIKey(string apiKey = null) public static async Task<string> GetAPIKey(string apiKey = null)
{ {
return apiKey ?? (await Consts.LocalAppDataPath.Combine("author-api-key.txt").ReadAllTextAsync()).Trim(); return apiKey ?? (await Consts.LocalAppDataPath.Combine(Consts.AuthorAPIKeyFile).ReadAllTextAsync()).Trim();
} }
public static Uri UploadURL => new Uri($"{Consts.WabbajackBuildServerUri}upload_file"); public static Uri UploadURL => new Uri($"{Consts.WabbajackBuildServerUri}upload_file");

5
Wabbajack.Lib/ModListRegistry/ModListMetadata.cs
View File

@ -1,13 +1,9 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Drawing;
using System.Linq; using System.Linq;
using System.Net.Http;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using Newtonsoft.Json; using Newtonsoft.Json;
using Wabbajack.Common; using Wabbajack.Common;
using File = System.IO.File;
using Game = Wabbajack.Common.Game; using Game = Wabbajack.Common.Game;
namespace Wabbajack.Lib.ModListRegistry namespace Wabbajack.Lib.ModListRegistry
@ -40,7 +36,6 @@ namespace Wabbajack.Lib.ModListRegistry
[JsonIgnore] [JsonIgnore]
public ModlistSummary ValidationSummary { get; set; } = new ModlistSummary(); public ModlistSummary ValidationSummary { get; set; } = new ModlistSummary();
[BsonIgnoreExtraElements]
public class LinksObject public class LinksObject
{ {
[JsonProperty("image")] [JsonProperty("image")]

6
Wabbajack.Lib/Wabbajack.Lib.csproj
View File

@ -36,12 +36,6 @@
<PackageReference Include="ModuleInit.Fody"> <PackageReference Include="ModuleInit.Fody">
<Version>2.1.0</Version> <Version>2.1.0</Version>
</PackageReference> </PackageReference>
<PackageReference Include="MongoDB.Bson">
<Version>2.10.1</Version>
</PackageReference>
<PackageReference Include="MongoDB.Bson">
<Version>2.10.1</Version>
</PackageReference>
<PackageReference Include="ReactiveUI"> <PackageReference Include="ReactiveUI">
<Version>11.2.3</Version> <Version>11.2.3</Version>
</PackageReference> </PackageReference>