mirror of
https://github.com/wabbajack-tools/wabbajack.git
synced 2024-08-30 18:42:17 +00:00
WIP
This commit is contained in:
parent
45a77425ec
commit
9ed98ace54
18
Wabbajack.BuildServer/Controllers/AControllerBase.cs
Normal file
18
Wabbajack.BuildServer/Controllers/AControllerBase.cs
Normal file
@ -0,0 +1,18 @@
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
public abstract class AControllerBase<T> : ControllerBase
|
||||
{
|
||||
protected readonly ILogger<T> Logger;
|
||||
protected readonly DBContext Db;
|
||||
|
||||
protected AControllerBase(ILogger<T> logger, DBContext db)
|
||||
{
|
||||
Db = db;
|
||||
Logger = logger;
|
||||
}
|
||||
}
|
||||
}
|
31
Wabbajack.BuildServer/Controllers/Jobs.cs
Normal file
31
Wabbajack.BuildServer/Controllers/Jobs.cs
Normal file
@ -0,0 +1,31 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Driver.Linq;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
[ApiController]
|
||||
[Route("/jobs")]
|
||||
public class Jobs : AControllerBase<Jobs>
|
||||
{
|
||||
public Jobs(ILogger<Jobs> logger, DBContext db) : base(logger, db)
|
||||
{
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("unfinished")]
|
||||
public async Task<IEnumerable<Job>> GetUnfinished()
|
||||
{
|
||||
return await Db.Jobs.AsQueryable()
|
||||
.Where(j => j.Ended == null)
|
||||
.OrderByDescending(j => j.Priority)
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
28
Wabbajack.BuildServer/Controllers/ListValidation.cs
Normal file
28
Wabbajack.BuildServer/Controllers/ListValidation.cs
Normal file
@ -0,0 +1,28 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Driver.Linq;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.Lib.ModListRegistry;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
[ApiController]
|
||||
[Route("/lists")]
|
||||
|
||||
public class ListValidation : AControllerBase<ListValidation>
|
||||
{
|
||||
public ListValidation(ILogger<ListValidation> logger, DBContext db) : base(logger, db)
|
||||
{
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("status")]
|
||||
public async Task<IList<ModlistSummary>> HandleGetLists()
|
||||
{
|
||||
return await Db.ModListStatus.AsQueryable().Select(m => m.Summary).ToListAsync();
|
||||
}
|
||||
}
|
||||
}
|
136
Wabbajack.BuildServer/Controllers/NexusCache.cs
Normal file
136
Wabbajack.BuildServer/Controllers/NexusCache.cs
Normal file
@ -0,0 +1,136 @@
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MongoDB.Driver;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.Lib.NexusApi;
|
||||
|
||||
namespace Wabbajack.BuildServer.Controllers
|
||||
{
|
||||
//[Authorize]
|
||||
[ApiController]
|
||||
[Route("/v1/games/")]
|
||||
public class NexusCache : AControllerBase<NexusCache>
|
||||
{
|
||||
|
||||
public NexusCache(ILogger<NexusCache> logger, DBContext db) : base(logger, db)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Looks up the mod details for a given Gamename/ModId pair. If the entry is not found in the cache it will
|
||||
/// be requested from the server (using the caller's Nexus API key if provided).
|
||||
/// </summary>
|
||||
/// <param name="db"></param>
|
||||
/// <param name="GameName">The Nexus game name</param>
|
||||
/// <param name="ModId">The Nexus mod id</param>
|
||||
/// <returns>A Mod Info result</returns>
|
||||
[HttpGet]
|
||||
[Route("{GameName}/mods/{ModId}.json")]
|
||||
public async Task<ModInfo> GetModInfo(string GameName, string ModId)
|
||||
{
|
||||
var result = await Db.NexusModInfos.FindOneAsync(info => info.Game == GameName && info.ModId == ModId);
|
||||
|
||||
string method = "CACHED";
|
||||
if (result == null)
|
||||
{
|
||||
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
|
||||
var path = $"/v1/games/{GameName}/mods/{ModId}.json";
|
||||
var body = await api.Get<ModInfo>(path);
|
||||
result = new NexusCacheData<ModInfo>
|
||||
{
|
||||
Data = body,
|
||||
Path = path,
|
||||
Game = GameName,
|
||||
ModId = ModId
|
||||
};
|
||||
try
|
||||
{
|
||||
await Db.NexusModInfos.InsertOneAsync(result);
|
||||
}
|
||||
catch (MongoWriteException)
|
||||
{
|
||||
}
|
||||
|
||||
method = "NOT_CACHED";
|
||||
}
|
||||
|
||||
Response.Headers.Add("x-cache-result", method);
|
||||
return result.Data;
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("{GameName}/mods/{ModId}/files.json")]
|
||||
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, string ModId)
|
||||
{
|
||||
var result = await Db.NexusModFiles.FindOneAsync(info => info.Game == GameName && info.ModId == ModId);
|
||||
|
||||
string method = "CACHED";
|
||||
if (result == null)
|
||||
{
|
||||
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
|
||||
var path = $"/v1/games/{GameName}/mods/{ModId}/files.json";
|
||||
var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
|
||||
result = new NexusCacheData<NexusApiClient.GetModFilesResponse>
|
||||
{
|
||||
Data = body,
|
||||
Path = path,
|
||||
Game = GameName,
|
||||
ModId = ModId
|
||||
};
|
||||
try
|
||||
{
|
||||
await Db.NexusModFiles.InsertOneAsync(result);
|
||||
}
|
||||
catch (MongoWriteException)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
method = "NOT_CACHED";
|
||||
}
|
||||
|
||||
Response.Headers.Add("x-cache-result", method);
|
||||
return result.Data;
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("{GameName}/mods/{ModId}/files/{FileId}.json")]
|
||||
public async Task<object> GetFileInfo(string GameName, string ModId, string FileId)
|
||||
{
|
||||
var result = await Db.NexusFileInfos.FindOneAsync(info => info.Game == GameName && info.ModId == ModId && info.FileId == FileId);
|
||||
|
||||
string method = "CACHED";
|
||||
if (result == null)
|
||||
{
|
||||
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
|
||||
var path = $"/v1/games/{GameName}/mods/{ModId}/files/{FileId}.json";
|
||||
var body = await api.Get<NexusFileInfo>(path);
|
||||
result = new NexusCacheData<NexusFileInfo>
|
||||
{
|
||||
Data = body,
|
||||
Path = path,
|
||||
Game = GameName,
|
||||
ModId = ModId,
|
||||
FileId = FileId
|
||||
};
|
||||
try
|
||||
{
|
||||
await Db.NexusFileInfos.InsertOneAsync(result);
|
||||
}
|
||||
catch (MongoWriteException)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
method = "NOT_CACHED";
|
||||
}
|
||||
|
||||
Response.Headers.Add("x-cache-method", method);
|
||||
return result.Data;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
17
Wabbajack.BuildServer/Extensions.cs
Normal file
17
Wabbajack.BuildServer/Extensions.cs
Normal file
@ -0,0 +1,17 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Driver.Linq;
|
||||
|
||||
namespace Wabbajack.BuildServer
|
||||
{
|
||||
public static class Extensions
|
||||
{
|
||||
public static async Task<T> FindOneAsync<T>(this IMongoCollection<T> coll, Expression<Func<T, bool>> expr)
|
||||
{
|
||||
return (await coll.AsQueryable().Where(expr).Take(1).ToListAsync()).FirstOrDefault();
|
||||
}
|
||||
}
|
||||
}
|
41
Wabbajack.BuildServer/Models/DBContext.cs
Normal file
41
Wabbajack.BuildServer/Models/DBContext.cs
Normal file
@ -0,0 +1,41 @@
|
||||
using System.Collections.Generic;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using MongoDB.Driver;
|
||||
using Wabbajack.Lib.NexusApi;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models
|
||||
{
|
||||
public class DBContext
|
||||
{
|
||||
private IConfiguration _configuration;
|
||||
private Settings _settings;
|
||||
public DBContext(IConfiguration configuration)
|
||||
{
|
||||
_configuration = configuration;
|
||||
|
||||
_settings = new Settings();
|
||||
_configuration.Bind("MongoDB", _settings);
|
||||
}
|
||||
|
||||
public IMongoCollection<NexusCacheData<ModInfo>> NexusModInfos => Client.GetCollection<NexusCacheData<ModInfo>>(_settings.Collections["NexusModInfos"]);
|
||||
public IMongoCollection<NexusCacheData<NexusFileInfo>> NexusFileInfos => Client.GetCollection<NexusCacheData<NexusFileInfo>>(_settings.Collections["NexusFileInfos"]);
|
||||
public IMongoCollection<ModListStatus> ModListStatus => Client.GetCollection<ModListStatus>(_settings.Collections["ModListStatus"]);
|
||||
|
||||
public IMongoCollection<Job> Jobs => Client.GetCollection<Job>(_settings.Collections["JobQueue"]);
|
||||
public IMongoCollection<DownloadState> DownloadStates => Client.GetCollection<DownloadState>(_settings.Collections["DownloadStates"]);
|
||||
public IMongoCollection<IndexedFile> IndexedFiles => Client.GetCollection<IndexedFile>(_settings.Collections["IndexedFiles"]);
|
||||
|
||||
public IMongoCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>> NexusModFiles =>
|
||||
Client.GetCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>>(
|
||||
_settings.Collections["NexusModFiles"]);
|
||||
|
||||
private IMongoDatabase Client => new MongoClient($"mongodb://{_settings.Host}").GetDatabase(_settings.Database);
|
||||
}
|
||||
public class Settings
|
||||
{
|
||||
public string Host { get; set; }
|
||||
public string Database { get; set; }
|
||||
public Dictionary<string, string> Collections { get; set; }
|
||||
}
|
||||
}
|
23
Wabbajack.BuildServer/Models/DownloadState.cs
Normal file
23
Wabbajack.BuildServer/Models/DownloadState.cs
Normal file
@ -0,0 +1,23 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models
|
||||
{
|
||||
public class DownloadState
|
||||
{
|
||||
[BsonId]
|
||||
public string Key { get; set; }
|
||||
public string Hash { get; set; }
|
||||
|
||||
public AbstractDownloadState State { get; set; }
|
||||
|
||||
public bool IsValid { get; set; }
|
||||
public DateTime LastValidationTime { get; set; } = DateTime.Now;
|
||||
public DateTime FirstValidationTime { get; set; } = DateTime.Now;
|
||||
}
|
||||
}
|
30
Wabbajack.BuildServer/Models/IndexedFile.cs
Normal file
30
Wabbajack.BuildServer/Models/IndexedFile.cs
Normal file
@ -0,0 +1,30 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using Wabbajack.VirtualFileSystem;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models
|
||||
{
|
||||
public class IndexedFile
|
||||
{
|
||||
[BsonId]
|
||||
public string Hash { get; set; }
|
||||
public string SHA256 { get; set; }
|
||||
public string SHA1 { get; set; }
|
||||
public string MD5 { get; set; }
|
||||
public string CRC { get; set; }
|
||||
public long Size { get; set; }
|
||||
public bool IsArchive { get; set; }
|
||||
public List<ChildFile> Children { get; set; } = new List<ChildFile>();
|
||||
}
|
||||
|
||||
public class ChildFile
|
||||
{
|
||||
public string Name;
|
||||
public string Extension;
|
||||
public string Hash;
|
||||
}
|
||||
}
|
36
Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs
Normal file
36
Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs
Normal file
@ -0,0 +1,36 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using Wabbajack.BuildServer.Models.Jobs;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.JobQueue
|
||||
{
|
||||
public abstract class AJobPayload
|
||||
{
|
||||
public static List<Type> KnownSubTypes = new List<Type>
|
||||
{
|
||||
typeof(IndexJob)
|
||||
|
||||
};
|
||||
public static Dictionary<Type, string> TypeToName { get; set; }
|
||||
public static Dictionary<string, Type> NameToType { get; set; }
|
||||
|
||||
|
||||
[BsonIgnore]
|
||||
public abstract string Description { get; }
|
||||
|
||||
public virtual bool UsesNexus { get; } = false;
|
||||
|
||||
public abstract Task<JobResult> Execute(DBContext db);
|
||||
|
||||
static AJobPayload()
|
||||
{
|
||||
NameToType = KnownSubTypes.ToDictionary(t => t.FullName.Substring(t.Namespace.Length + 1), t => t);
|
||||
TypeToName = NameToType.ToDictionary(k => k.Value, k => k.Key);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
67
Wabbajack.BuildServer/Models/JobQueue/Job.cs
Normal file
67
Wabbajack.BuildServer/Models/JobQueue/Job.cs
Normal file
@ -0,0 +1,67 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.JobQueue
|
||||
{
|
||||
public class Job
|
||||
{
|
||||
public enum JobPriority : int
|
||||
{
|
||||
Low,
|
||||
Normal,
|
||||
High,
|
||||
}
|
||||
|
||||
[BsonId]
|
||||
public Guid Id { get; set; }
|
||||
public DateTime? Started { get; set; }
|
||||
public DateTime? Ended { get; set; }
|
||||
public DateTime Created { get; set; } = DateTime.Now;
|
||||
public JobPriority Priority { get; set; } = JobPriority.Normal;
|
||||
|
||||
public JobResult Result { get; set; }
|
||||
public bool RequiresNexus { get; set; } = true;
|
||||
public AJobPayload Payload { get; set; }
|
||||
|
||||
public static async Task<Guid> Enqueue(DBContext db, Job job)
|
||||
{
|
||||
await db.Jobs.InsertOneAsync(job);
|
||||
return job.Id;
|
||||
}
|
||||
|
||||
public static async Task<Job> GetNext(DBContext db)
|
||||
{
|
||||
var filter = new BsonDocument
|
||||
{
|
||||
{"Started", BsonNull.Value}
|
||||
};
|
||||
var update = new BsonDocument
|
||||
{
|
||||
{"$set", new BsonDocument {{"Started", DateTime.Now}}}
|
||||
};
|
||||
var sort = new {Priority=-1, Created=1}.ToBsonDocument();
|
||||
var job = await db.Jobs.FindOneAndUpdateAsync<Job>(filter, update, new FindOneAndUpdateOptions<Job>{Sort = sort});
|
||||
return job;
|
||||
}
|
||||
|
||||
public static async Task<Job> Finish(DBContext db, Job job, JobResult jobResult)
|
||||
{
|
||||
var filter = new BsonDocument
|
||||
{
|
||||
{"query", new BsonDocument {{"Id", job.Id}}},
|
||||
};
|
||||
var update = new BsonDocument
|
||||
{
|
||||
{"$set", new BsonDocument {{"Ended", DateTime.Now}, {"Result", jobResult.ToBsonDocument()}}}
|
||||
};
|
||||
var result = await db.Jobs.FindOneAndUpdateAsync<Job>(filter, update);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
36
Wabbajack.BuildServer/Models/JobQueue/JobResult.cs
Normal file
36
Wabbajack.BuildServer/Models/JobQueue/JobResult.cs
Normal file
@ -0,0 +1,36 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.JobQueue
|
||||
{
|
||||
public class JobResult
|
||||
{
|
||||
public JobResultType ResultType { get; set; }
|
||||
[BsonIgnoreIfNull]
|
||||
public string Message { get; set; }
|
||||
|
||||
[BsonIgnoreIfNull]
|
||||
public string Stacktrace { get; set; }
|
||||
|
||||
public static JobResult Success()
|
||||
{
|
||||
return new JobResult { ResultType = JobResultType.Success };
|
||||
}
|
||||
|
||||
public static JobResult Error(Exception ex)
|
||||
{
|
||||
return new JobResult {ResultType = JobResultType.Error, Stacktrace = ex.ToString()};
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public enum JobResultType
|
||||
{
|
||||
Success,
|
||||
Error
|
||||
}
|
||||
}
|
110
Wabbajack.BuildServer/Models/Jobs/IndexJob.cs
Normal file
110
Wabbajack.BuildServer/Models/Jobs/IndexJob.cs
Normal file
@ -0,0 +1,110 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Alphaleonis.Win32.Filesystem;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Driver.Linq;
|
||||
using Wabbajack.BuildServer.Models;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using Wabbajack.VirtualFileSystem;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models.Jobs
|
||||
{
|
||||
|
||||
public class IndexJob : AJobPayload
|
||||
{
|
||||
public Archive Archive { get; set; }
|
||||
public override string Description { get; } = "Validate and index an archive";
|
||||
public override bool UsesNexus { get => Archive.State is NexusDownloader.State; }
|
||||
public override async Task<JobResult> Execute(DBContext db)
|
||||
{
|
||||
|
||||
/*
|
||||
|
||||
var pk = new List<object>();
|
||||
pk.Add(AbstractDownloadState.TypeToName[Archive.State.GetType()]);
|
||||
pk.AddRange(Archive.State.PrimaryKey);
|
||||
var pk_str = string.Join("|",pk.Select(p => p.ToString()));
|
||||
|
||||
var found = await db.DownloadStates.AsQueryable().Where(f => f.Key == pk_str).Take(1).ToListAsync();
|
||||
if (found.Count > 0)
|
||||
return JobResult.Success();
|
||||
|
||||
string fileName = Archive.Name;
|
||||
string folder = Guid.NewGuid().ToString();
|
||||
Utils.Log($"Indexer is downloading {fileName}");
|
||||
var downloadDest = Path.Combine(Server.Config.Indexer.DownloadDir, folder, fileName);
|
||||
await Archive.State.Download(downloadDest);
|
||||
|
||||
using (var queue = new WorkQueue())
|
||||
{
|
||||
var vfs = new Context(queue, true);
|
||||
await vfs.AddRoot(Path.Combine(Server.Config.Indexer.DownloadDir, folder));
|
||||
var archive = vfs.Index.ByRootPath.First();
|
||||
var converted = ConvertArchive(new List<IndexedFile>(), archive.Value);
|
||||
try
|
||||
{
|
||||
await db.IndexedFiles.InsertManyAsync(converted, new InsertManyOptions {IsOrdered = false});
|
||||
}
|
||||
catch (MongoBulkWriteException)
|
||||
{
|
||||
}
|
||||
|
||||
await db.DownloadStates.InsertOneAsync(new DownloadState
|
||||
{
|
||||
Key = pk_str,
|
||||
Hash = archive.Value.Hash,
|
||||
State = Archive.State,
|
||||
IsValid = true
|
||||
});
|
||||
|
||||
var to_path = Path.Combine(Server.Config.Indexer.ArchiveDir,
|
||||
$"{Path.GetFileName(fileName)}_{archive.Value.Hash.FromBase64().ToHex()}_{Path.GetExtension(fileName)}");
|
||||
if (File.Exists(to_path))
|
||||
File.Delete(downloadDest);
|
||||
else
|
||||
File.Move(downloadDest, to_path);
|
||||
Utils.DeleteDirectory(Path.Combine(Server.Config.Indexer.DownloadDir, folder));
|
||||
}
|
||||
*/
|
||||
return JobResult.Success();
|
||||
}
|
||||
|
||||
private List<IndexedFile> ConvertArchive(List<IndexedFile> files, VirtualFile file, bool isTop = true)
|
||||
{
|
||||
var name = isTop ? Path.GetFileName(file.Name) : file.Name;
|
||||
var ifile = new IndexedFile
|
||||
{
|
||||
Hash = file.Hash,
|
||||
SHA256 = file.ExtendedHashes.SHA256,
|
||||
SHA1 = file.ExtendedHashes.SHA1,
|
||||
MD5 = file.ExtendedHashes.MD5,
|
||||
CRC = file.ExtendedHashes.CRC,
|
||||
Size = file.Size,
|
||||
Children = file.Children != null ? file.Children.Select(
|
||||
f =>
|
||||
{
|
||||
ConvertArchive(files, f, false);
|
||||
|
||||
return new ChildFile
|
||||
{
|
||||
Hash = f.Hash,
|
||||
Name = f.Name.ToLowerInvariant(),
|
||||
Extension = Path.GetExtension(f.Name.ToLowerInvariant())
|
||||
};
|
||||
}).ToList() : new List<ChildFile>()
|
||||
};
|
||||
ifile.IsArchive = ifile.Children.Count > 0;
|
||||
files.Add(ifile);
|
||||
return files;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
62
Wabbajack.BuildServer/Models/ModListStatus.cs
Normal file
62
Wabbajack.BuildServer/Models/ModListStatus.cs
Normal file
@ -0,0 +1,62 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Driver.Linq;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.ModListRegistry;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models
|
||||
{
|
||||
public class ModListStatus
|
||||
{
|
||||
|
||||
[BsonId]
|
||||
public string Id { get; set; }
|
||||
public ModlistSummary Summary { get; set; }
|
||||
|
||||
public ModlistMetadata Metadata { get; set; }
|
||||
public DetailedStatus DetailedStatus { get; set; }
|
||||
|
||||
public static async Task Update(DBContext db, ModListStatus status)
|
||||
{
|
||||
var id = status.Metadata.Links.MachineURL;
|
||||
await db.ModListStatus.FindOneAndReplaceAsync<ModListStatus>(s => s.Id == id, status, new FindOneAndReplaceOptions<ModListStatus> {IsUpsert = true});
|
||||
}
|
||||
|
||||
public static IQueryable<ModListStatus> AllSummaries
|
||||
{
|
||||
get
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task<ModListStatus> ByName(DBContext db, string name)
|
||||
{
|
||||
var result = await db.ModListStatus
|
||||
.AsQueryable()
|
||||
.Where(doc => doc.Metadata.Links.MachineURL == name || doc.Metadata.Title == name)
|
||||
.ToListAsync();
|
||||
return result.First();
|
||||
}
|
||||
}
|
||||
|
||||
public class DetailedStatus
|
||||
{
|
||||
public string Name { get; set; }
|
||||
public DateTime Checked { get; set; } = DateTime.Now;
|
||||
public List<DetailedStatusItem> Archives { get; set; }
|
||||
public DownloadMetadata DownloadMetaData { get; set; }
|
||||
public bool HasFailures { get; set; }
|
||||
public string MachineName { get; set; }
|
||||
}
|
||||
|
||||
public class DetailedStatusItem
|
||||
{
|
||||
public bool IsFailing { get; set; }
|
||||
public Archive Archive { get; set; }
|
||||
}
|
||||
}
|
20
Wabbajack.BuildServer/Models/NexusCacheData.cs
Normal file
20
Wabbajack.BuildServer/Models/NexusCacheData.cs
Normal file
@ -0,0 +1,20 @@
|
||||
using System;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
|
||||
namespace Wabbajack.BuildServer.Models
|
||||
{
|
||||
public class NexusCacheData<T>
|
||||
{
|
||||
[BsonId]
|
||||
public string Path { get; set; }
|
||||
public T Data { get; set; }
|
||||
public string Game { get; set; }
|
||||
public string ModId { get; set; }
|
||||
|
||||
public DateTime LastCheckedUTC { get; set; } = DateTime.UtcNow;
|
||||
|
||||
[BsonIgnoreIfNull]
|
||||
public string FileId { get; set; }
|
||||
|
||||
}
|
||||
}
|
76
Wabbajack.BuildServer/SerializerSettings.cs
Normal file
76
Wabbajack.BuildServer/SerializerSettings.cs
Normal file
@ -0,0 +1,76 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.AspNetCore.Mvc.ModelBinding;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.IO;
|
||||
using MongoDB.Bson.Serialization;
|
||||
using MongoDB.Bson.Serialization.Conventions;
|
||||
using Wabbajack.BuildServer.Models.JobQueue;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
using Newtonsoft.Json.
|
||||
|
||||
namespace Wabbajack.BuildServer
|
||||
{
|
||||
public static class SerializerSettings
|
||||
{
|
||||
public static void Init()
|
||||
{
|
||||
var dis = new TypeDiscriminator(typeof(AbstractDownloadState), AbstractDownloadState.NameToType,
|
||||
AbstractDownloadState.TypeToName);
|
||||
BsonSerializer.RegisterDiscriminatorConvention(typeof(AbstractDownloadState), dis);
|
||||
BsonClassMap.RegisterClassMap<AbstractDownloadState>(cm => cm.SetIsRootClass(true));
|
||||
|
||||
dis = new TypeDiscriminator(typeof(AJobPayload), AJobPayload.NameToType, AJobPayload.TypeToName);
|
||||
BsonSerializer.RegisterDiscriminatorConvention(typeof(AJobPayload), dis);
|
||||
BsonClassMap.RegisterClassMap<AJobPayload>(cm => cm.SetIsRootClass(true));
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public class TypeDiscriminator : IDiscriminatorConvention
|
||||
{
|
||||
private readonly Type defaultType;
|
||||
private readonly Dictionary<string, Type> typeMap;
|
||||
private Dictionary<Type, string> revMap;
|
||||
|
||||
public TypeDiscriminator(Type defaultType,
|
||||
Dictionary<string, Type> typeMap, Dictionary<Type, string> revMap)
|
||||
{
|
||||
this.defaultType = defaultType;
|
||||
this.typeMap = typeMap;
|
||||
this.revMap = revMap;
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Element Name
|
||||
/// </summary>
|
||||
public string ElementName => "_wjType";
|
||||
|
||||
public Type GetActualType(IBsonReader bsonReader, Type nominalType)
|
||||
{
|
||||
Type type = null;
|
||||
var bookmark = bsonReader.GetBookmark();
|
||||
bsonReader.ReadStartDocument();
|
||||
if (bsonReader.FindElement(ElementName))
|
||||
{
|
||||
var value = bsonReader.ReadString();
|
||||
if (typeMap.ContainsKey(value))
|
||||
type = typeMap[value];
|
||||
}
|
||||
|
||||
bsonReader.ReturnToBookmark(bookmark);
|
||||
if (type == null)
|
||||
throw new Exception($"Type mis-configuration can't find bson type for ${nominalType}");
|
||||
return type;
|
||||
}
|
||||
|
||||
public BsonValue GetDiscriminator(Type nominalType, Type actualType)
|
||||
{
|
||||
return revMap[actualType];
|
||||
}
|
||||
}
|
||||
}
|
49
Wabbajack.BuildServer/Wabbajack.BuildServer.csproj
Normal file
49
Wabbajack.BuildServer/Wabbajack.BuildServer.csproj
Normal file
@ -0,0 +1,49 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<UserSecretsId>aspnet-Wabbajack.BuildServer-6E798B30-DB04-4436-BE65-F043AF37B314</UserSecretsId>
|
||||
<WebProject_DirectoryAccessLevelKey>0</WebProject_DirectoryAccessLevelKey>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.AzureAD.UI" Version="3.1.0" />
|
||||
<PackageReference Include="Microsoft.OpenApi" Version="1.1.4" />
|
||||
<PackageReference Include="MongoDB.Driver" Version="2.10.0" />
|
||||
<PackageReference Include="MongoDB.Driver.Core" Version="2.10.0" />
|
||||
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.0.0-rc5" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
|
||||
<ProjectReference Include="..\Wabbajack.Lib\Wabbajack.Lib.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Remove="chrome_elf.dll" />
|
||||
<None Remove="d3dcompiler_47.dll" />
|
||||
<None Remove="libGLESv2.dll" />
|
||||
<None Remove="CefSharp.dll" />
|
||||
<None Remove="v8_context_snapshot.bin" />
|
||||
<None Remove="CefSharp.Core.dll" />
|
||||
<None Remove="icudtl.dat" />
|
||||
<None Remove="innounp.exe" />
|
||||
<None Remove="CefSharp.Wpf.dll" />
|
||||
<None Remove="snapshot_blob.bin" />
|
||||
<None Remove="libEGL.dll" />
|
||||
<None Remove="libcef.dll" />
|
||||
<None Remove="natives_blob.bin" />
|
||||
<None Remove="CefSharp.OffScreen.dll" />
|
||||
<None Remove="devtools_resources.pak" />
|
||||
<None Remove="CefSharp.BrowserSubprocess.Core.dll" />
|
||||
<None Remove="CefSharp.BrowserSubprocess.exe" />
|
||||
<None Remove="cefsharp.7z" />
|
||||
<None Remove="cef_extensions.pak" />
|
||||
<None Remove="cef_200_percent.pak" />
|
||||
<None Remove="cef_100_percent.pak" />
|
||||
<None Remove="cef.pak" />
|
||||
<None Remove="7z.exe" />
|
||||
<None Remove="7z.dll" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
@ -122,7 +122,6 @@
|
||||
<Compile Include="ServerConfig\BuildServerConfig.cs" />
|
||||
<Compile Include="ServerConfig\IndexerConfig.cs" />
|
||||
<Compile Include="ServerConfig\MongoConfig.cs" />
|
||||
<Compile Include="ServerConfig\Settings.cs" />
|
||||
<Compile Include="TestingEndpoints.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
|
@ -109,12 +109,19 @@ namespace Wabbajack.Lib.ModListRegistry
|
||||
|
||||
public class ModlistSummary
|
||||
{
|
||||
public string Name;
|
||||
public DateTime Checked;
|
||||
public int Failed;
|
||||
public int Passed;
|
||||
[JsonProperty("name")]
|
||||
public string Name { get; set; }
|
||||
[JsonProperty("checked")]
|
||||
public DateTime Checked { get; set; }
|
||||
[JsonProperty("failed")]
|
||||
public int Failed { get; set; }
|
||||
[JsonProperty("passed")]
|
||||
public int Passed { get; set; }
|
||||
[JsonProperty("link")]
|
||||
public string Link => $"/lists/status/{Name}.json";
|
||||
[JsonProperty("report")]
|
||||
public string Report => $"/lists/status/{Name}.html";
|
||||
[JsonProperty("has_failures")]
|
||||
public bool HasFailures => Failed > 0;
|
||||
}
|
||||
|
||||
|
@ -15,35 +15,35 @@ namespace Wabbajack.Lib.NexusApi
|
||||
|
||||
public class NexusFileInfo
|
||||
{
|
||||
public ulong category_id;
|
||||
public string category_name;
|
||||
public string changelog_html;
|
||||
public string description;
|
||||
public string external_virus_scan_url;
|
||||
public ulong file_id;
|
||||
public string file_name;
|
||||
public bool is_primary;
|
||||
public string mod_version;
|
||||
public string name;
|
||||
public ulong size;
|
||||
public ulong size_kb;
|
||||
public DateTime uploaded_time;
|
||||
public ulong uploaded_timestamp;
|
||||
public string version;
|
||||
public ulong category_id { get; set; }
|
||||
public string category_name { get; set; }
|
||||
public string changelog_html { get; set; }
|
||||
public string description { get; set; }
|
||||
public string external_virus_scan_url { get; set; }
|
||||
public ulong file_id { get; set; }
|
||||
public string file_name { get; set; }
|
||||
public bool is_primary { get; set; }
|
||||
public string mod_version { get; set; }
|
||||
public string name { get; set; }
|
||||
public ulong size { get; set; }
|
||||
public ulong size_kb { get; set; }
|
||||
public DateTime uploaded_time { get; set; }
|
||||
public ulong uploaded_timestamp { get; set; }
|
||||
public string version { get; set; }
|
||||
}
|
||||
|
||||
public class ModInfo
|
||||
{
|
||||
public uint _internal_version;
|
||||
public string game_name;
|
||||
public string mod_id;
|
||||
public string name;
|
||||
public string summary;
|
||||
public string author;
|
||||
public string uploaded_by;
|
||||
public string uploaded_users_profile_url;
|
||||
public string picture_url;
|
||||
public bool contains_adult_content;
|
||||
public uint _internal_version { get; set; }
|
||||
public string game_name { get; set; }
|
||||
public string mod_id { get; set; }
|
||||
public string name { get; set; }
|
||||
public string summary { get; set; }
|
||||
public string author { get; set; }
|
||||
public string uploaded_by { get; set; }
|
||||
public string uploaded_users_profile_url { get; set; }
|
||||
public string picture_url { get; set; }
|
||||
public bool contains_adult_content { get; set; }
|
||||
}
|
||||
|
||||
public class MD5Response
|
||||
|
@ -272,7 +272,7 @@ namespace Wabbajack.Lib.NexusApi
|
||||
|
||||
public class GetModFilesResponse
|
||||
{
|
||||
public List<NexusFileInfo> files;
|
||||
public List<NexusFileInfo> files { get; set; }
|
||||
}
|
||||
|
||||
public async Task<GetModFilesResponse> GetModFiles(Game game, int modid)
|
||||
|
@ -36,6 +36,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.CacheServer", "Wa
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OMODExtractor", "OMODExtractor\OMODExtractor.csproj", "{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.BuildServer", "Wabbajack.BuildServer\Wabbajack.BuildServer.csproj", "{DE18D89E-39C5-48FD-8E42-16235E3C4593}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug (no commandargs)|Any CPU = Debug (no commandargs)|Any CPU
|
||||
@ -247,6 +249,24 @@ Global
|
||||
{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x86.Build.0 = Release|Any CPU
|
||||
{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.ActiveCfg = Release|x64
|
||||
{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.Build.0 = Release|x64
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|Any CPU.Build.0 = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|x64.ActiveCfg = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|x64.Build.0 = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|x86.ActiveCfg = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug (no commandargs)|x86.Build.0 = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.Build.0 = Release|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x86.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
Loading…
Reference in New Issue
Block a user