mirror of
https://github.com/wabbajack-tools/wabbajack.git
synced 2024-08-30 18:42:17 +00:00
Metrics and status are now in MongoDB
This commit is contained in:
parent
f9cdbbc6a1
commit
5a0e19f4b1
16
Wabbajack.CacheServer/DTOs/Metric.cs
Normal file
16
Wabbajack.CacheServer/DTOs/Metric.cs
Normal file
@ -0,0 +1,16 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using CouchDB.Driver.Types;
|
||||
|
||||
namespace Wabbajack.CacheServer.DTOs
|
||||
{
|
||||
public class Metric
|
||||
{
|
||||
public DateTime Timestamp;
|
||||
public string Action;
|
||||
public string Subject;
|
||||
}
|
||||
}
|
76
Wabbajack.CacheServer/DTOs/ModListStatus.cs
Normal file
76
Wabbajack.CacheServer/DTOs/ModListStatus.cs
Normal file
@ -0,0 +1,76 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using System.Windows.Media.Animation;
|
||||
using CouchDB.Driver.Extensions;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using MongoDB.Driver;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.ModListRegistry;
|
||||
|
||||
namespace Wabbajack.CacheServer.DTOs
|
||||
{
|
||||
public class ModListStatus
|
||||
{
|
||||
static ModListStatus()
|
||||
{
|
||||
SerializerSettings.Init();
|
||||
}
|
||||
|
||||
[BsonId]
|
||||
public string Id { get; set; }
|
||||
public ModlistSummary Summary { get; set; }
|
||||
|
||||
public ModlistMetadata Metadata { get; set; }
|
||||
public DetailedStatus DetailedStatus { get; set; }
|
||||
|
||||
public static async Task Update(ModListStatus status)
|
||||
{
|
||||
var id = status.Metadata.Links.MachineURL;
|
||||
await Server.Config.ListValidation.Connect().FindOneAndReplaceAsync<ModListStatus>(s => s.Id == id, status, new FindOneAndReplaceOptions<ModListStatus> {IsUpsert = true});
|
||||
}
|
||||
|
||||
public static IQueryable<ModListStatus> AllSummaries
|
||||
{
|
||||
get
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task<ModListStatus> ByName(string name)
|
||||
{
|
||||
var result = await Server.Config.ListValidation.Connect()
|
||||
.AsQueryable()
|
||||
.Where(doc => doc.Metadata.Links.MachineURL == name || doc.Metadata.Title == name)
|
||||
.ToListAsync();
|
||||
return result.First();
|
||||
}
|
||||
|
||||
public static IQueryable<ModListStatus> All
|
||||
{
|
||||
get
|
||||
{
|
||||
return Server.Config.ListValidation.Connect().AsQueryable();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class DetailedStatus
|
||||
{
|
||||
public string Name;
|
||||
public DateTime Checked = DateTime.Now;
|
||||
public List<DetailedStatusItem> Archives { get; set; }
|
||||
public DownloadMetadata DownloadMetaData { get; set; }
|
||||
public bool HasFailures { get; set; }
|
||||
public string MachineName { get; set; }
|
||||
}
|
||||
|
||||
public class DetailedStatusItem
|
||||
{
|
||||
public bool IsFailing { get; set; }
|
||||
public Archive Archive { get; set; }
|
||||
}
|
||||
}
|
14
Wabbajack.CacheServer/DTOs/MongoDoc.cs
Normal file
14
Wabbajack.CacheServer/DTOs/MongoDoc.cs
Normal file
@ -0,0 +1,14 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace Wabbajack.CacheServer.DTOs
|
||||
{
|
||||
public class MongoDoc
|
||||
{
|
||||
public ObjectId _id { get; set; } = ObjectId.Empty;
|
||||
}
|
||||
}
|
66
Wabbajack.CacheServer/DTOs/SerializerSettings.cs
Normal file
66
Wabbajack.CacheServer/DTOs/SerializerSettings.cs
Normal file
@ -0,0 +1,66 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.IO;
|
||||
using MongoDB.Bson.Serialization;
|
||||
using MongoDB.Bson.Serialization.Conventions;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
|
||||
namespace Wabbajack.CacheServer.DTOs
|
||||
{
|
||||
public static class SerializerSettings
|
||||
{
|
||||
public static void Init()
|
||||
{
|
||||
var dis = new TypeDiscriminator(typeof(AbstractDownloadState), AbstractDownloadState.NameToType,
|
||||
AbstractDownloadState.TypeToName);
|
||||
BsonSerializer.RegisterDiscriminatorConvention(typeof(AbstractDownloadState), dis);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public class TypeDiscriminator : IDiscriminatorConvention
|
||||
{
|
||||
private readonly Type defaultType;
|
||||
private readonly Dictionary<string, Type> typeMap;
|
||||
private Dictionary<Type, string> revMap;
|
||||
|
||||
public TypeDiscriminator(Type defaultType,
|
||||
Dictionary<string, Type> typeMap, Dictionary<Type, string> revMap)
|
||||
{
|
||||
this.defaultType = defaultType;
|
||||
this.typeMap = typeMap;
|
||||
this.revMap = revMap;
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Element Name
|
||||
/// </summary>
|
||||
public string ElementName => "_wjType";
|
||||
|
||||
public Type GetActualType(IBsonReader bsonReader, Type nominalType)
|
||||
{
|
||||
Type type = defaultType;
|
||||
var bookmark = bsonReader.GetBookmark();
|
||||
bsonReader.ReadStartDocument();
|
||||
if (bsonReader.FindElement(ElementName))
|
||||
{
|
||||
var value = bsonReader.ReadString();
|
||||
if (typeMap.ContainsKey(value))
|
||||
type = typeMap[value];
|
||||
}
|
||||
|
||||
bsonReader.ReturnToBookmark(bookmark);
|
||||
return type;
|
||||
}
|
||||
|
||||
public BsonValue GetDiscriminator(Type nominalType, Type actualType)
|
||||
{
|
||||
return revMap[actualType];
|
||||
}
|
||||
}
|
||||
}
|
@ -8,8 +8,10 @@ using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Alphaleonis.Win32.Filesystem;
|
||||
using CouchDB.Driver.Extensions;
|
||||
using Nancy;
|
||||
using Nancy.Responses;
|
||||
using Wabbajack.CacheServer.DTOs;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib;
|
||||
using Wabbajack.Lib.Downloaders;
|
||||
@ -19,18 +21,6 @@ namespace Wabbajack.CacheServer
|
||||
{
|
||||
public class ListValidationService : NancyModule
|
||||
{
|
||||
public class ModListStatus
|
||||
{
|
||||
public string Name;
|
||||
public DateTime Checked = DateTime.Now;
|
||||
public List<(Archive archive, bool)> Archives { get; set; }
|
||||
public DownloadMetadata DownloadMetaData { get; set; }
|
||||
public bool HasFailures { get; set; }
|
||||
public string MachineName { get; set; }
|
||||
}
|
||||
|
||||
public static Dictionary<string, ModListStatus> ModLists { get; set; }
|
||||
|
||||
public ListValidationService() : base("/lists")
|
||||
{
|
||||
Get("/status", HandleGetLists);
|
||||
@ -38,15 +28,9 @@ namespace Wabbajack.CacheServer
|
||||
Get("/status/{Name}.html", HandleGetListHtml);
|
||||
}
|
||||
|
||||
private object HandleGetLists(object arg)
|
||||
private async Task<string> HandleGetLists(object arg)
|
||||
{
|
||||
var summaries = ModLists.Values.Select(m => new ModlistSummary
|
||||
{
|
||||
Name = m.Name,
|
||||
Checked = m.Checked,
|
||||
Failed = m.Archives.Count(a => a.Item2),
|
||||
Passed = m.Archives.Count(a => !a.Item2),
|
||||
}).ToList();
|
||||
var summaries = await ModListStatus.All.Select(m => m.Summary).ToListAsync();
|
||||
return summaries.ToJSON();
|
||||
}
|
||||
|
||||
@ -63,44 +47,36 @@ namespace Wabbajack.CacheServer
|
||||
public List<ArchiveSummary> Passed;
|
||||
|
||||
}
|
||||
private object HandleGetListJson(dynamic arg)
|
||||
private async Task<string> HandleGetListJson(dynamic arg)
|
||||
{
|
||||
var lst = ModLists[(string)arg.Name];
|
||||
var summary = new DetailedSummary
|
||||
{
|
||||
Name = lst.Name,
|
||||
Checked = lst.Checked,
|
||||
Failed = lst.Archives.Where(a => a.Item2)
|
||||
.Select(a => new ArchiveSummary {Name = a.archive.Name, State = a.archive.State}).ToList(),
|
||||
Passed = lst.Archives.Where(a => !a.Item2)
|
||||
.Select(a => new ArchiveSummary { Name = a.archive.Name, State = a.archive.State }).ToList(),
|
||||
};
|
||||
return summary.ToJSON();
|
||||
var metric = Metrics.Log("list_validation.get_list_json", (string)arg.Name);
|
||||
var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
|
||||
return lst.ToJSON();
|
||||
}
|
||||
|
||||
private object HandleGetListHtml(dynamic arg)
|
||||
private async Task<Response> HandleGetListHtml(dynamic arg)
|
||||
{
|
||||
var lst = ModLists[(string)arg.Name];
|
||||
var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
|
||||
var sb = new StringBuilder();
|
||||
|
||||
sb.Append("<html><body>");
|
||||
sb.Append($"<h2>{lst.Name} - {lst.Checked}</h2>");
|
||||
|
||||
var failed_list = lst.Archives.Where(a => a.Item2).ToList();
|
||||
var failed_list = lst.Archives.Where(a => a.IsFailing).ToList();
|
||||
sb.Append($"<h3>Failed ({failed_list.Count}):</h3>");
|
||||
sb.Append("<ul>");
|
||||
foreach (var archive in failed_list)
|
||||
{
|
||||
sb.Append($"<li>{archive.archive.Name}</li>");
|
||||
sb.Append($"<li>{archive.Archive.Name}</li>");
|
||||
}
|
||||
sb.Append("</ul>");
|
||||
|
||||
var pased_list = lst.Archives.Where(a => !a.Item2).ToList();
|
||||
var pased_list = lst.Archives.Where(a => !a.IsFailing).ToList();
|
||||
sb.Append($"<h3>Passed ({pased_list.Count}):</h3>");
|
||||
sb.Append("<ul>");
|
||||
foreach (var archive in pased_list.OrderBy(f => f.archive.Name))
|
||||
foreach (var archive in pased_list.OrderBy(f => f.Archive.Name))
|
||||
{
|
||||
sb.Append($"<li>{archive.archive.Name}</li>");
|
||||
sb.Append($"<li>{archive.Archive.Name}</li>");
|
||||
}
|
||||
sb.Append("</ul>");
|
||||
|
||||
@ -134,75 +110,96 @@ namespace Wabbajack.CacheServer
|
||||
{
|
||||
Utils.Log("Cleaning Nexus Cache");
|
||||
var client = new HttpClient();
|
||||
await client.GetAsync("http://build.wabbajack.org/nexus_api_cache/update");
|
||||
//await client.GetAsync("http://build.wabbajack.org/nexus_api_cache/update");
|
||||
|
||||
Utils.Log("Starting Modlist Validation");
|
||||
var modlists = await ModlistMetadata.LoadFromGithub();
|
||||
|
||||
var statuses = new Dictionary<string, ModListStatus>();
|
||||
|
||||
using (var queue = new WorkQueue())
|
||||
{
|
||||
foreach (var list in modlists)
|
||||
{
|
||||
var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ExtensionManager.Extension);
|
||||
|
||||
if (list.NeedsDownload(modlist_path))
|
||||
try
|
||||
{
|
||||
if (File.Exists(modlist_path))
|
||||
File.Delete(modlist_path);
|
||||
|
||||
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
|
||||
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
|
||||
await state.Download(modlist_path);
|
||||
await ValidateList(list, queue);
|
||||
}
|
||||
else
|
||||
catch (Exception ex)
|
||||
{
|
||||
Utils.Log($"No changes detected from downloaded modlist");
|
||||
}
|
||||
|
||||
|
||||
Utils.Log($"Loading {modlist_path}");
|
||||
|
||||
var installer = AInstaller.LoadFromFile(modlist_path);
|
||||
|
||||
Utils.Log($"{installer.Archives.Count} archives to validate");
|
||||
|
||||
DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
|
||||
|
||||
var validated = (await installer.Archives
|
||||
.PMap(queue, async archive =>
|
||||
{
|
||||
Utils.Log($"Validating: {archive.Name}");
|
||||
bool is_failed;
|
||||
try
|
||||
{
|
||||
is_failed = !(await archive.State.Verify());
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
is_failed = false;
|
||||
}
|
||||
|
||||
return (archive, is_failed);
|
||||
}))
|
||||
.ToList();
|
||||
|
||||
|
||||
var status = new ModListStatus
|
||||
{
|
||||
Name = list.Title,
|
||||
Archives = validated.OrderBy(v => v.archive.Name).ToList(),
|
||||
DownloadMetaData = list.DownloadMetadata,
|
||||
HasFailures = validated.Any(v => v.is_failed)
|
||||
};
|
||||
|
||||
statuses.Add(status.Name, status);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Utils.Log($"Done validating {statuses.Count} lists");
|
||||
ModLists = statuses;
|
||||
Utils.Log($"Done validating {modlists.Count} lists");
|
||||
}
|
||||
|
||||
private static async Task ValidateList(ModlistMetadata list, WorkQueue queue)
|
||||
{
|
||||
var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ExtensionManager.Extension);
|
||||
|
||||
if (list.NeedsDownload(modlist_path))
|
||||
{
|
||||
if (File.Exists(modlist_path))
|
||||
File.Delete(modlist_path);
|
||||
|
||||
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
|
||||
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
|
||||
await state.Download(modlist_path);
|
||||
}
|
||||
else
|
||||
{
|
||||
Utils.Log($"No changes detected from downloaded modlist");
|
||||
}
|
||||
|
||||
|
||||
Utils.Log($"Loading {modlist_path}");
|
||||
|
||||
var installer = AInstaller.LoadFromFile(modlist_path);
|
||||
|
||||
Utils.Log($"{installer.Archives.Count} archives to validate");
|
||||
|
||||
DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
|
||||
|
||||
var validated = (await installer.Archives
|
||||
.PMap(queue, async archive =>
|
||||
{
|
||||
Utils.Log($"Validating: {archive.Name}");
|
||||
bool is_failed;
|
||||
try
|
||||
{
|
||||
is_failed = !(await archive.State.Verify());
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
is_failed = false;
|
||||
}
|
||||
|
||||
return new DetailedStatusItem {IsFailing = is_failed, Archive = archive};
|
||||
}))
|
||||
.ToList();
|
||||
|
||||
|
||||
var status = new DetailedStatus
|
||||
{
|
||||
Name = list.Title,
|
||||
Archives = validated.OrderBy(v => v.Archive.Name).ToList(),
|
||||
DownloadMetaData = list.DownloadMetadata,
|
||||
HasFailures = validated.Any(v => v.IsFailing)
|
||||
};
|
||||
|
||||
var dto = new ModListStatus
|
||||
{
|
||||
Id = list.Links.MachineURL,
|
||||
Summary = new ModlistSummary
|
||||
{
|
||||
Name = status.Name,
|
||||
Checked = status.Checked,
|
||||
Failed = status.Archives.Count(a => a.IsFailing),
|
||||
Passed = status.Archives.Count(a => !a.IsFailing),
|
||||
},
|
||||
DetailedStatus = status,
|
||||
Metadata = list
|
||||
};
|
||||
await ModListStatus.Update(dto);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -6,8 +6,10 @@ using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Alphaleonis.Win32.Filesystem;
|
||||
using CouchDB.Driver.Extensions;
|
||||
using MongoDB.Driver;
|
||||
using Nancy;
|
||||
using ReactiveUI;
|
||||
using Wabbajack.CacheServer.DTOs;
|
||||
using Wabbajack.Common;
|
||||
|
||||
namespace Wabbajack.CacheServer
|
||||
@ -19,19 +21,17 @@ namespace Wabbajack.CacheServer
|
||||
{
|
||||
private static SemaphoreSlim _lockObject = new SemaphoreSlim(1);
|
||||
|
||||
public static async Task Log(params object[] args)
|
||||
public static async Task Log(DateTime timestamp, string action, string subject)
|
||||
{
|
||||
var msg = new[] {string.Join("\t", args.Select(a => a.ToString()))};
|
||||
var msg = new[] {string.Join("\t", new[]{timestamp.ToString(), action, subject})};
|
||||
Utils.Log(msg.First());
|
||||
await _lockObject.WaitAsync();
|
||||
try
|
||||
{
|
||||
File.AppendAllLines("stats.tsv", msg);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lockObject.Release();
|
||||
}
|
||||
var db = Server.Config.Metrics.Connect();
|
||||
await db.InsertOneAsync(new Metric {Timestamp = timestamp, Action = action, Subject = subject});
|
||||
}
|
||||
|
||||
public static Task Log(string action, string subject)
|
||||
{
|
||||
return Log(DateTime.Now, action, subject);
|
||||
}
|
||||
|
||||
public Metrics() : base("/")
|
||||
@ -40,6 +40,26 @@ namespace Wabbajack.CacheServer
|
||||
Get("/metrics/chart/", HandleChart);
|
||||
Get("/metrics/chart/{Action}/", HandleChart);
|
||||
Get("/metrics/chart/{Action}/{Value}/", HandleChart);
|
||||
Get("/metrics/ingest/{filename}", HandleBulkIngest);
|
||||
}
|
||||
|
||||
private async Task<string> HandleBulkIngest(dynamic arg)
|
||||
{
|
||||
Log("Bulk Loading " + arg.filename.ToString());
|
||||
|
||||
var lines = File.ReadAllLines(Path.Combine(@"c:\tmp", (string)arg.filename));
|
||||
|
||||
var db = Server.Config.Metrics.Connect();
|
||||
|
||||
var data = lines.Select(line => line.Split('\t'))
|
||||
.Where(line => line.Length == 3)
|
||||
.Select(line => new Metric{ Timestamp = DateTime.Parse(line[0]), Action = line[1], Subject = line[2] })
|
||||
.ToList();
|
||||
|
||||
foreach (var metric in data)
|
||||
await db.InsertOneAsync(metric);
|
||||
|
||||
return $"Processed {lines.Length} records";
|
||||
}
|
||||
|
||||
private async Task<string> HandleMetrics(dynamic arg)
|
||||
@ -49,36 +69,33 @@ namespace Wabbajack.CacheServer
|
||||
return date.ToString();
|
||||
}
|
||||
|
||||
private static async Task<string[]> GetData()
|
||||
{
|
||||
await _lockObject.WaitAsync();
|
||||
try
|
||||
{
|
||||
return File.ReadAllLines("stats.tsv");
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lockObject.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<Response> HandleChart(dynamic arg)
|
||||
{
|
||||
var data = (await GetData()).Select(line => line.Split('\t'))
|
||||
/*var data = (await GetData()).Select(line => line.Split('\t'))
|
||||
.Where(line => line.Length == 3)
|
||||
.Select(line => new {date = DateTime.Parse(line[0]), Action = line[1], Value = line[2]});
|
||||
.Select(line => new {date = DateTime.Parse(line[0]), Action = line[1], Value = line[2]});*/
|
||||
|
||||
var q = (IQueryable<Metric>)Server.Config.Metrics.Connect().AsQueryable();
|
||||
|
||||
// Remove guids / Default, which come from testing
|
||||
data = data.Where(d => !Guid.TryParse(d.Value ?? "", out _) && (d.Value ?? "") != "Default");
|
||||
|
||||
if (arg?.Action != null)
|
||||
data = data.Where(d => d.Action == arg.Action);
|
||||
{
|
||||
var action = (string)arg.Action;
|
||||
q = q.Where(d => d.Action == action);
|
||||
}
|
||||
|
||||
|
||||
if (arg?.Value != null)
|
||||
data = data.Where(d => d.Value.StartsWith(arg.Value));
|
||||
{
|
||||
var value = (string)arg.Value;
|
||||
q = q.Where(d => d.Subject.StartsWith(value));
|
||||
}
|
||||
|
||||
var grouped_and_counted = data.GroupBy(d => d.date.ToString("yyyy-MM-dd"))
|
||||
var data = (await q.Take(Int32.MaxValue).ToListAsync()).AsEnumerable();
|
||||
data = data.Where(d => !Guid.TryParse(d.Subject ?? "", out Guid v) && (d.Subject ?? "") != "Default");
|
||||
|
||||
var grouped_and_counted = data.GroupBy(d => d.Timestamp.ToString("yyyy-MM-dd"))
|
||||
.OrderBy(d => d.Key)
|
||||
.Select(d => new {Day = d.Key, Count = d.Count()})
|
||||
.ToList();
|
||||
@ -116,5 +133,10 @@ namespace Wabbajack.CacheServer
|
||||
response.ContentType = "text/html";
|
||||
return response;
|
||||
}
|
||||
|
||||
public void Log(string l)
|
||||
{
|
||||
Utils.Log("Metrics: " + l);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -6,12 +6,15 @@ using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Alphaleonis.Win32.Filesystem;
|
||||
using Nancy;
|
||||
using Nancy.Bootstrapper;
|
||||
using Nancy.Configuration;
|
||||
using Nancy.Diagnostics;
|
||||
using Nancy.Hosting.Self;
|
||||
using Nancy.TinyIoc;
|
||||
using Wabbajack.CacheServer.ServerConfig;
|
||||
using Wabbajack.Common;
|
||||
|
||||
namespace Wabbajack.CacheServer
|
||||
{
|
||||
@ -19,6 +22,7 @@ namespace Wabbajack.CacheServer
|
||||
{
|
||||
private NancyHost _server;
|
||||
private HostConfiguration _config;
|
||||
public static BuildServerConfig Config;
|
||||
|
||||
public Server(string address)
|
||||
{
|
||||
@ -26,8 +30,8 @@ namespace Wabbajack.CacheServer
|
||||
_config = new HostConfiguration {MaximumConnectionCount = 24, RewriteLocalhost = true};
|
||||
//_config.UrlReservations.CreateAutomatically = true;
|
||||
_server = new NancyHost(_config, new Uri(address));
|
||||
|
||||
|
||||
|
||||
Config = File.ReadAllText("config.yaml").FromYaml<BuildServerConfig>();
|
||||
}
|
||||
|
||||
public string Address { get; }
|
||||
|
15
Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs
Normal file
15
Wabbajack.CacheServer/ServerConfig/BuildServerConfig.cs
Normal file
@ -0,0 +1,15 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Wabbajack.CacheServer.DTOs;
|
||||
|
||||
namespace Wabbajack.CacheServer.ServerConfig
|
||||
{
|
||||
public class BuildServerConfig
|
||||
{
|
||||
public MongoConfig<Metric> Metrics { get; set; }
|
||||
public MongoConfig<ModListStatus> ListValidation { get; set; }
|
||||
}
|
||||
}
|
34
Wabbajack.CacheServer/ServerConfig/MongoConfig.cs
Normal file
34
Wabbajack.CacheServer/ServerConfig/MongoConfig.cs
Normal file
@ -0,0 +1,34 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
using Wabbajack.CacheServer.DTOs;
|
||||
|
||||
namespace Wabbajack.CacheServer.ServerConfig
|
||||
{
|
||||
public class MongoConfig<T>
|
||||
{
|
||||
public string Host { get; set; }
|
||||
public string Database { get; set; }
|
||||
public string Collection { get; set; }
|
||||
public string Username { get; set; }
|
||||
public string Password { get; set; }
|
||||
|
||||
private IMongoDatabase Client
|
||||
{
|
||||
get
|
||||
{
|
||||
if (Username != null && Password != null)
|
||||
return new MongoClient($"mongodb://{Username}:{Password}@{Host}").GetDatabase(Database);
|
||||
return new MongoClient($"mongodb://{Host}").GetDatabase(Database);
|
||||
}
|
||||
}
|
||||
|
||||
public IMongoCollection<T> Connect()
|
||||
{
|
||||
return Client.GetCollection<T>(Collection);
|
||||
}
|
||||
}
|
||||
}
|
@ -73,6 +73,10 @@
|
||||
<Reference Include="WindowsBase" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="DTOs\Metric.cs" />
|
||||
<Compile Include="DTOs\ModListStatus.cs" />
|
||||
<Compile Include="DTOs\MongoDoc.cs" />
|
||||
<Compile Include="DTOs\SerializerSettings.cs" />
|
||||
<Compile Include="ListValidationService.cs" />
|
||||
<Compile Include="Metrics.cs" />
|
||||
<Compile Include="NexusCacheModule.cs" />
|
||||
@ -80,10 +84,15 @@
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
<Compile Include="Server.cs" />
|
||||
<Compile Include="Heartbeat.cs" />
|
||||
<Compile Include="ServerConfig\BuildServerConfig.cs" />
|
||||
<Compile Include="ServerConfig\MongoConfig.cs" />
|
||||
<Compile Include="TestingEndpoints.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="App.config" />
|
||||
<None Include="config.yaml">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj">
|
||||
@ -96,6 +105,12 @@
|
||||
</ProjectReference>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="CouchDB.NET">
|
||||
<Version>1.1.5</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="MongoDB.Driver">
|
||||
<Version>2.10.0</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="Nancy.Hosting.Self">
|
||||
<Version>2.0.0</Version>
|
||||
</PackageReference>
|
||||
@ -109,5 +124,6 @@
|
||||
<Version>4.3.2</Version>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
<ItemGroup />
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
</Project>
|
12
Wabbajack.CacheServer/config.yaml
Normal file
12
Wabbajack.CacheServer/config.yaml
Normal file
@ -0,0 +1,12 @@
|
||||
---
|
||||
Metrics:
|
||||
Host: internal.test.mongodb
|
||||
Database: wabbajack
|
||||
Collection: metrics
|
||||
ListValidation:
|
||||
Host: internal.test.mongodb
|
||||
Database: wabbajack
|
||||
Collection: mod_lists
|
||||
|
||||
|
||||
|
@ -95,7 +95,7 @@ namespace Wabbajack.Lib
|
||||
ModList.Readme = $"readme{readme.Extension}";
|
||||
}
|
||||
|
||||
ModList.ReadmeIsWebsite = ReadmeIsWebsite;
|
||||
//ModList.ReadmeIsWebsite = ReadmeIsWebsite;
|
||||
|
||||
ModList.ToCERAS(Path.Combine(ModListOutputFolder, "modlist"), CerasConfig.Config);
|
||||
|
||||
|
@ -32,7 +32,7 @@ namespace Wabbajack.Lib
|
||||
|
||||
},
|
||||
};
|
||||
Config.VersionTolerance.Mode = VersionToleranceMode.Standard;
|
||||
//Config.VersionTolerance.Mode = VersionToleranceMode.Standard;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -119,7 +119,7 @@ namespace Wabbajack.Lib
|
||||
/// <summary>
|
||||
/// Whether readme is a website
|
||||
/// </summary>
|
||||
public bool ReadmeIsWebsite;
|
||||
//public bool ReadmeIsWebsite;
|
||||
}
|
||||
|
||||
public class Directive
|
||||
|
@ -1,5 +1,9 @@
|
||||
using System.Threading.Tasks;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Alphaleonis.Win32.Filesystem;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using Wabbajack.Lib.Validation;
|
||||
|
||||
namespace Wabbajack.Lib.Downloaders
|
||||
@ -7,8 +11,37 @@ namespace Wabbajack.Lib.Downloaders
|
||||
/// <summary>
|
||||
/// Base for all abstract downloaders
|
||||
/// </summary>
|
||||
[BsonDiscriminator(RootClass = true)]
|
||||
[BsonKnownTypes(typeof(HTTPDownloader.State), typeof(GameFileSourceDownloader.State), typeof(GoogleDriveDownloader.State),
|
||||
typeof(LoversLabDownloader.State), typeof(ManualDownloader.State), typeof(MediaFireDownloader.State), typeof(MegaDownloader.State),
|
||||
typeof(ModDBDownloader.State), typeof(NexusDownloader.State), typeof(SteamWorkshopDownloader.State))]
|
||||
public abstract class AbstractDownloadState
|
||||
{
|
||||
|
||||
public static List<Type> KnownSubTypes = new List<Type>()
|
||||
{
|
||||
typeof(HTTPDownloader.State),
|
||||
typeof(GameFileSourceDownloader.State),
|
||||
typeof(GoogleDriveDownloader.State),
|
||||
typeof(LoversLabDownloader.State),
|
||||
typeof(ManualDownloader.State),
|
||||
typeof(MediaFireDownloader.State),
|
||||
typeof(MegaDownloader.State),
|
||||
typeof(ModDBDownloader.State),
|
||||
typeof(NexusDownloader.State),
|
||||
typeof(SteamWorkshopDownloader.State)
|
||||
};
|
||||
public static Dictionary<string, Type> NameToType { get; set; }
|
||||
public static Dictionary<Type, string> TypeToName { get; set; }
|
||||
|
||||
static AbstractDownloadState()
|
||||
{
|
||||
NameToType = KnownSubTypes.ToDictionary(t => t.FullName.Substring(t.Namespace.Length + 1), t => t);
|
||||
TypeToName = NameToType.ToDictionary(k => k.Value, k => k.Key);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if this file is allowed to be downloaded via whitelist
|
||||
/// </summary>
|
||||
|
@ -320,7 +320,7 @@ namespace Wabbajack.Lib
|
||||
|
||||
Utils.Log(
|
||||
$"Removing {remove.Count} archives from the compilation state, this is probably not an issue but reference this if you have compilation failures");
|
||||
remove.Do(r => Utils.Log($"Resolution failed for: {r.File}"));
|
||||
remove.Do(r => Utils.Log($"Resolution failed for: {r.File.FullPath}"));
|
||||
IndexedArchives.RemoveAll(a => remove.Contains(a));
|
||||
}
|
||||
|
||||
|
@ -57,6 +57,9 @@
|
||||
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="MongoDB.Bson">
|
||||
<HintPath>..\..\..\Users\tbald\.nuget\packages\mongodb.bson\2.10.0\lib\net452\MongoDB.Bson.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="PresentationCore" />
|
||||
<Reference Include="PresentationFramework" />
|
||||
<Reference Include="System" />
|
||||
@ -206,6 +209,9 @@
|
||||
<PackageReference Include="Microsoft.Toolkit.Wpf.UI.Controls.WebView">
|
||||
<Version>6.0.0</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="MongoDB.Bson">
|
||||
<Version>2.10.0</Version>
|
||||
</PackageReference>
|
||||
<PackageReference Include="Newtonsoft.Json">
|
||||
<Version>12.0.3</Version>
|
||||
</PackageReference>
|
||||
|
@ -87,7 +87,7 @@ namespace Wabbajack
|
||||
public void OpenReadmeWindow()
|
||||
{
|
||||
if (string.IsNullOrEmpty(Readme)) return;
|
||||
if (SourceModList.ReadmeIsWebsite)
|
||||
if (false) //SourceModList.ReadmeIsWebsite)
|
||||
{
|
||||
Process.Start(Readme);
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user