Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1193 from wabbajack-tools/improved-services

Browse Source 
Improve backend services, add watchdog service, and add utility_modli…
This commit is contained in:
Timothy Baldridge 2020-11-19 16:32:52 -07:00 committed by GitHub
commit 3a3206755a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 172 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Wabbajack.Common/Consts.cs
View File

@ -84,6 +84,7 @@ namespace Wabbajack.Common
public static string ServerWhitelistURL = "https://raw.githubusercontent.com/wabbajack-tools/opt-out-lists/master/ServerWhitelist.yml";
public static string ModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/modlists.json";
public static string UtilityModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/utility_modlists.json";
public static string UnlistedModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/unlisted_modlists.json";
public static string ModlistSummaryURL = "http://build.wabbajack.org/lists/status.json";
public static string UserAgent

5
Wabbajack.Lib/ModListRegistry/ModListMetadata.cs
View File

@ -38,6 +38,9 @@ namespace Wabbajack.Lib.ModListRegistry
[JsonProperty("utility_list")]
public bool UtilityList { get; set; }
[JsonProperty("force_down")]
public bool ForceDown { get; set; }
[JsonProperty("links")]
public LinksObject Links { get; set; } = new LinksObject();
@ -68,9 +71,11 @@ namespace Wabbajack.Lib.ModListRegistry
var client = new Http.Client();
Utils.Log("Loading ModLists from GitHub");
var metadataResult = client.GetStringAsync(Consts.ModlistMetadataURL);
var utilityResult = client.GetStringAsync(Consts.UtilityModlistMetadataURL);
var summaryResult = client.GetStringAsync(Consts.ModlistSummaryURL);
var metadata = (await metadataResult).FromJsonString<List<ModlistMetadata>>();
metadata = metadata.Concat((await utilityResult).FromJsonString<List<ModlistMetadata>>()).ToList();
try
{
var summaries = (await summaryResult).FromJsonString<List<ModListSummary>>().ToDictionary(d => d.MachineURL);

3
Wabbajack.Server.Test/ModListValidationTests.cs
View File

@ -189,6 +189,9 @@ namespace Wabbajack.BuildServer.Test
var statusRss = await _client.GetHtmlAsync(MakeURL("lists/status/test_list/broken.rss"));
Assert.Equal(failed, statusRss.DocumentNode.SelectNodes("//item")?.Count ?? 0);
var heartBeat = await _client.GetHtmlAsync(MakeURL("heartbeat/report"));
Assert.Contains(heartBeat.DocumentNode.Descendants(), c => c.InnerText.StartsWith("test_list"));
}

59
Wabbajack.Server/Controllers/Heartbeat.cs
View File

@ -1,12 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Nettle;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Server;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
{
@ -19,12 +23,18 @@ namespace Wabbajack.BuildServer.Controllers
}
private static DateTime _startTime;
private QuickSync _quickSync;
private ListValidator _listValidator;
public Heartbeat(ILogger<Heartbeat> logger, SqlService sql, GlobalInformation globalInformation)
public Heartbeat(ILogger<Heartbeat> logger, SqlService sql, GlobalInformation globalInformation, QuickSync quickSync, ListValidator listValidator)
{
_globalInformation = globalInformation;
_sql = sql;
_logger = logger;
_quickSync = quickSync;
_listValidator = listValidator;
}
private const int MAX_LOG_SIZE = 128;
@ -52,6 +62,53 @@ namespace Wabbajack.BuildServer.Controllers
LastNexusUpdate = _globalInformation.TimeSinceLastNexusSync,
});
}
private static readonly Func<object, string> HandleGetReport = NettleEngine.GetCompiler().Compile(@"
<html><body>
<h2>Server Status</h2>
<h3>Service Overview ({{services.Length}}):</h3>
<ul>
{{each $.services }}
{{if $.IsLate}}
<li><b>{{$.Name}} - {{$.Time}} - {{$.MaxTime}}</b></li>
{{else}}
<li>{{$.Name}} - {{$.Time}} - {{$.MaxTime}}</li>
{{/if}}
{{/each}}
</ul>
<h3>Lists ({{lists.Length}}):</h3>
<ul>
{{each $.lists }}
<li><a href='/lists/status/{{$.Name}}.html'>{{$.Name}}</a> - {{$.Time}}</li>
{{/each}}
</ul>
</body></html>
");
[HttpGet("report")]
public async Task<ContentResult> Report()
{
var response = HandleGetReport(new
{
services = (await _quickSync.Report())
.Select(s => new {Name = s.Key, Time = s.Value.LastRunTime, MaxTime = s.Value.Delay, IsLate = s.Value.LastRunTime > s.Value.Delay})
.OrderBy(s => s.Name)
.ToArray(),
lists = _listValidator.ValidationInfo.Select(s => new {Name = s.Key, Time = s.Value.ValidationTime})
.OrderBy(l => l.Name)
.ToArray()
});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}

19
Wabbajack.Server/Services/AbstractService.cs
View File

@ -10,14 +10,27 @@ namespace Wabbajack.Server.Services
{
public void Start();
}
public interface IReportingService
{
public TimeSpan Delay { get; }
public DateTime LastStart { get; }
public DateTime LastEnd { get; }
}
public abstract class AbstractService<TP, TR> : IStartable
public abstract class AbstractService<TP, TR> : IStartable, IReportingService
{
protected AppSettings _settings;
private TimeSpan _delay;
protected ILogger<TP> _logger;
protected QuickSync _quickSync;
public TimeSpan Delay => _delay;
public DateTime LastStart { get; private set; }
public DateTime LastEnd { get; private set; }
public AbstractService(ILogger<TP> logger, AppSettings settings, QuickSync quickSync, TimeSpan delay)
{
_settings = settings;
@ -40,7 +53,7 @@ namespace Wabbajack.Server.Services
Task.Run(async () =>
{
await Setup();
await _quickSync.Register(this);
while (true)
{
@ -48,7 +61,9 @@ namespace Wabbajack.Server.Services
try
{
_logger.LogInformation($"Running: {GetType().Name}");
LastStart = DateTime.UtcNow;
await Execute();
LastEnd = DateTime.UtcNow;
}
catch (Exception ex)
{

40
Wabbajack.Server/Services/ListValidator.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
@ -27,8 +28,9 @@ namespace Wabbajack.Server.Services
private NexusKeyMaintainance _nexus;
private ArchiveMaintainer _archives;
public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries { get; private set; } =
new (ModListSummary Summary, DetailedStatus Detailed)[0];
public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries => ValidationInfo.Values.Select(e => (e.Summary, e.Detailed));
public ConcurrentDictionary<string, (ModListSummary Summary, DetailedStatus Detailed, TimeSpan ValidationTime)> ValidationInfo = new ConcurrentDictionary<string, (ModListSummary Summary, DetailedStatus Detailed, TimeSpan ValidationTime)>();
public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives, QuickSync quickSync)
@ -50,14 +52,21 @@ namespace Wabbajack.Server.Services
var stopwatch = new Stopwatch();
stopwatch.Start();
var results = await data.ModLists.PMap(queue, async metadata =>
var results = await data.ModLists.Where(m => !m.ForceDown).PMap(queue, async metadata =>
{
var timer = new Stopwatch();
timer.Start();
var oldSummary =
oldSummaries.FirstOrDefault(s => s.Summary.MachineURL == metadata.Links.MachineURL);
var listArchives = await _sql.ModListArchives(metadata.Links.MachineURL);
var archives = await listArchives.PMap(queue, async archive =>
{
if (timer.Elapsed > Delay)
{
return (archive, ArchiveStatus.InValid);
}
try
{
var (_, result) = await ValidateArchive(data, archive);
@ -109,6 +118,24 @@ namespace Wabbajack.Server.Services
}).ToList()
};
if (timer.Elapsed > Delay)
{
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Embeds = new[]
{
new DiscordEmbed
{
Title =
$"Failing {summary.Name} (`{summary.MachineURL}`) because the max validation time expired",
Url = new Uri(
$"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html")
}
}
});
}
if (oldSummary != default && oldSummary.Summary.Failed != summary.Failed)
{
_logger.Log(LogLevel.Information, $"Number of failures {oldSummary.Summary.Failed} -> {summary.Failed}");
@ -151,9 +178,14 @@ namespace Wabbajack.Server.Services
}
timer.Stop();
ValidationInfo[summary.MachineURL] = (summary, detailed, timer.Elapsed);
return (summary, detailed);
});
Summaries = results;
stopwatch.Stop();
_logger.LogInformation($"Finished Validation in {stopwatch.Elapsed}");

16
Wabbajack.Server/Services/QuickSync.cs
View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@ -11,6 +12,7 @@ namespace Wabbajack.Server.Services
public class QuickSync
{
private Dictionary<Type, CancellationTokenSource> _syncs = new Dictionary<Type, CancellationTokenSource>();
private Dictionary<Type, IReportingService> _services = new Dictionary<Type, IReportingService>();
private AsyncLock _lock = new AsyncLock();
private ILogger<QuickSync> _logger;
@ -19,6 +21,20 @@ namespace Wabbajack.Server.Services
_logger = logger;
}
public async Task<Dictionary<Type, (TimeSpan Delay, TimeSpan LastRunTime)>> Report()
{
using var _ = await _lock.WaitAsync();
return _services.ToDictionary(s => s.Key,
s => (s.Value.Delay, DateTime.UtcNow - s.Value.LastEnd));
}
public async Task Register<T>(T service)
where T : IReportingService
{
using var _ = await _lock.WaitAsync();
_services[typeof(T)] = service;
}
public async Task<CancellationToken> GetToken<T>()
{
using var _ = await _lock.WaitAsync();

33
Wabbajack.Server/Services/Watchdog.cs Normal file
View File

@ -0,0 +1,33 @@
using System;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class Watchdog : AbstractService<Watchdog, int>
{
private DiscordWebHook _discord;
public Watchdog(ILogger<Watchdog> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discordWebHook) : base(logger, settings, quickSync, TimeSpan.FromMinutes(5))
{
_discord = discordWebHook;
}
public override async Task<int> Execute()
{
var report = await _quickSync.Report();
foreach (var service in report)
{
if (service.Value.LastRunTime >= service.Value.Delay * 2)
{
await _discord.Send(Channel.Spam,
new DiscordMessage {Content = $"Service {service.Key.Name} has missed it's scheduled execution window"});
}
}
return report.Count;
}
}
}

2
Wabbajack.Server/Startup.cs
View File

@ -71,6 +71,7 @@ namespace Wabbajack.Server
services.AddSingleton<NexusPermissionsUpdater>();
services.AddSingleton<MirrorUploader>();
services.AddSingleton<MirrorQueueService>();
services.AddSingleton<Watchdog>();
services.AddMvc();
services.AddControllers()
@ -129,6 +130,7 @@ namespace Wabbajack.Server
app.UseService<NexusPermissionsUpdater>();
app.UseService<MirrorUploader>();
app.UseService<MirrorQueueService>();
app.UseService<Watchdog>();
app.Use(next =>
{

2
Wabbajack/View Models/Gallery/ModListMetadataVM.cs
View File

@ -83,7 +83,7 @@ namespace Wabbajack
});
DownloadSizeText = "Download size : " + UIUtils.FormatBytes(Metadata.DownloadMetadata.SizeOfArchives);
InstallSizeText = "Installation size : " + UIUtils.FormatBytes(Metadata.DownloadMetadata.SizeOfInstalledFiles);
IsBroken = metadata.ValidationSummary.HasFailures;
IsBroken = metadata.ValidationSummary.HasFailures || metadata.ForceDown;
//https://www.wabbajack.org/#/modlists/info?machineURL=eldersouls
OpenWebsiteCommand = ReactiveCommand.Create(() => Utils.OpenWebsite(new Uri($"https://www.wabbajack.org/#/modlists/info?machineURL={Metadata.Links.MachineURL}")));
ExecuteCommand = ReactiveCommand.CreateFromObservable<Unit, Unit>(