2019-12-14 05:46:20 +00:00
|
|
|
|
using System;
|
|
|
|
|
|
using System.Collections.Generic;
|
|
|
|
|
|
using System.Linq;
|
2019-12-14 16:33:05 +00:00
|
|
|
|
using System.Net.Http;
|
2019-12-14 05:46:20 +00:00
|
|
|
|
using System.Threading.Tasks;
|
|
|
|
|
|
using Alphaleonis.Win32.Filesystem;
|
2019-12-30 04:35:54 +00:00
|
|
|
|
using MongoDB.Driver;
|
2019-12-14 05:46:20 +00:00
|
|
|
|
using Nancy;
|
2019-12-29 22:57:01 +00:00
|
|
|
|
using Wabbajack.CacheServer.DTOs;
|
2019-12-14 05:46:20 +00:00
|
|
|
|
using Wabbajack.Common;
|
|
|
|
|
|
using Wabbajack.Lib;
|
|
|
|
|
|
using Wabbajack.Lib.Downloaders;
|
|
|
|
|
|
using Wabbajack.Lib.ModListRegistry;
|
2019-12-30 04:35:54 +00:00
|
|
|
|
using MongoDB.Driver.Linq;
|
|
|
|
|
|
using Nettle;
|
2020-01-03 05:25:00 +00:00
|
|
|
|
using Nettle.Functions;
|
2019-12-14 05:46:20 +00:00
|
|
|
|
|
|
|
|
|
|
namespace Wabbajack.CacheServer
|
|
|
|
|
|
{
|
|
|
|
|
|
public class ListValidationService : NancyModule
|
|
|
|
|
|
{
|
|
|
|
|
|
public ListValidationService() : base("/lists")
|
|
|
|
|
|
{
|
|
|
|
|
|
Get("/status", HandleGetLists);
|
2020-01-02 19:21:54 +00:00
|
|
|
|
Get("/force_recheck", HandleForceRecheck);
|
2019-12-14 05:46:20 +00:00
|
|
|
|
Get("/status/{Name}.json", HandleGetListJson);
|
|
|
|
|
|
Get("/status/{Name}.html", HandleGetListHtml);
|
2020-01-03 05:25:00 +00:00
|
|
|
|
Get("/status/{Name}/broken.rss", HandleGetRSSFeed);
|
2019-12-30 04:35:54 +00:00
|
|
|
|
|
2019-12-14 05:46:20 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
2020-01-02 19:21:54 +00:00
|
|
|
|
private async Task<string> HandleForceRecheck(object arg)
|
|
|
|
|
|
{
|
|
|
|
|
|
await ValidateLists(false);
|
|
|
|
|
|
return "done";
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2019-12-29 22:57:01 +00:00
|
|
|
|
private async Task<string> HandleGetLists(object arg)
|
2019-12-14 05:46:20 +00:00
|
|
|
|
{
|
2019-12-29 22:57:01 +00:00
|
|
|
|
var summaries = await ModListStatus.All.Select(m => m.Summary).ToListAsync();
|
2019-12-14 05:46:20 +00:00
|
|
|
|
return summaries.ToJSON();
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public class ArchiveSummary
|
|
|
|
|
|
{
|
|
|
|
|
|
public string Name;
|
|
|
|
|
|
public AbstractDownloadState State;
|
|
|
|
|
|
}
|
|
|
|
|
|
public class DetailedSummary
|
|
|
|
|
|
{
|
|
|
|
|
|
public string Name;
|
|
|
|
|
|
public DateTime Checked;
|
|
|
|
|
|
public List<ArchiveSummary> Failed;
|
|
|
|
|
|
public List<ArchiveSummary> Passed;
|
|
|
|
|
|
|
|
|
|
|
|
}
|
2019-12-29 22:57:01 +00:00
|
|
|
|
private async Task<string> HandleGetListJson(dynamic arg)
|
2019-12-14 05:46:20 +00:00
|
|
|
|
{
|
2019-12-29 22:57:01 +00:00
|
|
|
|
var metric = Metrics.Log("list_validation.get_list_json", (string)arg.Name);
|
|
|
|
|
|
var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
|
|
|
|
|
|
return lst.ToJSON();
|
2019-12-14 05:46:20 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
2019-12-30 04:35:54 +00:00
|
|
|
|
|
|
|
|
|
|
private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
|
|
|
|
|
|
<html><body>
|
|
|
|
|
|
<h2>{{lst.Name}} - {{lst.Checked}}</h2>
|
|
|
|
|
|
<h3>Failed ({{failed.Count}}):</h3>
|
|
|
|
|
|
<ul>
|
|
|
|
|
|
{{each $.failed }}
|
|
|
|
|
|
<li>{{$.Archive.Name}}</li>
|
|
|
|
|
|
{{/each}}
|
|
|
|
|
|
</ul>
|
|
|
|
|
|
<h3>Passed ({{passed.Count}}):</h3>
|
|
|
|
|
|
<ul>
|
|
|
|
|
|
{{each $.passed }}
|
|
|
|
|
|
<li>{{$.Archive.Name}}</li>
|
|
|
|
|
|
{{/each}}
|
|
|
|
|
|
</ul>
|
|
|
|
|
|
</body></html>
|
|
|
|
|
|
");
|
|
|
|
|
|
|
2019-12-29 22:57:01 +00:00
|
|
|
|
private async Task<Response> HandleGetListHtml(dynamic arg)
|
2019-12-14 05:46:20 +00:00
|
|
|
|
{
|
2020-01-03 05:25:00 +00:00
|
|
|
|
|
2019-12-29 22:57:01 +00:00
|
|
|
|
var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
|
2019-12-30 04:35:54 +00:00
|
|
|
|
var response = (Response)HandleGetListTemplate(new
|
2019-12-14 05:46:20 +00:00
|
|
|
|
{
|
2019-12-30 04:35:54 +00:00
|
|
|
|
lst,
|
|
|
|
|
|
failed = lst.Archives.Where(a => a.IsFailing).ToList(),
|
|
|
|
|
|
passed = lst.Archives.Where(a => !a.IsFailing).ToList()
|
|
|
|
|
|
});
|
2019-12-14 05:46:20 +00:00
|
|
|
|
response.ContentType = "text/html";
|
|
|
|
|
|
return response;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2020-01-03 05:25:00 +00:00
|
|
|
|
private static readonly Func<object, string> HandleGetRSSFeedTemplate = NettleEngine.GetCompiler().Compile(@"
|
|
|
|
|
|
<?xml version=""1.0""?>
|
|
|
|
|
|
<rss version=""2.0"">
|
|
|
|
|
|
<channel>
|
|
|
|
|
|
<title>{{lst.Name}} - Broken Mods</title>
|
|
|
|
|
|
<link>http://build.wabbajack.org/status/{{lst.Name}}.html</link>
|
|
|
|
|
|
<description>These are mods that are broken and need updating</description>
|
|
|
|
|
|
{{ each $.failed }}
|
|
|
|
|
|
<item>
|
|
|
|
|
|
<title>{{$.Archive.Name}}</title>
|
|
|
|
|
|
<link>{{$.Archive.Name}}</link>
|
|
|
|
|
|
</item>
|
|
|
|
|
|
{{/each}}
|
|
|
|
|
|
</channel>
|
|
|
|
|
|
</rss>
|
|
|
|
|
|
");
|
|
|
|
|
|
|
|
|
|
|
|
public async Task<Response> HandleGetRSSFeed(dynamic arg)
|
|
|
|
|
|
{
|
|
|
|
|
|
var metric = Metrics.Log("failed_rss", arg.Name);
|
|
|
|
|
|
var lst = (await ModListStatus.ByName((string)arg.Name)).DetailedStatus;
|
|
|
|
|
|
var response = (Response)HandleGetRSSFeedTemplate(new
|
|
|
|
|
|
{
|
|
|
|
|
|
lst,
|
|
|
|
|
|
failed = lst.Archives.Where(a => a.IsFailing).ToList(),
|
|
|
|
|
|
passed = lst.Archives.Where(a => !a.IsFailing).ToList()
|
|
|
|
|
|
});
|
|
|
|
|
|
response.ContentType = "application/rss+xml";
|
|
|
|
|
|
await metric;
|
|
|
|
|
|
return response;
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2019-12-14 05:46:20 +00:00
|
|
|
|
public static void Start()
|
|
|
|
|
|
{
|
2019-12-22 00:26:51 +00:00
|
|
|
|
Task.Run(async () =>
|
2019-12-14 16:33:05 +00:00
|
|
|
|
{
|
|
|
|
|
|
while (true)
|
|
|
|
|
|
{
|
|
|
|
|
|
try
|
|
|
|
|
|
{
|
2019-12-22 00:26:51 +00:00
|
|
|
|
await ValidateLists();
|
2019-12-14 16:33:05 +00:00
|
|
|
|
}
|
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
|
{
|
|
|
|
|
|
Utils.Log(ex.ToString());
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Sleep for two hours
|
2019-12-22 00:26:51 +00:00
|
|
|
|
await Task.Delay(1000 * 60 * 60 * 2);
|
2019-12-14 16:33:05 +00:00
|
|
|
|
}
|
2019-12-22 00:26:51 +00:00
|
|
|
|
}).FireAndForget();
|
2019-12-14 05:46:20 +00:00
|
|
|
|
}
|
2020-01-02 02:04:57 +00:00
|
|
|
|
public static async Task ValidateLists(bool skipIfNewer = true)
|
2019-12-14 05:46:20 +00:00
|
|
|
|
{
|
2019-12-14 16:33:05 +00:00
|
|
|
|
Utils.Log("Cleaning Nexus Cache");
|
|
|
|
|
|
var client = new HttpClient();
|
2019-12-29 22:57:01 +00:00
|
|
|
|
//await client.GetAsync("http://build.wabbajack.org/nexus_api_cache/update");
|
2019-12-14 16:33:05 +00:00
|
|
|
|
|
2019-12-14 05:46:20 +00:00
|
|
|
|
Utils.Log("Starting Modlist Validation");
|
|
|
|
|
|
var modlists = await ModlistMetadata.LoadFromGithub();
|
|
|
|
|
|
|
|
|
|
|
|
using (var queue = new WorkQueue())
|
|
|
|
|
|
{
|
2020-01-06 00:21:05 +00:00
|
|
|
|
foreach (var list in modlists)
|
2019-12-14 05:46:20 +00:00
|
|
|
|
{
|
2019-12-29 22:57:01 +00:00
|
|
|
|
try
|
2019-12-14 05:46:20 +00:00
|
|
|
|
{
|
2020-01-02 19:21:54 +00:00
|
|
|
|
await ValidateList(list, queue, skipIfNewer);
|
2019-12-14 05:46:20 +00:00
|
|
|
|
}
|
2019-12-29 22:57:01 +00:00
|
|
|
|
catch (Exception ex)
|
2019-12-14 05:46:20 +00:00
|
|
|
|
{
|
2020-01-04 23:47:51 +00:00
|
|
|
|
Utils.Log(ex.ToString());
|
2019-12-14 05:46:20 +00:00
|
|
|
|
}
|
2019-12-29 22:57:01 +00:00
|
|
|
|
}
|
|
|
|
|
|
}
|
2019-12-14 05:46:20 +00:00
|
|
|
|
|
2019-12-29 22:57:01 +00:00
|
|
|
|
Utils.Log($"Done validating {modlists.Count} lists");
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2020-01-02 19:21:54 +00:00
|
|
|
|
private static async Task ValidateList(ModlistMetadata list, WorkQueue queue, bool skipIfNewer = true)
|
2019-12-29 22:57:01 +00:00
|
|
|
|
{
|
2020-01-02 02:04:57 +00:00
|
|
|
|
var existing = await Server.Config.ListValidation.Connect().FindOneAsync(l => l.Id == list.Links.MachineURL);
|
2020-01-05 04:01:47 +00:00
|
|
|
|
if (skipIfNewer && existing != null && DateTime.UtcNow - existing.DetailedStatus.Checked < TimeSpan.FromHours(2))
|
2020-01-02 02:04:57 +00:00
|
|
|
|
return;
|
|
|
|
|
|
|
2019-12-29 22:57:01 +00:00
|
|
|
|
var modlist_path = Path.Combine(Consts.ModListDownloadFolder, list.Links.MachineURL + ExtensionManager.Extension);
|
2019-12-14 05:46:20 +00:00
|
|
|
|
|
2019-12-29 22:57:01 +00:00
|
|
|
|
if (list.NeedsDownload(modlist_path))
|
|
|
|
|
|
{
|
|
|
|
|
|
if (File.Exists(modlist_path))
|
|
|
|
|
|
File.Delete(modlist_path);
|
|
|
|
|
|
|
|
|
|
|
|
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
|
|
|
|
|
|
Utils.Log($"Downloading {list.Links.MachineURL} - {list.Title}");
|
|
|
|
|
|
await state.Download(modlist_path);
|
|
|
|
|
|
}
|
|
|
|
|
|
else
|
|
|
|
|
|
{
|
|
|
|
|
|
Utils.Log($"No changes detected from downloaded modlist");
|
|
|
|
|
|
}
|
2019-12-14 05:46:20 +00:00
|
|
|
|
|
|
|
|
|
|
|
2019-12-29 22:57:01 +00:00
|
|
|
|
Utils.Log($"Loading {modlist_path}");
|
2019-12-14 05:46:20 +00:00
|
|
|
|
|
2019-12-29 22:57:01 +00:00
|
|
|
|
var installer = AInstaller.LoadFromFile(modlist_path);
|
2019-12-14 05:46:20 +00:00
|
|
|
|
|
2019-12-29 22:57:01 +00:00
|
|
|
|
Utils.Log($"{installer.Archives.Count} archives to validate");
|
2019-12-14 05:46:20 +00:00
|
|
|
|
|
2019-12-29 22:57:01 +00:00
|
|
|
|
DownloadDispatcher.PrepareAll(installer.Archives.Select(a => a.State));
|
2019-12-14 05:46:20 +00:00
|
|
|
|
|
2019-12-29 22:57:01 +00:00
|
|
|
|
var validated = (await installer.Archives
|
|
|
|
|
|
.PMap(queue, async archive =>
|
2019-12-14 05:46:20 +00:00
|
|
|
|
{
|
2019-12-29 22:57:01 +00:00
|
|
|
|
Utils.Log($"Validating: {archive.Name}");
|
|
|
|
|
|
bool is_failed;
|
|
|
|
|
|
try
|
|
|
|
|
|
{
|
|
|
|
|
|
is_failed = !(await archive.State.Verify());
|
|
|
|
|
|
}
|
|
|
|
|
|
catch (Exception)
|
|
|
|
|
|
{
|
|
|
|
|
|
is_failed = false;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
return new DetailedStatusItem {IsFailing = is_failed, Archive = archive};
|
|
|
|
|
|
}))
|
|
|
|
|
|
.ToList();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var status = new DetailedStatus
|
|
|
|
|
|
{
|
|
|
|
|
|
Name = list.Title,
|
|
|
|
|
|
Archives = validated.OrderBy(v => v.Archive.Name).ToList(),
|
|
|
|
|
|
DownloadMetaData = list.DownloadMetadata,
|
|
|
|
|
|
HasFailures = validated.Any(v => v.IsFailing)
|
|
|
|
|
|
};
|
2019-12-14 05:46:20 +00:00
|
|
|
|
|
2019-12-29 22:57:01 +00:00
|
|
|
|
var dto = new ModListStatus
|
|
|
|
|
|
{
|
|
|
|
|
|
Id = list.Links.MachineURL,
|
|
|
|
|
|
Summary = new ModlistSummary
|
|
|
|
|
|
{
|
|
|
|
|
|
Name = status.Name,
|
|
|
|
|
|
Checked = status.Checked,
|
|
|
|
|
|
Failed = status.Archives.Count(a => a.IsFailing),
|
|
|
|
|
|
Passed = status.Archives.Count(a => !a.IsFailing),
|
|
|
|
|
|
},
|
|
|
|
|
|
DetailedStatus = status,
|
|
|
|
|
|
Metadata = list
|
|
|
|
|
|
};
|
|
|
|
|
|
await ModListStatus.Update(dto);
|
2019-12-14 05:46:20 +00:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
