wabbajack/Wabbajack.Server/Services/ListValidator.cs

428 lines
18 KiB
C#
Raw Normal View History

2020-05-13 21:52:34 +00:00
using System;
using System.Collections.Concurrent;
2020-05-13 21:52:34 +00:00
using System.Collections.Generic;
using System.Diagnostics;
2020-05-13 21:52:34 +00:00
using System.Linq;
2020-05-14 22:21:56 +00:00
using System.Text.RegularExpressions;
2020-05-13 21:52:34 +00:00
using System.Threading.Tasks;
2020-05-14 22:21:56 +00:00
using Microsoft.AspNetCore.Mvc.Filters;
2020-05-13 21:52:34 +00:00
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
2020-05-14 22:21:56 +00:00
using Org.BouncyCastle.Crypto.Digests;
2020-05-13 21:52:34 +00:00
using RocksDbSharp;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
2020-05-13 22:48:33 +00:00
using Wabbajack.Lib.NexusApi;
2020-05-13 21:52:34 +00:00
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class ListValidator : AbstractService<ListValidator, int>
{
private SqlService _sql;
private DiscordWebHook _discord;
private NexusKeyMaintainance _nexus;
2020-05-20 12:18:47 +00:00
private ArchiveMaintainer _archives;
2020-05-13 21:52:34 +00:00
public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries => ValidationInfo.Values.Select(e => (e.Summary, e.Detailed));
public ConcurrentDictionary<string, (ModListSummary Summary, DetailedStatus Detailed, TimeSpan ValidationTime)> ValidationInfo = new ConcurrentDictionary<string, (ModListSummary Summary, DetailedStatus Detailed, TimeSpan ValidationTime)>();
2020-05-13 21:52:34 +00:00
public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives, QuickSync quickSync)
: base(logger, settings, quickSync, TimeSpan.FromMinutes(5))
2020-05-13 21:52:34 +00:00
{
_sql = sql;
_discord = discord;
_nexus = nexus;
2020-05-20 12:18:47 +00:00
_archives = archives;
2020-05-13 21:52:34 +00:00
}
public override async Task<int> Execute()
{
var data = await _sql.GetValidationData();
2020-05-13 21:52:34 +00:00
using var queue = new WorkQueue();
2020-05-18 12:40:55 +00:00
var oldSummaries = Summaries;
2020-05-13 21:52:34 +00:00
var stopwatch = new Stopwatch();
stopwatch.Start();
var results = await data.ModLists.Where(m => !m.ForceDown).PMap(queue, async metadata =>
2020-05-13 21:52:34 +00:00
{
var timer = new Stopwatch();
timer.Start();
2020-05-18 12:40:55 +00:00
var oldSummary =
2020-07-15 22:29:43 +00:00
oldSummaries.FirstOrDefault(s => s.Summary.MachineURL == metadata.Links.MachineURL);
2020-05-18 12:40:55 +00:00
2020-07-15 22:29:43 +00:00
var listArchives = await _sql.ModListArchives(metadata.Links.MachineURL);
var archives = await listArchives.PMap(queue, async archive =>
2020-05-13 21:52:34 +00:00
{
ReportStarting(archive.State.PrimaryKeyString);
if (timer.Elapsed > Delay)
{
return (archive, ArchiveStatus.InValid);
}
try
{
var (_, result) = await ValidateArchive(data, archive);
if (result == ArchiveStatus.InValid)
{
if (data.Mirrors.Contains(archive.Hash))
return (archive, ArchiveStatus.Mirrored);
return await TryToHeal(data, archive, metadata);
}
return (archive, result);
}
catch (Exception ex)
{
_logger.LogError(ex, $"During Validation of {archive.Hash} {archive.State.PrimaryKeyString}");
return (archive, ArchiveStatus.InValid);
}
finally
{
ReportEnding(archive.State.PrimaryKeyString);
}
2020-05-13 21:52:34 +00:00
});
var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid);
var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated);
var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating);
var mirroredCount = archives.Count(f => f.Item2 == ArchiveStatus.Mirrored);
2020-05-13 21:52:34 +00:00
var summary = new ModListSummary
{
Checked = DateTime.UtcNow,
Failed = failedCount,
Passed = passCount,
Updating = updatingCount,
Mirrored = mirroredCount,
2020-05-13 21:52:34 +00:00
MachineURL = metadata.Links.MachineURL,
Name = metadata.Title,
};
var detailed = new DetailedStatus
{
Name = metadata.Title,
Checked = DateTime.UtcNow,
DownloadMetaData = metadata.DownloadMetadata,
HasFailures = failedCount > 0,
MachineName = metadata.Links.MachineURL,
Archives = archives.Select(a => new DetailedStatusItem
{
2020-06-14 13:13:29 +00:00
Archive = a.Item1,
IsFailing = a.Item2 == ArchiveStatus.InValid,
2020-06-14 13:13:29 +00:00
ArchiveStatus = a.Item2
2020-05-13 21:52:34 +00:00
}).ToList()
};
if (timer.Elapsed > Delay)
{
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Embeds = new[]
{
new DiscordEmbed
{
Title =
$"Failing {summary.Name} (`{summary.MachineURL}`) because the max validation time expired",
Url = new Uri(
$"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html")
}
}
});
}
2020-05-18 12:40:55 +00:00
if (oldSummary != default && oldSummary.Summary.Failed != summary.Failed)
{
_logger.Log(LogLevel.Information, $"Number of failures {oldSummary.Summary.Failed} -> {summary.Failed}");
if (summary.HasFailures)
{
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Embeds = new[]
{
new DiscordEmbed
{
2020-06-20 22:51:47 +00:00
Title =
2020-05-18 12:40:55 +00:00
$"Number of failures in {summary.Name} (`{summary.MachineURL}`) was {oldSummary.Summary.Failed} is now {summary.Failed}",
Url = new Uri(
$"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html")
}
}
});
}
2020-05-22 20:56:58 +00:00
if (!summary.HasFailures && oldSummary.Summary.HasFailures)
2020-05-18 12:40:55 +00:00
{
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Embeds = new[]
{
new DiscordEmbed
{
2020-06-20 22:51:47 +00:00
Title = $"{summary.Name} (`{summary.MachineURL}`) is now passing.",
Url = new Uri(
$"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html")
2020-05-18 12:40:55 +00:00
}
}
});
}
}
timer.Stop();
ValidationInfo[summary.MachineURL] = (summary, detailed, timer.Elapsed);
2020-05-13 21:52:34 +00:00
return (summary, detailed);
});
stopwatch.Stop();
_logger.LogInformation($"Finished Validation in {stopwatch.Elapsed}");
2020-05-13 21:52:34 +00:00
return Summaries.Count(s => s.Summary.HasFailures);
}
2020-05-20 12:18:47 +00:00
private AsyncLock _healLock = new AsyncLock();
private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive, ModlistMetadata modList)
2020-05-20 12:18:47 +00:00
{
var srcDownload = await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size);
if (srcDownload == null || srcDownload.IsFailed == true)
{
2020-05-22 20:56:58 +00:00
_logger.Log(LogLevel.Information, $"Cannot heal {archive.State.PrimaryKeyString} Size: {archive.Size} Hash: {(long)archive.Hash} because it hasn't been previously successfully downloaded");
2020-05-20 12:18:47 +00:00
return (archive, ArchiveStatus.InValid);
}
var patches = await _sql.PatchesForSource(archive.Hash);
2020-05-20 12:18:47 +00:00
foreach (var patch in patches)
{
if (patch.Finished is null)
return (archive, ArchiveStatus.Updating);
if (patch.IsFailed == true)
2020-05-30 21:05:26 +00:00
return (archive, ArchiveStatus.InValid);
2020-05-20 12:18:47 +00:00
var (_, status) = await ValidateArchive(data, patch.Dest.Archive);
if (status == ArchiveStatus.Valid)
return (archive, ArchiveStatus.Updated);
}
using var _ = await _healLock.WaitAsync();
2020-05-20 12:18:47 +00:00
var upgradeTime = DateTime.UtcNow;
_logger.LogInformation($"Validator Finding Upgrade for {archive.Hash} {archive.State.PrimaryKeyString}");
2020-08-12 22:23:02 +00:00
Func<Archive, Task<AbsolutePath>> resolver = async findIt =>
{
_logger.LogInformation($"Quick find for {findIt.State.PrimaryKeyString}");
var foundArchive = await _sql.GetArchiveDownload(findIt.State.PrimaryKeyString);
if (foundArchive == null)
{
_logger.LogInformation($"No Quick find for {findIt.State.PrimaryKeyString}");
return default;
}
return _archives.TryGetPath(foundArchive.Archive.Hash, out var path) ? path : default;
};
2020-08-12 22:23:02 +00:00
var upgrade = await DownloadDispatcher.FindUpgrade(archive, resolver);
2020-05-20 12:18:47 +00:00
if (upgrade == default)
{
2020-05-23 21:03:25 +00:00
_logger.Log(LogLevel.Information, $"Cannot heal {archive.State.PrimaryKeyString} because an alternative wasn't found");
2020-05-20 12:18:47 +00:00
return (archive, ArchiveStatus.InValid);
}
_logger.LogInformation($"Upgrade {upgrade.Archive.State.PrimaryKeyString} found for {archive.State.PrimaryKeyString}");
{
}
2020-05-20 12:18:47 +00:00
var found = await _sql.GetArchiveDownload(upgrade.Archive.State.PrimaryKeyString, upgrade.Archive.Hash,
upgrade.Archive.Size);
Guid id;
if (found == null)
{
if (upgrade.NewFile.Path.Exists)
await _archives.Ingest(upgrade.NewFile.Path);
id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime);
}
else
{
id = found.Id;
}
2020-05-20 12:18:47 +00:00
var destDownload = await _sql.GetArchiveDownload(id);
if (destDownload.Archive.Hash == srcDownload.Archive.Hash && destDownload.Archive.State.PrimaryKeyString == srcDownload.Archive.State.PrimaryKeyString)
{
_logger.Log(LogLevel.Information, $"Can't heal because src and dest match");
return (archive, ArchiveStatus.InValid);
}
if (destDownload.Archive.Hash == default)
{
_logger.Log(LogLevel.Information, "Can't heal because we got back a default hash for the downloaded file");
return (archive, ArchiveStatus.InValid);
}
var existing = await _sql.FindPatch(srcDownload.Id, destDownload.Id);
if (existing == null)
{
await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload});
_logger.Log(LogLevel.Information,
$"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`"
});
}
2020-05-20 12:18:47 +00:00
await upgrade.NewFile.DisposeAsync();
2020-05-20 12:18:47 +00:00
_logger.LogInformation($"Patch in progress {archive.Hash} {archive.State.PrimaryKeyString}");
2020-05-20 12:18:47 +00:00
return (archive, ArchiveStatus.Updating);
}
2020-05-13 22:48:33 +00:00
private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(ValidationData data, Archive archive)
2020-05-13 21:52:34 +00:00
{
switch (archive.State)
{
case GoogleDriveDownloader.State _:
// Disabled for now due to GDrive rate-limiting the build server
return (archive, ArchiveStatus.Valid);
case NexusDownloader.State nexusState when data.NexusFiles.Contains((
nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)):
return (archive, ArchiveStatus.Valid);
2020-05-13 22:48:33 +00:00
case NexusDownloader.State ns:
return (archive, await FastNexusModStats(ns));
2020-05-13 21:52:34 +00:00
case ManualDownloader.State _:
return (archive, ArchiveStatus.Valid);
case ModDBDownloader.State _:
return (archive, ArchiveStatus.Valid);
case MediaFireDownloader.State _:
return (archive, ArchiveStatus.Valid);
2020-05-13 21:52:34 +00:00
default:
{
if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash),
out bool isValid))
{
return isValid ? (archive, ArchiveStatus.Valid) : (archive, ArchiveStatus.InValid);
}
2020-05-20 21:48:26 +00:00
return (archive, ArchiveStatus.Valid);
2020-05-13 21:52:34 +00:00
}
}
}
2020-05-14 22:21:56 +00:00
private AsyncLock _lock = new AsyncLock();
2020-05-14 22:21:56 +00:00
public async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns)
2020-05-13 22:48:33 +00:00
{
// Check if some other thread has added them
2020-05-13 22:48:33 +00:00
var mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID);
var files = await _sql.GetModFiles(ns.Game, ns.ModID);
if (mod == null || files == null)
2020-05-13 22:48:33 +00:00
{
// Aquire the lock
using var lck = await _lock.WaitAsync();
// Check again
mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID);
files = await _sql.GetModFiles(ns.Game, ns.ModID);
if (mod == null || files == null)
2020-05-13 22:48:33 +00:00
{
NexusApiClient nexusClient = await _nexus.GetClient();
var queryTime = DateTime.UtcNow;
2020-05-13 22:48:33 +00:00
try
{
if (mod == null)
{
_logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
try
{
mod = await nexusClient.GetModInfo(ns.Game, ns.ModID, false);
}
catch
{
mod = new ModInfo
{
mod_id = ns.ModID.ToString(),
game_id = ns.Game.MetaData().NexusGameId,
available = false
};
}
try
{
await _sql.AddNexusModInfo(ns.Game, ns.ModID, queryTime, mod);
}
2020-10-01 03:50:09 +00:00
catch (Exception)
{
// Could be a PK constraint failure
}
}
if (files == null)
{
_logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
try
{
files = await nexusClient.GetModFiles(ns.Game, ns.ModID, false);
}
catch
{
files = new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>()};
}
try
{
await _sql.AddNexusModFiles(ns.Game, ns.ModID, queryTime, files);
}
2020-10-01 03:50:09 +00:00
catch (Exception)
{
// Could be a PK constraint failure
}
}
2020-05-13 22:48:33 +00:00
}
2020-10-01 03:50:09 +00:00
catch (Exception)
2020-05-13 22:48:33 +00:00
{
return ArchiveStatus.InValid;
2020-05-13 22:48:33 +00:00
}
}
}
if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID))
return ArchiveStatus.Valid;
return ArchiveStatus.InValid;
}
2020-05-13 21:52:34 +00:00
}
}