using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Wabbajack.BuildServer; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.ModListRegistry; using Wabbajack.Lib.NexusApi; using Wabbajack.Server.DataLayer; using Wabbajack.Server.DTOs; using ArchiveStatus = Wabbajack.Server.DTOs.ArchiveStatus; namespace Wabbajack.Server.Services { public class ListValidator : AbstractService { private SqlService _sql; private DiscordWebHook _discord; private NexusKeyMaintainance _nexus; private ArchiveMaintainer _archives; public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries => ValidationInfo.Values.Select(e => (e.Summary, e.Detailed)); public ConcurrentDictionary ValidationInfo = new(); public ListValidator(ILogger logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromMinutes(5)) { _sql = sql; _discord = discord; _nexus = nexus; _archives = archives; } public override async Task Execute() { var data = await _sql.GetValidationData(); _logger.LogInformation("Found {count} nexus files", data.NexusFiles.Count); using var queue = new WorkQueue(); var oldSummaries = Summaries; var stopwatch = new Stopwatch(); stopwatch.Start(); var results = await data.ModLists.Where(m => !m.ForceDown).PMap(queue, async metadata => { var timer = new Stopwatch(); timer.Start(); var oldSummary = oldSummaries.FirstOrDefault(s => s.Summary.MachineURL == metadata.Links.MachineURL); var mainFile = await DownloadDispatcher.Infer(new Uri(metadata.Links.Download)); var mainArchive = new Archive(mainFile!) { Size = metadata.DownloadMetadata!.Size, Hash = metadata.DownloadMetadata!.Hash }; bool mainFailed = false; try { if (mainArchive.State is WabbajackCDNDownloader.State) { if (!await mainArchive.State.Verify(mainArchive)) { mainFailed = true; } } } catch (Exception ex) { mainFailed = true; } var listArchives = await _sql.ModListArchives(metadata.Links.MachineURL); var archives = await listArchives.PMap(queue, async archive => { if (mainFailed) return (archive, ArchiveStatus.InValid); try { ReportStarting(archive.State.PrimaryKeyString); if (timer.Elapsed > Delay) { return (archive, ArchiveStatus.InValid); } var (_, result) = await ValidateArchive(data, archive); if (result == ArchiveStatus.InValid) { if (data.Mirrors.TryGetValue(archive.Hash, out var done)) return (archive, done ? ArchiveStatus.Mirrored : ArchiveStatus.Updating); if ((await data.AllowedMirrors.Value).TryGetValue(archive.Hash, out var reason)) { await _sql.StartMirror((archive.Hash, reason)); return (archive, ArchiveStatus.Updating); } if (archive.State is NexusDownloader.State) return (archive, result); return await TryToHeal(data, archive, metadata); } return (archive, result); } catch (Exception ex) { _logger.LogError(ex, $"During Validation of {archive.Hash} {archive.State.PrimaryKeyString}"); Utils.Log($"Exception in validation of {archive.Hash} {archive.State.PrimaryKeyString} " + ex); return (archive, ArchiveStatus.InValid); } finally { ReportEnding(archive.State.PrimaryKeyString); } }); var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid || f.Item2 == ArchiveStatus.Updating); var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated); var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating); var mirroredCount = archives.Count(f => f.Item2 == ArchiveStatus.Mirrored); var summary = new ModListSummary { Checked = DateTime.UtcNow, Failed = failedCount, Passed = passCount, Updating = updatingCount, Mirrored = mirroredCount, MachineURL = metadata.Links.MachineURL, Name = metadata.Title, ModListIsMissing = mainFailed }; var detailed = new DetailedStatus { Name = metadata.Title, Checked = DateTime.UtcNow, DownloadMetaData = metadata.DownloadMetadata, HasFailures = failedCount > 0, MachineName = metadata.Links.MachineURL, Archives = archives.Select(a => new DetailedStatusItem { Archive = a.Item1, IsFailing = a.Item2 == ArchiveStatus.InValid, ArchiveStatus = a.Item2 }).ToList() }; if (timer.Elapsed > Delay) { await _discord.Send(Channel.Ham, new DiscordMessage { Embeds = new[] { new DiscordEmbed { Title = $"Failing {summary.Name} (`{summary.MachineURL}`) because the max validation time expired", Url = new Uri( $"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html") } } }); } if (oldSummary != default && oldSummary.Summary.Failed != summary.Failed) { _logger.Log(LogLevel.Information, $"Number of failures {oldSummary.Summary.Failed} -> {summary.Failed}"); if (summary.HasFailures) { await _discord.Send(Channel.Ham, new DiscordMessage { Embeds = new[] { new DiscordEmbed { Title = $"Number of failures in {summary.Name} (`{summary.MachineURL}`) was {oldSummary.Summary.Failed} is now {summary.Failed}", Url = new Uri( $"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html") } } }); } if (!summary.HasFailures && oldSummary.Summary.HasFailures) { await _discord.Send(Channel.Ham, new DiscordMessage { Embeds = new[] { new DiscordEmbed { Title = $"{summary.Name} (`{summary.MachineURL}`) is now passing.", Url = new Uri( $"https://build.wabbajack.org/lists/status/{summary.MachineURL}.html") } } }); } } timer.Stop(); ValidationInfo[summary.MachineURL] = (summary, detailed, timer.Elapsed); return (summary, detailed); }); stopwatch.Stop(); _logger.LogInformation($"Finished Validation in {stopwatch.Elapsed}"); return Summaries.Count(s => s.Summary.HasFailures); } private AsyncLock _healLock = new AsyncLock(); private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive, ModlistMetadata modList) { using var _ = await _healLock.WaitAsync(); var srcDownload = await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size); if (srcDownload == null || srcDownload.IsFailed == true) { _logger.Log(LogLevel.Information, $"Cannot heal {archive.State.PrimaryKeyString} Size: {archive.Size} Hash: {(long)archive.Hash} because it hasn't been previously successfully downloaded"); return (archive, ArchiveStatus.InValid); } var patches = await _sql.PatchesForSource(archive.Hash); foreach (var patch in patches) { if (patch.Finished is null) return (archive, ArchiveStatus.Updating); if (patch.IsFailed == true) return (archive, ArchiveStatus.InValid); var (_, status) = await ValidateArchive(data, patch.Dest.Archive); if (status == ArchiveStatus.Valid) return (archive, ArchiveStatus.Updated); } var upgradeTime = DateTime.UtcNow; _logger.LogInformation($"Validator Finding Upgrade for {archive.Hash} {archive.State.PrimaryKeyString}"); Func> resolver = async findIt => { _logger.LogInformation($"Quick find for {findIt.State.PrimaryKeyString}"); var foundArchive = await _sql.GetArchiveDownload(findIt.State.PrimaryKeyString); if (foundArchive == null) { _logger.LogInformation($"No Quick find for {findIt.State.PrimaryKeyString}"); return default; } return _archives.TryGetPath(foundArchive.Archive.Hash, out var path) ? path : default; }; if (archive.State is NexusDownloader.State) { DownloadDispatcher.GetInstance().Client = await _nexus.GetClient(); } var upgrade = await DownloadDispatcher.FindUpgrade(archive, resolver); if (upgrade == default) { _logger.Log(LogLevel.Information, $"Cannot heal {archive.State.PrimaryKeyString} because an alternative wasn't found"); return (archive, ArchiveStatus.InValid); } _logger.LogInformation($"Upgrade {upgrade.Archive.State.PrimaryKeyString} found for {archive.State.PrimaryKeyString}"); { } var found = await _sql.GetArchiveDownload(upgrade.Archive.State.PrimaryKeyString, upgrade.Archive.Hash, upgrade.Archive.Size); Guid id; if (found == null) { if (upgrade.NewFile.Path.Exists) await _archives.Ingest(upgrade.NewFile.Path); id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime); } else { id = found.Id; } var destDownload = await _sql.GetArchiveDownload(id); if (destDownload.Archive.Hash == srcDownload.Archive.Hash && destDownload.Archive.State.PrimaryKeyString == srcDownload.Archive.State.PrimaryKeyString) { _logger.Log(LogLevel.Information, $"Can't heal because src and dest match"); return (archive, ArchiveStatus.InValid); } if (destDownload.Archive.Hash == default) { _logger.Log(LogLevel.Information, "Can't heal because we got back a default hash for the downloaded file"); return (archive, ArchiveStatus.InValid); } var existing = await _sql.FindPatch(srcDownload.Id, destDownload.Id); if (existing == null) { if (await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload})) { _logger.Log(LogLevel.Information, $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}"); await _discord.Send(Channel.Ham, new DiscordMessage { Content = $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`" }); } } await upgrade.NewFile.DisposeAsync(); _logger.LogInformation($"Patch in progress {archive.Hash} {archive.State.PrimaryKeyString}"); return (archive, ArchiveStatus.Updating); } private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(ValidationData data, Archive archive) { switch (archive.State) { case GoogleDriveDownloader.State _: // Disabled for now due to GDrive rate-limiting the build server return (archive, ArchiveStatus.Valid); case NexusDownloader.State nexusState when data.NexusFiles.TryGetValue( (nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID), out var category): return (archive, category != null ? ArchiveStatus.Valid : ArchiveStatus.InValid); case NexusDownloader.State ns: return (archive, await FastNexusModStats(ns)); case ManualDownloader.State _: return (archive, ArchiveStatus.Valid); case ModDBDownloader.State _: return (archive, ArchiveStatus.Valid); case GameFileSourceDownloader.State _: return (archive, ArchiveStatus.Valid); case MediaFireDownloader.State _: return (archive, ArchiveStatus.Valid); case DeprecatedLoversLabDownloader.State _: return (archive, ArchiveStatus.InValid); case DeprecatedVectorPlexusDownloader.State _: return (archive, ArchiveStatus.InValid); default: { if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash), out bool isValid)) { return isValid ? (archive, ArchiveStatus.Valid) : (archive, ArchiveStatus.InValid); } return (archive, ArchiveStatus.Valid); } } } public async Task FastNexusModStats(NexusDownloader.State ns) { // Check if some other thread has added them var file = await _sql.GetModFile(ns.Game, ns.ModID, ns.FileID); if (file == null) { try { NexusApiClient nexusClient = await _nexus.GetClient(); var queryTime = DateTime.UtcNow; _logger.Log(LogLevel.Information, "Found missing Nexus file info {Game} {ModID} {FileID}", ns.Game, ns.ModID, ns.FileID); try { file = await nexusClient.GetModFile(ns.Game, ns.ModID, ns.FileID, false); } catch { file = new NexusFileInfo() {category_name = null}; } try { await _sql.AddNexusModFile(ns.Game, ns.ModID, ns.FileID, queryTime, file); } catch (Exception) { // Could be a PK constraint failure } } catch (Exception) { return ArchiveStatus.InValid; } } return file?.category_name != null ? ArchiveStatus.Valid : ArchiveStatus.InValid; } } }