wabbajack/Wabbajack.CLI/Verbs/ValidateLists.cs
2022-04-22 06:29:38 -06:00

669 lines
27 KiB
C#

using System;
using System.Collections.Generic;
using System.CommandLine;
using System.CommandLine.Invocation;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FluentFTP;
using Microsoft.Extensions.Logging;
using Wabbajack.CLI.Services;
using Wabbajack.Common;
using Wabbajack.Compression.Zip;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs;
using Wabbajack.DTOs.Configs;
using Wabbajack.DTOs.Directives;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.DTOs.ModListValidation;
using Wabbajack.DTOs.ServerResponses;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Installer;
using Wabbajack.Networking.Discord;
using Wabbajack.Networking.GitHub;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
using Wabbajack.Server.Lib.DTOs;
using Wabbajack.Server.Lib.TokenProviders;
namespace Wabbajack.CLI.Verbs;
public class ValidateLists : IVerb
{
private static readonly Uri MirrorPrefix = new("https://mirror.wabbajack.org");
private readonly WriteOnlyClient _discord;
private readonly DownloadDispatcher _dispatcher;
private readonly DTOSerializer _dtos;
private readonly IResource<IFtpSiteCredentials> _ftpRateLimiter;
private readonly IFtpSiteCredentials _ftpSiteCredentials;
private readonly Client _gitHubClient;
private readonly ILogger<ValidateLists> _logger;
private readonly ParallelOptions _parallelOptions;
private readonly Random _random;
private readonly TemporaryFileManager _temporaryFileManager;
private readonly Networking.WabbajackClientApi.Client _wjClient;
private readonly HttpClient _httpClient;
private readonly IResource<HttpClient> _httpLimiter;
public ValidateLists(ILogger<ValidateLists> logger, Networking.WabbajackClientApi.Client wjClient,
Client gitHubClient, TemporaryFileManager temporaryFileManager,
DownloadDispatcher dispatcher, DTOSerializer dtos, ParallelOptions parallelOptions,
IFtpSiteCredentials ftpSiteCredentials, IResource<IFtpSiteCredentials> ftpRateLimiter,
WriteOnlyClient discordClient, HttpClient httpClient, IResource<HttpClient> httpLimiter)
{
_logger = logger;
_wjClient = wjClient;
_gitHubClient = gitHubClient;
_temporaryFileManager = temporaryFileManager;
_dispatcher = dispatcher;
_dtos = dtos;
_parallelOptions = parallelOptions;
_ftpSiteCredentials = ftpSiteCredentials;
_ftpRateLimiter = ftpRateLimiter;
_discord = discordClient;
_random = new Random();
_httpClient = httpClient;
_httpLimiter = httpLimiter;
}
public Command MakeCommand()
{
var command = new Command("validate-lists");
command.Add(new Option<AbsolutePath>(new[] {"-r", "--reports"}, "Location to store validation report outputs"));
command.Add(new Option<AbsolutePath>(new[] {"--other-archives"},
"Look for files here before downloading (stored by hex hash name)")
{IsRequired = false});
command.Description = "Gets a list of modlists, validates them and exports a result list";
command.Handler = CommandHandler.Create(Run);
return command;
}
public async Task<int> Run(AbsolutePath reports, AbsolutePath otherArchives)
{
reports.CreateDirectory();
var token = CancellationToken.None;
_logger.LogInformation("Scanning for existing patches/mirrors");
var mirroredFiles = (await _wjClient.GetAllMirroredFileDefinitions(token)).Select(m => m.Hash).ToHashSet();
_logger.LogInformation("Found {Count} mirrored files", mirroredFiles.Count);
var patchFiles = await _wjClient.GetAllPatches(token);
_logger.LogInformation("Found {Count} patches", patchFiles.Length);
var forcedRemovals = (await _wjClient.GetForcedRemovals(token)).ToLookup(f => f.Hash);
_logger.LogInformation("Found {Count} forced removals", forcedRemovals.Count);
var validationCache = new LazyCache<string, Archive, (ArchiveStatus Status, Archive archive)>
(x => x.State.PrimaryKeyString + x.Hash,
archive => DownloadAndValidate(archive, forcedRemovals, token));
var stopWatch = Stopwatch.StartNew();
var listData = await _wjClient.LoadLists();
var validatedLists = await listData.PMapAll(async modList =>
{
var validatedList = new ValidatedModList
{
Name = modList.Title,
ModListHash = modList.DownloadMetadata?.Hash ?? default,
MachineURL = modList.NamespacedName,
Version = modList.Version
};
if (modList.ForceDown)
{
_logger.LogWarning("List is ForceDown, skipping");
validatedList.Status = ListStatus.ForcedDown;
return validatedList;
}
using var scope = _logger.BeginScope("MachineURL: {MachineUrl}", modList.NamespacedName);
_logger.LogInformation("Verifying {MachineUrl} - {Title}", modList.NamespacedName, modList.Title);
//await DownloadModList(modList, archiveManager, CancellationToken.None);
ModList modListData;
try
{
_logger.LogInformation("Loading Modlist");
modListData =
await StandardInstaller.Load(_dtos, _dispatcher, modList, token);
}
catch (Exception ex)
{
_logger.LogError(ex, "Forcing down {Modlist} due to error while loading: ", modList.NamespacedName);
validatedList.Status = ListStatus.ForcedDown;
return validatedList;
}
_logger.LogInformation("Verifying {Count} archives", modListData.Archives.Length);
var archives = await modListData.Archives.PMapAll(async archive =>
{
var result = await validationCache.Get(archive);
if (result.Status == ArchiveStatus.Valid)
{
return new ValidatedArchive
{
Status = ArchiveStatus.Valid,
Original = archive
};
}
if (result.Status == ArchiveStatus.InValid)
{
if (mirroredFiles.Contains(archive.Hash))
{
return new ValidatedArchive
{
Status = ArchiveStatus.Mirrored,
Original = archive,
PatchedFrom = new Archive
{
State = new WabbajackCDN
{
Url = _wjClient.GetMirrorUrl(archive.Hash)!
},
Size = archive.Size,
Name = archive.Name,
Hash = archive.Hash
}
};
}
}
if (result.Status == ArchiveStatus.InValid)
{
_logger.LogInformation("Looking for patch for {Hash}", archive.Hash);
foreach (var file in patchFiles.Where(p => p.Original.Hash == archive.Hash && p.Status == ArchiveStatus.Updated))
{
if (await _dispatcher.Verify(file.PatchedFrom!, token))
{
return new ValidatedArchive()
{
Original = archive,
Status = ArchiveStatus.Updated,
PatchUrl = file.PatchUrl,
PatchedFrom = file.PatchedFrom
};
}
}
}
return new ValidatedArchive()
{
Status = ArchiveStatus.InValid,
Original = archive
};
}).ToArray();
validatedList.Archives = archives;
validatedList.Status = archives.Any(a => a.Status == ArchiveStatus.InValid)
? ListStatus.Failed
: ListStatus.Available;
return validatedList;
}).ToArray();
var allArchives = validatedLists.SelectMany(l => l.Archives).ToList();
_logger.LogInformation("Validated {Count} lists in {Elapsed}", validatedLists.Length, stopWatch.Elapsed);
_logger.LogInformation(" - {Count} Valid", allArchives.Count(a => a.Status is ArchiveStatus.Valid));
_logger.LogInformation(" - {Count} Invalid", allArchives.Count(a => a.Status is ArchiveStatus.InValid));
_logger.LogInformation(" - {Count} Mirrored", allArchives.Count(a => a.Status is ArchiveStatus.Mirrored));
_logger.LogInformation(" - {Count} Updated", allArchives.Count(a => a.Status is ArchiveStatus.Updated));
foreach (var invalid in allArchives.Where(a => a.Status is ArchiveStatus.InValid)
.DistinctBy(a => a.Original.Hash))
{
_logger.LogInformation("-- Invalid {Hash}: {PrimaryKeyString}", invalid.Original.Hash.ToHex(),
invalid.Original.State.PrimaryKeyString);
}
await ExportReports(reports, validatedLists, token);
var usedMirroredFiles = validatedLists.SelectMany(a => a.Archives)
.Where(m => m.Status == ArchiveStatus.Mirrored)
.Select(m => m.Original.Hash)
.ToHashSet();
await DeleteOldMirrors(mirroredFiles, usedMirroredFiles);
return 0;
}
private async Task SendDefinitionToLoadOrderLibrary(ModlistMetadata metadata, ModList modListData, CancellationToken token)
{
var lolGame = modListData.GameType switch
{
Game.Morrowind => 1,
Game.Oblivion => 2,
Game.Skyrim => 3,
Game.SkyrimSpecialEdition => 4,
Game.SkyrimVR => 5,
Game.Fallout3 => 6,
Game.FalloutNewVegas => 7,
Game.Fallout4 => 8,
Game.Fallout4VR => 9,
_ => 0
};
if (lolGame == 0) return;
var files = (await GetFiles(modListData, metadata, token))
.Where(f => f.Key.Depth == 3)
.Where(f => f.Key.Parent.Parent == "profiles".ToRelativePath())
.GroupBy(f => f.Key.Parent.FileName.ToString())
.ToArray();
foreach (var profile in files)
{
var formData = new MultipartFormDataContent();
if (files.Length > 1)
{
formData.Add(new StringContent(modListData.Name + $"({metadata.NamespacedName})"), "name");
}
else
{
formData.Add(new StringContent(modListData.Name + $" - Profile: {profile.Key} ({metadata.NamespacedName})"), "name");
}
formData.Add(new StringContent(lolGame.ToString()), "game_id");
formData.Add(new StringContent(metadata.Description), "description");
formData.Add(new StringContent((metadata.Version ?? Version.Parse("0.0.0.0")).ToString()), "version");
formData.Add(new StringContent("0"), "is_private");
formData.Add(new StringContent("1hr"), "expires_at");
if (modListData.Website != null)
{
formData.Add(new StringContent(modListData.Website!.ToString()), "website");
}
if (metadata.Links.DiscordURL != null)
{
formData.Add(new StringContent(metadata.Links.DiscordURL), "discord");
}
if (metadata.Links.Readme != null)
{
formData.Add(new StringContent(metadata.Links.Readme), "readme");
}
foreach (var file in profile)
{
formData.Add(new ByteArrayContent(file.Value), "files[]", file.Key.FileName.ToString());
}
using var job = await _httpLimiter.Begin("Posting to load order library", 0, token);
var msg = new HttpRequestMessage(HttpMethod.Post, "https://api.loadorderlibrary.com/v1/lists");
msg.Content = formData;
using var result = await _httpClient.SendAsync(msg, token);
if (result.IsSuccessStatusCode)
return;
//var data = await result.Content.ReadFromJsonAsync<string>(token);
}
}
private static HashSet<RelativePath> LoadOrderFiles = new HashSet<string>()
{
"enblocal.ini",
"enbseries.ini",
"fallout.ini",
"falloutprefs.ini",
"fallout4.ini",
"fallout4custom.ini",
"fallout4prefs.ini",
"falloutcustom.ini",
"geckcustom.ini",
"geckprefs.ini",
"loadorder.txt",
"mge.ini",
"modlist.txt",
"morrowind.ini",
"mwse-version.ini",
"oblivion.ini",
"oblivionprefs.ini",
"plugins.txt",
"settings.ini",
"skyrim.ini",
"skyrimcustom.ini",
"skyrimprefs.ini",
"skyrimvr.ini",
}.Select(f => f.ToRelativePath()).ToHashSet();
private async Task<Dictionary<RelativePath, byte[]>> GetFiles(ModList modlist, ModlistMetadata metadata, CancellationToken token)
{
var archive = new Archive
{
State = _dispatcher.Parse(new Uri(metadata.Links.Download))!,
Size = metadata.DownloadMetadata!.Size,
Hash = metadata.DownloadMetadata.Hash
};
var stream = await _dispatcher.ChunkedSeekableStream(archive, token);
await using var reader = new ZipReader(stream);
var files = await reader.GetFiles();
var indexed = files.ToDictionary(f => f.FileName.ToRelativePath());
var entriesToGet = modlist.Directives.OfType<InlineFile>()
.Where(f => LoadOrderFiles.Contains(f.To.FileName))
.Select(f => (f, indexed[f.SourceDataID]))
.ToArray();
var fileData = new Dictionary<RelativePath, byte[]>();
foreach (var entry in entriesToGet)
{
var ms = new MemoryStream();
await reader.Extract(entry.Item2, ms, token);
fileData.Add(entry.f.To, ms.ToArray());
}
return fileData;
}
private async Task ExportReports(AbsolutePath reports, ValidatedModList[] validatedLists, CancellationToken token)
{
foreach (var validatedList in validatedLists)
{
var baseFolder = reports.Combine(validatedList.MachineURL);
baseFolder.CreateDirectory();
await using var jsonFile = baseFolder.Combine("status").WithExtension(Ext.Json)
.Open(FileMode.Create, FileAccess.Write, FileShare.None);
await _dtos.Serialize(validatedList, jsonFile, true);
await using var mdFile = baseFolder.Combine("status").WithExtension(Ext.Md)
.Open(FileMode.Create, FileAccess.Write, FileShare.None);
await using var sw = new StreamWriter(mdFile);
await sw.WriteLineAsync($"## Validation Report - {validatedList.Name} ({validatedList.MachineURL})");
await sw.WriteAsync("\n\n");
async Task WriteSection(TextWriter w, ArchiveStatus status, string sectionName)
{
var archives = validatedList.Archives.Where(a => a.Status == status).ToArray();
await w.WriteLineAsync($"### {sectionName} ({archives.Length})");
foreach (var archive in archives.OrderBy(a => a.Original.Name))
{
if (_dispatcher.TryGetDownloader(archive.Original, out var downloader) &&
downloader is IUrlDownloader u)
{
await w.WriteLineAsync(
$"* [{archive.Original.Name}]({u.UnParse(archive.Original.State)})");
}
else
{
await w.WriteLineAsync(
$"* {archive.Original.Name}");
}
}
}
await WriteSection(sw, ArchiveStatus.InValid, "Invalid");
await WriteSection(sw, ArchiveStatus.Updated, "Updated");
await WriteSection(sw, ArchiveStatus.Mirrored, "Mirrored");
await WriteSection(sw, ArchiveStatus.Valid, "Valid");
try
{
var oldSummary = await _wjClient.GetDetailedStatus(validatedList.MachineURL);
if (oldSummary.ModListHash != validatedList.ModListHash)
{
try
{
await SendDefinitionToLoadOrderLibrary(validatedList, token);
}
catch (Exception ex)
{
_logger.LogError("While uploading to load order library", ex);
}
await _discord.SendAsync(Channel.Ham,
$"Finished processing {validatedList.Name} ({validatedList.MachineURL}) v{validatedList.Version} ({oldSummary.ModListHash} -> {validatedList.ModListHash})",
token);
}
if (oldSummary.Failures != validatedList.Failures)
{
if (validatedList.Failures == 0)
{
await _discord.SendAsync(Channel.Ham,
new DiscordMessage
{
Embeds = new[]
{
new DiscordEmbed
{
Title =
$"{validatedList.Name} (`{validatedList.MachineURL}`) is now passing.",
Url = new Uri(
$"https://github.com/wabbajack-tools/mod-lists/blob/master/reports/{validatedList.MachineURL}/status.md")
}
}
}, token);
}
else
{
await _discord.SendAsync(Channel.Ham,
new DiscordMessage
{
Embeds = new[]
{
new DiscordEmbed
{
Title =
$"Number of failures in {validatedList.Name} (`{validatedList.MachineURL}`) was {oldSummary.Failures} is now {validatedList.Failures}",
Url = new Uri(
$"https://github.com/wabbajack-tools/mod-lists/blob/master/reports/{validatedList.MachineURL}/status.md")
}
}
}, token);
}
}
}
catch (Exception ex)
{
_logger.LogCritical(ex, "While sending discord message for {MachineURl}", validatedList.MachineURL);
}
}
var summaries = validatedLists.Select(l => new ModListSummary
{
Failed = l.Archives.Count(f => f.Status == ArchiveStatus.InValid),
Mirrored = l.Archives.Count(f => f.Status == ArchiveStatus.Mirrored),
Passed = l.Archives.Count(f => f.Status == ArchiveStatus.Valid),
MachineURL = l.MachineURL,
Name = l.Name,
Updating = 0
}).ToArray();
await using var summaryFile = reports.Combine("modListSummary.json")
.Open(FileMode.Create, FileAccess.Write, FileShare.None);
await _dtos.Serialize(summaries, summaryFile, true);
var upgradedMetas = validatedLists.SelectMany(v => v.Archives)
.Where(a => a.Status is ArchiveStatus.Mirrored or ArchiveStatus.Updated)
.DistinctBy(a => a.Original.Hash)
.OrderBy(a => a.Original.Hash)
.ToArray();
await using var upgradedMetasFile = reports.Combine("upgraded.json")
.Open(FileMode.Create, FileAccess.Write, FileShare.None);
await _dtos.Serialize(upgradedMetas, upgradedMetasFile, true);
}
private async Task SendDefinitionToLoadOrderLibrary(ValidatedModList validatedModList, CancellationToken token)
{
var modlistMetadata = (await _wjClient.LoadLists())
.First(l => l.NamespacedName == validatedModList.MachineURL);
var modList = await StandardInstaller.Load(_dtos, _dispatcher, modlistMetadata, token);
await SendDefinitionToLoadOrderLibrary(modlistMetadata, modList, token);
}
private async Task DeleteOldMirrors(IEnumerable<Hash> mirroredFiles, IReadOnlySet<Hash> usedMirroredFiles)
{
foreach (var file in mirroredFiles.Where(file => !usedMirroredFiles.Contains(file)))
{
await _wjClient.DeleteMirror(file);
}
}
private async Task<(ArchiveStatus, Archive)> DownloadAndValidate(Archive archive,
ILookup<Hash, ForcedRemoval> forcedRemovals, CancellationToken token)
{
if (forcedRemovals.Contains(archive.Hash))
return (ArchiveStatus.InValid, archive);
switch (archive.State)
{
case GameFileSource:
return (ArchiveStatus.Valid, archive);
case Manual:
return (ArchiveStatus.Valid, archive);
case TESAlliance:
return (ArchiveStatus.Valid, archive);
case Mega:
return (ArchiveStatus.Valid, archive);
case Nexus:
return (ArchiveStatus.Valid, archive);
case VectorPlexus:
return (ArchiveStatus.Valid, archive);
}
if (archive.State is Http http && http.Url.Host.EndsWith("github.com"))
return (ArchiveStatus.Valid, archive);
try
{
for (var attempts = 0; attempts < 3; attempts++)
{
var valid = await _dispatcher.Verify(archive, token);
if (valid)
return (ArchiveStatus.Valid, archive);
var delay = _random.Next(200, 1200);
_logger.LogWarning(
"Archive {primaryKeyString} is invalid retrying in {Delay} ms ({Attempt} of {MaxAttempts})",
archive.State.PrimaryKeyString, delay, attempts, 3);
await Task.Delay(delay, token);
}
_logger.LogWarning("Archive {primaryKeyString} is invalid", archive.State.PrimaryKeyString);
return (ArchiveStatus.InValid, archive);
}
catch (Exception ex)
{
_logger.LogCritical(ex, "While verifying {primaryKeyString}", archive.State.PrimaryKeyString);
return (ArchiveStatus.InValid, archive);
}
}
private async Task<Hash> DownloadModList(ModlistMetadata modList, ArchiveManager archiveManager,
CancellationToken token)
{
if (archiveManager.HaveArchive(modList.DownloadMetadata!.Hash))
{
_logger.LogInformation("Previously downloaded {hash} not re-downloading", modList.NamespacedName);
return modList.DownloadMetadata!.Hash;
}
else
{
_logger.LogInformation("Downloading {hash}", modList.NamespacedName);
await _discord.SendAsync(Channel.Ham,
$"Downloading and ingesting {modList.Title} ({modList.NamespacedName}) v{modList.Version}", token);
return await DownloadWabbajackFile(modList, archiveManager, token);
}
}
private async Task<Hash> DownloadWabbajackFile(ModlistMetadata modList, ArchiveManager archiveManager,
CancellationToken token)
{
var state = _dispatcher.Parse(new Uri(modList.Links.Download));
if (state == null)
_logger.LogCritical("Can't download {url}", modList.Links.Download);
var archive = new Archive
{
State = state!,
Size = modList.DownloadMetadata!.Size,
Hash = modList.DownloadMetadata.Hash
};
await using var tempFile = _temporaryFileManager.CreateFile(Ext.Wabbajack);
_logger.LogInformation("Downloading {primaryKeyString}", state.PrimaryKeyString);
var hash = await _dispatcher.Download(archive, tempFile.Path, token);
if (hash != modList.DownloadMetadata.Hash)
{
_logger.LogCritical("Downloaded modlist was {actual} expected {expected}", hash,
modList.DownloadMetadata.Hash);
throw new Exception();
}
_logger.LogInformation("Archiving {hash}", hash);
await archiveManager.Ingest(tempFile.Path, token);
return hash;
}
public async ValueTask<HashSet<Hash>> AllMirroredFiles(CancellationToken token)
{
using var client = await GetMirrorFtpClient(token);
using var job = await _ftpRateLimiter.Begin("Getting mirror list", 0, token);
var files = await client.GetListingAsync(token);
var parsed = files.TryKeep(f => (Hash.TryGetFromHex(f.Name, out var hash), hash)).ToHashSet();
return parsed;
}
public async ValueTask<HashSet<(Hash, Hash)>> AllPatchFiles(CancellationToken token)
{
using var client = await GetPatchesFtpClient(token);
using var job = await _ftpRateLimiter.Begin("Getting patches list", 0, token);
var files = await client.GetListingAsync(token);
var parsed = files.TryKeep(f =>
{
var parts = f.Name.Split("_");
return (parts.Length == 2, parts);
})
.TryKeep(p => (Hash.TryGetFromHex(p[0], out var fromHash) &
Hash.TryGetFromHex(p[1], out var toHash),
(fromHash, toHash)))
.ToHashSet();
return parsed;
}
private async Task<FtpClient> GetMirrorFtpClient(CancellationToken token)
{
var client = await (await _ftpSiteCredentials.Get())![StorageSpace.Mirrors].GetClient(_logger);
await client.ConnectAsync(token);
return client;
}
private async Task<FtpClient> GetPatchesFtpClient(CancellationToken token)
{
var client = await (await _ftpSiteCredentials.Get())![StorageSpace.Patches].GetClient(_logger);
await client.ConnectAsync(token);
return client;
}
}