From db3b441d19f03e6381b2262e7ce6512abcbb5e60 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 30 Dec 2020 23:44:42 -0700 Subject: [PATCH] #### Version - 2.3.6.1 - 12/31/2020 * When IPS4 (e.g. LL) sites based on CEF fail to validate, they no longer hang the app * If a IPS4 CEF site throws a 503, or 400 error, retry * Clean out the cookies during IPS4 CEF downloads so that they don't cause 400 errors * Limit the number of connections to IPS4 sites to 20 per minute (one per 6 seconds) * If a site *does* timeout, throw a log of the CEF state into `CEFStates` for easier debugging by the WJ team * Wrote a new CLI utility to stress test the Verification routines. * Ignore files that have `\Edit Scripts\Export\` in their path --- CHANGELOG.md | 9 ++ Wabbajack.CLI/OptionsDefinition.cs | 4 +- Wabbajack.CLI/Verbs/AllKnownDownloadStates.cs | 36 +++++++ Wabbajack.CLI/Verbs/VerifyAllDownloads.cs | 100 ++++++++++++++++++ Wabbajack.CLI/Wabbajack.CLI.csproj | 4 +- Wabbajack.Common/Consts.cs | 2 + Wabbajack.Launcher/Wabbajack.Launcher.csproj | 4 +- Wabbajack.Lib/ACompiler.cs | 6 +- Wabbajack.Lib/ClientAPI.cs | 9 ++ .../Downloaders/AbstractDownloadState.cs | 3 +- .../Downloaders/AbstractIPS4Downloader.cs | 85 ++++++++++----- .../Downloaders/GameFileSourceDownloader.cs | 5 +- .../Downloaders/GoogleDriveDownloader.cs | 5 +- Wabbajack.Lib/Downloaders/HTTPDownloader.cs | 11 +- Wabbajack.Lib/Downloaders/MEGADownloader.cs | 3 +- Wabbajack.Lib/Downloaders/ManualDownloader.cs | 3 +- .../Downloaders/MediaFireDownloader.cs | 9 +- Wabbajack.Lib/Downloaders/ModDBDownloader.cs | 9 +- Wabbajack.Lib/Downloaders/NexusDownloader.cs | 3 +- .../Downloaders/SteamWorkshopDownloader.cs | 2 +- .../Downloaders/WabbajackCDNDownloader.cs | 8 +- Wabbajack.Lib/Downloaders/YandexDownloader.cs | 5 +- Wabbajack.Lib/Http/Client.cs | 29 ++--- Wabbajack.Lib/LibCefHelpers/Init.cs | 28 ++++- Wabbajack.Lib/MO2Compiler.cs | 1 + .../WebAutomation/CefSharpWrapper.cs | 81 +++++++++++--- Wabbajack.Lib/WebAutomation/IWebDriver.cs | 3 +- Wabbajack.Lib/WebAutomation/WebAutomation.cs | 42 +++++++- Wabbajack.Server/Controllers/Heartbeat.cs | 41 ++++++- Wabbajack.Server/Services/AbstractService.cs | 22 ++++ .../Services/ArchiveDownloader.cs | 41 +++++-- Wabbajack.Server/Services/ListValidator.cs | 7 +- .../Services/NonNexusDownloadValidator.cs | 13 ++- Wabbajack.Server/Services/QuickSync.cs | 4 +- Wabbajack.Server/Services/Watchdog.cs | 4 +- Wabbajack.Server/Wabbajack.Server.csproj | 4 +- Wabbajack.Test/DownloaderTests.cs | 28 ++++- Wabbajack/Wabbajack.csproj | 4 +- 38 files changed, 558 insertions(+), 119 deletions(-) create mode 100644 Wabbajack.CLI/Verbs/AllKnownDownloadStates.cs create mode 100644 Wabbajack.CLI/Verbs/VerifyAllDownloads.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index eace137c..35093a3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ ### Changelog +#### Version - 2.3.6.1 - 12/31/2020 +* When IPS4 (e.g. LL) sites based on CEF fail to validate, they no longer hang the app +* If a IPS4 CEF site throws a 503, or 400 error, retry +* Clean out the cookies during IPS4 CEF downloads so that they don't cause 400 errors +* Limit the number of connections to IPS4 sites to 20 per minute (one per 6 seconds) +* If a site *does* timeout, throw a log of the CEF state into `CEFStates` for easier debugging by the WJ team +* Wrote a new CLI utility to stress test the Verification routines. +* Ignore files that have `\Edit Scripts\Export\` in their path + #### Version - 2.3.6.0 - 12/29/2020 * Move the LoversLab downloader to a CEF based backed making it interact with CloudFlare a bit better * Add support for No Man's Sky diff --git a/Wabbajack.CLI/OptionsDefinition.cs b/Wabbajack.CLI/OptionsDefinition.cs index cb61728d..64031c98 100644 --- a/Wabbajack.CLI/OptionsDefinition.cs +++ b/Wabbajack.CLI/OptionsDefinition.cs @@ -35,7 +35,9 @@ namespace Wabbajack.CLI typeof(HashGamefiles), typeof(Backup), typeof(Restore), - typeof(PurgeArchive) + typeof(PurgeArchive), + typeof(AllKnownDownloadStates), + typeof(VerifyAllDownloads) }; } } diff --git a/Wabbajack.CLI/Verbs/AllKnownDownloadStates.cs b/Wabbajack.CLI/Verbs/AllKnownDownloadStates.cs new file mode 100644 index 00000000..b7167c16 --- /dev/null +++ b/Wabbajack.CLI/Verbs/AllKnownDownloadStates.cs @@ -0,0 +1,36 @@ +using System; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.FileUploader; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("all-known-download-states", HelpText = "Print known Ini info for a given hash")] + public class AllKnownDownloadStates : AVerb + { + [Option('i', "input", Required = true, HelpText = "Input Hash")] + public string _input { get; set; } = ""; + + public Hash Input => Hash.Interpret(_input); + protected override async Task Run() + { + var states = await ClientAPI.InferAllDownloadStates(Input); + Console.WriteLine($"Found {states.Length} states"); + + foreach (var archive in states) + { + Console.WriteLine("----"); + Console.WriteLine($"Name : {archive.State.PrimaryKeyString}"); + Console.WriteLine($"Is Valid: {await archive.State.Verify(archive)}"); + Console.WriteLine("------ Begin INI--------"); + Console.WriteLine(archive.State.GetMetaIniString()); + Console.WriteLine("------ End INI --------"); + } + + return ExitCode.Ok; + } + } +} diff --git a/Wabbajack.CLI/Verbs/VerifyAllDownloads.cs b/Wabbajack.CLI/Verbs/VerifyAllDownloads.cs new file mode 100644 index 00000000..a6397803 --- /dev/null +++ b/Wabbajack.CLI/Verbs/VerifyAllDownloads.cs @@ -0,0 +1,100 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.LibCefHelpers; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("verify-all-downloads", HelpText = "Verify all downloads in a folder")] + public class VerifyAllDownloads : AVerb + { + [Option('i', "input", Required = true, HelpText = "Input Folder")] + public string _input { get; set; } = ""; + + public AbsolutePath Input => (AbsolutePath)_input; + + [Option('t', "type", Required = false, + HelpText = "Only verify files of this type of download state for example NexusDownloader+State")] + public string StateType { get; set; } = ""; + + protected override async Task Run() + { + var files = Input.EnumerateFiles() + .Where(f => f.WithExtension(Consts.MetaFileExtension).Exists) + .ToArray(); + + Console.WriteLine($"Found {files.Length} files to verify"); + + using var queue = new WorkQueue(); + + var states = (await files.PMap(queue, async f => + { + var ini = f.WithExtension(Consts.MetaFileExtension).LoadIniFile(); + var state = (AbstractDownloadState?)await DownloadDispatcher.ResolveArchive(ini, quickMode: true); + if (state == default) + { + Console.WriteLine($"[Skipping] {f.FileName} because no meta could be interpreted"); + } + + if (!string.IsNullOrWhiteSpace(StateType) && !state!.PrimaryKeyString.StartsWith(StateType + "|")) + { + Console.WriteLine( + $"[Skipping] {f.FileName} because type {state.PrimaryKeyString[0]} does not match filter"); + return (f, null); + } + + return (f, state); + + })).Where(s => s.state != null) + .Select(s => (s.f, s.state!)) + .ToArray(); + + await DownloadDispatcher.PrepareAll(states.Select(s => s.Item2)); + Helpers.Init(); + + Console.WriteLine($"Found {states.Length} states to verify"); + int timedOut = 0; + + await states.PMap(queue, async p=> + { + try + { + var (f, state) = p; + + + try + { + var cts = new CancellationTokenSource(); + cts.CancelAfter(TimeSpan.FromMinutes(10)); + var result = + await state!.Verify(new Archive(state) {Name = f.FileName.ToString(), Size = f.Size}, + cts.Token); + Console.WriteLine($"[{(result ? "Failed" : "Passed")}] {f.FileName}"); + + } + catch (TaskCanceledException) + { + Console.WriteLine($"[Timed Out] {f.FileName} {state!.PrimaryKeyString}"); + Interlocked.Increment(ref timedOut); + } + } + catch (Exception ex) + { + Console.WriteLine($"[Exception] {p.f.FileName} {ex.Message}"); + } + + + }); + + Console.WriteLine($"[Total TimedOut] {timedOut}"); + Console.WriteLine("[Done]"); + + return ExitCode.Ok; + } + } +} diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index 459afe36..921a97de 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -6,8 +6,8 @@ wabbajack-cli Wabbajack x64 - 2.3.6.0 - 2.3.6.0 + 2.3.6.1 + 2.3.6.1 Copyright © 2019-2020 An automated ModList installer true diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index ea905001..803cbe6a 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -131,6 +131,8 @@ namespace Wabbajack.Common public static AbsolutePath SettingsFile => LocalAppDataPath.Combine("settings.json"); public static RelativePath SettingsIni = (RelativePath)"settings.ini"; public static byte SettingsVersion => 2; + public static TimeSpan MaxVerifyTime => TimeSpan.FromMinutes(10); + public static RelativePath NativeSettingsJson = (RelativePath)"native_compiler_settings.json"; public static bool IsServer = false; diff --git a/Wabbajack.Launcher/Wabbajack.Launcher.csproj b/Wabbajack.Launcher/Wabbajack.Launcher.csproj index 0f7d4be2..4125eeb4 100644 --- a/Wabbajack.Launcher/Wabbajack.Launcher.csproj +++ b/Wabbajack.Launcher/Wabbajack.Launcher.csproj @@ -4,8 +4,8 @@ WinExe netcoreapp3.1 true - 2.3.6.0 - 2.3.6.0 + 2.3.6.1 + 2.3.6.1 Copyright © 2019-2020 Wabbajack Application Launcher true diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index bc263763..445ab884 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -8,7 +8,9 @@ using System.Linq; using System.Reactive.Subjects; using System.Reflection; using System.Text; +using System.Threading; using System.Threading.Tasks; +using Org.BouncyCastle.Asn1.Cms; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.Downloaders; @@ -511,7 +513,9 @@ namespace Wabbajack.Lib await result.State!.GetDownloader().Prepare(); - if (result.State != null && !await result.State.Verify(result)) + var token = new CancellationTokenSource(); + token.CancelAfter(Consts.MaxVerifyTime); + if (result.State != null && !await result.State.Verify(result, token.Token)) Error( $"Unable to resolve link for {archive.Name}. If this is hosted on the Nexus the file may have been removed."); diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index 3984284c..f866f53b 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -210,6 +210,15 @@ using Wabbajack.Lib.Downloaders; return null; } + public static async Task InferAllDownloadStates(Hash hash) + { + var client = await GetClient(); + + var results = await client.GetJsonAsync( + $"{Consts.WabbajackBuildServerUri}mod_files/by_hash/{hash.ToHex()}"); + return results; + } + public static async Task GetModUpgrades(Hash src) { var client = await GetClient(); diff --git a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs index f3954c40..4d5c8543 100644 --- a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs +++ b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json; using Wabbajack.Common; @@ -89,7 +90,7 @@ namespace Wabbajack.Lib.Downloaders /// Returns true if this link is still valid /// /// - public abstract Task Verify(Archive archive); + public abstract Task Verify(Archive archive, CancellationToken? token = null); public abstract IDownloader GetDownloader(); diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs index 9b77f50d..fc14035c 100644 --- a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Net; using System.Net.Http; using System.Text.RegularExpressions; +using System.Threading; using System.Threading.Tasks; using System.Web; using F23.StringSimilarity; @@ -13,17 +14,27 @@ using Newtonsoft.Json; using Wabbajack.Common; using Wabbajack.Lib.LibCefHelpers; using Wabbajack.Lib.Validation; +using Wabbajack.Lib.WebAutomation; namespace Wabbajack.Lib.Downloaders { + + interface IWaitForWindowDownloader + { + public Task WaitForNextRequestWindow(); + } // IPS4 is the site used by LoversLab, VectorPlexus, etc. the general mechanics of each site are the // same, so we can fairly easily abstract the state. // Pass in the state type via TState - public abstract class AbstractIPS4Downloader : AbstractNeedsLoginDownloader, IDownloader + public abstract class AbstractIPS4Downloader : AbstractNeedsLoginDownloader, IDownloader, IWaitForWindowDownloader where TState : AbstractIPS4Downloader.State, new() where TDownloader : IDownloader { + private DateTime LastRequestTime = default; + protected long RequestsPerMinute = 20; + private TimeSpan RequestDelay => TimeSpan.FromMinutes(1) / RequestsPerMinute; + protected AbstractIPS4Downloader(Uri loginUri, string encryptedKeyName, string cookieDomain, string loginCookie = "ips4_member_id") : base(loginUri, encryptedKeyName, cookieDomain, loginCookie) { @@ -35,6 +46,27 @@ namespace Wabbajack.Lib.Downloaders return await GetDownloaderStateFromUrl(url, quickMode); } + public async Task WaitForNextRequestWindow() + { + TimeSpan delay; + lock (this) + { + if (LastRequestTime < DateTime.UtcNow - RequestDelay) + { + LastRequestTime = DateTime.UtcNow; + delay = TimeSpan.Zero; + } + else + { + LastRequestTime += RequestDelay; + delay = LastRequestTime - DateTime.UtcNow; + } + } + + Utils.Log($"Waiting for {delay.TotalSeconds} to make request via {typeof(TDownloader).Name}"); + await Task.Delay(delay); + } + public async Task GetDownloaderStateFromUrl(Uri url, bool quickMode) { var absolute = true; @@ -150,10 +182,11 @@ namespace Wabbajack.Lib.Downloaders public override async Task Download(Archive a, AbsolutePath destination) { + await ((IWaitForWindowDownloader)Downloader).WaitForNextRequestWindow(); return await ResolveDownloadStream(a, destination, false); } - private async Task ResolveDownloadStream(Archive a, AbsolutePath path, bool quickMode) + private async Task ResolveDownloadStream(Archive a, AbsolutePath path, bool quickMode, CancellationToken? token = null) { @@ -168,7 +201,7 @@ namespace Wabbajack.Lib.Downloaders var csrfURL = string.IsNullOrWhiteSpace(FileID) ? $"{Site}/files/file/{FileName}/?do=download" : $"{Site}/files/file/{FileName}/?do=download&r={FileID}"; - var html = await GetStringAsync(new Uri(csrfURL)); + var html = await GetStringAsync(new Uri(csrfURL), token); var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])"); var matches = pattern.Matches(html).Cast(); @@ -182,25 +215,15 @@ namespace Wabbajack.Lib.Downloaders } var sep = Site.EndsWith("?") ? "&" : "?"; - if (!Downloader.IsCloudFlareProtected) - { - - url = FileID == null - ? $"{Site}/files/file/{FileName}/{sep}do=download&confirm=1&t=1&csrfKey={csrfKey}" - : $"{Site}/files/file/{FileName}/{sep}do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}"; - } - else - { - url = FileID == null - ? $"{Site}/files/file/{FileName}/{sep}do=download&confirm=1&t=1" - : $"{Site}/files/file/{FileName}/{sep}do=download&r={FileID}&confirm=1&t=1"; - } + url = FileID == null + ? $"{Site}/files/file/{FileName}/{sep}do=download&confirm=1&t=1&csrfKey={csrfKey}" + : $"{Site}/files/file/{FileName}/{sep}do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}"; } if (Downloader.IsCloudFlareProtected) { using var driver = await Downloader.GetAuthedDriver(); - var size = await driver.NavigateToAndDownload(new Uri(url), path, quickMode: quickMode); + var size = await driver.NavigateToAndDownload(new Uri(url), path, quickMode: quickMode, token); if (a.Size == 0 || size == 0 || a.Size == size) return true; @@ -268,6 +291,11 @@ namespace Wabbajack.Lib.Downloaders goto TOP; } + private async Task DeleteOldDownloadCookies(Driver driver) + { + await driver.DeleteCookiesWhere(c => c.Name.StartsWith("ips4_downloads_delay_") && c.Value == "-1"); + } + private class WaitResponse { [JsonProperty("download")] @@ -276,10 +304,11 @@ namespace Wabbajack.Lib.Downloaders public int CurrentTime { get; set; } } - public override async Task Verify(Archive a) + public override async Task Verify(Archive a, CancellationToken? token) { + await ((IWaitForWindowDownloader)Downloader).WaitForNextRequestWindow(); await using var tp = new TempFile(); - var isValid = await ResolveDownloadStream(a, tp.Path, true); + var isValid = await ResolveDownloadStream(a, tp.Path, true, token: token); return isValid; } @@ -290,6 +319,7 @@ namespace Wabbajack.Lib.Downloaders public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func> downloadResolver) { + await ((IWaitForWindowDownloader)Downloader).WaitForNextRequestWindow(); var files = await GetFilesInGroup(); var nl = new Levenshtein(); @@ -321,7 +351,10 @@ namespace Wabbajack.Lib.Downloaders public async Task> GetFilesInGroup() { - var others = await GetHtmlAsync(new Uri($"{Site}/files/file/{FileName}?do=download")); + var token = new CancellationTokenSource(); + token.CancelAfter(TimeSpan.FromMinutes(10)); + + var others = await GetHtmlAsync(new Uri($"{Site}/files/file/{FileName}?do=download"), token.Token); var pairs = others.DocumentNode.SelectNodes("//a[@data-action='download']") .Select(item => (item.GetAttributeValue("href", ""), @@ -348,18 +381,22 @@ namespace Wabbajack.Lib.Downloaders return IsAttachment ? FullURL : $"{Site}/files/file/{FileName}/?do=download&r={FileID}"; } - public async Task GetStringAsync(Uri uri) + public async Task GetStringAsync(Uri uri, CancellationToken? token = null) { if (!Downloader.IsCloudFlareProtected) return await Downloader.AuthedClient.GetStringAsync(uri); using var driver = await Downloader.GetAuthedDriver(); + await DeleteOldDownloadCookies(driver); + //var drivercookies = await Helpers.GetCookies("loverslab.com"); //var cookies = await ClientAPI.GetAuthInfo("loverslabcookies"); //await Helpers.IngestCookies(uri.ToString(), cookies); - await driver.NavigateTo(uri); + await driver.NavigateTo(uri, token); + if ((token ?? CancellationToken.None).IsCancellationRequested) + return ""; var source = await driver.GetSourceAsync(); @@ -380,9 +417,9 @@ namespace Wabbajack.Lib.Downloaders return source; } - public async Task GetHtmlAsync(Uri s) + public async Task GetHtmlAsync(Uri s, CancellationToken? token = null) { - var body = await GetStringAsync(s); + var body = await GetStringAsync(s, token); var doc = new HtmlDocument(); doc.LoadHtml(body); return doc; diff --git a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs index 28f29ccc..0753e010 100644 --- a/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +using System.Threading; +using System.Threading.Tasks; using Newtonsoft.Json; using Wabbajack.Common; using Wabbajack.Common.Serialization.Json; @@ -79,7 +80,7 @@ namespace Wabbajack.Lib.Downloaders return true; } - public override async Task Verify(Archive a) + public override async Task Verify(Archive a, CancellationToken? token) { return SourcePath.Exists && await SourcePath.FileHashCachedAsync() == Hash; } diff --git a/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs b/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs index 1c8f0d3f..2aedfe21 100644 --- a/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs +++ b/Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs @@ -1,4 +1,5 @@ using System.Text.RegularExpressions; +using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json; using Wabbajack.Common; @@ -70,10 +71,10 @@ namespace Wabbajack.Lib.Downloaders return httpState; } - public override async Task Verify(Archive a) + public override async Task Verify(Archive a, CancellationToken? token) { var state = await ToHttpState(); - return await state.Verify(a); + return await state.Verify(a, token); } public override IDownloader GetDownloader() diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index d8956f10..78dbd700 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Net.Http; using System.Net.Http.Headers; +using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json; using Wabbajack.Common; @@ -74,7 +75,7 @@ namespace Wabbajack.Lib.Downloaders return DoDownload(a, destination, true); } - public async Task DoDownload(Archive a, AbsolutePath destination, bool download) + public async Task DoDownload(Archive a, AbsolutePath destination, bool download, CancellationToken? token = null) { if (download) { @@ -98,7 +99,7 @@ namespace Wabbajack.Lib.Downloaders var bufferSize = 1024 * 32 * 8; Utils.Status($"Starting Download {a.Name ?? Url}", Percent.Zero); - var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false); + var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false, token:token); TOP: if (!response.IsSuccessStatusCode) @@ -143,7 +144,7 @@ TOP: var buffer = new byte[bufferSize]; int readThisCycle = 0; - while (true) + while (!(token ?? CancellationToken.None).IsCancellationRequested) { int read = 0; try @@ -188,9 +189,9 @@ TOP: } } - public override async Task Verify(Archive a) + public override async Task Verify(Archive a, CancellationToken? token) { - return await DoDownload(a, ((RelativePath)"").RelativeToEntryPoint(), false); + return await DoDownload(a, ((RelativePath)"").RelativeToEntryPoint(), false, token: token); } public override IDownloader GetDownloader() diff --git a/Wabbajack.Lib/Downloaders/MEGADownloader.cs b/Wabbajack.Lib/Downloaders/MEGADownloader.cs index c1404cc0..ae4fd48f 100644 --- a/Wabbajack.Lib/Downloaders/MEGADownloader.cs +++ b/Wabbajack.Lib/Downloaders/MEGADownloader.cs @@ -2,6 +2,7 @@ using System.Reactive; using System.Reactive.Linq; using System.Security; +using System.Threading; using System.Threading.Tasks; using CG.Web.MegaApiClient; using Newtonsoft.Json; @@ -178,7 +179,7 @@ namespace Wabbajack.Lib.Downloaders return true; } - public override async Task Verify(Archive a) + public override async Task Verify(Archive a, CancellationToken? token) { await MegaLogin(); diff --git a/Wabbajack.Lib/Downloaders/ManualDownloader.cs b/Wabbajack.Lib/Downloaders/ManualDownloader.cs index 197ed996..40adf6c6 100644 --- a/Wabbajack.Lib/Downloaders/ManualDownloader.cs +++ b/Wabbajack.Lib/Downloaders/ManualDownloader.cs @@ -1,6 +1,7 @@ using System.IO; using System.Linq; using System.Reactive.Subjects; +using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json; using Wabbajack.Common; @@ -93,7 +94,7 @@ namespace Wabbajack.Lib.Downloaders return await state.Download(a, destination); } - public override async Task Verify(Archive a) + public override async Task Verify(Archive a, CancellationToken? token) { return true; } diff --git a/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs b/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs index d6517dfa..f5a34673 100644 --- a/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs +++ b/Wabbajack.Lib/Downloaders/MediaFireDownloader.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Net.Http; +using System.Threading; using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Common.Serialization.Json; @@ -43,15 +44,15 @@ namespace Wabbajack.Lib.Downloaders return await result.Download(a, destination); } - public override async Task Verify(Archive a) + public override async Task Verify(Archive a, CancellationToken? token) { - return await Resolve() != null; + return await Resolve(token) != null; } - private async Task Resolve() + private async Task Resolve(CancellationToken? token = null) { var client = new Http.Client(); - var result = await client.GetAsync(Url, HttpCompletionOption.ResponseHeadersRead); + var result = await client.GetAsync(Url, HttpCompletionOption.ResponseHeadersRead, token:token); if (!result.IsSuccessStatusCode) return null; diff --git a/Wabbajack.Lib/Downloaders/ModDBDownloader.cs b/Wabbajack.Lib/Downloaders/ModDBDownloader.cs index 9d0022ae..9c20455b 100644 --- a/Wabbajack.Lib/Downloaders/ModDBDownloader.cs +++ b/Wabbajack.Lib/Downloaders/ModDBDownloader.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Threading; using System.Threading.Tasks; using HtmlAgilityPack; using Newtonsoft.Json; @@ -71,12 +72,12 @@ namespace Wabbajack.Lib.Downloaders return false; } - private async Task GetDownloadUrls() + private async Task GetDownloadUrls(CancellationToken? token = null) { var uri = new Uri(Url); var modId = uri.AbsolutePath.Split('/').Reverse().First(f => int.TryParse(f, out int _)); var mirrorUrl = $"https://www.moddb.com/downloads/start/{modId}/all"; - var doc = await new HtmlWeb().LoadFromWebAsync($"https://www.moddb.com/downloads/start/{modId}/all"); + var doc = await new HtmlWeb().LoadFromWebAsync($"https://www.moddb.com/downloads/start/{modId}/all", token ?? CancellationToken.None); var mirrors = doc.DocumentNode.Descendants().Where(d => d.NodeType == HtmlNodeType.Element && d.HasClass("row")) .Select(d => new { @@ -98,9 +99,9 @@ namespace Wabbajack.Lib.Downloaders return mirrors.Select(d => d.Link).ToArray(); } - public override async Task Verify(Archive a) + public override async Task Verify(Archive a, CancellationToken? token) { - await GetDownloadUrls(); + await GetDownloadUrls(token); return true; } diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index c16476a0..cefd68d0 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -4,6 +4,7 @@ using System.ComponentModel.DataAnnotations; using System.Linq; using System.Reactive; using System.Reactive.Linq; +using System.Threading; using System.Threading.Tasks; using F23.StringSimilarity; using Newtonsoft.Json; @@ -213,7 +214,7 @@ namespace Wabbajack.Lib.Downloaders return await new HTTPDownloader.State(url).Download(a, destination); } - public override async Task Verify(Archive a) + public override async Task Verify(Archive a, CancellationToken? token = null) { try { diff --git a/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs b/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs index 20db6b44..32dcda85 100644 --- a/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs +++ b/Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs @@ -87,7 +87,7 @@ namespace Wabbajack.Lib.Downloaders return true; } - public override async Task Verify(Archive a) + public override async Task Verify(Archive a, CancellationToken? token) { //TODO: find a way to verify steam workshop items throw new NotImplementedException(); diff --git a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs index fa3430ae..7d95abbf 100644 --- a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs +++ b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs @@ -108,9 +108,9 @@ namespace Wabbajack.Lib.Downloaders return true; } - public override async Task Verify(Archive archive) + public override async Task Verify(Archive archive, CancellationToken? token) { - var definition = await GetDefinition(); + var definition = await GetDefinition(token); return true; } @@ -151,13 +151,13 @@ namespace Wabbajack.Lib.Downloaders return builder.ToString(); } - private async Task GetDefinition() + private async Task GetDefinition(CancellationToken? token = null) { var client = new Wabbajack.Lib.Http.Client(); if (DomainRemaps.TryGetValue(Url.Host, out var remap)) { var builder = new UriBuilder(Url) {Host = remap}; - using var data = await client.GetAsync(builder + "/definition.json.gz"); + using var data = await client.GetAsync(builder + "/definition.json.gz", token: token); await using var gz = new GZipStream(await data.Content.ReadAsStreamAsync(), CompressionMode.Decompress); return gz.FromJson(); diff --git a/Wabbajack.Lib/Downloaders/YandexDownloader.cs b/Wabbajack.Lib/Downloaders/YandexDownloader.cs index b309495f..6b80bd3b 100644 --- a/Wabbajack.Lib/Downloaders/YandexDownloader.cs +++ b/Wabbajack.Lib/Downloaders/YandexDownloader.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Common.Serialization.Json; @@ -59,10 +60,10 @@ namespace Wabbajack.Lib.Downloaders return await new HTTPDownloader.State(uri!.ToString()).Download(destination); } - public override async Task Verify(Archive archive) + public override async Task Verify(Archive archive, CancellationToken? token) { var client = new Wabbajack.Lib.Http.Client(); - var result = await client.GetAsync(Url, errorsAsExceptions: false); + var result = await client.GetAsync(Url, errorsAsExceptions: false, token: token); return result.IsSuccessStatusCode; } diff --git a/Wabbajack.Lib/Http/Client.cs b/Wabbajack.Lib/Http/Client.cs index ee37589b..3fdf2613 100644 --- a/Wabbajack.Lib/Http/Client.cs +++ b/Wabbajack.Lib/Http/Client.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Net; using System.Net.Http; +using System.Threading; using System.Threading.Tasks; using HtmlAgilityPack; using Wabbajack.Common; @@ -16,16 +17,16 @@ namespace Wabbajack.Lib.Http { public List<(string, string?)> Headers = new List<(string, string?)>(); public List Cookies = new List(); - public async Task GetAsync(string url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, bool retry = true) + public async Task GetAsync(string url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, bool retry = true, CancellationToken? token = null) { var request = new HttpRequestMessage(HttpMethod.Get, url); - return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions, retry: retry); + return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions, retry: retry, token: token); } - public async Task GetAsync(Uri url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true) + public async Task GetAsync(Uri url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, CancellationToken? token = null) { var request = new HttpRequestMessage(HttpMethod.Get, url); - return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions); + return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions, token:token); } @@ -41,10 +42,10 @@ namespace Wabbajack.Lib.Http return await SendAsync(request, responseHeadersRead); } - public async Task GetStringAsync(string url) + public async Task GetStringAsync(string url, CancellationToken? token = null) { var request = new HttpRequestMessage(HttpMethod.Get, url); - return await SendStringAsync(request); + return await SendStringAsync(request, token: token); } public async Task GetStringAsync(Uri url) @@ -59,9 +60,9 @@ namespace Wabbajack.Lib.Http return await SendStringAsync(request); } - private async Task SendStringAsync(HttpRequestMessage request) + private async Task SendStringAsync(HttpRequestMessage request, CancellationToken? token = null) { - using var result = await SendAsync(request); + using var result = await SendAsync(request, token: token); if (!result.IsSuccessStatusCode) { Utils.Log("Internal Error"); @@ -72,7 +73,7 @@ namespace Wabbajack.Lib.Http return await result.Content.ReadAsStringAsync(); } - public async Task SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, bool retry = true) + public async Task SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, bool retry = true, CancellationToken? token = null) { foreach (var (k, v) in Headers) msg.Headers.Add(k, v); @@ -83,7 +84,7 @@ namespace Wabbajack.Lib.Http TOP: try { - response = await ClientFactory.Client.SendAsync(msg, responseHeadersRead); + response = await ClientFactory.Client.SendAsync(msg, responseHeadersRead, token ?? CancellationToken.None); if (response.IsSuccessStatusCode) return response; if (errorsAsExceptions) @@ -105,7 +106,7 @@ namespace Wabbajack.Lib.Http var ms = Utils.NextRandom(100, 1000); Utils.Log($"Got a {http.Code} from {msg.RequestUri} retrying in {ms}ms"); - await Task.Delay(ms); + await Task.Delay(ms, token ?? CancellationToken.None); msg = CloneMessage(msg); goto TOP; } @@ -114,7 +115,7 @@ namespace Wabbajack.Lib.Http retries++; Utils.LogStraightToFile(ex.ToString()); Utils.Log($"Http Connect error to {msg.RequestUri} retry {retries}"); - await Task.Delay(100 * retries); + await Task.Delay(100 * retries, token ?? CancellationToken.None); msg = CloneMessage(msg); goto TOP; @@ -138,9 +139,9 @@ namespace Wabbajack.Lib.Http return result.FromJsonString(); } - public async Task GetHtmlAsync(string s) + public async Task GetHtmlAsync(string s, CancellationToken? token = null) { - var body = await GetStringAsync(s); + var body = await GetStringAsync(s, token: token); var doc = new HtmlDocument(); doc.LoadHtml(body); return doc; diff --git a/Wabbajack.Lib/LibCefHelpers/Init.cs b/Wabbajack.Lib/LibCefHelpers/Init.cs index 259c33e7..c14067d4 100644 --- a/Wabbajack.Lib/LibCefHelpers/Init.cs +++ b/Wabbajack.Lib/LibCefHelpers/Init.cs @@ -103,17 +103,43 @@ namespace Wabbajack.Lib.LibCefHelpers var visitor = new CookieDeleter(); manager.VisitAllCookies(visitor); } + + public static async Task DeleteCookiesWhere(Func filter) + { + var manager = Cef.GetGlobalCookieManager(); + var visitor = new CookieDeleter(filter); + manager.VisitAllCookies(visitor); + } } class CookieDeleter : ICookieVisitor { + private Func? _filter; + + public CookieDeleter(Func? filter = null) + { + _filter = filter; + } public void Dispose() { } public bool Visit(Cookie cookie, int count, int total, ref bool deleteCookie) { - deleteCookie = true; + if (_filter == null) + { + deleteCookie = true; + } + else + { + var conv = new Helpers.Cookie + { + Name = cookie.Name, Domain = cookie.Domain, Value = cookie.Value, Path = cookie.Path + }; + if (_filter(conv)) + deleteCookie = true; + } + return true; } } diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index f0340d12..e8be9553 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -474,6 +474,7 @@ namespace Wabbajack.Lib new IncludeAllConfigs(this), new zEditIntegration.IncludeZEditPatches(this), new IncludeTaggedMods(this, Consts.WABBAJACK_NOMATCH_INCLUDE), + new IgnorePathContains(this,@"\Edit Scripts\Export\"), new DropAll(this) }; } diff --git a/Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs b/Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs index 815a1356..0edc34fb 100644 --- a/Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs +++ b/Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs @@ -3,10 +3,13 @@ using System.Collections.Generic; using System.Linq; using System.Net; using System.Net.Http; +using System.Net.Http.Headers; using System.Text; +using System.Threading; using System.Threading.Tasks; using CefSharp; using Wabbajack.Common; +using Wabbajack.Common.Exceptions; using Wabbajack.Lib.LibCefHelpers; namespace Wabbajack.Lib.WebAutomation @@ -23,36 +26,78 @@ namespace Wabbajack.Lib.WebAutomation _browser.LifeSpanHandler = new PopupBlocker(this); } - public Task NavigateTo(Uri uri) + public Task NavigateTo(Uri uri, CancellationToken? token = null) { var tcs = new TaskCompletionSource(); EventHandler? handler = null; handler = (sender, e) => { - if (!e.IsLoading) - { - _browser.LoadingStateChanged -= handler; - tcs.SetResult(true); - } + if (e.IsLoading) return; + + _browser.LoadingStateChanged -= handler; + tcs.SetResult(true); }; _browser.LoadingStateChanged += handler; _browser.Load(uri.ToString()); - + token?.Register(() => tcs.TrySetCanceled()); + return tcs.Task; } - public async Task NavigateToAndDownload(Uri uri, AbsolutePath dest, bool quickMode = false) + private readonly string[] KnownServerLoadStrings = + { + "

Temporarily Unavailable

", + "
Request Header Or Cookie Too Large
", + //"" + }; + private readonly (string, int)[] KnownErrorStrings = + { + ("

400 Bad Request

", 400), + ("We could not locate the item you are trying to view.", 404) + }; + private static readonly Random RetryRandom = new Random(); + + public async Task NavigateToAndDownload(Uri uri, AbsolutePath dest, bool quickMode = false, CancellationToken? token = null) { var oldCB = _browser.DownloadHandler; - var handler = new ReroutingDownloadHandler(this, dest, quickMode: quickMode); + var handler = new ReroutingDownloadHandler(this, dest, quickMode: quickMode, token); _browser.DownloadHandler = handler; try { - await NavigateTo(uri); - return await handler.Task; + int retryCount = 0; + RETRY: + await NavigateTo(uri, token); + var source = await _browser.GetSourceAsync(); + foreach (var err in KnownServerLoadStrings) + { + if (!source.Contains(err)) + continue; + + if ((token?.IsCancellationRequested) == true) + { + throw new TimeoutException(); + } + else + { + retryCount += 1; + var retry = RetryRandom.Next(retryCount * 5000, retryCount * 5000 * 2); + Utils.Log($"Got server load error from {uri} retying in {retry}ms [{err}]"); + await Task.Delay(TimeSpan.FromMilliseconds(retry)); + goto RETRY; + } + } + + foreach (var (err, httpCode) in KnownErrorStrings) + { + if (source.Contains(err)) + throw new HttpException(httpCode,$"Web driver failed: {err}"); + } + + Utils.Log($"Loaded page {uri} starting download..."); + return await handler.TaskResult; } finally { _browser.DownloadHandler = oldCB; @@ -83,6 +128,7 @@ namespace Wabbajack.Lib.WebAutomation } public string Location => _browser.Address; + } public class PopupBlocker : ILifeSpanHandler @@ -123,19 +169,24 @@ namespace Wabbajack.Lib.WebAutomation private AbsolutePath _path; public TaskCompletionSource _tcs = new TaskCompletionSource(); private bool _quickMode; - public Task Task => _tcs.Task; + private CancellationToken? _cancelationToken; + private TimeSpan _downloadTimeout; + public Task TaskResult => _tcs.Task; - public ReroutingDownloadHandler(CefSharpWrapper wrapper, AbsolutePath path, bool quickMode) + public ReroutingDownloadHandler(CefSharpWrapper wrapper, AbsolutePath path, bool quickMode, CancellationToken? token) { _wrapper = wrapper; _path = path; _quickMode = quickMode; + _cancelationToken = token; + token?.Register(() => _tcs.TrySetCanceled()); } public void OnBeforeDownload(IWebBrowser chromiumWebBrowser, IBrowser browser, DownloadItem downloadItem, IBeforeDownloadCallback callback) { if (_quickMode) return; + callback.Continue(_path.ToString(), false); } @@ -145,12 +196,12 @@ namespace Wabbajack.Lib.WebAutomation if (_quickMode) { callback.Cancel(); - _tcs.SetResult(downloadItem.TotalBytes); + _tcs.TrySetResult(downloadItem.TotalBytes); return; } if (downloadItem.IsComplete) - _tcs.SetResult(downloadItem.TotalBytes); + _tcs.TrySetResult(downloadItem.TotalBytes); callback.Resume(); } } diff --git a/Wabbajack.Lib/WebAutomation/IWebDriver.cs b/Wabbajack.Lib/WebAutomation/IWebDriver.cs index 529a04cb..3ec08fa6 100644 --- a/Wabbajack.Lib/WebAutomation/IWebDriver.cs +++ b/Wabbajack.Lib/WebAutomation/IWebDriver.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Text; +using System.Threading; using System.Threading.Tasks; using Wabbajack.Lib.LibCefHelpers; @@ -10,7 +11,7 @@ namespace Wabbajack.Lib.WebAutomation { public interface IWebDriver { - Task NavigateTo(Uri uri); + Task NavigateTo(Uri uri, CancellationToken? token = null); Task EvaluateJavaScript(string text); Task GetCookies(string domainPrefix); public Action? DownloadHandler { get; set; } diff --git a/Wabbajack.Lib/WebAutomation/WebAutomation.cs b/Wabbajack.Lib/WebAutomation/WebAutomation.cs index 4e1cd770..b1c7d22f 100644 --- a/Wabbajack.Lib/WebAutomation/WebAutomation.cs +++ b/Wabbajack.Lib/WebAutomation/WebAutomation.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Windows; @@ -28,15 +29,41 @@ namespace Wabbajack.Lib.WebAutomation return driver; } - public async Task NavigateTo(Uri uri) + public async Task NavigateTo(Uri uri, CancellationToken? token = null) { - await _driver.NavigateTo(uri); - return await GetLocation(); + try + { + await _driver.NavigateTo(uri, token); + return await GetLocation(); + } + catch (TaskCanceledException ex) + { + await DumpState(uri, ex); + throw; + } } - public async Task NavigateToAndDownload(Uri uri, AbsolutePath absolutePath, bool quickMode = false) + private async Task DumpState(Uri uri, Exception ex) { - return await _driver.NavigateToAndDownload(uri, absolutePath, quickMode: quickMode); + var file = AbsolutePath.EntryPoint.Combine("CEFStates", DateTime.UtcNow.ToFileTimeUtc().ToString()) + .WithExtension(new Extension(".html")); + file.Parent.CreateDirectory(); + var source = await GetSourceAsync(); + var cookies = await Helpers.GetCookies(); + var cookiesString = string.Join('\n', cookies.Select(c => c.Name + " - " + c.Value)); + await file.WriteAllTextAsync(uri + "\n " + source + "\n" + ex + "\n" + cookiesString); + } + + public async Task NavigateToAndDownload(Uri uri, AbsolutePath absolutePath, bool quickMode = false, CancellationToken? token = null) + { + try + { + return await _driver.NavigateToAndDownload(uri, absolutePath, quickMode: quickMode, token: token); + } + catch (TaskCanceledException ex) { + await DumpState(uri, ex); + throw; + } } public async ValueTask GetLocation() @@ -79,5 +106,10 @@ namespace Wabbajack.Lib.WebAutomation { Helpers.ClearCookies(); } + + public async Task DeleteCookiesWhere(Func filter) + { + await Helpers.DeleteCookiesWhere(filter); + } } } diff --git a/Wabbajack.Server/Controllers/Heartbeat.cs b/Wabbajack.Server/Controllers/Heartbeat.cs index a46aacbf..2ed43793 100644 --- a/Wabbajack.Server/Controllers/Heartbeat.cs +++ b/Wabbajack.Server/Controllers/Heartbeat.cs @@ -71,9 +71,9 @@ namespace Wabbajack.BuildServer.Controllers @@ -109,8 +109,45 @@ namespace Wabbajack.BuildServer.Controllers }; } + + private static readonly Func HandleGetServiceReport = NettleEngine.GetCompiler().Compile(@" + +

Service Status: {{Name}} {{TimeSinceLastRun}}

+

Service Overview ({{ActiveWorkQueue.Length}}):

+
    + {{each $.ActiveWorkQueue }} +
  • {{$.Name}} {{$.Time}}
    • + {{/each}} +
+ + "); + [HttpGet("report/services/{serviceName}.html")] + public async Task ReportServiceStatus(string serviceName) + { + var services = await _quickSync.Report(); + var info = services.First(kvp => kvp.Key.Name == serviceName); + + var response = HandleGetServiceReport(new + { + Name = info.Key.Name, + TimeSinceLastRun = DateTime.UtcNow - info.Value.LastRunTime, + ActiveWorkQueue = info.Value.ActiveWork.Select(p => new + { + Name = p.Item1, + Time = DateTime.UtcNow - p.Item2 + }).OrderByDescending(kp => kp.Time) + .ToArray() + + }); + return new ContentResult + { + ContentType = "text/html", + StatusCode = (int) HttpStatusCode.OK, + Content = response + }; + } } } diff --git a/Wabbajack.Server/Services/AbstractService.cs b/Wabbajack.Server/Services/AbstractService.cs index 5f854c07..8e91b7eb 100644 --- a/Wabbajack.Server/Services/AbstractService.cs +++ b/Wabbajack.Server/Services/AbstractService.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Logging; @@ -16,6 +17,8 @@ namespace Wabbajack.Server.Services public TimeSpan Delay { get; } public DateTime LastStart { get; } public DateTime LastEnd { get; } + + public (String, DateTime)[] ActiveWorkStatus { get; } } @@ -30,6 +33,7 @@ namespace Wabbajack.Server.Services public TimeSpan Delay => _delay; public DateTime LastStart { get; private set; } public DateTime LastEnd { get; private set; } + public (String, DateTime)[] ActiveWorkStatus { get; private set; }= { }; public AbstractService(ILogger logger, AppSettings settings, QuickSync quickSync, TimeSpan delay) { @@ -85,6 +89,22 @@ namespace Wabbajack.Server.Services } public abstract Task Execute(); + + protected void ReportStarting(string value) + { + lock (this) + { + ActiveWorkStatus = ActiveWorkStatus.Cons((value, DateTime.UtcNow)).ToArray(); + } + } + + protected void ReportEnding(string value) + { + lock (this) + { + ActiveWorkStatus = ActiveWorkStatus.Where(x => x.Item1 != value).ToArray(); + } + } } public static class AbstractServiceExtensions @@ -96,4 +116,6 @@ namespace Wabbajack.Server.Services } } + + } diff --git a/Wabbajack.Server/Services/ArchiveDownloader.cs b/Wabbajack.Server/Services/ArchiveDownloader.cs index 122df26b..ccd804df 100644 --- a/Wabbajack.Server/Services/ArchiveDownloader.cs +++ b/Wabbajack.Server/Services/ArchiveDownloader.cs @@ -63,23 +63,30 @@ namespace Wabbajack.Server.Services try { _logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"); - if (!(nextDownload.Archive.State is GameFileSourceDownloader.State)) - await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"}); + ReportStarting(nextDownload.Archive.State.PrimaryKeyString); + if (!(nextDownload.Archive.State is GameFileSourceDownloader.State)) + await _discord.Send(Channel.Spam, + new DiscordMessage + { + Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}" + }); await DownloadDispatcher.PrepareAll(new[] {nextDownload.Archive.State}); await using var tempPath = new TempFile(); if (!await nextDownload.Archive.State.Download(nextDownload.Archive, tempPath.Path)) { - _logger.LogError($"Downloader returned false for {nextDownload.Archive.State.PrimaryKeyString}"); + _logger.LogError( + $"Downloader returned false for {nextDownload.Archive.State.PrimaryKeyString}"); await nextDownload.Fail(_sql, "Downloader returned false"); continue; } var hash = await tempPath.Path.FileHashAsync(); - + if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash) { - _logger.Log(LogLevel.Warning, $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}"); + _logger.Log(LogLevel.Warning, + $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}"); await nextDownload.Fail(_sql, "Invalid Hash"); continue; } @@ -90,17 +97,23 @@ namespace Wabbajack.Server.Services await nextDownload.Fail(_sql, "Invalid Size"); continue; } + nextDownload.Archive.Hash = hash; nextDownload.Archive.Size = tempPath.Path.Size; _logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}"); await _archiveMaintainer.Ingest(tempPath.Path); - _logger.Log(LogLevel.Information, $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}"); + _logger.Log(LogLevel.Information, + $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}"); await nextDownload.Finish(_sql); - - if (!(nextDownload.Archive.State is GameFileSourceDownloader.State)) - await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}"}); + + if (!(nextDownload.Archive.State is GameFileSourceDownloader.State)) + await _discord.Send(Channel.Spam, + new DiscordMessage + { + Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}" + }); } @@ -108,7 +121,15 @@ namespace Wabbajack.Server.Services { _logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}"); await nextDownload.Fail(_sql, ex.ToString()); - await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}"}); + await _discord.Send(Channel.Spam, + new DiscordMessage + { + Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}" + }); + } + finally + { + ReportEnding(nextDownload.Archive.State.PrimaryKeyString); } count++; diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 49acc614..8621495b 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -62,6 +62,7 @@ namespace Wabbajack.Server.Services var listArchives = await _sql.ModListArchives(metadata.Links.MachineURL); var archives = await listArchives.PMap(queue, async archive => { + ReportStarting(archive.State.PrimaryKeyString); if (timer.Elapsed > Delay) { return (archive, ArchiveStatus.InValid); @@ -77,7 +78,7 @@ namespace Wabbajack.Server.Services return await TryToHeal(data, archive, metadata); } - + return (archive, result); } catch (Exception ex) @@ -85,6 +86,10 @@ namespace Wabbajack.Server.Services _logger.LogError(ex, $"During Validation of {archive.Hash} {archive.State.PrimaryKeyString}"); return (archive, ArchiveStatus.InValid); } + finally + { + ReportEnding(archive.State.PrimaryKeyString); + } }); var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid); diff --git a/Wabbajack.Server/Services/NonNexusDownloadValidator.cs b/Wabbajack.Server/Services/NonNexusDownloadValidator.cs index 6b929e42..2ff52979 100644 --- a/Wabbajack.Server/Services/NonNexusDownloadValidator.cs +++ b/Wabbajack.Server/Services/NonNexusDownloadValidator.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Dapper; using Microsoft.Extensions.Logging; @@ -33,18 +34,22 @@ namespace Wabbajack.Server.Services { try { + var token = new CancellationTokenSource(); + token.CancelAfter(TimeSpan.FromMinutes(10)); + + ReportStarting(archive.State.PrimaryKeyString); bool isValid = false; switch (archive.State) { case WabbajackCDNDownloader.State _: case GoogleDriveDownloader.State _: case ManualDownloader.State _: - case ModDBDownloader.State _: + case ModDBDownloader.State _: case HTTPDownloader.State h when h.Url.StartsWith("https://wabbajack"): isValid = true; break; default: - isValid = await archive.State.Verify(archive); + isValid = await archive.State.Verify(archive, token.Token); break; } return (Archive: archive, IsValid: isValid); @@ -54,6 +59,10 @@ namespace Wabbajack.Server.Services _logger.Log(LogLevel.Warning, $"Error for {archive.Name} {archive.State.PrimaryKeyString} {ex}"); return (Archive: archive, IsValid: false); } + finally + { + ReportEnding(archive.State.PrimaryKeyString); + } }); diff --git a/Wabbajack.Server/Services/QuickSync.cs b/Wabbajack.Server/Services/QuickSync.cs index fa4308d5..954f1058 100644 --- a/Wabbajack.Server/Services/QuickSync.cs +++ b/Wabbajack.Server/Services/QuickSync.cs @@ -21,11 +21,11 @@ namespace Wabbajack.Server.Services _logger = logger; } - public async Task> Report() + public async Task> Report() { using var _ = await _lock.WaitAsync(); return _services.ToDictionary(s => s.Key, - s => (s.Value.Delay, DateTime.UtcNow - s.Value.LastEnd)); + s => (s.Value.Delay, DateTime.UtcNow - s.Value.LastEnd, s.Value.ActiveWorkStatus)); } public async Task Register(T service) diff --git a/Wabbajack.Server/Services/Watchdog.cs b/Wabbajack.Server/Services/Watchdog.cs index ef7e873c..215140fa 100644 --- a/Wabbajack.Server/Services/Watchdog.cs +++ b/Wabbajack.Server/Services/Watchdog.cs @@ -20,10 +20,10 @@ namespace Wabbajack.Server.Services var report = await _quickSync.Report(); foreach (var service in report) { - if (service.Value.LastRunTime != default && service.Value.LastRunTime >= service.Value.Delay * 2) + if (service.Value.LastRunTime != default && service.Value.LastRunTime >= service.Value.Delay * 4) { await _discord.Send(Channel.Spam, - new DiscordMessage {Content = $"Service {service.Key.Name} has missed it's scheduled execution window"}); + new DiscordMessage {Content = $"Service {service.Key.Name} has missed it's scheduled execution window. \n Current work: \n {string.Join("\n", service.Value.ActiveWork)}"}); } } diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index e51b87fa..b53855b9 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -3,8 +3,8 @@ Exe netcoreapp3.1 - 2.3.6.0 - 2.3.6.0 + 2.3.6.1 + 2.3.6.1 Copyright © 2019-2020 Wabbajack Server win-x64 diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 7e353595..d83a5e50 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -291,9 +291,33 @@ namespace Wabbajack.Test var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); var otherfiles = await ((LoversLabDownloader.State)state).GetFilesInGroup(); - + } + + [Fact] + public async Task CanCancelLLValidation() + { + await using var filename = new TempFile(); + await DownloadDispatcher.GetInstance().Prepare(); + + var state = new LoversLabDownloader.State + { + FileName = "14424-books-of-dibella-se-alternate-start-plugin", FileID = "870820", + }; + + using var queue = new WorkQueue(); + var tcs = new CancellationTokenSource(); + tcs.CancelAfter(2); + await Assert.ThrowsAsync(async () => + { + await Enumerable.Range(0, 2).PMap(queue, + async x => + { + Assert.True(await state.Verify(new Archive(state: null!) {Size = 252269}, tcs.Token)); + }); + }); + + - } [Fact] diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 761449be..7c97b8bc 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -6,8 +6,8 @@ true x64 win10-x64 - 2.3.6.0 - 2.3.6.0 + 2.3.6.1 + 2.3.6.1 Copyright © 2019-2020 An automated ModList installer true