Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

#### Version - 2.3.6.1 - 12/31/2020

Browse Source 
* When IPS4 (e.g. LL) sites based on CEF fail to validate, they no longer hang the app
* If a IPS4 CEF site throws a 503, or 400 error, retry
* Clean out the cookies during IPS4 CEF downloads so that they don't cause 400 errors
* Limit the number of connections to IPS4 sites to 20 per minute (one per 6 seconds)
* If a site *does* timeout, throw a log of the CEF state into `CEFStates` for easier debugging by the WJ team
* Wrote a new CLI utility to stress test the Verification routines.
* Ignore files that have `\Edit Scripts\Export\` in their path
This commit is contained in:
Timothy Baldridge 2020-12-30 23:44:42 -07:00
parent c4998e44be
commit db3b441d19
38 changed files with 558 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CHANGELOG.md
View File

@ -1,5 +1,14 @@
### Changelog
#### Version - 2.3.6.1 - 12/31/2020
* When IPS4 (e.g. LL) sites based on CEF fail to validate, they no longer hang the app
* If a IPS4 CEF site throws a 503, or 400 error, retry
* Clean out the cookies during IPS4 CEF downloads so that they don't cause 400 errors
* Limit the number of connections to IPS4 sites to 20 per minute (one per 6 seconds)
* If a site *does* timeout, throw a log of the CEF state into `CEFStates` for easier debugging by the WJ team
* Wrote a new CLI utility to stress test the Verification routines.
* Ignore files that have `\Edit Scripts\Export\` in their path
#### Version - 2.3.6.0 - 12/29/2020
* Move the LoversLab downloader to a CEF based backed making it interact with CloudFlare a bit better
* Add support for No Man's Sky

4
Wabbajack.CLI/OptionsDefinition.cs
View File

@ -35,7 +35,9 @@ namespace Wabbajack.CLI
typeof(HashGamefiles),
typeof(Backup),
typeof(Restore),
typeof(PurgeArchive)
typeof(PurgeArchive),
typeof(AllKnownDownloadStates),
typeof(VerifyAllDownloads)
};
}
}

36
Wabbajack.CLI/Verbs/AllKnownDownloadStates.cs Normal file
View File

@ -0,0 +1,36 @@
using System;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.FileUploader;
namespace Wabbajack.CLI.Verbs
{
[Verb("all-known-download-states", HelpText = "Print known Ini info for a given hash")]
public class AllKnownDownloadStates : AVerb
{
[Option('i', "input", Required = true, HelpText = "Input Hash")]
public string _input { get; set; } = "";
public Hash Input => Hash.Interpret(_input);
protected override async Task<ExitCode> Run()
{
var states = await ClientAPI.InferAllDownloadStates(Input);
Console.WriteLine($"Found {states.Length} states");
foreach (var archive in states)
{
Console.WriteLine("----");
Console.WriteLine($"Name : {archive.State.PrimaryKeyString}");
Console.WriteLine($"Is Valid: {await archive.State.Verify(archive)}");
Console.WriteLine("------ Begin INI--------");
Console.WriteLine(archive.State.GetMetaIniString());
Console.WriteLine("------ End INI --------");
}
return ExitCode.Ok;
}
}
}

100
Wabbajack.CLI/Verbs/VerifyAllDownloads.cs Normal file
View File

@ -0,0 +1,100 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.LibCefHelpers;
namespace Wabbajack.CLI.Verbs
{
[Verb("verify-all-downloads", HelpText = "Verify all downloads in a folder")]
public class VerifyAllDownloads : AVerb
{
[Option('i', "input", Required = true, HelpText = "Input Folder")]
public string _input { get; set; } = "";
public AbsolutePath Input => (AbsolutePath)_input;
[Option('t', "type", Required = false,
HelpText = "Only verify files of this type of download state for example NexusDownloader+State")]
public string StateType { get; set; } = "";
protected override async Task<ExitCode> Run()
{
var files = Input.EnumerateFiles()
.Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
.ToArray();
Console.WriteLine($"Found {files.Length} files to verify");
using var queue = new WorkQueue();
var states = (await files.PMap(queue, async f =>
{
var ini = f.WithExtension(Consts.MetaFileExtension).LoadIniFile();
var state = (AbstractDownloadState?)await DownloadDispatcher.ResolveArchive(ini, quickMode: true);
if (state == default)
{
Console.WriteLine($"[Skipping] {f.FileName} because no meta could be interpreted");
}
if (!string.IsNullOrWhiteSpace(StateType) && !state!.PrimaryKeyString.StartsWith(StateType + "|"))
{
Console.WriteLine(
$"[Skipping] {f.FileName} because type {state.PrimaryKeyString[0]} does not match filter");
return (f, null);
}
return (f, state);
})).Where(s => s.state != null)
.Select(s => (s.f, s.state!))
.ToArray();
await DownloadDispatcher.PrepareAll(states.Select(s => s.Item2));
Helpers.Init();
Console.WriteLine($"Found {states.Length} states to verify");
int timedOut = 0;
await states.PMap(queue, async p=>
{
try
{
var (f, state) = p;
try
{
var cts = new CancellationTokenSource();
cts.CancelAfter(TimeSpan.FromMinutes(10));
var result =
await state!.Verify(new Archive(state) {Name = f.FileName.ToString(), Size = f.Size},
cts.Token);
Console.WriteLine($"[{(result ? "Failed" : "Passed")}] {f.FileName}");
}
catch (TaskCanceledException)
{
Console.WriteLine($"[Timed Out] {f.FileName} {state!.PrimaryKeyString}");
Interlocked.Increment(ref timedOut);
}
}
catch (Exception ex)
{
Console.WriteLine($"[Exception] {p.f.FileName} {ex.Message}");
}
});
Console.WriteLine($"[Total TimedOut] {timedOut}");
Console.WriteLine("[Done]");
return ExitCode.Ok;
}
}
}

4
Wabbajack.CLI/Wabbajack.CLI.csproj
View File

@ -6,8 +6,8 @@
<AssemblyName>wabbajack-cli</AssemblyName>
<Company>Wabbajack</Company>
<Platforms>x64</Platforms>
<AssemblyVersion>2.3.6.0</AssemblyVersion>
<FileVersion>2.3.6.0</FileVersion>
<AssemblyVersion>2.3.6.1</AssemblyVersion>
<FileVersion>2.3.6.1</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>An automated ModList installer</Description>
<PublishReadyToRun>true</PublishReadyToRun>

2
Wabbajack.Common/Consts.cs
View File

@ -131,6 +131,8 @@ namespace Wabbajack.Common
public static AbsolutePath SettingsFile => LocalAppDataPath.Combine("settings.json");
public static RelativePath SettingsIni = (RelativePath)"settings.ini";
public static byte SettingsVersion => 2;
public static TimeSpan MaxVerifyTime => TimeSpan.FromMinutes(10);
public static RelativePath NativeSettingsJson = (RelativePath)"native_compiler_settings.json";
public static bool IsServer = false;

4
Wabbajack.Launcher/Wabbajack.Launcher.csproj
View File

@ -4,8 +4,8 @@
<OutputType>WinExe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
<UseWPF>true</UseWPF>
<AssemblyVersion>2.3.6.0</AssemblyVersion>
<FileVersion>2.3.6.0</FileVersion>
<AssemblyVersion>2.3.6.1</AssemblyVersion>
<FileVersion>2.3.6.1</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>Wabbajack Application Launcher</Description>
<PublishReadyToRun>true</PublishReadyToRun>

6
Wabbajack.Lib/ACompiler.cs
View File

@ -8,7 +8,9 @@ using System.Linq;
using System.Reactive.Subjects;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Org.BouncyCastle.Asn1.Cms;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.Downloaders;
@ -511,7 +513,9 @@ namespace Wabbajack.Lib
await result.State!.GetDownloader().Prepare();
if (result.State != null && !await result.State.Verify(result))
var token = new CancellationTokenSource();
token.CancelAfter(Consts.MaxVerifyTime);
if (result.State != null && !await result.State.Verify(result, token.Token))
Error(
$"Unable to resolve link for {archive.Name}. If this is hosted on the Nexus the file may have been removed.");

9
Wabbajack.Lib/ClientAPI.cs
View File

@ -210,6 +210,15 @@ using Wabbajack.Lib.Downloaders;
return null;
}
public static async Task<Archive[]> InferAllDownloadStates(Hash hash)
{
var client = await GetClient();
var results = await client.GetJsonAsync<Archive[]>(
$"{Consts.WabbajackBuildServerUri}mod_files/by_hash/{hash.ToHex()}");
return results;
}
public static async Task<Archive[]> GetModUpgrades(Hash src)
{
var client = await GetClient();

3
Wabbajack.Lib/Downloaders/AbstractDownloadState.cs
View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Wabbajack.Common;
@ -89,7 +90,7 @@ namespace Wabbajack.Lib.Downloaders
/// Returns true if this link is still valid
/// </summary>
/// <returns></returns>
public abstract Task<bool> Verify(Archive archive);
public abstract Task<bool> Verify(Archive archive, CancellationToken? token = null);
public abstract IDownloader GetDownloader();

85
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -5,6 +5,7 @@ using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using F23.StringSimilarity;
@ -13,17 +14,27 @@ using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib.LibCefHelpers;
using Wabbajack.Lib.Validation;
using Wabbajack.Lib.WebAutomation;
namespace Wabbajack.Lib.Downloaders
{
interface IWaitForWindowDownloader
{
public Task WaitForNextRequestWindow();
}
// IPS4 is the site used by LoversLab, VectorPlexus, etc. the general mechanics of each site are the
// same, so we can fairly easily abstract the state.
// Pass in the state type via TState
public abstract class AbstractIPS4Downloader<TDownloader, TState> : AbstractNeedsLoginDownloader, IDownloader
public abstract class AbstractIPS4Downloader<TDownloader, TState> : AbstractNeedsLoginDownloader, IDownloader, IWaitForWindowDownloader
where TState : AbstractIPS4Downloader<TDownloader, TState>.State<TDownloader>, new()
where TDownloader : IDownloader
{
private DateTime LastRequestTime = default;
protected long RequestsPerMinute = 20;
private TimeSpan RequestDelay => TimeSpan.FromMinutes(1) / RequestsPerMinute;
protected AbstractIPS4Downloader(Uri loginUri, string encryptedKeyName, string cookieDomain, string loginCookie = "ips4_member_id")
: base(loginUri, encryptedKeyName, cookieDomain, loginCookie)
{
@ -35,6 +46,27 @@ namespace Wabbajack.Lib.Downloaders
return await GetDownloaderStateFromUrl(url, quickMode);
}
public async Task WaitForNextRequestWindow()
{
TimeSpan delay;
lock (this)
{
if (LastRequestTime < DateTime.UtcNow - RequestDelay)
{
LastRequestTime = DateTime.UtcNow;
delay = TimeSpan.Zero;
}
else
{
LastRequestTime += RequestDelay;
delay = LastRequestTime - DateTime.UtcNow;
}
}
Utils.Log($"Waiting for {delay.TotalSeconds} to make request via {typeof(TDownloader).Name}");
await Task.Delay(delay);
}
public async Task<AbstractDownloadState?> GetDownloaderStateFromUrl(Uri url, bool quickMode)
{
var absolute = true;
@ -150,10 +182,11 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Download(Archive a, AbsolutePath destination)
{
await ((IWaitForWindowDownloader)Downloader).WaitForNextRequestWindow();
return await ResolveDownloadStream(a, destination, false);
}
private async Task<bool> ResolveDownloadStream(Archive a, AbsolutePath path, bool quickMode)
private async Task<bool> ResolveDownloadStream(Archive a, AbsolutePath path, bool quickMode, CancellationToken? token = null)
{
@ -168,7 +201,7 @@ namespace Wabbajack.Lib.Downloaders
var csrfURL = string.IsNullOrWhiteSpace(FileID)
? $"{Site}/files/file/{FileName}/?do=download"
: $"{Site}/files/file/{FileName}/?do=download&r={FileID}";
var html = await GetStringAsync(new Uri(csrfURL));
var html = await GetStringAsync(new Uri(csrfURL), token);
var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])");
var matches = pattern.Matches(html).Cast<Match>();
@ -182,25 +215,15 @@ namespace Wabbajack.Lib.Downloaders
}
var sep = Site.EndsWith("?") ? "&" : "?";
if (!Downloader.IsCloudFlareProtected)
{
url = FileID == null
? $"{Site}/files/file/{FileName}/{sep}do=download&confirm=1&t=1&csrfKey={csrfKey}"
: $"{Site}/files/file/{FileName}/{sep}do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}";
}
else
{
url = FileID == null
? $"{Site}/files/file/{FileName}/{sep}do=download&confirm=1&t=1"
: $"{Site}/files/file/{FileName}/{sep}do=download&r={FileID}&confirm=1&t=1";
}
url = FileID == null
? $"{Site}/files/file/{FileName}/{sep}do=download&confirm=1&t=1&csrfKey={csrfKey}"
: $"{Site}/files/file/{FileName}/{sep}do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}";
}
if (Downloader.IsCloudFlareProtected)
{
using var driver = await Downloader.GetAuthedDriver();
var size = await driver.NavigateToAndDownload(new Uri(url), path, quickMode: quickMode);
var size = await driver.NavigateToAndDownload(new Uri(url), path, quickMode: quickMode, token);
if (a.Size == 0 || size == 0 || a.Size == size) return true;
@ -268,6 +291,11 @@ namespace Wabbajack.Lib.Downloaders
goto TOP;
}
private async Task DeleteOldDownloadCookies(Driver driver)
{
await driver.DeleteCookiesWhere(c => c.Name.StartsWith("ips4_downloads_delay_") && c.Value == "-1");
}
private class WaitResponse
{
[JsonProperty("download")]
@ -276,10 +304,11 @@ namespace Wabbajack.Lib.Downloaders
public int CurrentTime { get; set; }
}
public override async Task<bool> Verify(Archive a)
public override async Task<bool> Verify(Archive a, CancellationToken? token)
{
await ((IWaitForWindowDownloader)Downloader).WaitForNextRequestWindow();
await using var tp = new TempFile();
var isValid = await ResolveDownloadStream(a, tp.Path, true);
var isValid = await ResolveDownloadStream(a, tp.Path, true, token: token);
return isValid;
}
@ -290,6 +319,7 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
{
await ((IWaitForWindowDownloader)Downloader).WaitForNextRequestWindow();
var files = await GetFilesInGroup();
var nl = new Levenshtein();
@ -321,7 +351,10 @@ namespace Wabbajack.Lib.Downloaders
public async Task<List<Archive>> GetFilesInGroup()
{
var others = await GetHtmlAsync(new Uri($"{Site}/files/file/{FileName}?do=download"));
var token = new CancellationTokenSource();
token.CancelAfter(TimeSpan.FromMinutes(10));
var others = await GetHtmlAsync(new Uri($"{Site}/files/file/{FileName}?do=download"), token.Token);
var pairs = others.DocumentNode.SelectNodes("//a[@data-action='download']")
.Select(item => (item.GetAttributeValue("href", ""),
@ -348,18 +381,22 @@ namespace Wabbajack.Lib.Downloaders
return IsAttachment ? FullURL : $"{Site}/files/file/{FileName}/?do=download&r={FileID}";
}
public async Task<string> GetStringAsync(Uri uri)
public async Task<string> GetStringAsync(Uri uri, CancellationToken? token = null)
{
if (!Downloader.IsCloudFlareProtected)
return await Downloader.AuthedClient.GetStringAsync(uri);
using var driver = await Downloader.GetAuthedDriver();
await DeleteOldDownloadCookies(driver);
//var drivercookies = await Helpers.GetCookies("loverslab.com");
//var cookies = await ClientAPI.GetAuthInfo<Helpers.Cookie[]>("loverslabcookies");
//await Helpers.IngestCookies(uri.ToString(), cookies);
await driver.NavigateTo(uri);
await driver.NavigateTo(uri, token);
if ((token ?? CancellationToken.None).IsCancellationRequested)
return "";
var source = await driver.GetSourceAsync();
@ -380,9 +417,9 @@ namespace Wabbajack.Lib.Downloaders
return source;
}
public async Task<HtmlDocument> GetHtmlAsync(Uri s)
public async Task<HtmlDocument> GetHtmlAsync(Uri s, CancellationToken? token = null)
{
var body = await GetStringAsync(s);
var body = await GetStringAsync(s, token);
var doc = new HtmlDocument();
doc.LoadHtml(body);
return doc;

5
Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
View File

@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
@ -79,7 +80,7 @@ namespace Wabbajack.Lib.Downloaders
return true;
}
public override async Task<bool> Verify(Archive a)
public override async Task<bool> Verify(Archive a, CancellationToken? token)
{
return SourcePath.Exists && await SourcePath.FileHashCachedAsync() == Hash;
}

5
Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs
View File

@ -1,4 +1,5 @@
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Wabbajack.Common;
@ -70,10 +71,10 @@ namespace Wabbajack.Lib.Downloaders
return httpState;
}
public override async Task<bool> Verify(Archive a)
public override async Task<bool> Verify(Archive a, CancellationToken? token)
{
var state = await ToHttpState();
return await state.Verify(a);
return await state.Verify(a, token);
}
public override IDownloader GetDownloader()

11
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -4,6 +4,7 @@ using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Wabbajack.Common;
@ -74,7 +75,7 @@ namespace Wabbajack.Lib.Downloaders
return DoDownload(a, destination, true);
}
public async Task<bool> DoDownload(Archive a, AbsolutePath destination, bool download)
public async Task<bool> DoDownload(Archive a, AbsolutePath destination, bool download, CancellationToken? token = null)
{
if (download)
{
@ -98,7 +99,7 @@ namespace Wabbajack.Lib.Downloaders
var bufferSize = 1024 * 32 * 8;
Utils.Status($"Starting Download {a.Name ?? Url}", Percent.Zero);
var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false);
var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false, token:token);
TOP:
if (!response.IsSuccessStatusCode)
@ -143,7 +144,7 @@ TOP:
var buffer = new byte[bufferSize];
int readThisCycle = 0;
while (true)
while (!(token ?? CancellationToken.None).IsCancellationRequested)
{
int read = 0;
try
@ -188,9 +189,9 @@ TOP:
}
}
public override async Task<bool> Verify(Archive a)
public override async Task<bool> Verify(Archive a, CancellationToken? token)
{
return await DoDownload(a, ((RelativePath)"").RelativeToEntryPoint(), false);
return await DoDownload(a, ((RelativePath)"").RelativeToEntryPoint(), false, token: token);
}
public override IDownloader GetDownloader()

3
Wabbajack.Lib/Downloaders/MEGADownloader.cs
View File

@ -2,6 +2,7 @@
using System.Reactive;
using System.Reactive.Linq;
using System.Security;
using System.Threading;
using System.Threading.Tasks;
using CG.Web.MegaApiClient;
using Newtonsoft.Json;
@ -178,7 +179,7 @@ namespace Wabbajack.Lib.Downloaders
return true;
}
public override async Task<bool> Verify(Archive a)
public override async Task<bool> Verify(Archive a, CancellationToken? token)
{
await MegaLogin();

3
Wabbajack.Lib/Downloaders/ManualDownloader.cs
View File

@ -1,6 +1,7 @@
using System.IO;
using System.Linq;
using System.Reactive.Subjects;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Wabbajack.Common;
@ -93,7 +94,7 @@ namespace Wabbajack.Lib.Downloaders
return await state.Download(a, destination);
}
public override async Task<bool> Verify(Archive a)
public override async Task<bool> Verify(Archive a, CancellationToken? token)
{
return true;
}

9
Wabbajack.Lib/Downloaders/MediaFireDownloader.cs
View File

@ -1,6 +1,7 @@
using System;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
@ -43,15 +44,15 @@ namespace Wabbajack.Lib.Downloaders
return await result.Download(a, destination);
}
public override async Task<bool> Verify(Archive a)
public override async Task<bool> Verify(Archive a, CancellationToken? token)
{
return await Resolve() != null;
return await Resolve(token) != null;
}
private async Task<HTTPDownloader.State?> Resolve()
private async Task<HTTPDownloader.State?> Resolve(CancellationToken? token = null)
{
var client = new Http.Client();
var result = await client.GetAsync(Url, HttpCompletionOption.ResponseHeadersRead);
var result = await client.GetAsync(Url, HttpCompletionOption.ResponseHeadersRead, token:token);
if (!result.IsSuccessStatusCode)
return null;

9
Wabbajack.Lib/Downloaders/ModDBDownloader.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using HtmlAgilityPack;
using Newtonsoft.Json;
@ -71,12 +72,12 @@ namespace Wabbajack.Lib.Downloaders
return false;
}
private async Task<string[]> GetDownloadUrls()
private async Task<string[]> GetDownloadUrls(CancellationToken? token = null)
{
var uri = new Uri(Url);
var modId = uri.AbsolutePath.Split('/').Reverse().First(f => int.TryParse(f, out int _));
var mirrorUrl = $"https://www.moddb.com/downloads/start/{modId}/all";
var doc = await new HtmlWeb().LoadFromWebAsync($"https://www.moddb.com/downloads/start/{modId}/all");
var doc = await new HtmlWeb().LoadFromWebAsync($"https://www.moddb.com/downloads/start/{modId}/all", token ?? CancellationToken.None);
var mirrors = doc.DocumentNode.Descendants().Where(d => d.NodeType == HtmlNodeType.Element && d.HasClass("row"))
.Select(d => new
{
@ -98,9 +99,9 @@ namespace Wabbajack.Lib.Downloaders
return mirrors.Select(d => d.Link).ToArray();
}
public override async Task<bool> Verify(Archive a)
public override async Task<bool> Verify(Archive a, CancellationToken? token)
{
await GetDownloadUrls();
await GetDownloadUrls(token);
return true;
}

3
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -4,6 +4,7 @@ using System.ComponentModel.DataAnnotations;
using System.Linq;
using System.Reactive;
using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using F23.StringSimilarity;
using Newtonsoft.Json;
@ -213,7 +214,7 @@ namespace Wabbajack.Lib.Downloaders
return await new HTTPDownloader.State(url).Download(a, destination);
}
public override async Task<bool> Verify(Archive a)
public override async Task<bool> Verify(Archive a, CancellationToken? token = null)
{
try
{

2
Wabbajack.Lib/Downloaders/SteamWorkshopDownloader.cs
View File

@ -87,7 +87,7 @@ namespace Wabbajack.Lib.Downloaders
return true;
}
public override async Task<bool> Verify(Archive a)
public override async Task<bool> Verify(Archive a, CancellationToken? token)
{
//TODO: find a way to verify steam workshop items
throw new NotImplementedException();

8
Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs
View File

@ -108,9 +108,9 @@ namespace Wabbajack.Lib.Downloaders
return true;
}
public override async Task<bool> Verify(Archive archive)
public override async Task<bool> Verify(Archive archive, CancellationToken? token)
{
var definition = await GetDefinition();
var definition = await GetDefinition(token);
return true;
}
@ -151,13 +151,13 @@ namespace Wabbajack.Lib.Downloaders
return builder.ToString();
}
private async Task<CDNFileDefinition> GetDefinition()
private async Task<CDNFileDefinition> GetDefinition(CancellationToken? token = null)
{
var client = new Wabbajack.Lib.Http.Client();
if (DomainRemaps.TryGetValue(Url.Host, out var remap))
{
var builder = new UriBuilder(Url) {Host = remap};
using var data = await client.GetAsync(builder + "/definition.json.gz");
using var data = await client.GetAsync(builder + "/definition.json.gz", token: token);
await using var gz = new GZipStream(await data.Content.ReadAsStreamAsync(),
CompressionMode.Decompress);
return gz.FromJson<CDNFileDefinition>();

5
Wabbajack.Lib/Downloaders/YandexDownloader.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
@ -59,10 +60,10 @@ namespace Wabbajack.Lib.Downloaders
return await new HTTPDownloader.State(uri!.ToString()).Download(destination);
}
public override async Task<bool> Verify(Archive archive)
public override async Task<bool> Verify(Archive archive, CancellationToken? token)
{
var client = new Wabbajack.Lib.Http.Client();
var result = await client.GetAsync(Url, errorsAsExceptions: false);
var result = await client.GetAsync(Url, errorsAsExceptions: false, token: token);
return result.IsSuccessStatusCode;
}

29
Wabbajack.Lib/Http/Client.cs
View File

@ -4,6 +4,7 @@ using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using HtmlAgilityPack;
using Wabbajack.Common;
@ -16,16 +17,16 @@ namespace Wabbajack.Lib.Http
{
public List<(string, string?)> Headers = new List<(string, string?)>();
public List<Cookie> Cookies = new List<Cookie>();
public async Task<HttpResponseMessage> GetAsync(string url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, bool retry = true)
public async Task<HttpResponseMessage> GetAsync(string url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, bool retry = true, CancellationToken? token = null)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions, retry: retry);
return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions, retry: retry, token: token);
}
public async Task<HttpResponseMessage> GetAsync(Uri url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true)
public async Task<HttpResponseMessage> GetAsync(Uri url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, CancellationToken? token = null)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions);
return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions, token:token);
}
@ -41,10 +42,10 @@ namespace Wabbajack.Lib.Http
return await SendAsync(request, responseHeadersRead);
}
public async Task<string> GetStringAsync(string url)
public async Task<string> GetStringAsync(string url, CancellationToken? token = null)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
return await SendStringAsync(request);
return await SendStringAsync(request, token: token);
}
public async Task<string> GetStringAsync(Uri url)
@ -59,9 +60,9 @@ namespace Wabbajack.Lib.Http
return await SendStringAsync(request);
}
private async Task<string> SendStringAsync(HttpRequestMessage request)
private async Task<string> SendStringAsync(HttpRequestMessage request, CancellationToken? token = null)
{
using var result = await SendAsync(request);
using var result = await SendAsync(request, token: token);
if (!result.IsSuccessStatusCode)
{
Utils.Log("Internal Error");
@ -72,7 +73,7 @@ namespace Wabbajack.Lib.Http
return await result.Content.ReadAsStringAsync();
}
public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, bool retry = true)
public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, bool retry = true, CancellationToken? token = null)
{
foreach (var (k, v) in Headers)
msg.Headers.Add(k, v);
@ -83,7 +84,7 @@ namespace Wabbajack.Lib.Http
TOP:
try
{
response = await ClientFactory.Client.SendAsync(msg, responseHeadersRead);
response = await ClientFactory.Client.SendAsync(msg, responseHeadersRead, token ?? CancellationToken.None);
if (response.IsSuccessStatusCode) return response;
if (errorsAsExceptions)
@ -105,7 +106,7 @@ namespace Wabbajack.Lib.Http
var ms = Utils.NextRandom(100, 1000);
Utils.Log($"Got a {http.Code} from {msg.RequestUri} retrying in {ms}ms");
await Task.Delay(ms);
await Task.Delay(ms, token ?? CancellationToken.None);
msg = CloneMessage(msg);
goto TOP;
}
@ -114,7 +115,7 @@ namespace Wabbajack.Lib.Http
retries++;
Utils.LogStraightToFile(ex.ToString());
Utils.Log($"Http Connect error to {msg.RequestUri} retry {retries}");
await Task.Delay(100 * retries);
await Task.Delay(100 * retries, token ?? CancellationToken.None);
msg = CloneMessage(msg);
goto TOP;
@ -138,9 +139,9 @@ namespace Wabbajack.Lib.Http
return result.FromJsonString<T>();
}
public async Task<HtmlDocument> GetHtmlAsync(string s)
public async Task<HtmlDocument> GetHtmlAsync(string s, CancellationToken? token = null)
{
var body = await GetStringAsync(s);
var body = await GetStringAsync(s, token: token);
var doc = new HtmlDocument();
doc.LoadHtml(body);
return doc;

28
Wabbajack.Lib/LibCefHelpers/Init.cs
View File

@ -103,17 +103,43 @@ namespace Wabbajack.Lib.LibCefHelpers
var visitor = new CookieDeleter();
manager.VisitAllCookies(visitor);
}
public static async Task DeleteCookiesWhere(Func<Cookie,bool> filter)
{
var manager = Cef.GetGlobalCookieManager();
var visitor = new CookieDeleter(filter);
manager.VisitAllCookies(visitor);
}
}
class CookieDeleter : ICookieVisitor
{
private Func<Helpers.Cookie, bool>? _filter;
public CookieDeleter(Func<Helpers.Cookie, bool>? filter = null)
{
_filter = filter;
}
public void Dispose()
{
}
public bool Visit(Cookie cookie, int count, int total, ref bool deleteCookie)
{
deleteCookie = true;
if (_filter == null)
{
deleteCookie = true;
}
else
{
var conv = new Helpers.Cookie
{
Name = cookie.Name, Domain = cookie.Domain, Value = cookie.Value, Path = cookie.Path
};
if (_filter(conv))
deleteCookie = true;
}
return true;
}
}

1
Wabbajack.Lib/MO2Compiler.cs
View File

@ -474,6 +474,7 @@ namespace Wabbajack.Lib
new IncludeAllConfigs(this),
new zEditIntegration.IncludeZEditPatches(this),
new IncludeTaggedMods(this, Consts.WABBAJACK_NOMATCH_INCLUDE),
new IgnorePathContains(this,@"\Edit Scripts\Export\"),
new DropAll(this)
};
}

81
Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs
View File

@ -3,10 +3,13 @@ using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using CefSharp;
using Wabbajack.Common;
using Wabbajack.Common.Exceptions;
using Wabbajack.Lib.LibCefHelpers;
namespace Wabbajack.Lib.WebAutomation
@ -23,36 +26,78 @@ namespace Wabbajack.Lib.WebAutomation
_browser.LifeSpanHandler = new PopupBlocker(this);
}
public Task NavigateTo(Uri uri)
public Task NavigateTo(Uri uri, CancellationToken? token = null)
{
var tcs = new TaskCompletionSource<bool>();
EventHandler<LoadingStateChangedEventArgs>? handler = null;
handler = (sender, e) =>
{
if (!e.IsLoading)
{
_browser.LoadingStateChanged -= handler;
tcs.SetResult(true);
}
if (e.IsLoading) return;
_browser.LoadingStateChanged -= handler;
tcs.SetResult(true);
};
_browser.LoadingStateChanged += handler;
_browser.Load(uri.ToString());
token?.Register(() => tcs.TrySetCanceled());
return tcs.Task;
}
public async Task<long> NavigateToAndDownload(Uri uri, AbsolutePath dest, bool quickMode = false)
private readonly string[] KnownServerLoadStrings =
{
"<h1>Temporarily Unavailable</h1>",
"<center>Request Header Or Cookie Too Large</center>",
//"<html><head></head><body></body></html>"
};
private readonly (string, int)[] KnownErrorStrings =
{
("<h1>400 Bad Request</h1>", 400),
("We could not locate the item you are trying to view.", 404)
};
private static readonly Random RetryRandom = new Random();
public async Task<long> NavigateToAndDownload(Uri uri, AbsolutePath dest, bool quickMode = false, CancellationToken? token = null)
{
var oldCB = _browser.DownloadHandler;
var handler = new ReroutingDownloadHandler(this, dest, quickMode: quickMode);
var handler = new ReroutingDownloadHandler(this, dest, quickMode: quickMode, token);
_browser.DownloadHandler = handler;
try
{
await NavigateTo(uri);
return await handler.Task;
int retryCount = 0;
RETRY:
await NavigateTo(uri, token);
var source = await _browser.GetSourceAsync();
foreach (var err in KnownServerLoadStrings)
{
if (!source.Contains(err))
continue;
if ((token?.IsCancellationRequested) == true)
{
throw new TimeoutException();
}
else
{
retryCount += 1;
var retry = RetryRandom.Next(retryCount * 5000, retryCount * 5000 * 2);
Utils.Log($"Got server load error from {uri} retying in {retry}ms [{err}]");
await Task.Delay(TimeSpan.FromMilliseconds(retry));
goto RETRY;
}
}
foreach (var (err, httpCode) in KnownErrorStrings)
{
if (source.Contains(err))
throw new HttpException(httpCode,$"Web driver failed: {err}");
}
Utils.Log($"Loaded page {uri} starting download...");
return await handler.TaskResult;
}
finally {
_browser.DownloadHandler = oldCB;
@ -83,6 +128,7 @@ namespace Wabbajack.Lib.WebAutomation
}
public string Location => _browser.Address;
}
public class PopupBlocker : ILifeSpanHandler
@ -123,19 +169,24 @@ namespace Wabbajack.Lib.WebAutomation
private AbsolutePath _path;
public TaskCompletionSource<long> _tcs = new TaskCompletionSource<long>();
private bool _quickMode;
public Task<long> Task => _tcs.Task;
private CancellationToken? _cancelationToken;
private TimeSpan _downloadTimeout;
public Task<long> TaskResult => _tcs.Task;
public ReroutingDownloadHandler(CefSharpWrapper wrapper, AbsolutePath path, bool quickMode)
public ReroutingDownloadHandler(CefSharpWrapper wrapper, AbsolutePath path, bool quickMode, CancellationToken? token)
{
_wrapper = wrapper;
_path = path;
_quickMode = quickMode;
_cancelationToken = token;
token?.Register(() => _tcs.TrySetCanceled());
}
public void OnBeforeDownload(IWebBrowser chromiumWebBrowser, IBrowser browser, DownloadItem downloadItem,
IBeforeDownloadCallback callback)
{
if (_quickMode) return;
callback.Continue(_path.ToString(), false);
}
@ -145,12 +196,12 @@ namespace Wabbajack.Lib.WebAutomation
if (_quickMode)
{
callback.Cancel();
_tcs.SetResult(downloadItem.TotalBytes);
_tcs.TrySetResult(downloadItem.TotalBytes);
return;
}
if (downloadItem.IsComplete)
_tcs.SetResult(downloadItem.TotalBytes);
_tcs.TrySetResult(downloadItem.TotalBytes);
callback.Resume();
}
}

3
Wabbajack.Lib/WebAutomation/IWebDriver.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.Lib.LibCefHelpers;
@ -10,7 +11,7 @@ namespace Wabbajack.Lib.WebAutomation
{
public interface IWebDriver
{
Task NavigateTo(Uri uri);
Task NavigateTo(Uri uri, CancellationToken? token = null);
Task<string> EvaluateJavaScript(string text);
Task<Helpers.Cookie[]> GetCookies(string domainPrefix);
public Action<Uri>? DownloadHandler { get; set; }

42
Wabbajack.Lib/WebAutomation/WebAutomation.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Windows;
@ -28,15 +29,41 @@ namespace Wabbajack.Lib.WebAutomation
return driver;
}
public async Task<Uri?> NavigateTo(Uri uri)
public async Task<Uri?> NavigateTo(Uri uri, CancellationToken? token = null)
{
await _driver.NavigateTo(uri);
return await GetLocation();
try
{
await _driver.NavigateTo(uri, token);
return await GetLocation();
}
catch (TaskCanceledException ex)
{
await DumpState(uri, ex);
throw;
}
}
public async Task<long> NavigateToAndDownload(Uri uri, AbsolutePath absolutePath, bool quickMode = false)
private async Task DumpState(Uri uri, Exception ex)
{
return await _driver.NavigateToAndDownload(uri, absolutePath, quickMode: quickMode);
var file = AbsolutePath.EntryPoint.Combine("CEFStates", DateTime.UtcNow.ToFileTimeUtc().ToString())
.WithExtension(new Extension(".html"));
file.Parent.CreateDirectory();
var source = await GetSourceAsync();
var cookies = await Helpers.GetCookies();
var cookiesString = string.Join('\n', cookies.Select(c => c.Name + " - " + c.Value));
await file.WriteAllTextAsync(uri + "\n " + source + "\n" + ex + "\n" + cookiesString);
}
public async Task<long> NavigateToAndDownload(Uri uri, AbsolutePath absolutePath, bool quickMode = false, CancellationToken? token = null)
{
try
{
return await _driver.NavigateToAndDownload(uri, absolutePath, quickMode: quickMode, token: token);
}
catch (TaskCanceledException ex) {
await DumpState(uri, ex);
throw;
}
}
public async ValueTask<Uri?> GetLocation()
@ -79,5 +106,10 @@ namespace Wabbajack.Lib.WebAutomation
{
Helpers.ClearCookies();
}
public async Task DeleteCookiesWhere(Func<Helpers.Cookie, bool> filter)
{
await Helpers.DeleteCookiesWhere(filter);
}
}
}

41
Wabbajack.Server/Controllers/Heartbeat.cs
View File

@ -71,9 +71,9 @@ namespace Wabbajack.BuildServer.Controllers
<ul>
{{each $.services }}
{{if $.IsLate}}
<li><b>{{$.Name}} - {{$.Time}} - {{$.MaxTime}}</b></li>
<li><a href='/heartbeat/report/services/{{$.Name}}.html'><b>{{$.Name}} - {{$.Time}} - {{$.MaxTime}}</b></a></li>
{{else}}
<li>{{$.Name}} - {{$.Time}} - {{$.MaxTime}}</li>
<li><a href='/heartbeat/report/services/{{$.Name}}.html'>{{$.Name}} - {{$.Time}} - {{$.MaxTime}}</a></li>
{{/if}}
{{/each}}
</ul>
@ -109,8 +109,45 @@ namespace Wabbajack.BuildServer.Controllers
};
}
private static readonly Func<object, string> HandleGetServiceReport = NettleEngine.GetCompiler().Compile(@"
<html><body>
<h2>Service Status: {{Name}} {{TimeSinceLastRun}}</h2>
<h3>Service Overview ({{ActiveWorkQueue.Length}}):</h3>
<ul>
{{each $.ActiveWorkQueue }}
<li>{{$.Name}} {{$.Time}}</li>
{{/each}}
</ul>
</body></html>
");
[HttpGet("report/services/{serviceName}.html")]
public async Task<ContentResult> ReportServiceStatus(string serviceName)
{
var services = await _quickSync.Report();
var info = services.First(kvp => kvp.Key.Name == serviceName);
var response = HandleGetServiceReport(new
{
Name = info.Key.Name,
TimeSinceLastRun = DateTime.UtcNow - info.Value.LastRunTime,
ActiveWorkQueue = info.Value.ActiveWork.Select(p => new
{
Name = p.Item1,
Time = DateTime.UtcNow - p.Item2
}).OrderByDescending(kp => kp.Time)
.ToArray()
});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
}
}

22
Wabbajack.Server/Services/AbstractService.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Logging;
@ -16,6 +17,8 @@ namespace Wabbajack.Server.Services
public TimeSpan Delay { get; }
public DateTime LastStart { get; }
public DateTime LastEnd { get; }
public (String, DateTime)[] ActiveWorkStatus { get; }
}
@ -30,6 +33,7 @@ namespace Wabbajack.Server.Services
public TimeSpan Delay => _delay;
public DateTime LastStart { get; private set; }
public DateTime LastEnd { get; private set; }
public (String, DateTime)[] ActiveWorkStatus { get; private set; }= { };
public AbstractService(ILogger<TP> logger, AppSettings settings, QuickSync quickSync, TimeSpan delay)
{
@ -85,6 +89,22 @@ namespace Wabbajack.Server.Services
}
public abstract Task<TR> Execute();
protected void ReportStarting(string value)
{
lock (this)
{
ActiveWorkStatus = ActiveWorkStatus.Cons((value, DateTime.UtcNow)).ToArray();
}
}
protected void ReportEnding(string value)
{
lock (this)
{
ActiveWorkStatus = ActiveWorkStatus.Where(x => x.Item1 != value).ToArray();
}
}
}
public static class AbstractServiceExtensions
@ -96,4 +116,6 @@ namespace Wabbajack.Server.Services
}
}
}

41
Wabbajack.Server/Services/ArchiveDownloader.cs
View File

@ -63,23 +63,30 @@ namespace Wabbajack.Server.Services
try
{
_logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}");
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"});
ReportStarting(nextDownload.Archive.State.PrimaryKeyString);
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"
});
await DownloadDispatcher.PrepareAll(new[] {nextDownload.Archive.State});
await using var tempPath = new TempFile();
if (!await nextDownload.Archive.State.Download(nextDownload.Archive, tempPath.Path))
{
_logger.LogError($"Downloader returned false for {nextDownload.Archive.State.PrimaryKeyString}");
_logger.LogError(
$"Downloader returned false for {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, "Downloader returned false");
continue;
}
var hash = await tempPath.Path.FileHashAsync();
if (nextDownload.Archive.Hash != default && hash != nextDownload.Archive.Hash)
{
_logger.Log(LogLevel.Warning, $"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}");
_logger.Log(LogLevel.Warning,
$"Downloaded archive hashes don't match for {nextDownload.Archive.State.PrimaryKeyString} {nextDownload.Archive.Hash} {nextDownload.Archive.Size} vs {hash} {tempPath.Path.Size}");
await nextDownload.Fail(_sql, "Invalid Hash");
continue;
}
@ -90,17 +97,23 @@ namespace Wabbajack.Server.Services
await nextDownload.Fail(_sql, "Invalid Size");
continue;
}
nextDownload.Archive.Hash = hash;
nextDownload.Archive.Size = tempPath.Path.Size;
_logger.Log(LogLevel.Information, $"Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await _archiveMaintainer.Ingest(tempPath.Path);
_logger.Log(LogLevel.Information, $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
_logger.Log(LogLevel.Information,
$"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Finish(_sql);
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}"});
if (!(nextDownload.Archive.State is GameFileSourceDownloader.State))
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}"
});
}
@ -108,7 +121,15 @@ namespace Wabbajack.Server.Services
{
_logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}");
await nextDownload.Fail(_sql, ex.ToString());
await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}"});
await _discord.Send(Channel.Spam,
new DiscordMessage
{
Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}"
});
}
finally
{
ReportEnding(nextDownload.Archive.State.PrimaryKeyString);
}
count++;

7
Wabbajack.Server/Services/ListValidator.cs
View File

@ -62,6 +62,7 @@ namespace Wabbajack.Server.Services
var listArchives = await _sql.ModListArchives(metadata.Links.MachineURL);
var archives = await listArchives.PMap(queue, async archive =>
{
ReportStarting(archive.State.PrimaryKeyString);
if (timer.Elapsed > Delay)
{
return (archive, ArchiveStatus.InValid);
@ -77,7 +78,7 @@ namespace Wabbajack.Server.Services
return await TryToHeal(data, archive, metadata);
}
return (archive, result);
}
catch (Exception ex)
@ -85,6 +86,10 @@ namespace Wabbajack.Server.Services
_logger.LogError(ex, $"During Validation of {archive.Hash} {archive.State.PrimaryKeyString}");
return (archive, ArchiveStatus.InValid);
}
finally
{
ReportEnding(archive.State.PrimaryKeyString);
}
});
var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid);

13
Wabbajack.Server/Services/NonNexusDownloadValidator.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Dapper;
using Microsoft.Extensions.Logging;
@ -33,18 +34,22 @@ namespace Wabbajack.Server.Services
{
try
{
var token = new CancellationTokenSource();
token.CancelAfter(TimeSpan.FromMinutes(10));
ReportStarting(archive.State.PrimaryKeyString);
bool isValid = false;
switch (archive.State)
{
case WabbajackCDNDownloader.State _:
case GoogleDriveDownloader.State _:
case ManualDownloader.State _:
case ModDBDownloader.State _:
case ModDBDownloader.State _:
case HTTPDownloader.State h when h.Url.StartsWith("https://wabbajack"):
isValid = true;
break;
default:
isValid = await archive.State.Verify(archive);
isValid = await archive.State.Verify(archive, token.Token);
break;
}
return (Archive: archive, IsValid: isValid);
@ -54,6 +59,10 @@ namespace Wabbajack.Server.Services
_logger.Log(LogLevel.Warning, $"Error for {archive.Name} {archive.State.PrimaryKeyString} {ex}");
return (Archive: archive, IsValid: false);
}
finally
{
ReportEnding(archive.State.PrimaryKeyString);
}
});

4
Wabbajack.Server/Services/QuickSync.cs
View File

@ -21,11 +21,11 @@ namespace Wabbajack.Server.Services
_logger = logger;
}
public async Task<Dictionary<Type, (TimeSpan Delay, TimeSpan LastRunTime)>> Report()
public async Task<Dictionary<Type, (TimeSpan Delay, TimeSpan LastRunTime, (String, DateTime)[] ActiveWork)>> Report()
{
using var _ = await _lock.WaitAsync();
return _services.ToDictionary(s => s.Key,
s => (s.Value.Delay, DateTime.UtcNow - s.Value.LastEnd));
s => (s.Value.Delay, DateTime.UtcNow - s.Value.LastEnd, s.Value.ActiveWorkStatus));
}
public async Task Register<T>(T service)

4
Wabbajack.Server/Services/Watchdog.cs
View File

@ -20,10 +20,10 @@ namespace Wabbajack.Server.Services
var report = await _quickSync.Report();
foreach (var service in report)
{
if (service.Value.LastRunTime != default && service.Value.LastRunTime >= service.Value.Delay * 2)
if (service.Value.LastRunTime != default && service.Value.LastRunTime >= service.Value.Delay * 4)
{
await _discord.Send(Channel.Spam,
new DiscordMessage {Content = $"Service {service.Key.Name} has missed it's scheduled execution window"});
new DiscordMessage {Content = $"Service {service.Key.Name} has missed it's scheduled execution window. \n Current work: \n {string.Join("\n", service.Value.ActiveWork)}"});
}
}

4
Wabbajack.Server/Wabbajack.Server.csproj
View File

@ -3,8 +3,8 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
<AssemblyVersion>2.3.6.0</AssemblyVersion>
<FileVersion>2.3.6.0</FileVersion>
<AssemblyVersion>2.3.6.1</AssemblyVersion>
<FileVersion>2.3.6.1</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>Wabbajack Server</Description>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>

28
Wabbajack.Test/DownloaderTests.cs
View File

@ -291,9 +291,33 @@ namespace Wabbajack.Test
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
var otherfiles = await ((LoversLabDownloader.State)state).GetFilesInGroup();
}
[Fact]
public async Task CanCancelLLValidation()
{
await using var filename = new TempFile();
await DownloadDispatcher.GetInstance<LoversLabDownloader>().Prepare();
var state = new LoversLabDownloader.State
{
FileName = "14424-books-of-dibella-se-alternate-start-plugin", FileID = "870820",
};
using var queue = new WorkQueue();
var tcs = new CancellationTokenSource();
tcs.CancelAfter(2);
await Assert.ThrowsAsync<TaskCanceledException>(async () =>
{
await Enumerable.Range(0, 2).PMap(queue,
async x =>
{
Assert.True(await state.Verify(new Archive(state: null!) {Size = 252269}, tcs.Token));
});
});
}
[Fact]

4
Wabbajack/Wabbajack.csproj
View File

@ -6,8 +6,8 @@
<UseWPF>true</UseWPF>
<Platforms>x64</Platforms>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
<AssemblyVersion>2.3.6.0</AssemblyVersion>
<FileVersion>2.3.6.0</FileVersion>
<AssemblyVersion>2.3.6.1</AssemblyVersion>
<FileVersion>2.3.6.1</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>An automated ModList installer</Description>
<PublishReadyToRun>true</PublishReadyToRun>