wabbajack/Wabbajack.CLI/Verbs/VerifyAllDownloads.cs
Timothy Baldridge db3b441d19 #### Version - 2.3.6.1 - 12/31/2020
* When IPS4 (e.g. LL) sites based on CEF fail to validate, they no longer hang the app
* If a IPS4 CEF site throws a 503, or 400 error, retry
* Clean out the cookies during IPS4 CEF downloads so that they don't cause 400 errors
* Limit the number of connections to IPS4 sites to 20 per minute (one per 6 seconds)
* If a site *does* timeout, throw a log of the CEF state into `CEFStates` for easier debugging by the WJ team
* Wrote a new CLI utility to stress test the Verification routines.
* Ignore files that have `\Edit Scripts\Export\` in their path
2020-12-30 23:44:58 -07:00

101 lines
3.5 KiB
C#

using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.LibCefHelpers;
namespace Wabbajack.CLI.Verbs
{
[Verb("verify-all-downloads", HelpText = "Verify all downloads in a folder")]
public class VerifyAllDownloads : AVerb
{
[Option('i', "input", Required = true, HelpText = "Input Folder")]
public string _input { get; set; } = "";
public AbsolutePath Input => (AbsolutePath)_input;
[Option('t', "type", Required = false,
HelpText = "Only verify files of this type of download state for example NexusDownloader+State")]
public string StateType { get; set; } = "";
protected override async Task<ExitCode> Run()
{
var files = Input.EnumerateFiles()
.Where(f => f.WithExtension(Consts.MetaFileExtension).Exists)
.ToArray();
Console.WriteLine($"Found {files.Length} files to verify");
using var queue = new WorkQueue();
var states = (await files.PMap(queue, async f =>
{
var ini = f.WithExtension(Consts.MetaFileExtension).LoadIniFile();
var state = (AbstractDownloadState?)await DownloadDispatcher.ResolveArchive(ini, quickMode: true);
if (state == default)
{
Console.WriteLine($"[Skipping] {f.FileName} because no meta could be interpreted");
}
if (!string.IsNullOrWhiteSpace(StateType) && !state!.PrimaryKeyString.StartsWith(StateType + "|"))
{
Console.WriteLine(
$"[Skipping] {f.FileName} because type {state.PrimaryKeyString[0]} does not match filter");
return (f, null);
}
return (f, state);
})).Where(s => s.state != null)
.Select(s => (s.f, s.state!))
.ToArray();
await DownloadDispatcher.PrepareAll(states.Select(s => s.Item2));
Helpers.Init();
Console.WriteLine($"Found {states.Length} states to verify");
int timedOut = 0;
await states.PMap(queue, async p=>
{
try
{
var (f, state) = p;
try
{
var cts = new CancellationTokenSource();
cts.CancelAfter(TimeSpan.FromMinutes(10));
var result =
await state!.Verify(new Archive(state) {Name = f.FileName.ToString(), Size = f.Size},
cts.Token);
Console.WriteLine($"[{(result ? "Failed" : "Passed")}] {f.FileName}");
}
catch (TaskCanceledException)
{
Console.WriteLine($"[Timed Out] {f.FileName} {state!.PrimaryKeyString}");
Interlocked.Increment(ref timedOut);
}
}
catch (Exception ex)
{
Console.WriteLine($"[Exception] {p.f.FileName} {ex.Message}");
}
});
Console.WriteLine($"[Total TimedOut] {timedOut}");
Console.WriteLine("[Done]");
return ExitCode.Ok;
}
}
}