Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Pass the csrf token into the IPS4 downloader page

Browse Source
This commit is contained in:
Timothy Baldridge 2021-01-10 19:08:14 -07:00
parent 783dd76209
commit e7db21d0fe
3 changed files with 71 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Wabbajack.CLI/OptionsDefinition.cs
View File

@ -37,7 +37,8 @@ namespace Wabbajack.CLI
typeof(Restore),
typeof(PurgeArchive),
typeof(AllKnownDownloadStates),
typeof(VerifyAllDownloads)
typeof(VerifyAllDownloads),
typeof(HashBenchmark)
};
}
}

46
Wabbajack.CLI/Verbs/HashBenchmark.cs Normal file
View File

@ -0,0 +1,46 @@
using System;
using System.Diagnostics;
using System.Linq;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Common;
namespace Wabbajack.CLI.Verbs
{
[Verb("hash-benchmark", HelpText = "Perform a benchmark of the hash routines by benchmarking all files in a folder")]
public class HashBenchmark : AVerb
{
[Option('i', "input", Required = true, HelpText = "Input Folder")]
public string _input { get; set; } = "";
public AbsolutePath Input => (AbsolutePath)_input;
[Option('t', "threads", Required = false, HelpText = "Thread Count (defaults to number of logical cores)")]
public int ThreadCount { get; set; } = Environment.ProcessorCount;
protected override async Task<ExitCode> Run()
{
using var queue = new WorkQueue(ThreadCount);
var files = Input.EnumerateFiles().Select(f => (f, f.Size)).Take(1000).ToArray();
var totalSize = files.Sum(f => f.Size);
Console.WriteLine($"Found {files.Length} files and {totalSize.ToFileSizeString()} to hash");
var stopwatch = new Stopwatch();
stopwatch.Start();
var su = new StatusUpdateTracker(1);
await files.PMap(queue, su, async f =>
{
await f.f.FileHashAsync();
});
stopwatch.Stop();
Console.WriteLine($"Hashed {totalSize.ToFileSizeString()} in {stopwatch.Elapsed.TotalSeconds} or {((long)(totalSize/stopwatch.Elapsed.TotalSeconds)).ToFileSizeString()}/sec");
return ExitCode.Ok;
}
}
}

33
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -198,15 +198,7 @@ namespace Wabbajack.Lib.Downloaders
}
else
{
var csrfURL = string.IsNullOrWhiteSpace(FileID)
? $"{Site}/files/file/{FileName}/?do=download"
: $"{Site}/files/file/{FileName}/?do=download&r={FileID}";
var html = await GetStringAsync(new Uri(csrfURL), token);
var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])");
var matches = pattern.Matches(html).Cast<Match>();
var csrfKey = matches.Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault();
var csrfKey = await GetCsrfKey(token);
if (!Downloader.IsCloudFlareProtected && csrfKey == null)
{
@ -291,6 +283,20 @@ namespace Wabbajack.Lib.Downloaders
goto TOP;
}
private async Task<string?> GetCsrfKey(CancellationToken? token)
{
var csrfURL = string.IsNullOrWhiteSpace(FileID)
? $"{Site}/files/file/{FileName}/?do=download"
: $"{Site}/files/file/{FileName}/?do=download&r={FileID}";
var html = await GetStringAsync(new Uri(csrfURL), token);
var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])");
var matches = pattern.Matches(html).Cast<Match>();
var csrfKey = matches.Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault();
return csrfKey;
}
private async Task DeleteOldDownloadCookies(Driver driver)
{
await driver.DeleteCookiesWhere(c => c.Name.StartsWith("ips4_downloads_delay_") && c.Value == "-1");
@ -354,8 +360,15 @@ namespace Wabbajack.Lib.Downloaders
{
var token = new CancellationTokenSource();
token.CancelAfter(TimeSpan.FromMinutes(10));
var csrfKey = await GetCsrfKey(token.Token);
if (csrfKey == null)
{
Utils.Log($"Could not load CRSF key");
return new List<Archive>();
}
var others = await GetHtmlAsync(new Uri($"{Site}/files/file/{FileName}?do=download"), token.Token);
var others = await GetHtmlAsync(new Uri($"{Site}/files/file/{FileName}?do=download&csrfKey={csrfKey}"), token.Token);
var pairs = others.DocumentNode.SelectNodes("//a[@data-action='download']")
.Select(item => (item.GetAttributeValue("href", ""),