Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #436 from erri120/more-downloader-tests

Browse Source 
More downloader tests
This commit is contained in:
Timothy Baldridge 2020-01-27 14:27:08 -07:00 committed by GitHub
commit 1ac0210746
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 96 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -25,10 +25,21 @@ namespace Wabbajack.Lib.Downloaders
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
{
Uri url = DownloaderUtils.GetDirectURL(archiveINI);
if (url == null || url.Host != SiteURL.Host || !url.AbsolutePath.StartsWith("/files/file/")) return null;
var id = HttpUtility.ParseQueryString(url.Query)["r"];
var file = url.AbsolutePath.Split('/').Last(s => s != "");
var absolute = true;
if (url == null || url.Host != SiteURL.Host) return null;
if (!url.PathAndQuery.StartsWith("/files/file/"))
{
if (string.IsNullOrWhiteSpace(url.Query)) return null;
if (!url.Query.Substring(1).StartsWith("/files/file/")) return null;
absolute = false;
}
var id = HttpUtility.ParseQueryString(url.Query)["r"];
var file = absolute
? url.AbsolutePath.Split('/').Last(s => s != "")
: url.Query.Substring(1).Split('/').Last(s => s != "");
return new TState
{
FileID = id,
@ -45,13 +56,17 @@ namespace Wabbajack.Lib.Downloaders
private static bool IsHTTPS => Downloader.SiteURL.AbsolutePath.StartsWith("https://");
private static string URLPrefix => IsHTTPS ? "https://" : "http://";
private static string Site => string.IsNullOrWhiteSpace(Downloader.SiteURL.Query)
? $"{URLPrefix}{Downloader.SiteURL.Host}"
: Downloader.SiteURL.ToString();
public override object[] PrimaryKey
{
get
{
if (FileID == null)
return new object[] {Downloader.SiteURL, FileName};
return new object[] {Downloader.SiteURL, FileName, FileID};
return FileID == null
? new object[] {Downloader.SiteURL, FileName}
: new object[] {Downloader.SiteURL, FileName, FileID};
}
}
@ -75,15 +90,9 @@ namespace Wabbajack.Lib.Downloaders
var downloader = (AbstractNeedsLoginDownloader)(object)DownloadDispatcher.GetInstance<TDownloader>();
TOP:
string csrfurl;
if (FileID == null)
{
csrfurl = $"{URLPrefix}{downloader.SiteURL.Host}/files/file/{FileName}/?do=download";
}
else
{
csrfurl = $"{URLPrefix}{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID}";
}
var csrfurl = FileID == null
? $"{Site}/files/file/{FileName}/?do=download"
: $"{Site}/files/file/{FileName}/?do=download&r={FileID}";
var html = await downloader.AuthedClient.GetStringAsync(csrfurl);
var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])");
@ -94,11 +103,9 @@ namespace Wabbajack.Lib.Downloaders
if (csrfKey == null)
return null;
string url;
if (FileID == null)
url = $"{URLPrefix}{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&confirm=1&t=1&csrfKey={csrfKey}";
else
url = $"{URLPrefix}{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}";
var url = FileID == null
? $"{Site}/files/file/{FileName}/?do=download&confirm=1&t=1&csrfKey={csrfKey}"
: $"{Site}/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}";
var streamResult = await downloader.AuthedClient.GetAsync(url);
@ -107,23 +114,21 @@ namespace Wabbajack.Lib.Downloaders
Utils.Error(new InvalidOperationException(), $"{downloader.SiteName} servers reported an error for file: {FileID}");
}
var content_type = streamResult.Content.Headers.ContentType;
var contentType = streamResult.Content.Headers.ContentType;
if (content_type.MediaType == "application/json")
if (contentType.MediaType != "application/json")
return await streamResult.Content.ReadAsStreamAsync();
// Sometimes LL hands back a json object telling us to wait until a certain time
var times = (await streamResult.Content.ReadAsStringAsync()).FromJSONString<WaitResponse>();
var secs = times.Download - times.CurrentTime;
for (int x = 0; x < secs; x++)
{
// Sometimes LL hands back a json object telling us to wait until a certain time
var times = (await streamResult.Content.ReadAsStringAsync()).FromJSONString<WaitResponse>();
var secs = times.Download - times.CurrentTime;
for (int x = 0; x < secs; x++)
{
Utils.Status($"Waiting for {secs} at the request of {downloader.SiteName}", x * 100 / secs);
await Task.Delay(1000);
}
Utils.Status("Retrying download");
goto TOP;
Utils.Status($"Waiting for {secs} at the request of {downloader.SiteName}", x * 100 / secs);
await Task.Delay(1000);
}
return await streamResult.Content.ReadAsStreamAsync();
Utils.Status("Retrying download");
goto TOP;
}
private class WaitResponse
@ -138,9 +143,7 @@ namespace Wabbajack.Lib.Downloaders
{
var stream = await ResolveDownloadStream();
if (stream == null)
{
return false;
}
stream.Close();
return true;
@ -153,24 +156,21 @@ namespace Wabbajack.Lib.Downloaders
public override string GetReportEntry(Archive a)
{
var downloader = (INeedsLogin)GetDownloader();
return $"* {((INeedsLogin)GetDownloader()).SiteName} - [{a.Name}]({URLPrefix}{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID})";
return $"* {((INeedsLogin)GetDownloader()).SiteName} - [{a.Name}]({Site}/files/file/{FileName}/?do=download&r={FileID})";
}
public override string[] GetMetaIni()
{
var downloader = Downloader;
if (FileID != null)
return new[]
{
"[General]",
$"directURL={URLPrefix}{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1"
$"directURL={Site}/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1"
};
return new[]
{
"[General]",
$"directURL={URLPrefix}{downloader.SiteURL.Host}/files/file/{FileName}"
$"directURL={Site}/files/file/{FileName}"
};
}

60
Wabbajack.Test/DownloaderTests.cs
View File

@ -1,11 +1,8 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using System.Reactive.Disposables;
using System.Reactive.Linq;
using Alphaleonis.Win32.Filesystem;
using CefSharp;
using CefSharp.OffScreen;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Common;
using Wabbajack.Common.StatusFeed;
@ -14,7 +11,6 @@ using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.LibCefHelpers;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using Wabbajack.Lib.WebAutomation;
using File = Alphaleonis.Win32.Filesystem.File;
using Game = Wabbajack.Common.Game;
@ -304,9 +300,9 @@ namespace Wabbajack.Test
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
await converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
await converted.Download(new Archive { Name = "LoversLab Test.txt" }, filename);
Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
Assert.AreEqual("eSIyd+KOG3s=", filename.FileHash());
Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
}
@ -334,11 +330,61 @@ namespace Wabbajack.Test
await converted.Download(new Archive { Name = "Vector Plexus Test.zip" }, filename);
Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
Assert.AreEqual("eSIyd+KOG3s=", filename.FileHash());
Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
}
[TestMethod]
public async Task TESAllianceDownload()
{
await DownloadDispatcher.GetInstance<TESAllianceDownloader>().Prepare();
const string ini = "[General]\n" +
"directURL=http://tesalliance.org/forums/index.php?/files/file/2035-wabbajack-test-file/";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.IsNotNull(state);
var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
await converted.Download(new Archive { Name = "TESAlliance Test.zip" }, filename);
Assert.AreEqual("eSIyd+KOG3s=", filename.FileHash());
Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
}
/* WAITING FOR APPROVAL BY MODERATOR
[TestMethod]
public async Task DeadlyStreamDownloader()
{
await DownloadDispatcher.GetInstance<DeadlyStreamDownloader>().Prepare();
const string ini = "[General]\n" +
"directURL=https://deadlystream.com/files/file/1550-wabbajack-test-file/";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.IsNotNull(state);
var converted = await state.RoundTripState();
Assert.IsTrue(await converted.Verify(new Archive{Size = 20}));
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
await converted.Download(new Archive { Name = "DeadlyStream Test.zip" }, filename);
Assert.AreEqual("eSIyd+KOG3s=", filename.FileHash());
Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
}*/
[TestMethod]
public async Task GameFileSourceDownload()
{

1
extra/WABBAJACK_TEST_FILE.zip Normal file
View File

@ -0,0 +1 @@
Cheese for Everyone!