Merge pull request #354 from wabbajack-tools/vector-plus-redux

Implement VectorPlexus support as a downloader
This commit is contained in:
Timothy Baldridge 2020-01-05 16:40:34 -08:00 committed by GitHub
commit b8a6db5211
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 214 additions and 110 deletions

View File

@ -158,7 +158,7 @@ namespace Wabbajack.CacheServer
using (var queue = new WorkQueue())
{
foreach (var list in modlists.Skip(2).Take(1))
foreach (var list in modlists)
{
try
{

View File

@ -0,0 +1,155 @@
using System;
using System.IO;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Web;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib.Validation;
using File = System.IO.File;
namespace Wabbajack.Lib.Downloaders
{
// IPS4 is the site used by LoversLab, VectorPlexus, etc. the general mechanics of each site are the
// same, so we can fairly easily abstract the state.
// Pass in the state type via TState
public abstract class AbstractIPS4Downloader<TDownloader, TState> : AbstractNeedsLoginDownloader, IDownloader
where TState : AbstractIPS4Downloader<TDownloader, TState>.State<TDownloader>, new()
where TDownloader : IDownloader
{
public override string SiteName { get; }
public override Uri SiteURL { get; }
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
{
Uri url = DownloaderUtils.GetDirectURL(archiveINI);
if (url == null || url.Host != SiteURL.Host || !url.AbsolutePath.StartsWith("/files/file/")) return null;
var id = HttpUtility.ParseQueryString(url.Query)["r"];
var file = url.AbsolutePath.Split('/').Last(s => s != "");
return new TState
{
FileID = id,
FileName = file
};
}
public class State<TDownloader> : AbstractDownloadState where TDownloader : IDownloader
{
public string FileID { get; set; }
public string FileName { get; set; }
public override object[] PrimaryKey { get => new object[] {FileID, FileName}; }
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
return true;
}
public override async Task Download(Archive a, string destination)
{
var stream = await ResolveDownloadStream();
using (var file = File.OpenWrite(destination))
{
stream.CopyTo(file);
}
}
private async Task<Stream> ResolveDownloadStream()
{
var downloader = (AbstractNeedsLoginDownloader)(object)DownloadDispatcher.GetInstance<TDownloader>();
TOP:
string csrfurl;
if (FileID == null)
{
csrfurl = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download";
}
else
{
csrfurl = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID}";
}
var html = await downloader.AuthedClient.GetStringAsync(csrfurl);
var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])");
var matches = pattern.Matches(html).Cast<Match>();
var csrfKey = matches.Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault();
if (csrfKey == null)
return null;
string url;
if (FileID == null)
url = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&confirm=1&t=1&csrfKey={csrfKey}";
else
url = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}";
var streamResult = await downloader.AuthedClient.GetAsync(url);
if (streamResult.StatusCode != HttpStatusCode.OK)
{
Utils.Error(new InvalidOperationException(), $"{downloader.SiteName} servers reported an error for file: {FileID}");
}
var content_type = streamResult.Content.Headers.ContentType;
if (content_type.MediaType == "application/json")
{
// Sometimes LL hands back a json object telling us to wait until a certain time
var times = (await streamResult.Content.ReadAsStringAsync()).FromJSONString<WaitResponse>();
var secs = times.Download - times.CurrentTime;
for (int x = 0; x < secs; x++)
{
Utils.Status($"Waiting for {secs} at the request of {downloader.SiteName}", x * 100 / secs);
await Task.Delay(1000);
}
Utils.Status("Retrying download");
goto TOP;
}
return await streamResult.Content.ReadAsStreamAsync();
}
private class WaitResponse
{
[JsonProperty("download")]
public int Download { get; set; }
[JsonProperty("currentTime")]
public int CurrentTime { get; set; }
}
public override async Task<bool> Verify()
{
var stream = await ResolveDownloadStream();
if (stream == null)
{
return false;
}
stream.Close();
return true;
}
public override IDownloader GetDownloader()
{
return DownloadDispatcher.GetInstance<TDownloader>();
}
public override string GetReportEntry(Archive a)
{
var downloader = (INeedsLogin)GetDownloader();
return $"* {((INeedsLogin)GetDownloader()).SiteName} - [{a.Name}](https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID})";
}
}
protected AbstractIPS4Downloader(Uri loginUri, string encryptedKeyName, string cookieDomain) :
base(loginUri, encryptedKeyName, cookieDomain, "ips4_member_id")
{
}
}
}

View File

@ -20,7 +20,7 @@ namespace Wabbajack.Lib.Downloaders
private readonly string _encryptedKeyName;
private readonly string _cookieDomain;
private readonly string _cookieName;
protected HttpClient AuthedClient;
internal HttpClient AuthedClient;
/// <summary>
/// Sets up all the login facilites needed for a INeedsLogin downloader based on having the user log

View File

@ -18,6 +18,7 @@ namespace Wabbajack.Lib.Downloaders
new NexusDownloader(),
new MediaFireDownloader(),
new LoversLabDownloader(),
new VectorPlexusDownloader(),
new HTTPDownloader(),
new ManualDownloader(),
};

View File

@ -21,33 +21,18 @@ using File = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.Lib.Downloaders
{
public class LoversLabDownloader : AbstractNeedsLoginDownloader, IDownloader
public class LoversLabDownloader : AbstractIPS4Downloader<LoversLabDownloader, LoversLabDownloader.State>
{
#region INeedsDownload
public override string SiteName => "Lovers Lab";
public override Uri SiteURL => new Uri("https://loverslab.com");
public override Uri SiteURL => new Uri("https://www.loverslab.com");
public override Uri IconUri => new Uri("https://www.loverslab.com/favicon.ico");
#endregion
public LoversLabDownloader() : base(new Uri("https://www.loverslab.com/login"),
"loverslabcookies", "loverslab.com", "ips4_member_id")
"loverslabcookies", "loverslab.com")
{
}
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archive_ini)
{
Uri url = DownloaderUtils.GetDirectURL(archive_ini);
if (url == null || url.Host != "www.loverslab.com" || !url.AbsolutePath.StartsWith("/files/file/")) return null;
var id = HttpUtility.ParseQueryString(url.Query)["r"];
var file = url.AbsolutePath.Split('/').Last(s => s != "");
return new State
{
FileID = id,
FileName = file
};
}
protected override async Task WhileWaiting(IWebDriver browser)
{
try
@ -60,97 +45,8 @@ namespace Wabbajack.Lib.Downloaders
Utils.Error(ex);
}
}
public class State : AbstractDownloadState
public class State : State<LoversLabDownloader>
{
public string FileID { get; set; }
public string FileName { get; set; }
public override object[] PrimaryKey { get => new object[] {FileID, FileName}; }
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
return true;
}
public override async Task Download(Archive a, string destination)
{
var stream = await ResolveDownloadStream();
using (var file = File.OpenWrite(destination))
{
stream.CopyTo(file);
}
}
private async Task<Stream> ResolveDownloadStream()
{
var result = DownloadDispatcher.GetInstance<LoversLabDownloader>();
TOP:
var html = await result.AuthedClient.GetStringAsync(
$"https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID}");
var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])");
var csrfKey = pattern.Matches(html).Cast<Match>().Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault();
if (csrfKey == null)
return null;
var url =
$"https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}";
var streamResult = await result.AuthedClient.GetAsync(url);
if (streamResult.StatusCode != HttpStatusCode.OK)
{
Utils.Error(new InvalidOperationException(), $"LoversLab servers reported an error for file: {FileID}");
}
var content_type = streamResult.Content.Headers.ContentType;
if (content_type.MediaType == "application/json")
{
// Sometimes LL hands back a json object telling us to wait until a certain time
var times = (await streamResult.Content.ReadAsStringAsync()).FromJSONString<WaitResponse>();
var secs = times.download - times.currentTime;
for (int x = 0; x < secs; x++)
{
Utils.Status($"Waiting for {secs} at the request of LoversLab", x * 100 / secs);
await Task.Delay(1000);
}
Utils.Status("Retrying download");
goto TOP;
}
return await streamResult.Content.ReadAsStreamAsync();
}
internal class WaitResponse
{
public int download { get; set; }
public int currentTime { get; set; }
}
public override async Task<bool> Verify()
{
var stream = await ResolveDownloadStream();
if (stream == null)
{
return false;
}
stream.Close();
return true;
}
public override IDownloader GetDownloader()
{
return DownloadDispatcher.GetInstance<LoversLabDownloader>();
}
public override string GetReportEntry(Archive a)
{
return $"* Lovers Lab - [{a.Name}](https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID})";
}
}
}
}

View File

@ -0,0 +1,22 @@
using System;
using System.Threading.Tasks;
namespace Wabbajack.Lib.Downloaders
{
public class VectorPlexusDownloader : AbstractIPS4Downloader<VectorPlexusDownloader, VectorPlexusDownloader.State>
{
#region INeedsDownload
public override string SiteName => "Vector Plexus";
public override Uri SiteURL => new Uri("https://vectorplexus.com");
public override Uri IconUri => new Uri("https://www.vectorplexus.com/favicon.ico");
#endregion
public VectorPlexusDownloader() : base(new Uri("https://vectorplexus.com/login"),
"vectorplexus", "vectorplexus.com")
{
}
public class State : State<VectorPlexusDownloader>
{
}
}
}

View File

@ -123,11 +123,13 @@
<Compile Include="CompilationSteps\IStackStep.cs" />
<Compile Include="CompilationSteps\PatchStockESMs.cs" />
<Compile Include="CompilationSteps\Serialization.cs" />
<Compile Include="Downloaders\AbstractIPS4Downloader.cs" />
<Compile Include="Downloaders\AbstractNeedsLoginDownloader.cs" />
<Compile Include="Downloaders\GameFileSourceDownloader.cs" />
<Compile Include="Downloaders\INeedsLogin.cs" />
<Compile Include="Downloaders\LoversLabDownloader.cs" />
<Compile Include="Downloaders\SteamWorkshopDownloader.cs" />
<Compile Include="Downloaders\VectorPlexusDownloader.cs" />
<Compile Include="Extensions\ReactiveUIExt.cs" />
<Compile Include="LibCefHelpers\Init.cs" />
<Compile Include="MO2Compiler.cs" />

View File

@ -310,6 +310,34 @@ namespace Wabbajack.Test
Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
}
[TestMethod]
public async Task VectorPlexusDownload()
{
await DownloadDispatcher.GetInstance<VectorPlexusDownloader>().Prepare();
var ini = @"[General]
directURL=https://vectorplexus.com/files/file/290-wabbajack-test-file";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.IsNotNull(state);
/*var url_state = DownloadDispatcher.ResolveArchive("https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1");
Assert.AreEqual("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt",
((HTTPDownloader.State)url_state).Url);
*/
var converted = state.ViaJSON();
Assert.IsTrue(await converted.Verify());
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
await converted.Download(new Archive { Name = "Vector Plexus Test.zip" }, filename);
Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
}
[TestMethod]
public async Task GameFileSourceDownload()