mirror of
https://github.com/wabbajack-tools/wabbajack.git
synced 2024-08-30 18:42:17 +00:00
Merge pull request #354 from wabbajack-tools/vector-plus-redux
Implement VectorPlexus support as a downloader
This commit is contained in:
commit
b8a6db5211
@ -158,7 +158,7 @@ namespace Wabbajack.CacheServer
|
||||
|
||||
using (var queue = new WorkQueue())
|
||||
{
|
||||
foreach (var list in modlists.Skip(2).Take(1))
|
||||
foreach (var list in modlists)
|
||||
{
|
||||
try
|
||||
{
|
||||
|
155
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
Normal file
155
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
Normal file
@ -0,0 +1,155 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
using System.Web;
|
||||
using Newtonsoft.Json;
|
||||
using Wabbajack.Common;
|
||||
using Wabbajack.Lib.Validation;
|
||||
using File = System.IO.File;
|
||||
|
||||
namespace Wabbajack.Lib.Downloaders
|
||||
{
|
||||
// IPS4 is the site used by LoversLab, VectorPlexus, etc. the general mechanics of each site are the
|
||||
// same, so we can fairly easily abstract the state.
|
||||
// Pass in the state type via TState
|
||||
public abstract class AbstractIPS4Downloader<TDownloader, TState> : AbstractNeedsLoginDownloader, IDownloader
|
||||
where TState : AbstractIPS4Downloader<TDownloader, TState>.State<TDownloader>, new()
|
||||
where TDownloader : IDownloader
|
||||
{
|
||||
public override string SiteName { get; }
|
||||
public override Uri SiteURL { get; }
|
||||
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
|
||||
{
|
||||
Uri url = DownloaderUtils.GetDirectURL(archiveINI);
|
||||
if (url == null || url.Host != SiteURL.Host || !url.AbsolutePath.StartsWith("/files/file/")) return null;
|
||||
var id = HttpUtility.ParseQueryString(url.Query)["r"];
|
||||
var file = url.AbsolutePath.Split('/').Last(s => s != "");
|
||||
|
||||
return new TState
|
||||
{
|
||||
FileID = id,
|
||||
FileName = file
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
public class State<TDownloader> : AbstractDownloadState where TDownloader : IDownloader
|
||||
{
|
||||
public string FileID { get; set; }
|
||||
public string FileName { get; set; }
|
||||
|
||||
public override object[] PrimaryKey { get => new object[] {FileID, FileName}; }
|
||||
|
||||
public override bool IsWhitelisted(ServerWhitelist whitelist)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
public override async Task Download(Archive a, string destination)
|
||||
{
|
||||
var stream = await ResolveDownloadStream();
|
||||
using (var file = File.OpenWrite(destination))
|
||||
{
|
||||
stream.CopyTo(file);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<Stream> ResolveDownloadStream()
|
||||
{
|
||||
var downloader = (AbstractNeedsLoginDownloader)(object)DownloadDispatcher.GetInstance<TDownloader>();
|
||||
|
||||
TOP:
|
||||
string csrfurl;
|
||||
if (FileID == null)
|
||||
{
|
||||
csrfurl = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download";
|
||||
}
|
||||
else
|
||||
{
|
||||
csrfurl = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID}";
|
||||
}
|
||||
var html = await downloader.AuthedClient.GetStringAsync(csrfurl);
|
||||
|
||||
var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])");
|
||||
var matches = pattern.Matches(html).Cast<Match>();
|
||||
|
||||
var csrfKey = matches.Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault();
|
||||
|
||||
if (csrfKey == null)
|
||||
return null;
|
||||
|
||||
string url;
|
||||
if (FileID == null)
|
||||
url = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&confirm=1&t=1&csrfKey={csrfKey}";
|
||||
else
|
||||
url = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}";
|
||||
|
||||
|
||||
var streamResult = await downloader.AuthedClient.GetAsync(url);
|
||||
if (streamResult.StatusCode != HttpStatusCode.OK)
|
||||
{
|
||||
Utils.Error(new InvalidOperationException(), $"{downloader.SiteName} servers reported an error for file: {FileID}");
|
||||
}
|
||||
|
||||
var content_type = streamResult.Content.Headers.ContentType;
|
||||
|
||||
if (content_type.MediaType == "application/json")
|
||||
{
|
||||
// Sometimes LL hands back a json object telling us to wait until a certain time
|
||||
var times = (await streamResult.Content.ReadAsStringAsync()).FromJSONString<WaitResponse>();
|
||||
var secs = times.Download - times.CurrentTime;
|
||||
for (int x = 0; x < secs; x++)
|
||||
{
|
||||
Utils.Status($"Waiting for {secs} at the request of {downloader.SiteName}", x * 100 / secs);
|
||||
await Task.Delay(1000);
|
||||
}
|
||||
Utils.Status("Retrying download");
|
||||
goto TOP;
|
||||
}
|
||||
|
||||
return await streamResult.Content.ReadAsStreamAsync();
|
||||
}
|
||||
|
||||
private class WaitResponse
|
||||
{
|
||||
[JsonProperty("download")]
|
||||
public int Download { get; set; }
|
||||
[JsonProperty("currentTime")]
|
||||
public int CurrentTime { get; set; }
|
||||
}
|
||||
|
||||
public override async Task<bool> Verify()
|
||||
{
|
||||
var stream = await ResolveDownloadStream();
|
||||
if (stream == null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
stream.Close();
|
||||
return true;
|
||||
}
|
||||
|
||||
public override IDownloader GetDownloader()
|
||||
{
|
||||
return DownloadDispatcher.GetInstance<TDownloader>();
|
||||
}
|
||||
|
||||
public override string GetReportEntry(Archive a)
|
||||
{
|
||||
var downloader = (INeedsLogin)GetDownloader();
|
||||
return $"* {((INeedsLogin)GetDownloader()).SiteName} - [{a.Name}](https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID})";
|
||||
}
|
||||
}
|
||||
|
||||
protected AbstractIPS4Downloader(Uri loginUri, string encryptedKeyName, string cookieDomain) :
|
||||
base(loginUri, encryptedKeyName, cookieDomain, "ips4_member_id")
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
@ -20,7 +20,7 @@ namespace Wabbajack.Lib.Downloaders
|
||||
private readonly string _encryptedKeyName;
|
||||
private readonly string _cookieDomain;
|
||||
private readonly string _cookieName;
|
||||
protected HttpClient AuthedClient;
|
||||
internal HttpClient AuthedClient;
|
||||
|
||||
/// <summary>
|
||||
/// Sets up all the login facilites needed for a INeedsLogin downloader based on having the user log
|
||||
|
@ -18,6 +18,7 @@ namespace Wabbajack.Lib.Downloaders
|
||||
new NexusDownloader(),
|
||||
new MediaFireDownloader(),
|
||||
new LoversLabDownloader(),
|
||||
new VectorPlexusDownloader(),
|
||||
new HTTPDownloader(),
|
||||
new ManualDownloader(),
|
||||
};
|
||||
|
@ -21,33 +21,18 @@ using File = Alphaleonis.Win32.Filesystem.File;
|
||||
|
||||
namespace Wabbajack.Lib.Downloaders
|
||||
{
|
||||
public class LoversLabDownloader : AbstractNeedsLoginDownloader, IDownloader
|
||||
public class LoversLabDownloader : AbstractIPS4Downloader<LoversLabDownloader, LoversLabDownloader.State>
|
||||
{
|
||||
#region INeedsDownload
|
||||
public override string SiteName => "Lovers Lab";
|
||||
public override Uri SiteURL => new Uri("https://loverslab.com");
|
||||
public override Uri SiteURL => new Uri("https://www.loverslab.com");
|
||||
public override Uri IconUri => new Uri("https://www.loverslab.com/favicon.ico");
|
||||
#endregion
|
||||
|
||||
public LoversLabDownloader() : base(new Uri("https://www.loverslab.com/login"),
|
||||
"loverslabcookies", "loverslab.com", "ips4_member_id")
|
||||
"loverslabcookies", "loverslab.com")
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archive_ini)
|
||||
{
|
||||
Uri url = DownloaderUtils.GetDirectURL(archive_ini);
|
||||
if (url == null || url.Host != "www.loverslab.com" || !url.AbsolutePath.StartsWith("/files/file/")) return null;
|
||||
var id = HttpUtility.ParseQueryString(url.Query)["r"];
|
||||
var file = url.AbsolutePath.Split('/').Last(s => s != "");
|
||||
|
||||
return new State
|
||||
{
|
||||
FileID = id,
|
||||
FileName = file
|
||||
};
|
||||
}
|
||||
protected override async Task WhileWaiting(IWebDriver browser)
|
||||
{
|
||||
try
|
||||
@ -60,97 +45,8 @@ namespace Wabbajack.Lib.Downloaders
|
||||
Utils.Error(ex);
|
||||
}
|
||||
}
|
||||
|
||||
public class State : AbstractDownloadState
|
||||
public class State : State<LoversLabDownloader>
|
||||
{
|
||||
public string FileID { get; set; }
|
||||
public string FileName { get; set; }
|
||||
|
||||
public override object[] PrimaryKey { get => new object[] {FileID, FileName}; }
|
||||
|
||||
public override bool IsWhitelisted(ServerWhitelist whitelist)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
public override async Task Download(Archive a, string destination)
|
||||
{
|
||||
var stream = await ResolveDownloadStream();
|
||||
using (var file = File.OpenWrite(destination))
|
||||
{
|
||||
stream.CopyTo(file);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<Stream> ResolveDownloadStream()
|
||||
{
|
||||
var result = DownloadDispatcher.GetInstance<LoversLabDownloader>();
|
||||
TOP:
|
||||
var html = await result.AuthedClient.GetStringAsync(
|
||||
$"https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID}");
|
||||
|
||||
var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])");
|
||||
var csrfKey = pattern.Matches(html).Cast<Match>().Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault();
|
||||
|
||||
if (csrfKey == null)
|
||||
return null;
|
||||
|
||||
var url =
|
||||
$"https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}";
|
||||
|
||||
var streamResult = await result.AuthedClient.GetAsync(url);
|
||||
if (streamResult.StatusCode != HttpStatusCode.OK)
|
||||
{
|
||||
Utils.Error(new InvalidOperationException(), $"LoversLab servers reported an error for file: {FileID}");
|
||||
}
|
||||
|
||||
var content_type = streamResult.Content.Headers.ContentType;
|
||||
|
||||
if (content_type.MediaType == "application/json")
|
||||
{
|
||||
// Sometimes LL hands back a json object telling us to wait until a certain time
|
||||
var times = (await streamResult.Content.ReadAsStringAsync()).FromJSONString<WaitResponse>();
|
||||
var secs = times.download - times.currentTime;
|
||||
for (int x = 0; x < secs; x++)
|
||||
{
|
||||
Utils.Status($"Waiting for {secs} at the request of LoversLab", x * 100 / secs);
|
||||
await Task.Delay(1000);
|
||||
}
|
||||
Utils.Status("Retrying download");
|
||||
goto TOP;
|
||||
}
|
||||
|
||||
return await streamResult.Content.ReadAsStreamAsync();
|
||||
}
|
||||
|
||||
internal class WaitResponse
|
||||
{
|
||||
public int download { get; set; }
|
||||
public int currentTime { get; set; }
|
||||
}
|
||||
|
||||
public override async Task<bool> Verify()
|
||||
{
|
||||
var stream = await ResolveDownloadStream();
|
||||
if (stream == null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
stream.Close();
|
||||
return true;
|
||||
}
|
||||
|
||||
public override IDownloader GetDownloader()
|
||||
{
|
||||
return DownloadDispatcher.GetInstance<LoversLabDownloader>();
|
||||
}
|
||||
|
||||
public override string GetReportEntry(Archive a)
|
||||
{
|
||||
return $"* Lovers Lab - [{a.Name}](https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID})";
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
22
Wabbajack.Lib/Downloaders/VectorPlexusDownloader.cs
Normal file
22
Wabbajack.Lib/Downloaders/VectorPlexusDownloader.cs
Normal file
@ -0,0 +1,22 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Wabbajack.Lib.Downloaders
|
||||
{
|
||||
public class VectorPlexusDownloader : AbstractIPS4Downloader<VectorPlexusDownloader, VectorPlexusDownloader.State>
|
||||
{
|
||||
#region INeedsDownload
|
||||
public override string SiteName => "Vector Plexus";
|
||||
public override Uri SiteURL => new Uri("https://vectorplexus.com");
|
||||
public override Uri IconUri => new Uri("https://www.vectorplexus.com/favicon.ico");
|
||||
#endregion
|
||||
|
||||
public VectorPlexusDownloader() : base(new Uri("https://vectorplexus.com/login"),
|
||||
"vectorplexus", "vectorplexus.com")
|
||||
{
|
||||
}
|
||||
public class State : State<VectorPlexusDownloader>
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
@ -123,11 +123,13 @@
|
||||
<Compile Include="CompilationSteps\IStackStep.cs" />
|
||||
<Compile Include="CompilationSteps\PatchStockESMs.cs" />
|
||||
<Compile Include="CompilationSteps\Serialization.cs" />
|
||||
<Compile Include="Downloaders\AbstractIPS4Downloader.cs" />
|
||||
<Compile Include="Downloaders\AbstractNeedsLoginDownloader.cs" />
|
||||
<Compile Include="Downloaders\GameFileSourceDownloader.cs" />
|
||||
<Compile Include="Downloaders\INeedsLogin.cs" />
|
||||
<Compile Include="Downloaders\LoversLabDownloader.cs" />
|
||||
<Compile Include="Downloaders\SteamWorkshopDownloader.cs" />
|
||||
<Compile Include="Downloaders\VectorPlexusDownloader.cs" />
|
||||
<Compile Include="Extensions\ReactiveUIExt.cs" />
|
||||
<Compile Include="LibCefHelpers\Init.cs" />
|
||||
<Compile Include="MO2Compiler.cs" />
|
||||
|
@ -310,6 +310,34 @@ namespace Wabbajack.Test
|
||||
|
||||
Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task VectorPlexusDownload()
|
||||
{
|
||||
await DownloadDispatcher.GetInstance<VectorPlexusDownloader>().Prepare();
|
||||
var ini = @"[General]
|
||||
directURL=https://vectorplexus.com/files/file/290-wabbajack-test-file";
|
||||
|
||||
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
|
||||
|
||||
Assert.IsNotNull(state);
|
||||
|
||||
/*var url_state = DownloadDispatcher.ResolveArchive("https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1");
|
||||
Assert.AreEqual("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt",
|
||||
((HTTPDownloader.State)url_state).Url);
|
||||
*/
|
||||
var converted = state.ViaJSON();
|
||||
Assert.IsTrue(await converted.Verify());
|
||||
var filename = Guid.NewGuid().ToString();
|
||||
|
||||
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
|
||||
|
||||
await converted.Download(new Archive { Name = "Vector Plexus Test.zip" }, filename);
|
||||
|
||||
Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
|
||||
|
||||
Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task GameFileSourceDownload()
|
||||
|
Loading…
Reference in New Issue
Block a user