diff --git a/Wabbajack.CacheServer/ListValidationService.cs b/Wabbajack.CacheServer/ListValidationService.cs index bcf18a93..0c717b1a 100644 --- a/Wabbajack.CacheServer/ListValidationService.cs +++ b/Wabbajack.CacheServer/ListValidationService.cs @@ -158,7 +158,7 @@ namespace Wabbajack.CacheServer using (var queue = new WorkQueue()) { - foreach (var list in modlists.Skip(2).Take(1)) + foreach (var list in modlists) { try { diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs new file mode 100644 index 00000000..3635b549 --- /dev/null +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs @@ -0,0 +1,155 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Web; +using Newtonsoft.Json; +using Wabbajack.Common; +using Wabbajack.Lib.Validation; +using File = System.IO.File; + +namespace Wabbajack.Lib.Downloaders +{ + // IPS4 is the site used by LoversLab, VectorPlexus, etc. the general mechanics of each site are the + // same, so we can fairly easily abstract the state. + // Pass in the state type via TState + public abstract class AbstractIPS4Downloader : AbstractNeedsLoginDownloader, IDownloader + where TState : AbstractIPS4Downloader.State, new() + where TDownloader : IDownloader + { + public override string SiteName { get; } + public override Uri SiteURL { get; } + public async Task GetDownloaderState(dynamic archiveINI) + { + Uri url = DownloaderUtils.GetDirectURL(archiveINI); + if (url == null || url.Host != SiteURL.Host || !url.AbsolutePath.StartsWith("/files/file/")) return null; + var id = HttpUtility.ParseQueryString(url.Query)["r"]; + var file = url.AbsolutePath.Split('/').Last(s => s != ""); + + return new TState + { + FileID = id, + FileName = file + }; + } + + + public class State : AbstractDownloadState where TDownloader : IDownloader + { + public string FileID { get; set; } + public string FileName { get; set; } + + public override object[] PrimaryKey { get => new object[] {FileID, FileName}; } + + public override bool IsWhitelisted(ServerWhitelist whitelist) + { + return true; + } + + public override async Task Download(Archive a, string destination) + { + var stream = await ResolveDownloadStream(); + using (var file = File.OpenWrite(destination)) + { + stream.CopyTo(file); + } + } + + private async Task ResolveDownloadStream() + { + var downloader = (AbstractNeedsLoginDownloader)(object)DownloadDispatcher.GetInstance(); + + TOP: + string csrfurl; + if (FileID == null) + { + csrfurl = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download"; + } + else + { + csrfurl = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID}"; + } + var html = await downloader.AuthedClient.GetStringAsync(csrfurl); + + var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])"); + var matches = pattern.Matches(html).Cast(); + + var csrfKey = matches.Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault(); + + if (csrfKey == null) + return null; + + string url; + if (FileID == null) + url = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&confirm=1&t=1&csrfKey={csrfKey}"; + else + url = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}"; + + + var streamResult = await downloader.AuthedClient.GetAsync(url); + if (streamResult.StatusCode != HttpStatusCode.OK) + { + Utils.Error(new InvalidOperationException(), $"{downloader.SiteName} servers reported an error for file: {FileID}"); + } + + var content_type = streamResult.Content.Headers.ContentType; + + if (content_type.MediaType == "application/json") + { + // Sometimes LL hands back a json object telling us to wait until a certain time + var times = (await streamResult.Content.ReadAsStringAsync()).FromJSONString(); + var secs = times.Download - times.CurrentTime; + for (int x = 0; x < secs; x++) + { + Utils.Status($"Waiting for {secs} at the request of {downloader.SiteName}", x * 100 / secs); + await Task.Delay(1000); + } + Utils.Status("Retrying download"); + goto TOP; + } + + return await streamResult.Content.ReadAsStreamAsync(); + } + + private class WaitResponse + { + [JsonProperty("download")] + public int Download { get; set; } + [JsonProperty("currentTime")] + public int CurrentTime { get; set; } + } + + public override async Task Verify() + { + var stream = await ResolveDownloadStream(); + if (stream == null) + { + return false; + } + + stream.Close(); + return true; + } + + public override IDownloader GetDownloader() + { + return DownloadDispatcher.GetInstance(); + } + + public override string GetReportEntry(Archive a) + { + var downloader = (INeedsLogin)GetDownloader(); + return $"* {((INeedsLogin)GetDownloader()).SiteName} - [{a.Name}](https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID})"; + } + } + + protected AbstractIPS4Downloader(Uri loginUri, string encryptedKeyName, string cookieDomain) : + base(loginUri, encryptedKeyName, cookieDomain, "ips4_member_id") + { + } + + + } +} diff --git a/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs b/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs index 61a83a81..1bead261 100644 --- a/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs @@ -20,7 +20,7 @@ namespace Wabbajack.Lib.Downloaders private readonly string _encryptedKeyName; private readonly string _cookieDomain; private readonly string _cookieName; - protected HttpClient AuthedClient; + internal HttpClient AuthedClient; /// /// Sets up all the login facilites needed for a INeedsLogin downloader based on having the user log diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index f2524f95..9ef2f0ce 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -18,6 +18,7 @@ namespace Wabbajack.Lib.Downloaders new NexusDownloader(), new MediaFireDownloader(), new LoversLabDownloader(), + new VectorPlexusDownloader(), new HTTPDownloader(), new ManualDownloader(), }; diff --git a/Wabbajack.Lib/Downloaders/LoversLabDownloader.cs b/Wabbajack.Lib/Downloaders/LoversLabDownloader.cs index 87d4edf4..9e142ad1 100644 --- a/Wabbajack.Lib/Downloaders/LoversLabDownloader.cs +++ b/Wabbajack.Lib/Downloaders/LoversLabDownloader.cs @@ -21,33 +21,18 @@ using File = Alphaleonis.Win32.Filesystem.File; namespace Wabbajack.Lib.Downloaders { - public class LoversLabDownloader : AbstractNeedsLoginDownloader, IDownloader + public class LoversLabDownloader : AbstractIPS4Downloader { #region INeedsDownload public override string SiteName => "Lovers Lab"; - public override Uri SiteURL => new Uri("https://loverslab.com"); + public override Uri SiteURL => new Uri("https://www.loverslab.com"); public override Uri IconUri => new Uri("https://www.loverslab.com/favicon.ico"); #endregion public LoversLabDownloader() : base(new Uri("https://www.loverslab.com/login"), - "loverslabcookies", "loverslab.com", "ips4_member_id") + "loverslabcookies", "loverslab.com") { } - - - public async Task GetDownloaderState(dynamic archive_ini) - { - Uri url = DownloaderUtils.GetDirectURL(archive_ini); - if (url == null || url.Host != "www.loverslab.com" || !url.AbsolutePath.StartsWith("/files/file/")) return null; - var id = HttpUtility.ParseQueryString(url.Query)["r"]; - var file = url.AbsolutePath.Split('/').Last(s => s != ""); - - return new State - { - FileID = id, - FileName = file - }; - } protected override async Task WhileWaiting(IWebDriver browser) { try @@ -60,97 +45,8 @@ namespace Wabbajack.Lib.Downloaders Utils.Error(ex); } } - - public class State : AbstractDownloadState + public class State : State { - public string FileID { get; set; } - public string FileName { get; set; } - - public override object[] PrimaryKey { get => new object[] {FileID, FileName}; } - - public override bool IsWhitelisted(ServerWhitelist whitelist) - { - return true; - } - - public override async Task Download(Archive a, string destination) - { - var stream = await ResolveDownloadStream(); - using (var file = File.OpenWrite(destination)) - { - stream.CopyTo(file); - } - } - - private async Task ResolveDownloadStream() - { - var result = DownloadDispatcher.GetInstance(); - TOP: - var html = await result.AuthedClient.GetStringAsync( - $"https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID}"); - - var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])"); - var csrfKey = pattern.Matches(html).Cast().Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault(); - - if (csrfKey == null) - return null; - - var url = - $"https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}"; - - var streamResult = await result.AuthedClient.GetAsync(url); - if (streamResult.StatusCode != HttpStatusCode.OK) - { - Utils.Error(new InvalidOperationException(), $"LoversLab servers reported an error for file: {FileID}"); - } - - var content_type = streamResult.Content.Headers.ContentType; - - if (content_type.MediaType == "application/json") - { - // Sometimes LL hands back a json object telling us to wait until a certain time - var times = (await streamResult.Content.ReadAsStringAsync()).FromJSONString(); - var secs = times.download - times.currentTime; - for (int x = 0; x < secs; x++) - { - Utils.Status($"Waiting for {secs} at the request of LoversLab", x * 100 / secs); - await Task.Delay(1000); - } - Utils.Status("Retrying download"); - goto TOP; - } - - return await streamResult.Content.ReadAsStreamAsync(); - } - - internal class WaitResponse - { - public int download { get; set; } - public int currentTime { get; set; } - } - - public override async Task Verify() - { - var stream = await ResolveDownloadStream(); - if (stream == null) - { - return false; - } - - stream.Close(); - return true; - } - - public override IDownloader GetDownloader() - { - return DownloadDispatcher.GetInstance(); - } - - public override string GetReportEntry(Archive a) - { - return $"* Lovers Lab - [{a.Name}](https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID})"; - } } - } } diff --git a/Wabbajack.Lib/Downloaders/VectorPlexusDownloader.cs b/Wabbajack.Lib/Downloaders/VectorPlexusDownloader.cs new file mode 100644 index 00000000..43a5d551 --- /dev/null +++ b/Wabbajack.Lib/Downloaders/VectorPlexusDownloader.cs @@ -0,0 +1,22 @@ +using System; +using System.Threading.Tasks; + +namespace Wabbajack.Lib.Downloaders +{ + public class VectorPlexusDownloader : AbstractIPS4Downloader + { + #region INeedsDownload + public override string SiteName => "Vector Plexus"; + public override Uri SiteURL => new Uri("https://vectorplexus.com"); + public override Uri IconUri => new Uri("https://www.vectorplexus.com/favicon.ico"); + #endregion + + public VectorPlexusDownloader() : base(new Uri("https://vectorplexus.com/login"), + "vectorplexus", "vectorplexus.com") + { + } + public class State : State + { + } + } +} diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 93863e14..44af5def 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -123,11 +123,13 @@ + + diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 395a0be8..9d6fd0c0 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -310,6 +310,34 @@ namespace Wabbajack.Test Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!"); } + + [TestMethod] + public async Task VectorPlexusDownload() + { + await DownloadDispatcher.GetInstance().Prepare(); + var ini = @"[General] + directURL=https://vectorplexus.com/files/file/290-wabbajack-test-file"; + + var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); + + Assert.IsNotNull(state); + + /*var url_state = DownloadDispatcher.ResolveArchive("https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1"); + Assert.AreEqual("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt", + ((HTTPDownloader.State)url_state).Url); + */ + var converted = state.ViaJSON(); + Assert.IsTrue(await converted.Verify()); + var filename = Guid.NewGuid().ToString(); + + Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); + + await converted.Download(new Archive { Name = "Vector Plexus Test.zip" }, filename); + + Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename)); + + Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!"); + } [TestMethod] public async Task GameFileSourceDownload()