From c443016638c2df209b2202d8a2c02dcbf942320c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 28 May 2020 16:31:01 -0600 Subject: [PATCH] Add support for downloading via Yandex --- .../Downloaders/AbstractDownloadState.cs | 1 + .../Downloaders/DownloadDispatcher.cs | 1 + Wabbajack.Lib/Downloaders/YandexDownloader.cs | 85 +++++++++++++++++++ Wabbajack.Lib/WebAutomation/WebAutomation.cs | 9 ++ Wabbajack.Test/DownloaderTests.cs | 24 ++++++ 5 files changed, 120 insertions(+) create mode 100644 Wabbajack.Lib/Downloaders/YandexDownloader.cs diff --git a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs index 9e0c2e63..d4d9a618 100644 --- a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs +++ b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs @@ -40,6 +40,7 @@ namespace Wabbajack.Lib.Downloaders typeof(TESAllianceDownloader.State), typeof(BethesdaNetDownloader.State), typeof(YouTubeDownloader.State), + typeof(YandexDownloader.State), typeof(WabbajackCDNDownloader.State) }; public static Dictionary NameToType { get; set; } diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index f350fb93..e643d3cb 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -28,6 +28,7 @@ namespace Wabbajack.Lib.Downloaders new TESAllianceDownloader(), new YouTubeDownloader(), new WabbajackCDNDownloader(), + new YandexDownloader(), new HTTPDownloader(), new ManualDownloader(), }; diff --git a/Wabbajack.Lib/Downloaders/YandexDownloader.cs b/Wabbajack.Lib/Downloaders/YandexDownloader.cs new file mode 100644 index 00000000..1bc5acfd --- /dev/null +++ b/Wabbajack.Lib/Downloaders/YandexDownloader.cs @@ -0,0 +1,85 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Common.Serialization.Json; +using Wabbajack.Lib.Validation; + +namespace Wabbajack.Lib.Downloaders +{ + public class YandexDownloader : IUrlDownloader + { + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode = false) + { + var uri = (Uri?)DownloaderUtils.GetDirectURL(archiveINI); + if (uri == null) return null; + + return uri.Host == "yadi.sk" ? new State(uri) : null; + } + + public async Task Prepare() + { + } + + public AbstractDownloadState? GetDownloaderState(string url) + { + var uri = new Uri(url); + if (uri.Host == "yadi.sk") + { + return new State(uri); + } + + return null; + } + + [JsonName("YandexDownloader+State")] + public class State : AbstractDownloadState + { + public Uri Url { get; set; } + + public State(Uri url) + { + Url = url; + } + public override object[] PrimaryKey => new object[] {Url}; + public override bool IsWhitelisted(ServerWhitelist whitelist) + { + var url = Url.ToString(); + return whitelist.AllowedPrefixes.Any(p => url.StartsWith(p)); + } + + public override async Task Download(Archive a, AbsolutePath destination) + { + using var driver = await WebAutomation.Driver.Create(); + var tcs = new TaskCompletionSource(); + driver.DownloadHandler = uri => tcs.SetResult(uri); + await driver.NavigateTo(Url); + await driver.EvalJavascript("document.getElementsByClassName(\"download-button\")[0].click();"); + var uri = await tcs.Task; + return await new HTTPDownloader.State(uri!.ToString()).Download(destination); + } + + public override async Task Verify(Archive archive) + { + var client = new Common.Http.Client(); + var result = await client.GetAsync(Url, errorsAsExceptions: false); + return result.IsSuccessStatusCode; + } + + public override IDownloader GetDownloader() + { + return DownloadDispatcher.GetInstance(); + } + + public override string? GetManifestURL(Archive a) + { + return Url.ToString(); + } + + public override string[] GetMetaIni() + { + return new[] {"[General]", $"directURL={Url}"}; + } + } + } +} diff --git a/Wabbajack.Lib/WebAutomation/WebAutomation.cs b/Wabbajack.Lib/WebAutomation/WebAutomation.cs index 34999d22..3ae0276f 100644 --- a/Wabbajack.Lib/WebAutomation/WebAutomation.cs +++ b/Wabbajack.Lib/WebAutomation/WebAutomation.cs @@ -43,12 +43,21 @@ namespace Wabbajack.Lib.WebAutomation return null; } } + + public Action DownloadHandler { + set => _driver.DownloadHandler = value; + } public Task GetAttr(string selector, string attr) { return _driver.EvaluateJavaScript($"document.querySelector(\"{selector}\").{attr}"); } + public Task EvalJavascript(string js) + { + return _driver.EvaluateJavaScript(js); + } + public void Dispose() { _browser.Dispose(); diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index b5a1aac8..06c3e28c 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -337,6 +337,30 @@ namespace Wabbajack.Test Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); } + + [Fact] + public async Task YandexDownloader() + { + await DownloadDispatcher.GetInstance().Prepare(); + var ini = @"[General] + directURL=https://yadi.sk/d/jqwQT4ByYtC9Tw"; + + var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); + + Assert.NotNull(state); + + var converted = RoundTripState(state); + Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); + await using var filename = new TempFile(); + + Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() {"https://yadi.sk"} })); + + await converted.Download(new Archive(state: null!) { Name = "WABBAJACK_TEST_FILE.txt" }, filename.Path); + + Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); + + Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); + } [Fact]