Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move LoversLab backend to LibCef

Browse Source
This commit is contained in:
Timothy Baldridge 2020-12-29 16:15:47 -07:00
parent b22943c2b5
commit 17bb79784e
12 changed files with 224 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -1,5 +1,9 @@
### Changelog
#### Version - 2.3.6.0 - ???
* Move the LoversLab downloader to a CEF based backed making it interact with CloudFlare a bit better
#### Version - 2.3.5.1 - 12/23/2020
* HOTFIX : Recover from errors in the EGS location detector

17
Wabbajack.Launcher/MainWindowVM.cs
View File

@ -83,7 +83,7 @@ namespace Wabbajack.Launcher
var wc = new WebClient();
wc.DownloadProgressChanged += UpdateProgress;
Status = $"Downloading {_version.Tag} ...";
var data = await wc.DownloadDataTaskAsync(asset.BrowserDownloadUrl);
var data = await wc.DownloadDataTaskAsync(asset.BrowserDownloadUrlFast);
using (var zip = new ZipArchive(new MemoryStream(data), ZipArchiveMode.Read))
{
@ -142,6 +142,7 @@ namespace Wabbajack.Launcher
[JsonProperty("assets")]
public Asset[] Assets { get; set; }
}
class Asset
@ -149,6 +150,20 @@ namespace Wabbajack.Launcher
[JsonProperty("browser_download_url")]
public Uri BrowserDownloadUrl { get; set; }
[JsonIgnore]
public Uri BrowserDownloadUrlFast {
get
{
if (BrowserDownloadUrl.ToString()
.StartsWith("https://github.com/wabbajack-tools/wabbajack/releases/"))
return new Uri(BrowserDownloadUrl.ToString()
.Replace("https://github.com/wabbajack-tools/wabbajack/releases/",
"https://releases.wabbajack.org/"));
return BrowserDownloadUrl;
}
}
[JsonProperty("name")]
public string Name { get; set; }
}

126
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -11,6 +11,7 @@ using F23.StringSimilarity;
using HtmlAgilityPack;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib.LibCefHelpers;
using Wabbajack.Lib.Validation;
namespace Wabbajack.Lib.Downloaders
@ -22,6 +23,7 @@ namespace Wabbajack.Lib.Downloaders
where TState : AbstractIPS4Downloader<TDownloader, TState>.State<TDownloader>, new()
where TDownloader : IDownloader
{
protected AbstractIPS4Downloader(Uri loginUri, string encryptedKeyName, string cookieDomain, string loginCookie = "ips4_member_id")
: base(loginUri, encryptedKeyName, cookieDomain, loginCookie)
{
@ -148,17 +150,13 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Download(Archive a, AbsolutePath destination)
{
var (isValid, istream) = await ResolveDownloadStream(a, false);
if (!isValid) return false;
using var stream = istream!;
await using var fromStream = await stream.Content.ReadAsStreamAsync();
await using var toStream = await destination.Create();
await fromStream.CopyToAsync(toStream);
return true;
return await ResolveDownloadStream(a, destination, false);
}
private async Task<(bool, HttpResponseMessage?)> ResolveDownloadStream(Archive a, bool quickMode)
private async Task<bool> ResolveDownloadStream(Archive a, AbsolutePath path, bool quickMode)
{
TOP:
string url;
if (IsAttachment)
@ -170,7 +168,7 @@ namespace Wabbajack.Lib.Downloaders
var csrfURL = string.IsNullOrWhiteSpace(FileID)
? $"{Site}/files/file/{FileName}/?do=download"
: $"{Site}/files/file/{FileName}/?do=download&r={FileID}";
var html = await Downloader.AuthedClient.GetStringAsync(csrfURL);
var html = await GetStringAsync(new Uri(csrfURL));
var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])");
var matches = pattern.Matches(html).Cast<Match>();
@ -180,7 +178,7 @@ namespace Wabbajack.Lib.Downloaders
if (csrfKey == null)
{
Utils.Log($"Returning null from IPS4 Downloader because no csrfKey was found");
return (false, null);
return false;
}
var sep = Site.EndsWith("?") ? "&" : "?";
@ -188,8 +186,20 @@ namespace Wabbajack.Lib.Downloaders
? $"{Site}/files/file/{FileName}/{sep}do=download&confirm=1&t=1&csrfKey={csrfKey}"
: $"{Site}/files/file/{FileName}/{sep}do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}";
}
if (Downloader.IsCloudFlareProtected)
{
using var driver = await Downloader.GetAuthedDriver();
var size = await driver.NavigateToAndDownload(new Uri(url), path, quickMode: quickMode);
var streamResult = await Downloader.AuthedClient.GetAsync(url);
if (a.Size == 0 || size == 0 || a.Size == size) return true;
Utils.Log($"Bad Header Content sizes {a.Size} vs {size}");
return false;
}
var streamResult = await GetDownloadAsync(new Uri(url));
if (streamResult.StatusCode != HttpStatusCode.OK)
{
Utils.ErrorThrow(new InvalidOperationException(), $"{Downloader.SiteName} servers reported an error for file: {FileID}");
@ -211,10 +221,26 @@ namespace Wabbajack.Lib.Downloaders
if (a.Size != 0 && headerContentSize != 0 && a.Size != headerContentSize)
{
Utils.Log($"Bad Header Content sizes {a.Size} vs {headerContentSize}");
return (false, null);
return false;
}
return (true, streamResult);
await using (var os = await path.Create())
await using (var ins = await streamResult.Content.ReadAsStreamAsync())
{
if (a.Size == 0)
{
Utils.Status($"Downloading {a.Name}");
await ins.CopyToAsync(os);
}
else
{
await ins.CopyToWithStatusAsync(headerContentSize, os, $"Downloading {a.Name}");
}
}
streamResult.Dispose();
return true;
}
// Sometimes LL hands back a json object telling us to wait until a certain time
@ -222,8 +248,9 @@ namespace Wabbajack.Lib.Downloaders
var secs = times.Download - times.CurrentTime;
for (int x = 0; x < secs; x++)
{
if (quickMode) return (true, default);
if (quickMode) return true;
Utils.Status($"Waiting for {secs} at the request of {Downloader.SiteName}", Percent.FactoryPutInRange(x, secs));
Utils.Log($"Waiting for {secs} at the request of {Downloader.SiteName}, {secs - x} remaining");
await Task.Delay(1000);
}
streamResult.Dispose();
@ -241,13 +268,9 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Verify(Archive a)
{
var (isValid, stream) = await ResolveDownloadStream(a, true);
if (!isValid) return false;
if (stream == null)
return false;
stream.Dispose();
return true;
await using var tp = new TempFile();
var isValid = await ResolveDownloadStream(a, tp.Path, true);
return isValid;
}
public override IDownloader GetDownloader()
@ -288,7 +311,7 @@ namespace Wabbajack.Lib.Downloaders
public async Task<List<Archive>> GetFilesInGroup()
{
var others = await Downloader.AuthedClient.GetHtmlAsync($"{Site}/files/file/{FileName}?do=download");
var others = await GetHtmlAsync(new Uri($"{Site}/files/file/{FileName}?do=download"));
var pairs = others.DocumentNode.SelectNodes("//a[@data-action='download']")
.Select(item => (item.GetAttributeValue("href", ""),
@ -315,6 +338,65 @@ namespace Wabbajack.Lib.Downloaders
return IsAttachment ? FullURL : $"{Site}/files/file/{FileName}/?do=download&r={FileID}";
}
public async Task<string> GetStringAsync(Uri uri)
{
if (!Downloader.IsCloudFlareProtected)
return await Downloader.AuthedClient.GetStringAsync(uri);
using var driver = await Downloader.GetAuthedDriver();
//var drivercookies = await Helpers.GetCookies("loverslab.com");
//var cookies = await ClientAPI.GetAuthInfo<Helpers.Cookie[]>("loverslabcookies");
//await Helpers.IngestCookies(uri.ToString(), cookies);
await driver.NavigateTo(uri);
var source = await driver.GetSourceAsync();
/*
Downloader.AuthedClient.Cookies.Add(drivercookies.Where(dc => dc.Name == "cf_clearance")
.Select(dc => new Cookie
{
Name = dc.Name,
Domain = dc.Domain,
Value = dc.Value,
Path = dc.Path
})
.FirstOrDefault());
var source = await Downloader.AuthedClient.GetStringAsync(uri);
*/
return source;
}
public async Task<HtmlDocument> GetHtmlAsync(Uri s)
{
var body = await GetStringAsync(s);
var doc = new HtmlDocument();
doc.LoadHtml(body);
return doc;
}
public async Task<HttpResponseMessage> GetDownloadAsync(Uri uri)
{
if (!Downloader.IsCloudFlareProtected)
return await Downloader.AuthedClient.GetAsync(uri);
using var driver = await Downloader.GetAuthedDriver();
TaskCompletionSource<Uri?> promise = new TaskCompletionSource<Uri?>();
driver.DownloadHandler = uri1 =>
{
promise.SetResult(uri);
};
await driver.NavigateTo(uri);
var url = await promise.Task;
if (url == null) throw new Exception("No Url to download");
var location = await driver.GetLocation();
return await Helpers.GetClient(await Helpers.GetCookies(), location!.ToString()).GetAsync(uri);
}
public override string[] GetMetaIni()
{
if (IsAttachment)

14
Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs
View File

@ -7,6 +7,7 @@ using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Input;
using Org.BouncyCastle.Crypto.Parameters;
using ReactiveUI;
using Wabbajack.Common;
using Wabbajack.Common.StatusFeed;
@ -17,6 +18,8 @@ namespace Wabbajack.Lib.Downloaders
{
public abstract class AbstractNeedsLoginDownloader : INeedsLogin
{
public bool IsCloudFlareProtected = false;
private readonly Uri _loginUri;
private readonly string _encryptedKeyName;
private readonly string _cookieDomain;
@ -104,7 +107,8 @@ namespace Wabbajack.Lib.Downloaders
}
cookies = await ClientAPI.GetAuthInfo<Helpers.Cookie[]>(_encryptedKeyName);
return Helpers.GetClient(cookies, SiteURL.ToString());
var client = Helpers.GetClient(cookies, SiteURL.ToString());
return client;
}
public async Task Prepare()
@ -114,6 +118,12 @@ namespace Wabbajack.Lib.Downloaders
_isPrepared = true;
}
public async Task<Driver> GetAuthedDriver()
{
var driver = await Driver.Create();
return driver;
}
public class NotLoggedInError : Exception
{
public AbstractNeedsLoginDownloader Downloader { get; }
@ -124,7 +134,7 @@ namespace Wabbajack.Lib.Downloaders
}
}
public class RequestSiteLogin : AUserIntervention
{
public AbstractNeedsLoginDownloader Downloader { get; }

2
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -95,7 +95,7 @@ namespace Wabbajack.Lib.Downloaders
}
long totalRead = 0;
var bufferSize = 1024 * 32;
var bufferSize = 1024 * 32 * 8;
Utils.Status($"Starting Download {a.Name ?? Url}", Percent.Zero);
var response = await client.GetAsync(Url, errorsAsExceptions:false, retry:false);

1
Wabbajack.Lib/Downloaders/LoversLabDownloader.cs
View File

@ -20,6 +20,7 @@ namespace Wabbajack.Lib.Downloaders
public LoversLabDownloader() : base(new Uri("https://www.loverslab.com/login"),
"loverslabcookies", "loverslab.com")
{
IsCloudFlareProtected = true;
}
protected override async Task WhileWaiting(IWebDriver browser)
{

15
Wabbajack.Lib/ModListRegistry/ModListMetadata.cs
View File

@ -56,8 +56,19 @@ namespace Wabbajack.Lib.ModListRegistry
[JsonName("Links")]
public class LinksObject
{
[JsonProperty("image")]
public string ImageUri { get; set; } = string.Empty;
[JsonProperty("image")] public string ImageUri { get; set; } = string.Empty;
[JsonIgnore]
public string ImageUrlFast
{
get
{
if (ImageUri.StartsWith("https://raw.githubusercontent.com/wabbajack-tools/mod-lists/"))
return ImageUri.Replace("https://raw.githubusercontent.com/wabbajack-tools/mod-lists/",
"https://mod-lists.wabbajack.org/");
return ImageUri;
}
}
[JsonProperty("readme")]
public string Readme { get; set; } = string.Empty;

56
Wabbajack.Lib/WebAutomation/CefSharpWrapper.cs
View File

@ -42,6 +42,24 @@ namespace Wabbajack.Lib.WebAutomation
return tcs.Task;
}
public async Task<long> NavigateToAndDownload(Uri uri, AbsolutePath dest, bool quickMode = false)
{
var oldCB = _browser.DownloadHandler;
var handler = new ReroutingDownloadHandler(this, dest, quickMode: quickMode);
_browser.DownloadHandler = handler;
try
{
await NavigateTo(uri);
return await handler.Task;
}
finally {
_browser.DownloadHandler = oldCB;
}
}
public async Task<string> EvaluateJavaScript(string text)
{
var result = await _browser.EvaluateScriptAsync(text);
@ -99,6 +117,44 @@ namespace Wabbajack.Lib.WebAutomation
}
}
public class ReroutingDownloadHandler : IDownloadHandler
{
private CefSharpWrapper _wrapper;
private AbsolutePath _path;
public TaskCompletionSource<long> _tcs = new TaskCompletionSource<long>();
private bool _quickMode;
public Task<long> Task => _tcs.Task;
public ReroutingDownloadHandler(CefSharpWrapper wrapper, AbsolutePath path, bool quickMode)
{
_wrapper = wrapper;
_path = path;
_quickMode = quickMode;
}
public void OnBeforeDownload(IWebBrowser chromiumWebBrowser, IBrowser browser, DownloadItem downloadItem,
IBeforeDownloadCallback callback)
{
if (_quickMode) return;
callback.Continue(_path.ToString(), false);
}
public void OnDownloadUpdated(IWebBrowser chromiumWebBrowser, IBrowser browser, DownloadItem downloadItem,
IDownloadItemCallback callback)
{
if (_quickMode)
{
callback.Cancel();
_tcs.SetResult(downloadItem.TotalBytes);
return;
}
if (downloadItem.IsComplete)
_tcs.SetResult(downloadItem.TotalBytes);
callback.Resume();
}
}
public class DownloadHandler : IDownloadHandler
{
private CefSharpWrapper _wrapper;

11
Wabbajack.Lib/WebAutomation/WebAutomation.cs
View File

@ -4,6 +4,7 @@ using System.Threading.Tasks;
using System.Windows;
using CefSharp;
using CefSharp.OffScreen;
using Wabbajack.Common;
using Wabbajack.Lib.LibCefHelpers;
namespace Wabbajack.Lib.WebAutomation
@ -33,6 +34,11 @@ namespace Wabbajack.Lib.WebAutomation
return await GetLocation();
}
public async Task<long> NavigateToAndDownload(Uri uri, AbsolutePath absolutePath, bool quickMode = false)
{
return await _driver.NavigateToAndDownload(uri, absolutePath, quickMode: quickMode);
}
public async ValueTask<Uri?> GetLocation()
{
try
@ -44,6 +50,11 @@ namespace Wabbajack.Lib.WebAutomation
return null;
}
}
public async ValueTask<string> GetSourceAsync()
{
return await _browser.GetSourceAsync();
}
public Action<Uri?> DownloadHandler {
set => _driver.DownloadHandler = value;

4
Wabbajack.Test/DownloaderTests.cs
View File

@ -11,9 +11,11 @@ using Wabbajack.Common;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.Http;
using Wabbajack.Lib.LibCefHelpers;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using Wabbajack.Lib.WebAutomation;
using Xunit;
using Xunit.Abstractions;
using Directory = System.IO.Directory;
@ -282,6 +284,8 @@ namespace Wabbajack.Test
[Fact]
public async Task LoversLabDownload()
{
await DownloadDispatcher.GetInstance<LoversLabDownloader>().Prepare();
var ini = @"[General]
directURL=https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1";

2
Wabbajack/View Models/Gallery/ModListMetadataVM.cs
View File

@ -161,7 +161,7 @@ namespace Wabbajack
})
.ToGuiProperty(this, nameof(Exists));
var imageObs = Observable.Return(Metadata.Links.ImageUri)
var imageObs = Observable.Return(Metadata.Links.ImageUrlFast)
.DownloadBitmapImage((ex) => Utils.Log($"Error downloading modlist image {Metadata.Title}"));
_Image = imageObs

2
Wabbajack/Views/ModListTileView.xaml
View File

@ -89,7 +89,7 @@
HorizontalAlignment="Center"
VerticalAlignment="Center"
Stretch="UniformToFill">
<Image x:Name="ModListImage" Source="{Binding Metadata.Links.ImageUri}">
<Image x:Name="ModListImage" Source="{Binding Metadata.Links.ImageUriFast}">
<Image.Style>
<Style TargetType="Image">
<Style.Triggers>