From ae7402be581389caa63374d067968f1dfad66bf1 Mon Sep 17 00:00:00 2001 From: UrbanCMC Date: Wed, 9 Aug 2023 21:03:56 +0200 Subject: [PATCH 1/3] Fix manual downloads sometimes launching in browser --- .../UserIntervention/ManualDownloadHandler.cs | 15 ++++++--------- .../View Models/BrowserWindowViewModel.cs | 4 +++- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Wabbajack.App.Wpf/UserIntervention/ManualDownloadHandler.cs b/Wabbajack.App.Wpf/UserIntervention/ManualDownloadHandler.cs index 6bf219c1..003099bf 100644 --- a/Wabbajack.App.Wpf/UserIntervention/ManualDownloadHandler.cs +++ b/Wabbajack.App.Wpf/UserIntervention/ManualDownloadHandler.cs @@ -1,10 +1,7 @@ -using System.Security.Policy; using System.Threading; using System.Threading.Tasks; -using Wabbajack.DTOs; using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.Interventions; -using Wabbajack.Paths; namespace Wabbajack.UserIntervention; @@ -17,18 +14,18 @@ public class ManualDownloadHandler : BrowserWindowViewModel //await WaitForReady(); var archive = Intervention.Archive; var md = Intervention.Archive.State as Manual; - + HeaderText = $"Manual download ({md.Url.Host})"; - + Instructions = string.IsNullOrWhiteSpace(md.Prompt) ? $"Please download {archive.Name}" : md.Prompt; - await NavigateTo(md.Url); - - var uri = await WaitForDownloadUri(token, async () => + var task = WaitForDownloadUri(token, async () => { await RunJavaScript("Array.from(document.getElementsByTagName(\"iframe\")).forEach(f => f.remove())"); }); - + await NavigateTo(md.Url); + var uri = await task; + Intervention.Finish(uri); } } \ No newline at end of file diff --git a/Wabbajack.App.Wpf/View Models/BrowserWindowViewModel.cs b/Wabbajack.App.Wpf/View Models/BrowserWindowViewModel.cs index b74cdd99..bd05085c 100644 --- a/Wabbajack.App.Wpf/View Models/BrowserWindowViewModel.cs +++ b/Wabbajack.App.Wpf/View Models/BrowserWindowViewModel.cs @@ -112,11 +112,13 @@ public abstract class BrowserWindowViewModel : ViewModel { var source = new TaskCompletionSource(); var referer = _browser.Source; + while (_browser.CoreWebView2 == null) + await Task.Delay(10, token); + _browser.CoreWebView2.DownloadStarting += (sender, args) => { try { - source.SetResult(new Uri(args.DownloadOperation.Uri)); } catch (Exception) From 6399ef07d501c9746f01ebb471d32c0419be2cde Mon Sep 17 00:00:00 2001 From: UrbanCMC Date: Wed, 9 Aug 2023 21:05:20 +0200 Subject: [PATCH 2/3] Fix manual downloads from secure servers --- .../View Models/BrowserWindowViewModel.cs | 23 +++++++++++++------ Wabbajack.Common/HttpExtensions.cs | 11 ++------- .../Interventions/ManualDownload.cs | 6 ++--- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/Wabbajack.App.Wpf/View Models/BrowserWindowViewModel.cs b/Wabbajack.App.Wpf/View Models/BrowserWindowViewModel.cs index bd05085c..d0aa4a4c 100644 --- a/Wabbajack.App.Wpf/View Models/BrowserWindowViewModel.cs +++ b/Wabbajack.App.Wpf/View Models/BrowserWindowViewModel.cs @@ -23,7 +23,7 @@ public abstract class BrowserWindowViewModel : ViewModel [Reactive] public string HeaderText { get; set; } [Reactive] public string Instructions { get; set; } - + [Reactive] public string Address { get; set; } public BrowserWindow? Browser { get; set; } @@ -83,6 +83,11 @@ public abstract class BrowserWindowViewModel : ViewModel public async Task GetCookies(string domainEnding, CancellationToken token) { + // Strip www. before searching for cookies on a domain to handle websites saving their cookies like .example.org + if (domainEnding.StartsWith("www.")) + { + domainEnding = domainEnding[4..]; + } var cookies = (await _browser.CoreWebView2.CookieManager.GetCookiesAsync("")) .Where(c => c.Domain.EndsWith(domainEnding)); return cookies.Select(c => new Cookie @@ -125,7 +130,7 @@ public abstract class BrowserWindowViewModel : ViewModel { source.SetCanceled(); } - + args.Cancel = true; args.Handled = true; }; @@ -146,12 +151,16 @@ public abstract class BrowserWindowViewModel : ViewModel } var cookies = await GetCookies(uri.Host, token); - return new ManualDownload.BrowserDownloadState(uri, cookies, new[] - { - ("Referer", referer.ToString()) - }); + return new ManualDownload.BrowserDownloadState( + uri, + cookies, + new[] + { + ("Referer", referer?.ToString() ?? uri.ToString()) + }, + _browser.CoreWebView2.Settings.UserAgent); } - + public async Task WaitForDownload(AbsolutePath path, CancellationToken token) { var source = new TaskCompletionSource(); diff --git a/Wabbajack.Common/HttpExtensions.cs b/Wabbajack.Common/HttpExtensions.cs index a0900120..35ff173c 100644 --- a/Wabbajack.Common/HttpExtensions.cs +++ b/Wabbajack.Common/HttpExtensions.cs @@ -19,7 +19,7 @@ public static class HttpExtensions msg.Headers.Add("Cookie", string.Join(";", cookies.Select(c => $"{c.Name}={c.Value}"))); return msg; } - + public static HttpRequestMessage AddHeaders(this HttpRequestMessage msg, IEnumerable<(string Key, string Value)> headers) { foreach (var header in headers) @@ -29,17 +29,10 @@ public static class HttpExtensions return msg; } - public static HttpRequestMessage AddChromeAgent(this HttpRequestMessage msg) - { - msg.Headers.Add("User-Agent", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"); - return msg; - } - public static HttpRequestMessage ToHttpRequestMessage(this ManualDownload.BrowserDownloadState browserState) { var msg = new HttpRequestMessage(HttpMethod.Get, browserState.Uri); - msg.AddChromeAgent(); + msg.Headers.Add("User-Agent", browserState.UserAgent); msg.AddCookies(browserState.Cookies); msg.AddHeaders(browserState.Headers); return msg; diff --git a/Wabbajack.DTOs/Interventions/ManualDownload.cs b/Wabbajack.DTOs/Interventions/ManualDownload.cs index 15a3be24..84dda1dd 100644 --- a/Wabbajack.DTOs/Interventions/ManualDownload.cs +++ b/Wabbajack.DTOs/Interventions/ManualDownload.cs @@ -10,14 +10,14 @@ namespace Wabbajack.DTOs.Interventions; public class ManualDownload : AUserIntervention { public Archive Archive { get; } - + public ManualDownload(Archive archive) { Archive = archive; } - public record BrowserDownloadState(Uri Uri, Cookie[] Cookies, (string Key, string Value)[] Headers) + public record BrowserDownloadState(Uri Uri, Cookie[] Cookies, (string Key, string Value)[] Headers, string UserAgent) { - + } } \ No newline at end of file From 63eaffd9e3f7576d727fd905df61a70da0544331 Mon Sep 17 00:00:00 2001 From: UrbanCMC Date: Wed, 9 Aug 2023 21:29:34 +0200 Subject: [PATCH 3/3] Remove duplicate user agent code --- Wabbajack.App.Wpf/LauncherUpdater.cs | 19 ++++++++-------- .../UserIntervention/OAuth2LoginHandler.cs | 10 ++++----- Wabbajack.Common/HttpExtensions.cs | 11 +++++++++- .../GoogleDriveDownloader.cs | 22 +++++++++---------- Wabbajack.Networking.Http/Extensions.cs | 7 ------ 5 files changed, 36 insertions(+), 33 deletions(-) diff --git a/Wabbajack.App.Wpf/LauncherUpdater.cs b/Wabbajack.App.Wpf/LauncherUpdater.cs index 94e7031e..96d3fd6b 100644 --- a/Wabbajack.App.Wpf/LauncherUpdater.cs +++ b/Wabbajack.App.Wpf/LauncherUpdater.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.VisualBasic.CompilerServices; using Newtonsoft.Json; +using Wabbajack.Common; using Wabbajack.Downloaders; using Wabbajack.DTOs; using Wabbajack.DTOs.DownloadStates; @@ -27,9 +28,9 @@ namespace Wabbajack private readonly HttpClient _client; private readonly Client _wjclient; private readonly DTOSerializer _dtos; - + private readonly DownloadDispatcher _downloader; - + private static Uri GITHUB_REPO_RELEASES = new("https://api.github.com/repos/wabbajack-tools/wabbajack/releases"); public LauncherUpdater(ILogger logger, HttpClient client, Client wjclient, DTOSerializer dtos, @@ -41,8 +42,8 @@ namespace Wabbajack _dtos = dtos; _downloader = downloader; } - - + + public static Lazy CommonFolder = new (() => { var entryPoint = KnownFolders.EntryPoint; @@ -105,7 +106,7 @@ namespace Wabbajack var launcherFolder = KnownFolders.EntryPoint.Parent; var exePath = launcherFolder.Combine("Wabbajack.exe"); - + var launcherVersion = FileVersionInfo.GetVersionInfo(exePath.ToString()); if (release != default && release.version > Version.Parse(launcherVersion.FileVersion!)) @@ -119,7 +120,7 @@ namespace Wabbajack Name = release.asset.Name, Size = release.asset.Size }, tempPath, CancellationToken.None); - + if (tempPath.Size() != release.asset.Size) { _logger.LogInformation( @@ -130,12 +131,12 @@ namespace Wabbajack if (exePath.FileExists()) exePath.Delete(); await tempPath.MoveToAsync(exePath, true, CancellationToken.None); - + _logger.LogInformation("Finished updating wabbajack"); await _wjclient.SendMetric("updated_launcher", $"{launcherVersion.FileVersion} -> {release.version}"); } } - + private async Task GetReleases() { _logger.LogInformation("Getting new Wabbajack version list"); @@ -146,7 +147,7 @@ namespace Wabbajack private HttpRequestMessage MakeMessage(Uri uri) { var msg = new HttpRequestMessage(HttpMethod.Get, uri); - msg.UseChromeUserAgent(); + msg.AddChromeAgent(); return msg; } diff --git a/Wabbajack.App.Wpf/UserIntervention/OAuth2LoginHandler.cs b/Wabbajack.App.Wpf/UserIntervention/OAuth2LoginHandler.cs index 1d287022..51f45c7b 100644 --- a/Wabbajack.App.Wpf/UserIntervention/OAuth2LoginHandler.cs +++ b/Wabbajack.App.Wpf/UserIntervention/OAuth2LoginHandler.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using System.Web; using Microsoft.Extensions.Logging; using ReactiveUI; +using Wabbajack.Common; using Wabbajack.DTOs.Interventions; using Wabbajack.DTOs.Logins; using Wabbajack.Messages; @@ -40,7 +41,7 @@ public abstract class OAuth2LoginHandler : BrowserWindowViewModel var tcs = new TaskCompletionSource(); await NavigateTo(tlogin.AuthorizationEndpoint); - + Browser!.Browser.CoreWebView2.Settings.UserAgent = "Wabbajack"; Browser!.Browser.NavigationStarting += (sender, args) => { @@ -50,7 +51,7 @@ public abstract class OAuth2LoginHandler : BrowserWindowViewModel tcs.TrySetResult(uri); } }; - + Instructions = $"Please log in and allow Wabbajack to access your {tlogin.SiteName} account"; var scopes = string.Join(" ", tlogin.Scopes); @@ -88,8 +89,7 @@ public abstract class OAuth2LoginHandler : BrowserWindowViewModel var msg = new HttpRequestMessage(); msg.Method = HttpMethod.Post; msg.RequestUri = tlogin.TokenEndpoint; - msg.Headers.Add("User-Agent", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"); + msg.AddChromeAgent(); msg.Headers.Add("Cookie", string.Join(";", cookies.Select(c => $"{c.Name}={c.Value}"))); msg.Content = new FormUrlEncodedContent(formData.ToList()); @@ -101,6 +101,6 @@ public abstract class OAuth2LoginHandler : BrowserWindowViewModel Cookies = cookies, ResultState = data! }); - + } } \ No newline at end of file diff --git a/Wabbajack.Common/HttpExtensions.cs b/Wabbajack.Common/HttpExtensions.cs index 35ff173c..b2ca8d39 100644 --- a/Wabbajack.Common/HttpExtensions.cs +++ b/Wabbajack.Common/HttpExtensions.cs @@ -14,12 +14,21 @@ namespace Wabbajack.Common; public static class HttpExtensions { + private const string ChromeUserAgent = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"; public static HttpRequestMessage AddCookies(this HttpRequestMessage msg, Cookie[] cookies) { msg.Headers.Add("Cookie", string.Join(";", cookies.Select(c => $"{c.Name}={c.Value}"))); return msg; } + public static HttpRequestMessage AddChromeAgent(this HttpRequestMessage msg, string? overrideUserAgent = null) + { + msg.Headers.UserAgent.Clear(); + msg.Headers.Add("User-Agent", overrideUserAgent ?? ChromeUserAgent); + return msg; + } + public static HttpRequestMessage AddHeaders(this HttpRequestMessage msg, IEnumerable<(string Key, string Value)> headers) { foreach (var header in headers) @@ -32,7 +41,7 @@ public static class HttpExtensions public static HttpRequestMessage ToHttpRequestMessage(this ManualDownload.BrowserDownloadState browserState) { var msg = new HttpRequestMessage(HttpMethod.Get, browserState.Uri); - msg.Headers.Add("User-Agent", browserState.UserAgent); + msg.AddChromeAgent(browserState.UserAgent); msg.AddCookies(browserState.Cookies); msg.AddHeaders(browserState.Headers); return msg; diff --git a/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs b/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs index 2dede342..c4906694 100644 --- a/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs +++ b/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs @@ -65,7 +65,7 @@ public class GoogleDriveDownloader : ADownloader iniData) { if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"].CleanIniString(), UriKind.Absolute, out var uri)) @@ -74,8 +74,8 @@ public class GoogleDriveDownloader : ADownloader Priority.Normal; - - + + public async Task DownloadStream(Archive archive, Func> fn, CancellationToken token) { var state = archive.State as DTOs.DownloadStates.GoogleDrive; @@ -112,8 +112,8 @@ public class GoogleDriveDownloader : ADownloader c.Key.StartsWith("download_warning_")); @@ -124,7 +124,7 @@ public class GoogleDriveDownloader : ADownloaderGoogle Drive - Quota exceeded")) throw new Exception("Google Drive - Quota Exceeded"); - + doc.LoadHtml(txt); var action = doc.DocumentNode.DescendantsAndSelf() @@ -133,7 +133,7 @@ public class GoogleDriveDownloader : ADownloader d.GetAttributeValue("action", "")) .FirstOrDefault(); - if (action != null) + if (action != null) warning = ("download_warning_", "t"); } @@ -145,18 +145,18 @@ public class GoogleDriveDownloader : ADownloader GetJsonFromSendAsync(this HttpClient client, HttpRequestMessage msg, JsonSerializerOptions opts, CancellationToken? token = null) {