Merge remote-tracking branch 'origin/webview-download-fix' into pre-release

Wabbajack.App.Wpf/LauncherUpdater.cs

@@ -8,6 +8,7 @@ using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Microsoft.VisualBasic.CompilerServices;
 using Newtonsoft.Json;
+using Wabbajack.Common;
 using Wabbajack.Downloaders;
 using Wabbajack.DTOs;
 using Wabbajack.DTOs.DownloadStates;
@@ -27,9 +28,9 @@ namespace Wabbajack
         private readonly HttpClient _client;
         private readonly Client _wjclient;
         private readonly DTOSerializer _dtos;
-        
+
         private readonly DownloadDispatcher _downloader;
-        
+
         private static Uri GITHUB_REPO_RELEASES = new("https://api.github.com/repos/wabbajack-tools/wabbajack/releases");
 
         public LauncherUpdater(ILogger<LauncherUpdater> logger, HttpClient client, Client wjclient, DTOSerializer dtos,
@@ -41,8 +42,8 @@ namespace Wabbajack
             _dtos = dtos;
             _downloader = downloader;
         }
-            
+
-            
+
         public static Lazy<AbsolutePath> CommonFolder = new (() =>
         {
             var entryPoint = KnownFolders.EntryPoint;
@@ -105,7 +106,7 @@ namespace Wabbajack
 
             var launcherFolder = KnownFolders.EntryPoint.Parent;
             var exePath = launcherFolder.Combine("Wabbajack.exe");
-            
+
             var launcherVersion = FileVersionInfo.GetVersionInfo(exePath.ToString());
 
             if (release != default && release.version > Version.Parse(launcherVersion.FileVersion!))
@@ -119,7 +120,7 @@ namespace Wabbajack
                     Name = release.asset.Name,
                     Size = release.asset.Size
                 }, tempPath, CancellationToken.None);
-                    
+
                 if (tempPath.Size() != release.asset.Size)
                 {
                     _logger.LogInformation(
@@ -130,12 +131,12 @@ namespace Wabbajack
                 if (exePath.FileExists())
                     exePath.Delete();
                 await tempPath.MoveToAsync(exePath, true, CancellationToken.None);
-                
+
                 _logger.LogInformation("Finished updating wabbajack");
                 await _wjclient.SendMetric("updated_launcher", $"{launcherVersion.FileVersion} -> {release.version}");
             }
         }
-        
+
         private async Task<Release[]> GetReleases()
         {
             _logger.LogInformation("Getting new Wabbajack version list");
@@ -146,7 +147,7 @@ namespace Wabbajack
         private HttpRequestMessage MakeMessage(Uri uri)
         {
             var msg =  new HttpRequestMessage(HttpMethod.Get, uri);
-            msg.UseChromeUserAgent();
+            msg.AddChromeAgent();
             return msg;
         }
 

Wabbajack.App.Wpf/UserIntervention/ManualDownloadHandler.cs

@@ -1,10 +1,7 @@
-using System.Security.Policy;
 using System.Threading;
 using System.Threading.Tasks;
-using Wabbajack.DTOs;
 using Wabbajack.DTOs.DownloadStates;
 using Wabbajack.DTOs.Interventions;
-using Wabbajack.Paths;
 
 namespace Wabbajack.UserIntervention;
 
@@ -17,18 +14,18 @@ public class ManualDownloadHandler : BrowserWindowViewModel
         //await WaitForReady();
         var archive = Intervention.Archive;
         var md = Intervention.Archive.State as Manual;
-        
+
         HeaderText = $"Manual download ({md.Url.Host})";
-        
+
         Instructions = string.IsNullOrWhiteSpace(md.Prompt) ? $"Please download {archive.Name}" : md.Prompt;
-        await NavigateTo(md.Url);
 
-
+        var task = WaitForDownloadUri(token, async () =>
-        var uri = await WaitForDownloadUri(token, async () =>
         {
             await RunJavaScript("Array.from(document.getElementsByTagName(\"iframe\")).forEach(f => f.remove())");
         });
-        
+        await NavigateTo(md.Url);
+        var uri = await task;
+
         Intervention.Finish(uri);
     }
 }

Wabbajack.App.Wpf/UserIntervention/OAuth2LoginHandler.cs

@@ -8,6 +8,7 @@ using System.Threading.Tasks;
 using System.Web;
 using Microsoft.Extensions.Logging;
 using ReactiveUI;
+using Wabbajack.Common;
 using Wabbajack.DTOs.Interventions;
 using Wabbajack.DTOs.Logins;
 using Wabbajack.Messages;
@@ -40,7 +41,7 @@ public abstract class OAuth2LoginHandler<TLoginType> : BrowserWindowViewModel
 
         var tcs = new TaskCompletionSource<Uri>();
         await NavigateTo(tlogin.AuthorizationEndpoint);
-        
+
         Browser!.Browser.CoreWebView2.Settings.UserAgent = "Wabbajack";
         Browser!.Browser.NavigationStarting += (sender, args) =>
         {
@@ -50,7 +51,7 @@ public abstract class OAuth2LoginHandler<TLoginType> : BrowserWindowViewModel
                 tcs.TrySetResult(uri);
             }
         };
-        
+
         Instructions = $"Please log in and allow Wabbajack to access your {tlogin.SiteName} account";
 
         var scopes = string.Join(" ", tlogin.Scopes);
@@ -88,8 +89,7 @@ public abstract class OAuth2LoginHandler<TLoginType> : BrowserWindowViewModel
         var msg = new HttpRequestMessage();
         msg.Method = HttpMethod.Post;
         msg.RequestUri = tlogin.TokenEndpoint;
-        msg.Headers.Add("User-Agent",
+        msg.AddChromeAgent();
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36");
         msg.Headers.Add("Cookie", string.Join(";", cookies.Select(c => $"{c.Name}={c.Value}")));
         msg.Content = new FormUrlEncodedContent(formData.ToList());
 
@@ -101,6 +101,6 @@ public abstract class OAuth2LoginHandler<TLoginType> : BrowserWindowViewModel
             Cookies = cookies,
             ResultState = data!
         });
-        
+
     }
 }

Wabbajack.App.Wpf/View Models/BrowserWindowViewModel.cs

@@ -23,7 +23,7 @@ public abstract class BrowserWindowViewModel : ViewModel
     [Reactive] public string HeaderText { get; set; }
 
     [Reactive] public string Instructions { get; set; }
-    
+
     [Reactive] public string Address { get; set; }
 
     public BrowserWindow? Browser { get; set; }
@@ -83,6 +83,11 @@ public abstract class BrowserWindowViewModel : ViewModel
 
     public async Task<Cookie[]> GetCookies(string domainEnding, CancellationToken token)
     {
+        // Strip www. before searching for cookies on a domain to handle websites saving their cookies like .example.org
+        if (domainEnding.StartsWith("www."))
+        {
+            domainEnding = domainEnding[4..];
+        }
         var cookies = (await _browser.CoreWebView2.CookieManager.GetCookiesAsync(""))
             .Where(c => c.Domain.EndsWith(domainEnding));
         return cookies.Select(c => new Cookie
@@ -112,18 +117,20 @@ public abstract class BrowserWindowViewModel : ViewModel
     {
         var source = new TaskCompletionSource<Uri>();
         var referer = _browser.Source;
+        while (_browser.CoreWebView2 == null)
+            await Task.Delay(10, token);
+
         _browser.CoreWebView2.DownloadStarting += (sender, args) =>
         {
             try
             {
-                
                 source.SetResult(new Uri(args.DownloadOperation.Uri));
             }
             catch (Exception)
             {
                 source.SetCanceled();
             }
-            
+
             args.Cancel = true;
             args.Handled = true;
         };
@@ -144,12 +151,16 @@ public abstract class BrowserWindowViewModel : ViewModel
         }
 
         var cookies = await GetCookies(uri.Host, token);
-        return new ManualDownload.BrowserDownloadState(uri, cookies, new[]
+        return new ManualDownload.BrowserDownloadState(
-        {
+            uri,
-            ("Referer", referer.ToString())
+            cookies,
-        });
+            new[]
+            {
+                ("Referer", referer?.ToString() ?? uri.ToString())
+            },
+            _browser.CoreWebView2.Settings.UserAgent);
     }
-    
+
     public async Task<Hash> WaitForDownload(AbsolutePath path, CancellationToken token)
     {
         var source = new TaskCompletionSource();

Wabbajack.Common/HttpExtensions.cs

@@ -14,12 +14,21 @@ namespace Wabbajack.Common;
 
 public static class HttpExtensions
 {
+    private const string ChromeUserAgent =
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36";
     public static HttpRequestMessage AddCookies(this HttpRequestMessage msg, Cookie[] cookies)
     {
         msg.Headers.Add("Cookie", string.Join(";", cookies.Select(c => $"{c.Name}={c.Value}")));
         return msg;
     }
-    
+
+    public static HttpRequestMessage AddChromeAgent(this HttpRequestMessage msg, string? overrideUserAgent = null)
+    {
+        msg.Headers.UserAgent.Clear();
+        msg.Headers.Add("User-Agent", overrideUserAgent ?? ChromeUserAgent);
+        return msg;
+    }
+
     public static HttpRequestMessage AddHeaders(this HttpRequestMessage msg, IEnumerable<(string Key, string Value)> headers)
     {
         foreach (var header in headers)
@@ -29,17 +38,10 @@ public static class HttpExtensions
         return msg;
     }
 
-    public static HttpRequestMessage AddChromeAgent(this HttpRequestMessage msg)
-    {
-        msg.Headers.Add("User-Agent",
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36");
-        return msg;
-    }
-
     public static HttpRequestMessage ToHttpRequestMessage(this ManualDownload.BrowserDownloadState browserState)
     {
         var msg = new HttpRequestMessage(HttpMethod.Get, browserState.Uri);
-        msg.AddChromeAgent();
+        msg.AddChromeAgent(browserState.UserAgent);
         msg.AddCookies(browserState.Cookies);
         msg.AddHeaders(browserState.Headers);
         return msg;

Wabbajack.DTOs/Interventions/ManualDownload.cs

@@ -10,14 +10,14 @@ namespace Wabbajack.DTOs.Interventions;
 public class ManualDownload : AUserIntervention<ManualDownload.BrowserDownloadState>
 {
     public Archive Archive { get; }
-    
+
     public ManualDownload(Archive archive)
     {
         Archive = archive;
     }
 
-    public record BrowserDownloadState(Uri Uri, Cookie[] Cookies, (string Key, string Value)[] Headers)
+    public record BrowserDownloadState(Uri Uri, Cookie[] Cookies, (string Key, string Value)[] Headers, string UserAgent)
     {
-        
+
     }
 }

Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs

@@ -65,7 +65,7 @@ public class GoogleDriveDownloader : ADownloader<DTOs.DownloadStates.GoogleDrive
         return new Uri(
             $"https://drive.google.com/uc?id={(state as DTOs.DownloadStates.GoogleDrive)?.Id}&export=download");
     }
-    
+
     public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
     {
         if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"].CleanIniString(), UriKind.Absolute, out var uri))
@@ -74,8 +74,8 @@ public class GoogleDriveDownloader : ADownloader<DTOs.DownloadStates.GoogleDrive
     }
 
     public override Priority Priority => Priority.Normal;
-    
+
-    
+
     public async Task<T> DownloadStream<T>(Archive archive, Func<Stream, Task<T>> fn, CancellationToken token)
     {
         var state = archive.State as DTOs.DownloadStates.GoogleDrive;
@@ -112,8 +112,8 @@ public class GoogleDriveDownloader : ADownloader<DTOs.DownloadStates.GoogleDrive
         {
             var initialUrl = $"https://drive.google.com/uc?id={state.Id}&export=download";
             var msg = new HttpRequestMessage(HttpMethod.Get, initialUrl);
-            msg.UseChromeUserAgent();
+            msg.AddChromeAgent();
-            
+
             using var response = await _client.SendAsync(msg, token);
             var cookies = response.GetSetCookies();
             var warning = cookies.FirstOrDefault(c => c.Key.StartsWith("download_warning_"));
@@ -124,7 +124,7 @@ public class GoogleDriveDownloader : ADownloader<DTOs.DownloadStates.GoogleDrive
                 var txt = await response.Content.ReadAsStringAsync(token);
                 if (txt.Contains("<title>Google Drive - Quota exceeded</title>"))
                     throw new Exception("Google Drive - Quota Exceeded");
-                
+
                 doc.LoadHtml(txt);
 
                 var action = doc.DocumentNode.DescendantsAndSelf()
@@ -133,7 +133,7 @@ public class GoogleDriveDownloader : ADownloader<DTOs.DownloadStates.GoogleDrive
                     .Select(d => d.GetAttributeValue("action", ""))
                     .FirstOrDefault();
 
-                if (action != null) 
+                if (action != null)
                     warning = ("download_warning_", "t");
 
             }
@@ -145,18 +145,18 @@ public class GoogleDriveDownloader : ADownloader<DTOs.DownloadStates.GoogleDrive
 
             var url = $"https://drive.google.com/uc?export=download&confirm={warning.Value}&id={state.Id}";
             var httpState = new HttpRequestMessage(HttpMethod.Get, url);
-            httpState.UseChromeUserAgent();
+            httpState.AddChromeAgent();
             return httpState;
         }
         else
         {
             var url = $"https://drive.google.com/file/d/{state.Id}/edit";
             var msg = new HttpRequestMessage(HttpMethod.Get, url);
-            msg.UseChromeUserAgent();
+            msg.AddChromeAgent();
-            
+
             using var response = await _client.SendAsync(msg, token);
             msg = new HttpRequestMessage(HttpMethod.Get, url);
-            msg.UseChromeUserAgent();
+            msg.AddChromeAgent();
             return !response.IsSuccessStatusCode ? null : msg;
         }
     }

Wabbajack.Networking.Http/Extensions.cs

@@ -10,13 +10,6 @@ namespace Wabbajack.Networking.Http;
 
 public static class Extensions
 {
-    public static HttpRequestMessage UseChromeUserAgent(this HttpRequestMessage msg)
-    {
-        msg.Headers.UserAgent.Clear();
-        msg.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36");
-        return msg;
-    }
-
     public static async Task<T> GetJsonFromSendAsync<T>(this HttpClient client, HttpRequestMessage msg,
         JsonSerializerOptions opts, CancellationToken? token = null)
     {