diff --git a/Wabbajack.Common/StatusFeed/Errors/GenericException.cs b/Wabbajack.Common/StatusFeed/Errors/GenericException.cs index b091636c..55aabf90 100644 --- a/Wabbajack.Common/StatusFeed/Errors/GenericException.cs +++ b/Wabbajack.Common/StatusFeed/Errors/GenericException.cs @@ -12,7 +12,7 @@ namespace Wabbajack.Common.StatusFeed.Errors public DateTime Timestamp { get; } = DateTime.Now; - public string ShortDescription => ExtraMessage ?? Exception?.Message; + public string ShortDescription => ExtraMessage + " - " + Exception?.Message; public string ExtendedDescription => $"{ExtraMessage}: {Exception?.ToString()}"; diff --git a/Wabbajack.Lib/CerasConfig.cs b/Wabbajack.Lib/CerasConfig.cs index ae8f2e86..1b08961f 100644 --- a/Wabbajack.Lib/CerasConfig.cs +++ b/Wabbajack.Lib/CerasConfig.cs @@ -23,7 +23,8 @@ namespace Wabbajack.Lib typeof(MegaDownloader.State), typeof(ModDBDownloader.State), typeof(NexusDownloader.State), typeof(BSAStateObject), typeof(BSAFileStateObject), typeof(BA2StateObject), typeof(BA2DX10EntryState), typeof(BA2FileEntryState), typeof(MediaFireDownloader.State), typeof(ArchiveMeta), - typeof(PropertyFile), typeof(SteamMeta), typeof(SteamWorkshopDownloader), typeof(SteamWorkshopDownloader.State) + typeof(PropertyFile), typeof(SteamMeta), typeof(SteamWorkshopDownloader), typeof(SteamWorkshopDownloader.State), + typeof(LoversLabDownloader.State) } }; diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index 4ae2efd1..c1e7c229 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -15,6 +15,7 @@ namespace Wabbajack.Lib.Downloaders new ModDBDownloader(), new NexusDownloader(), new MediaFireDownloader(), + new LoversLabDownloader(), new HTTPDownloader(), new ManualDownloader(), }; @@ -26,9 +27,11 @@ namespace Wabbajack.Lib.Downloaders IndexedDownloaders = Downloaders.ToDictionary(d => d.GetType()); } - public static T GetInstance() + public static T GetInstance() where T : IDownloader { - return (T)IndexedDownloaders[typeof(T)]; + var inst = (T)IndexedDownloaders[typeof(T)]; + inst.Prepare(); + return inst; } public static AbstractDownloadState ResolveArchive(dynamic ini) diff --git a/Wabbajack.Lib/Downloaders/LoversLabDownloader.cs b/Wabbajack.Lib/Downloaders/LoversLabDownloader.cs new file mode 100644 index 00000000..9e45f2fd --- /dev/null +++ b/Wabbajack.Lib/Downloaders/LoversLabDownloader.cs @@ -0,0 +1,202 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Web; +using Wabbajack.Common; +using Wabbajack.Common.StatusFeed; +using Wabbajack.Common.StatusFeed.Errors; +using Wabbajack.Lib.LibCefHelpers; +using Wabbajack.Lib.Validation; +using Wabbajack.Lib.WebAutomation; +using Xilium.CefGlue.Common; +using File = Alphaleonis.Win32.Filesystem.File; + +namespace Wabbajack.Lib.Downloaders +{ + public class LoversLabDownloader : IDownloader + { + internal HttpClient _authedClient; + + public AbstractDownloadState GetDownloaderState(dynamic archive_ini) + { + + Uri url = DownloaderUtils.GetDirectURL(archive_ini); + if (url == null || url.Host != "www.loverslab.com" && url.AbsolutePath.StartsWith("/files/file/")) return null; + var id = HttpUtility.ParseQueryString(url.Query)["r"]; + var file = url.AbsolutePath.Split('/').Last(s => s != ""); + + return new State + { + FileID = id, + FileName = file + }; + } + + public void Prepare() + { + _authedClient = GetAuthedClient().Result ?? throw new Exception("not logged into LL, TODO"); + } + + public static async Task GetAndCacheLoversLabCookies(BaseCefBrowser browser, Action updateStatus) + { + updateStatus("Please Log Into Lovers Lab"); + browser.Address = "https://www.loverslab.com/login"; + + async Task CleanAds() + { + try + { + // await browser.EvaluateJavaScript( + // "document.querySelectorAll(\"iframe\").forEach(function(itm) { itm.src = \"about:blank\"})"); + await browser.EvaluateJavaScript( + "document.querySelectorAll(\".ad\").forEach(function (itm) { itm.innerHTML = \"\";});"); + } + catch (Exception ex) + { + } + return false; + } + var cookies = new Helpers.Cookie[0]; + while (true) + { + await CleanAds(); + cookies = (await Helpers.GetCookies("loverslab.com")); + if (cookies.FirstOrDefault(c => c.Name == "ips4_member_id") != null) + break; + await Task.Delay(500); + } + + cookies.ToEcryptedJson("loverslabcookies"); + + return cookies; + } + + public async Task GetAuthedClient() + { + Helpers.Cookie[] cookies; + try + { + cookies = Utils.FromEncryptedJson("loverslabcookies"); + if (cookies != null) + return Helpers.GetClient(cookies, "https://www.loverslab.com"); + } + catch (FileNotFoundException) { } + + cookies = Utils.Log(new RequestLoversLabLogin()).Task.Result; + return Helpers.GetClient(cookies, "https://www.loverslab.com"); + } + + public class State : AbstractDownloadState + { + public string FileID { get; set; } + public string FileName { get; set; } + + public override bool IsWhitelisted(ServerWhitelist whitelist) + { + return true; + } + + public override void Download(Archive a, string destination) + { + var stream = ResolveDownloadStream().Result; + using (var file = File.OpenWrite(destination)) + { + stream.CopyTo(file); + } + } + + private async Task ResolveDownloadStream() + { + var result = DownloadDispatcher.GetInstance(); + TOP: + var html = await result._authedClient.GetStringAsync( + $"https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID}"); + + var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])"); + var csrfKey = pattern.Matches(html).Cast().Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault(); + + if (csrfKey == null) + return null; + + var url = + $"https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}"; + + var streamResult = await result._authedClient.GetAsync(url); + if (streamResult.StatusCode != HttpStatusCode.OK) + { + Utils.Error(new InvalidOperationException(), $"LoversLab servers reported an error for file: {FileID}"); + } + + var content_type = streamResult.Content.Headers.ContentType; + + if (content_type.MediaType == "application/json") + { + // Sometimes LL hands back a json object telling us to wait until a certain time + var times = (await streamResult.Content.ReadAsStringAsync()).FromJSONString(); + var secs = times.download - times.currentTime; + for (int x = 0; x < secs; x++) + { + Utils.Status($"Waiting for {secs} at the request of LoversLab", x * 100 / secs); + await Task.Delay(1000); + } + Utils.Status("Retrying download"); + goto TOP; + } + + return await streamResult.Content.ReadAsStreamAsync(); + } + + internal class WaitResponse + { + public int download { get; set; } + public int currentTime { get; set; } + } + + public override bool Verify() + { + var stream = ResolveDownloadStream().Result; + if (stream == null) + { + return false; + } + + stream.Close(); + return true; + + } + + public override IDownloader GetDownloader() + { + return DownloadDispatcher.GetInstance(); + } + + public override string GetReportEntry(Archive a) + { + return $"* Lovers Lab - [{a.Name}](https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID})"; + } + } + } + + public class RequestLoversLabLogin : AStatusMessage, IUserIntervention + { + public override string ShortDescription => "Getting LoversLab information"; + public override string ExtendedDescription { get; } + + private readonly TaskCompletionSource _source = new TaskCompletionSource(); + public Task Task => _source.Task; + + public void Resume(Helpers.Cookie[] cookies) + { + _source.SetResult(cookies); + } + public void Cancel() + { + _source.SetCanceled(); + } + } +} diff --git a/Wabbajack.Lib/LibCefHelpers/Init.cs b/Wabbajack.Lib/LibCefHelpers/Init.cs index 27cc7885..ff75da3f 100644 --- a/Wabbajack.Lib/LibCefHelpers/Init.cs +++ b/Wabbajack.Lib/LibCefHelpers/Init.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net; +using System.Net.Http; using System.Reflection; using System.Text; using System.Threading; @@ -31,6 +33,27 @@ namespace Wabbajack.Lib.LibCefHelpers FileExtractor.ExtractAll(wq, "cefglue.7z", "."); } + public static HttpClient GetClient(IEnumerable cookies, string referer) + { + var container = ToCookieContainer(cookies); + var handler = new HttpClientHandler { CookieContainer = container }; + var client = new HttpClient(handler); + client.DefaultRequestHeaders.Referrer = new Uri(referer); + return client; + } + + private static CookieContainer ToCookieContainer(IEnumerable cookies) + { + var container = new CookieContainer(); + cookies + .Do(cookie => + { + container.Add(new System.Net.Cookie(cookie.Name, cookie.Value, cookie.Path, cookie.Domain)); + }); + + return container; + } + public static async Task GetCookies(string domainEnding) { var manager = CefCookieManager.GetGlobal(null); @@ -42,6 +65,7 @@ namespace Wabbajack.Lib.LibCefHelpers return (await visitor.Task).Where(c => c.Domain.EndsWith(domainEnding)).ToArray(); } + private class CookieVisitor : CefCookieVisitor { TaskCompletionSource> _source = new TaskCompletionSource>(); @@ -68,6 +92,8 @@ namespace Wabbajack.Lib.LibCefHelpers if (disposing) _source.SetResult(Cookies); } + + } public class Cookie diff --git a/Wabbajack.Lib/MO2Installer.cs b/Wabbajack.Lib/MO2Installer.cs index f55e4d84..01e1f16a 100644 --- a/Wabbajack.Lib/MO2Installer.cs +++ b/Wabbajack.Lib/MO2Installer.cs @@ -40,6 +40,10 @@ namespace Wabbajack.Lib ConfigureProcessor(18, RecommendQueueSize()); var game = GameRegistry.Games[ModList.GameType]; + + // TODO: Remove + DownloadDispatcher.GetInstance().Prepare(); + if (GameFolder == null) GameFolder = game.GameLocation(SteamHandler.Instance.Games.Any(g => g.Game == game.Game)); diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 1f1e1a40..ce951b6c 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -116,6 +116,7 @@ + diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 8dfa1da8..0480f5a6 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -1,7 +1,10 @@ using System; using System.Collections.Generic; +using System.Reactive.Disposables; +using System.Reactive.Linq; using Microsoft.VisualStudio.TestTools.UnitTesting; using Wabbajack.Common; +using Wabbajack.Common.StatusFeed; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.LibCefHelpers; @@ -14,10 +17,17 @@ namespace Wabbajack.Test [TestClass] public class DownloaderTests { + + public TestContext TestContext { get; set; } + [TestInitialize] public void Setup() { Helpers.ExtractLibs(); + Utils.LogMessages.OfType().Subscribe(onNext: msg => TestContext.WriteLine(msg.ShortDescription)); + Utils.LogMessages.OfType().Subscribe(msg => + TestContext.WriteLine("ERROR: User intervetion required: " + msg.ShortDescription)); + } [TestMethod] @@ -261,6 +271,35 @@ namespace Wabbajack.Test Assert.AreEqual("2lZt+1h6wxM=", filename.FileHash()); } + + [TestMethod] + public void LoversLabDownload() + { + DownloadDispatcher.GetInstance().Prepare(); + var ini = @"[General] + directURL=https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1"; + + var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString()); + + Assert.IsNotNull(state); + + /*var url_state = DownloadDispatcher.ResolveArchive("https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1"); + Assert.AreEqual("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt", + ((HTTPDownloader.State)url_state).Url); + */ + var converted = state.ViaJSON(); + Assert.IsTrue(converted.Verify()); + var filename = Guid.NewGuid().ToString(); + + Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); + + converted.Download(new Archive { Name = "MEGA Test.txt" }, filename); + + Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename)); + + Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!"); + + } } diff --git a/Wabbajack/View Models/MainWindowVM.cs b/Wabbajack/View Models/MainWindowVM.cs index bfebbbc3..09ddd38f 100644 --- a/Wabbajack/View Models/MainWindowVM.cs +++ b/Wabbajack/View Models/MainWindowVM.cs @@ -12,6 +12,7 @@ using System.Windows.Threading; using Wabbajack.Common; using Wabbajack.Common.StatusFeed; using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.StatusMessages; @@ -37,6 +38,7 @@ namespace Wabbajack public readonly Lazy Gallery; public readonly ModeSelectionVM ModeSelectionVM; public readonly WebBrowserVM WebBrowserVM; + public readonly UserInterventionHandlers UserInterventionHandlers; public Dispatcher ViewDispatcher { get; set; } public MainWindowVM(MainWindow mainWindow, MainSettings settings) @@ -49,6 +51,7 @@ namespace Wabbajack Gallery = new Lazy(() => new ModListGalleryVM(this)); ModeSelectionVM = new ModeSelectionVM(this); WebBrowserVM = new WebBrowserVM(); + UserInterventionHandlers = new UserInterventionHandlers {MainWindow = this, ViewDispatcher = mainWindow.Dispatcher}; // Set up logging Utils.LogMessages @@ -63,12 +66,8 @@ namespace Wabbajack .DisposeWith(CompositeDisposable); Utils.LogMessages - .OfType() - .Subscribe(msg => ConfirmUpdate(msg)); - - Utils.LogMessages - .OfType() - .Subscribe(HandleRequestNexusAuthorization); + .OfType() + .Subscribe(msg => UserInterventionHandlers.Handle(msg)); if (IsStartingFromModlist(out var path)) { @@ -82,47 +81,6 @@ namespace Wabbajack } } - private void HandleRequestNexusAuthorization(RequestNexusAuthorization msg) - { - ViewDispatcher.InvokeAsync(async () => - { - var oldPane = ActivePane; - var vm = new WebBrowserVM(); - ActivePane = vm; - try - { - vm.BackCommand = ReactiveCommand.Create(() => - { - ActivePane = oldPane; - msg.Cancel(); - }); - } - catch (Exception e) - { } - - try - { - var key = await NexusApiClient.SetupNexusLogin(vm.Browser, m => vm.Instructions = m); - msg.Resume(key); - } - catch (Exception ex) - { - msg.Cancel(); - } - ActivePane = oldPane; - - }); - } - - private void ConfirmUpdate(ConfirmUpdateOfExistingInstall msg) - { - var result = MessageBox.Show(msg.ExtendedDescription, msg.ShortDescription, MessageBoxButton.OKCancel); - if (result == MessageBoxResult.OK) - msg.Confirm(); - else - msg.Cancel(); - } - private static bool IsStartingFromModlist(out string modlistPath) { string[] args = Environment.GetCommandLineArgs(); diff --git a/Wabbajack/View Models/SlideShow.cs b/Wabbajack/View Models/SlideShow.cs index 4a838259..053dfd87 100644 --- a/Wabbajack/View Models/SlideShow.cs +++ b/Wabbajack/View Models/SlideShow.cs @@ -98,7 +98,7 @@ namespace Wabbajack _targetMod = Observable.CombineLatest( modVMs.QueryWhenChanged(), selectedIndex, - resultSelector: (query, selected) => query.Items.ElementAtOrDefault(selected % query.Count)) + resultSelector: (query, selected) => query.Items.ElementAtOrDefault(selected % (query.Count == 0 ? 1 : query.Count))) .StartWith(default(ModVM)) .ObserveOn(RxApp.MainThreadScheduler) .ToProperty(this, nameof(TargetMod)); diff --git a/Wabbajack/View Models/UserInterventionHandlers.cs b/Wabbajack/View Models/UserInterventionHandlers.cs new file mode 100644 index 00000000..e28d4e04 --- /dev/null +++ b/Wabbajack/View Models/UserInterventionHandlers.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows; +using System.Windows.Threading; +using ReactiveUI; +using Wabbajack.Common.StatusFeed; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.NexusApi; +using Wabbajack.Lib.StatusMessages; + +namespace Wabbajack +{ + public class UserInterventionHandlers + { + public Dispatcher ViewDispatcher { get; set; } + public MainWindowVM MainWindow { get; set; } + internal void Handle(RequestLoversLabLogin msg) + { + ViewDispatcher.InvokeAsync(async () => + { + var oldPane = MainWindow.ActivePane; + var vm = new WebBrowserVM(); + MainWindow.ActivePane = vm; + try + { + vm.BackCommand = ReactiveCommand.Create(() => + { + MainWindow.ActivePane = oldPane; + msg.Cancel(); + }); + } + catch (Exception e) + { } + + try + { + var data = await LoversLabDownloader.GetAndCacheLoversLabCookies(vm.Browser, m => vm.Instructions = m); + msg.Resume(data); + } + catch (Exception ex) + { + msg.Cancel(); + } + MainWindow.ActivePane = oldPane; + + }); + } + + internal void Handle(RequestNexusAuthorization msg) + { + ViewDispatcher.InvokeAsync(async () => + { + var oldPane = MainWindow.ActivePane; + var vm = new WebBrowserVM(); + MainWindow.ActivePane = vm; + try + { + vm.BackCommand = ReactiveCommand.Create(() => + { + MainWindow.ActivePane = oldPane; + msg.Cancel(); + }); + } + catch (Exception e) + { } + + try + { + var key = await NexusApiClient.SetupNexusLogin(vm.Browser, m => vm.Instructions = m); + msg.Resume(key); + } + catch (Exception ex) + { + msg.Cancel(); + } + MainWindow.ActivePane = oldPane; + + }); + } + + internal void Handle(ConfirmUpdateOfExistingInstall msg) + { + var result = MessageBox.Show(msg.ExtendedDescription, msg.ShortDescription, MessageBoxButton.OKCancel); + if (result == MessageBoxResult.OK) + msg.Confirm(); + else + msg.Cancel(); + } + + public void Handle(IUserIntervention msg) + { + switch (msg) + { + case ConfirmUpdateOfExistingInstall c: + Handle(c); + break; + case RequestNexusAuthorization c: + Handle(c); + break; + case RequestLoversLabLogin c: + Handle(c); + break; + default: + throw new NotImplementedException($"No handler for {msg}"); + } + } + } +} diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 0dd03091..e24a46de 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -173,6 +173,7 @@ Designer + MO2InstallerConfigView.xaml