Merge pull request #242 from wabbajack-tools/ll-redux-redux

Add LoversLab support

Wabbajack.Common/StatusFeed/Errors/GenericException.cs

@@ -12,7 +12,7 @@ namespace Wabbajack.Common.StatusFeed.Errors
 
         public DateTime Timestamp { get; } = DateTime.Now;
 
-        public string ShortDescription => ExtraMessage ?? Exception?.Message;
+        public string ShortDescription => ExtraMessage + " - " + Exception?.Message;
 
         public string ExtendedDescription => $"{ExtraMessage}: {Exception?.ToString()}";
 

Wabbajack.Lib/CerasConfig.cs

@@ -23,7 +23,8 @@ namespace Wabbajack.Lib
                 typeof(MegaDownloader.State), typeof(ModDBDownloader.State), typeof(NexusDownloader.State),
                 typeof(BSAStateObject), typeof(BSAFileStateObject), typeof(BA2StateObject), typeof(BA2DX10EntryState),
                 typeof(BA2FileEntryState), typeof(MediaFireDownloader.State), typeof(ArchiveMeta),
-                typeof(PropertyFile), typeof(SteamMeta), typeof(SteamWorkshopDownloader), typeof(SteamWorkshopDownloader.State)
+                typeof(PropertyFile), typeof(SteamMeta), typeof(SteamWorkshopDownloader), typeof(SteamWorkshopDownloader.State),
+                typeof(LoversLabDownloader.State)
 
             }
         };

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -15,6 +15,7 @@ namespace Wabbajack.Lib.Downloaders
             new ModDBDownloader(),
             new NexusDownloader(),
             new MediaFireDownloader(),
+            new LoversLabDownloader(),
             new HTTPDownloader(),
             new ManualDownloader(),
         };
@@ -26,9 +27,11 @@ namespace Wabbajack.Lib.Downloaders
             IndexedDownloaders = Downloaders.ToDictionary(d => d.GetType());
         }
 
-        public static T GetInstance<T>()
+        public static T GetInstance<T>() where T : IDownloader
         {
-            return (T)IndexedDownloaders[typeof(T)];
+            var inst = (T)IndexedDownloaders[typeof(T)];
+            inst.Prepare();
+            return inst;
         }
 
         public static AbstractDownloadState ResolveArchive(dynamic ini)

Wabbajack.Lib/Downloaders/LoversLabDownloader.cs

@@ -0,0 +1,200 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using System.Web;
+using Wabbajack.Common;
+using Wabbajack.Common.StatusFeed;
+using Wabbajack.Common.StatusFeed.Errors;
+using Wabbajack.Lib.LibCefHelpers;
+using Wabbajack.Lib.Validation;
+using Wabbajack.Lib.WebAutomation;
+using Xilium.CefGlue.Common;
+using File = Alphaleonis.Win32.Filesystem.File;
+
+namespace Wabbajack.Lib.Downloaders
+{
+    public class LoversLabDownloader : IDownloader
+    {
+        internal HttpClient _authedClient;
+
+        public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
+        {
+
+            Uri url = DownloaderUtils.GetDirectURL(archive_ini);
+            if (url == null || url.Host != "www.loverslab.com" || !url.AbsolutePath.StartsWith("/files/file/")) return null;
+            var id = HttpUtility.ParseQueryString(url.Query)["r"];
+            var file = url.AbsolutePath.Split('/').Last(s => s != "");
+
+            return new State
+            {
+                FileID = id,
+                FileName = file
+            };
+        }
+
+        public void Prepare()
+        {
+            _authedClient = GetAuthedClient().Result ?? throw new Exception("not logged into LL, TODO");
+        }
+
+        public static async Task<Helpers.Cookie[]> GetAndCacheLoversLabCookies(BaseCefBrowser browser, Action<string> updateStatus)
+        {
+            updateStatus("Please Log Into Lovers Lab");
+            browser.Address = "https://www.loverslab.com/login";
+
+            async Task<bool> CleanAds()
+            {
+                try
+                {
+                    await browser.EvaluateJavaScript<string>(
+                        "document.querySelectorAll(\".ll_adblock\").forEach(function (itm) { itm.innerHTML = \"\";});");
+                }
+                catch (Exception ex)
+                {
+                }
+                return false;
+            }
+            var cookies = new Helpers.Cookie[0];
+            while (true)
+            {
+                await CleanAds();
+                cookies = (await Helpers.GetCookies("loverslab.com"));
+                if (cookies.FirstOrDefault(c => c.Name == "ips4_member_id") != null)
+                    break;
+                await Task.Delay(500);
+            }
+
+            cookies.ToEcryptedJson("loverslabcookies");
+
+            return cookies;
+        }
+
+        public async Task<HttpClient> GetAuthedClient()
+        {
+            Helpers.Cookie[] cookies;
+            try
+            {
+                cookies = Utils.FromEncryptedJson<Helpers.Cookie[]>("loverslabcookies");
+                if (cookies != null)
+                    return Helpers.GetClient(cookies, "https://www.loverslab.com");
+            }
+            catch (FileNotFoundException) { }
+
+            cookies = Utils.Log(new RequestLoversLabLogin()).Task.Result;
+            return Helpers.GetClient(cookies, "https://www.loverslab.com");
+        }
+
+        public class State : AbstractDownloadState
+        {
+            public string FileID { get; set; }
+            public string FileName { get; set; }
+
+            public override bool IsWhitelisted(ServerWhitelist whitelist)
+            {
+                return true;
+            }
+
+            public override void Download(Archive a, string destination)
+            {
+                var stream = ResolveDownloadStream().Result;
+                using (var file = File.OpenWrite(destination))
+                {
+                    stream.CopyTo(file);
+                }
+            }
+
+            private async Task<Stream> ResolveDownloadStream()
+            {
+                var result = DownloadDispatcher.GetInstance<LoversLabDownloader>();
+                TOP:
+                var html = await result._authedClient.GetStringAsync(
+                    $"https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID}");
+
+                var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])");
+                var csrfKey = pattern.Matches(html).Cast<Match>().Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault();
+
+                if (csrfKey == null)
+                    return null;
+
+                var url =
+                    $"https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}";
+
+                var streamResult = await result._authedClient.GetAsync(url);
+                if (streamResult.StatusCode != HttpStatusCode.OK)
+                {
+                    Utils.Error(new InvalidOperationException(), $"LoversLab servers reported an error for file: {FileID}");
+                }
+
+                var content_type = streamResult.Content.Headers.ContentType;
+
+                if (content_type.MediaType == "application/json")
+                {
+                    // Sometimes LL hands back a json object telling us to wait until a certain time
+                    var times = (await streamResult.Content.ReadAsStringAsync()).FromJSONString<WaitResponse>();
+                    var secs = times.download - times.currentTime;
+                    for (int x = 0; x < secs; x++)
+                    {
+                        Utils.Status($"Waiting for {secs} at the request of LoversLab", x * 100 / secs);
+                        await Task.Delay(1000);
+                    }
+                    Utils.Status("Retrying download");
+                    goto TOP;
+                }
+
+                return await streamResult.Content.ReadAsStreamAsync();
+            }
+
+            internal class WaitResponse
+            {
+                public int download { get; set; }
+                public int currentTime { get; set; }
+            }
+
+            public override bool Verify()
+            {
+                var stream = ResolveDownloadStream().Result;
+                if (stream == null)
+                {
+                    return false;
+                }
+
+                stream.Close();
+                return true;
+
+            }
+
+            public override IDownloader GetDownloader()
+            {
+                return DownloadDispatcher.GetInstance<LoversLabDownloader>();
+            }
+
+            public override string GetReportEntry(Archive a)
+            {
+                return $"* Lovers Lab - [{a.Name}](https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID})";
+            }
+        }
+    }
+
+    public class RequestLoversLabLogin : AStatusMessage, IUserIntervention
+    {
+        public override string ShortDescription => "Getting LoversLab information";
+        public override string ExtendedDescription { get; }
+
+        private readonly TaskCompletionSource<Helpers.Cookie[]> _source = new TaskCompletionSource<Helpers.Cookie[]>();
+        public Task<Helpers.Cookie[]> Task => _source.Task;
+
+        public void Resume(Helpers.Cookie[] cookies)
+        {
+            _source.SetResult(cookies);
+        }
+        public void Cancel()
+        {
+            _source.SetCanceled();
+        }
+    }
+}

Wabbajack.Lib/LibCefHelpers/Init.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net;
+using System.Net.Http;
 using System.Reflection;
 using System.Text;
 using System.Threading;
@@ -31,6 +33,27 @@ namespace Wabbajack.Lib.LibCefHelpers
                 FileExtractor.ExtractAll(wq, "cefglue.7z", ".");
 
         }
+        public static HttpClient GetClient(IEnumerable<Cookie> cookies, string referer)
+        {
+            var container = ToCookieContainer(cookies);
+            var handler = new HttpClientHandler { CookieContainer = container };
+            var client = new HttpClient(handler);
+            client.DefaultRequestHeaders.Referrer = new Uri(referer);
+            return client;
+        }
+
+        private static CookieContainer ToCookieContainer(IEnumerable<Cookie> cookies)
+        {
+            var container = new CookieContainer();
+            cookies
+                .Do(cookie =>
+                {
+                    container.Add(new System.Net.Cookie(cookie.Name, cookie.Value, cookie.Path, cookie.Domain));
+                });
+
+            return container;
+        }
+
         public static async Task<Cookie[]> GetCookies(string domainEnding)
         {
             var manager = CefCookieManager.GetGlobal(null);
@@ -42,6 +65,7 @@ namespace Wabbajack.Lib.LibCefHelpers
             return (await visitor.Task).Where(c => c.Domain.EndsWith(domainEnding)).ToArray();
         }
 
+
         private class CookieVisitor : CefCookieVisitor
         {
             TaskCompletionSource<List<Cookie>> _source = new TaskCompletionSource<List<Cookie>>();
@@ -68,6 +92,8 @@ namespace Wabbajack.Lib.LibCefHelpers
                 if (disposing)
                     _source.SetResult(Cookies);
             }
+
+
         }
 
         public class Cookie

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -116,6 +116,7 @@
     <Compile Include="CompilationSteps\IStackStep.cs" />
     <Compile Include="CompilationSteps\PatchStockESMs.cs" />
     <Compile Include="CompilationSteps\Serialization.cs" />
+    <Compile Include="Downloaders\LoversLabDownloader.cs" />
     <Compile Include="Downloaders\SteamWorkshopDownloader.cs" />
     <Compile Include="LibCefHelpers\Init.cs" />
     <Compile Include="MO2Compiler.cs" />

Wabbajack.Test/DownloaderTests.cs

@@ -1,7 +1,10 @@
 using System;
 using System.Collections.Generic;
+using System.Reactive.Disposables;
+using System.Reactive.Linq;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Wabbajack.Common;
+using Wabbajack.Common.StatusFeed;
 using Wabbajack.Lib;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.LibCefHelpers;
@@ -14,10 +17,17 @@ namespace Wabbajack.Test
     [TestClass]
     public class DownloaderTests
     {
+
+        public TestContext TestContext { get; set; }
+
         [TestInitialize]
         public void Setup()
         {
             Helpers.ExtractLibs();
+            Utils.LogMessages.OfType<IInfo>().Subscribe(onNext: msg => TestContext.WriteLine(msg.ShortDescription));
+            Utils.LogMessages.OfType<IUserIntervention>().Subscribe(msg =>
+                TestContext.WriteLine("ERROR: User intervetion required: " + msg.ShortDescription));
+
         }
 
         [TestMethod]
@@ -261,6 +271,35 @@ namespace Wabbajack.Test
 
             Assert.AreEqual("2lZt+1h6wxM=", filename.FileHash());
         }
+
+        [TestMethod]
+        public void LoversLabDownload()
+        {
+            DownloadDispatcher.GetInstance<LoversLabDownloader>().Prepare();
+            var ini = @"[General]
+                        directURL=https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1";
+
+            var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString());
+
+            Assert.IsNotNull(state);
+
+            /*var url_state = DownloadDispatcher.ResolveArchive("https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1");
+            Assert.AreEqual("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt",
+                ((HTTPDownloader.State)url_state).Url);
+                */
+            var converted = state.ViaJSON();
+            Assert.IsTrue(converted.Verify());
+            var filename = Guid.NewGuid().ToString();
+
+            Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
+
+            converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
+
+            Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
+
+            Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
+
+        }
     }
 
 

Wabbajack/View Models/MainWindowVM.cs

@@ -12,6 +12,7 @@ using System.Windows.Threading;
 using Wabbajack.Common;
 using Wabbajack.Common.StatusFeed;
 using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.NexusApi;
 using Wabbajack.Lib.StatusMessages;
 
@@ -37,6 +38,7 @@ namespace Wabbajack
         public readonly Lazy<ModListGalleryVM> Gallery;
         public readonly ModeSelectionVM ModeSelectionVM;
         public readonly WebBrowserVM WebBrowserVM;
+        public readonly UserInterventionHandlers UserInterventionHandlers;
         public Dispatcher ViewDispatcher { get; set; }
 
         public MainWindowVM(MainWindow mainWindow, MainSettings settings)
@@ -49,6 +51,7 @@ namespace Wabbajack
             Gallery = new Lazy<ModListGalleryVM>(() => new ModListGalleryVM(this));
             ModeSelectionVM = new ModeSelectionVM(this);
             WebBrowserVM = new WebBrowserVM();
+            UserInterventionHandlers = new UserInterventionHandlers {MainWindow = this, ViewDispatcher = mainWindow.Dispatcher};
 
             // Set up logging
             Utils.LogMessages
@@ -63,12 +66,8 @@ namespace Wabbajack
                 .DisposeWith(CompositeDisposable);
 
             Utils.LogMessages
-                .OfType<ConfirmUpdateOfExistingInstall>()
+                .OfType<IUserIntervention>()
-                .Subscribe(msg => ConfirmUpdate(msg));
+                .Subscribe(msg => UserInterventionHandlers.Handle(msg));
-
-            Utils.LogMessages
-                .OfType<RequestNexusAuthorization>()
-                .Subscribe(HandleRequestNexusAuthorization);
 
             if (IsStartingFromModlist(out var path))
             {
@@ -82,47 +81,6 @@ namespace Wabbajack
             }
         }
 
-        private void HandleRequestNexusAuthorization(RequestNexusAuthorization msg)
-        {
-            ViewDispatcher.InvokeAsync(async () =>
-            {
-                var oldPane = ActivePane;
-                var vm = new WebBrowserVM();
-                ActivePane = vm;
-                try
-                {
-                    vm.BackCommand = ReactiveCommand.Create(() =>
-                    {
-                        ActivePane = oldPane;
-                        msg.Cancel();
-                    });
-                }
-                catch (Exception e)
-                { }
-
-                try
-                {
-                    var key = await NexusApiClient.SetupNexusLogin(vm.Browser, m => vm.Instructions = m);
-                    msg.Resume(key);
-                }
-                catch (Exception ex)
-                {
-                    msg.Cancel();
-                }
-                ActivePane = oldPane;
-
-            });
-        }
-
-        private void ConfirmUpdate(ConfirmUpdateOfExistingInstall msg)
-        {
-            var result = MessageBox.Show(msg.ExtendedDescription, msg.ShortDescription, MessageBoxButton.OKCancel);
-            if (result == MessageBoxResult.OK)
-                msg.Confirm();
-            else
-                msg.Cancel();
-        }
-
         private static bool IsStartingFromModlist(out string modlistPath)
         {
             string[] args = Environment.GetCommandLineArgs();

Wabbajack/View Models/SlideShow.cs

@@ -98,7 +98,7 @@ namespace Wabbajack
             _targetMod = Observable.CombineLatest(
                     modVMs.QueryWhenChanged(),
                     selectedIndex,
-                    resultSelector: (query, selected) => query.Items.ElementAtOrDefault(selected % query.Count))
+                    resultSelector: (query, selected) => query.Items.ElementAtOrDefault(selected % (query.Count == 0 ? 1 : query.Count)))
                 .StartWith(default(ModVM))
                 .ObserveOn(RxApp.MainThreadScheduler)
                 .ToProperty(this, nameof(TargetMod));

Wabbajack/View Models/UserInterventionHandlers.cs

@@ -0,0 +1,111 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Windows;
+using System.Windows.Threading;
+using ReactiveUI;
+using Wabbajack.Common.StatusFeed;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.NexusApi;
+using Wabbajack.Lib.StatusMessages;
+
+namespace Wabbajack
+{
+    public class UserInterventionHandlers
+    {
+        public Dispatcher ViewDispatcher { get; set; }
+        public MainWindowVM MainWindow { get; set; }
+        internal void Handle(RequestLoversLabLogin msg)
+        {
+            ViewDispatcher.InvokeAsync(async () =>
+            {
+                var oldPane = MainWindow.ActivePane;
+                var vm = new WebBrowserVM();
+                MainWindow.ActivePane = vm;
+                try
+                {
+                    vm.BackCommand = ReactiveCommand.Create(() =>
+                    {
+                        MainWindow.ActivePane = oldPane;
+                        msg.Cancel();
+                    });
+                }
+                catch (Exception e)
+                { }
+
+                try
+                {
+                    var data = await LoversLabDownloader.GetAndCacheLoversLabCookies(vm.Browser, m => vm.Instructions = m);
+                    msg.Resume(data);
+                }
+                catch (Exception ex)
+                {
+                    msg.Cancel();
+                }
+                MainWindow.ActivePane = oldPane;
+
+            });
+        }
+
+        internal void Handle(RequestNexusAuthorization msg)
+        {
+            ViewDispatcher.InvokeAsync(async () =>
+            {
+                var oldPane = MainWindow.ActivePane;
+                var vm = new WebBrowserVM();
+                MainWindow.ActivePane = vm;
+                try
+                {
+                    vm.BackCommand = ReactiveCommand.Create(() =>
+                    {
+                        MainWindow.ActivePane = oldPane;
+                        msg.Cancel();
+                    });
+                }
+                catch (Exception e)
+                { }
+
+                try
+                {
+                    var key = await NexusApiClient.SetupNexusLogin(vm.Browser, m => vm.Instructions = m);
+                    msg.Resume(key);
+                }
+                catch (Exception ex)
+                {
+                    msg.Cancel();
+                }
+                MainWindow.ActivePane = oldPane;
+
+            });
+        }
+
+        internal void Handle(ConfirmUpdateOfExistingInstall msg)
+        {
+            var result = MessageBox.Show(msg.ExtendedDescription, msg.ShortDescription, MessageBoxButton.OKCancel);
+            if (result == MessageBoxResult.OK)
+                msg.Confirm();
+            else
+                msg.Cancel();
+        }
+
+        public void Handle(IUserIntervention msg)
+        {
+            switch (msg)
+            {
+                case ConfirmUpdateOfExistingInstall c: 
+                    Handle(c);
+                    break;
+                case RequestNexusAuthorization c:
+                    Handle(c);
+                    break;
+                case RequestLoversLabLogin c:
+                    Handle(c);
+                    break;
+                default:
+                    throw new NotImplementedException($"No handler for {msg}");
+            }
+        }
+    }
+}

Wabbajack/Wabbajack.csproj

@@ -173,6 +173,7 @@
       <SubType>Designer</SubType>
     </ApplicationDefinition>
     <Compile Include="Converters\EqualsToBoolConverter.cs" />
+    <Compile Include="View Models\UserInterventionHandlers.cs" />
     <Compile Include="View Models\WebBrowserVM.cs" />
     <Compile Include="Views\Installers\MO2InstallerConfigView.xaml.cs">
       <DependentUpon>MO2InstallerConfigView.xaml</DependentUpon>