Merge pull request #354 from wabbajack-tools/vector-plus-redux

Implement VectorPlexus support as a downloader

Wabbajack.CacheServer/ListValidationService.cs

@@ -158,7 +158,7 @@ namespace Wabbajack.CacheServer
 
             using (var queue = new WorkQueue())
             {
-                foreach (var list in modlists.Skip(2).Take(1))
+                foreach (var list in modlists)
                 {
                     try
                     {

Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs

@@ -0,0 +1,155 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using System.Web;
+using Newtonsoft.Json;
+using Wabbajack.Common;
+using Wabbajack.Lib.Validation;
+using File = System.IO.File;
+
+namespace Wabbajack.Lib.Downloaders
+{
+    // IPS4 is the site used by LoversLab, VectorPlexus, etc. the general mechanics of each site are the 
+    // same, so we can fairly easily abstract the state.
+    // Pass in the state type via TState
+    public abstract class AbstractIPS4Downloader<TDownloader, TState> : AbstractNeedsLoginDownloader, IDownloader 
+        where TState : AbstractIPS4Downloader<TDownloader, TState>.State<TDownloader>, new() 
+        where TDownloader : IDownloader
+    {
+        public override string SiteName { get; }
+        public override Uri SiteURL { get; }
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
+        {
+            Uri url = DownloaderUtils.GetDirectURL(archiveINI);
+            if (url == null || url.Host != SiteURL.Host || !url.AbsolutePath.StartsWith("/files/file/")) return null;
+            var id = HttpUtility.ParseQueryString(url.Query)["r"];
+            var file = url.AbsolutePath.Split('/').Last(s => s != "");
+
+            return new TState
+            {
+                FileID = id,
+                FileName = file
+            };
+        }
+
+
+        public class State<TDownloader> : AbstractDownloadState where TDownloader : IDownloader
+        {
+            public string FileID { get; set; }
+            public string FileName { get; set; }
+
+            public override object[] PrimaryKey { get => new object[] {FileID, FileName}; }
+
+            public override bool IsWhitelisted(ServerWhitelist whitelist)
+            {
+                return true;
+            }
+
+            public override async Task Download(Archive a, string destination)
+            {
+                var stream = await ResolveDownloadStream();
+                using (var file = File.OpenWrite(destination))
+                {
+                    stream.CopyTo(file);
+                }
+            }
+
+            private async Task<Stream> ResolveDownloadStream()
+            {
+                var downloader = (AbstractNeedsLoginDownloader)(object)DownloadDispatcher.GetInstance<TDownloader>();
+
+                TOP:
+                string csrfurl;
+                if (FileID == null)
+                {
+                    csrfurl = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download";
+                }
+                else
+                {
+                    csrfurl = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID}";
+                }
+                var html = await downloader.AuthedClient.GetStringAsync(csrfurl);
+
+                var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])");
+                var matches = pattern.Matches(html).Cast<Match>();
+                    
+                    var csrfKey = matches.Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault();
+
+                if (csrfKey == null)
+                    return null;
+
+                string url;
+                if (FileID == null)
+                    url = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&confirm=1&t=1&csrfKey={csrfKey}";
+                else
+                    url = $"https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}";
+                    
+
+                var streamResult = await downloader.AuthedClient.GetAsync(url);
+                if (streamResult.StatusCode != HttpStatusCode.OK)
+                {
+                    Utils.Error(new InvalidOperationException(), $"{downloader.SiteName} servers reported an error for file: {FileID}");
+                }
+
+                var content_type = streamResult.Content.Headers.ContentType;
+
+                if (content_type.MediaType == "application/json")
+                {
+                    // Sometimes LL hands back a json object telling us to wait until a certain time
+                    var times = (await streamResult.Content.ReadAsStringAsync()).FromJSONString<WaitResponse>();
+                    var secs = times.Download - times.CurrentTime;
+                    for (int x = 0; x < secs; x++)
+                    {
+                        Utils.Status($"Waiting for {secs} at the request of {downloader.SiteName}", x * 100 / secs);
+                        await Task.Delay(1000);
+                    }
+                    Utils.Status("Retrying download");
+                    goto TOP;
+                }
+
+                return await streamResult.Content.ReadAsStreamAsync();
+            }
+
+            private class WaitResponse
+            {
+                [JsonProperty("download")]
+                public int Download { get; set; }
+                [JsonProperty("currentTime")]
+                public int CurrentTime { get; set; }
+            }
+
+            public override async Task<bool> Verify()
+            {
+                var stream = await ResolveDownloadStream();
+                if (stream == null)
+                {
+                    return false;
+                }
+
+                stream.Close();
+                return true;
+            }
+
+            public override IDownloader GetDownloader()
+            {
+                return DownloadDispatcher.GetInstance<TDownloader>();
+            }
+
+            public override string GetReportEntry(Archive a)
+            {
+                var downloader = (INeedsLogin)GetDownloader();
+                return $"* {((INeedsLogin)GetDownloader()).SiteName} - [{a.Name}](https://{downloader.SiteURL.Host}/files/file/{FileName}/?do=download&r={FileID})";
+            }
+        }
+
+        protected AbstractIPS4Downloader(Uri loginUri, string encryptedKeyName, string cookieDomain) : 
+            base(loginUri, encryptedKeyName, cookieDomain, "ips4_member_id")
+        {
+        }
+
+
+    }
+}

Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs

@@ -20,7 +20,7 @@ namespace Wabbajack.Lib.Downloaders
         private readonly string _encryptedKeyName;
         private readonly string _cookieDomain;
         private readonly string _cookieName;
-        protected HttpClient AuthedClient;
+        internal HttpClient AuthedClient;
 
         /// <summary>
         /// Sets up all the login facilites needed for a INeedsLogin downloader based on having the user log

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -18,6 +18,7 @@ namespace Wabbajack.Lib.Downloaders
             new NexusDownloader(),
             new MediaFireDownloader(),
             new LoversLabDownloader(),
+            new VectorPlexusDownloader(),
             new HTTPDownloader(),
             new ManualDownloader(),
         };

Wabbajack.Lib/Downloaders/LoversLabDownloader.cs

@@ -21,33 +21,18 @@ using File = Alphaleonis.Win32.Filesystem.File;
 
 namespace Wabbajack.Lib.Downloaders
 {
-    public class LoversLabDownloader : AbstractNeedsLoginDownloader, IDownloader
+    public class LoversLabDownloader : AbstractIPS4Downloader<LoversLabDownloader, LoversLabDownloader.State>
     {
         #region INeedsDownload
         public override string SiteName => "Lovers Lab";
-        public override Uri SiteURL => new Uri("https://loverslab.com");
+        public override Uri SiteURL => new Uri("https://www.loverslab.com");
         public override Uri IconUri => new Uri("https://www.loverslab.com/favicon.ico");
         #endregion
 
         public LoversLabDownloader() : base(new Uri("https://www.loverslab.com/login"), 
-            "loverslabcookies", "loverslab.com", "ips4_member_id")
+            "loverslabcookies", "loverslab.com")
         {
         }
-
-
-        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archive_ini)
-        {
-            Uri url = DownloaderUtils.GetDirectURL(archive_ini);
-            if (url == null || url.Host != "www.loverslab.com" || !url.AbsolutePath.StartsWith("/files/file/")) return null;
-            var id = HttpUtility.ParseQueryString(url.Query)["r"];
-            var file = url.AbsolutePath.Split('/').Last(s => s != "");
-
-            return new State
-            {
-                FileID = id,
-                FileName = file
-            };
-        }
         protected override async Task WhileWaiting(IWebDriver browser)
         {
             try
@@ -60,97 +45,8 @@ namespace Wabbajack.Lib.Downloaders
                 Utils.Error(ex);
             }
         }
-
+        public class State : State<LoversLabDownloader>
-        public class State : AbstractDownloadState
         {
-            public string FileID { get; set; }
-            public string FileName { get; set; }
-
-            public override object[] PrimaryKey { get => new object[] {FileID, FileName}; }
-
-            public override bool IsWhitelisted(ServerWhitelist whitelist)
-            {
-                return true;
-            }
-
-            public override async Task Download(Archive a, string destination)
-            {
-                var stream = await ResolveDownloadStream();
-                using (var file = File.OpenWrite(destination))
-                {
-                    stream.CopyTo(file);
-                }
-            }
-
-            private async Task<Stream> ResolveDownloadStream()
-            {
-                var result = DownloadDispatcher.GetInstance<LoversLabDownloader>();
-                TOP:
-                var html = await result.AuthedClient.GetStringAsync(
-                    $"https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID}");
-
-                var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])");
-                var csrfKey = pattern.Matches(html).Cast<Match>().Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault();
-
-                if (csrfKey == null)
-                    return null;
-
-                var url =
-                    $"https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}";
-
-                var streamResult = await result.AuthedClient.GetAsync(url);
-                if (streamResult.StatusCode != HttpStatusCode.OK)
-                {
-                    Utils.Error(new InvalidOperationException(), $"LoversLab servers reported an error for file: {FileID}");
-                }
-
-                var content_type = streamResult.Content.Headers.ContentType;
-
-                if (content_type.MediaType == "application/json")
-                {
-                    // Sometimes LL hands back a json object telling us to wait until a certain time
-                    var times = (await streamResult.Content.ReadAsStringAsync()).FromJSONString<WaitResponse>();
-                    var secs = times.download - times.currentTime;
-                    for (int x = 0; x < secs; x++)
-                    {
-                        Utils.Status($"Waiting for {secs} at the request of LoversLab", x * 100 / secs);
-                        await Task.Delay(1000);
-                    }
-                    Utils.Status("Retrying download");
-                    goto TOP;
-                }
-
-                return await streamResult.Content.ReadAsStreamAsync();
-            }
-
-            internal class WaitResponse
-            {
-                public int download { get; set; }
-                public int currentTime { get; set; }
-            }
-
-            public override async Task<bool> Verify()
-            {
-                var stream = await ResolveDownloadStream();
-                if (stream == null)
-                {
-                    return false;
-                }
-
-                stream.Close();
-                return true;
-            }
-
-            public override IDownloader GetDownloader()
-            {
-                return DownloadDispatcher.GetInstance<LoversLabDownloader>();
-            }
-
-            public override string GetReportEntry(Archive a)
-            {
-                return $"* Lovers Lab - [{a.Name}](https://www.loverslab.com/files/file/{FileName}/?do=download&r={FileID})";
-            }
         }
-
     }
 }

Wabbajack.Lib/Downloaders/VectorPlexusDownloader.cs

@@ -0,0 +1,22 @@
+using System;
+using System.Threading.Tasks;
+
+namespace Wabbajack.Lib.Downloaders
+{
+    public class VectorPlexusDownloader : AbstractIPS4Downloader<VectorPlexusDownloader, VectorPlexusDownloader.State>
+    {
+        #region INeedsDownload
+        public override string SiteName => "Vector Plexus";
+        public override Uri SiteURL => new Uri("https://vectorplexus.com");
+        public override Uri IconUri => new Uri("https://www.vectorplexus.com/favicon.ico");
+        #endregion
+
+        public VectorPlexusDownloader() : base(new Uri("https://vectorplexus.com/login"), 
+            "vectorplexus", "vectorplexus.com")
+        {
+        }
+        public class State : State<VectorPlexusDownloader>
+        {
+        }
+    }
+}

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -123,11 +123,13 @@
     <Compile Include="CompilationSteps\IStackStep.cs" />
     <Compile Include="CompilationSteps\PatchStockESMs.cs" />
     <Compile Include="CompilationSteps\Serialization.cs" />
+    <Compile Include="Downloaders\AbstractIPS4Downloader.cs" />
     <Compile Include="Downloaders\AbstractNeedsLoginDownloader.cs" />
     <Compile Include="Downloaders\GameFileSourceDownloader.cs" />
     <Compile Include="Downloaders\INeedsLogin.cs" />
     <Compile Include="Downloaders\LoversLabDownloader.cs" />
     <Compile Include="Downloaders\SteamWorkshopDownloader.cs" />
+    <Compile Include="Downloaders\VectorPlexusDownloader.cs" />
     <Compile Include="Extensions\ReactiveUIExt.cs" />
     <Compile Include="LibCefHelpers\Init.cs" />
     <Compile Include="MO2Compiler.cs" />

Wabbajack.Test/DownloaderTests.cs

@@ -311,6 +311,34 @@ namespace Wabbajack.Test
             Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
         }
         
+        [TestMethod]
+        public async Task VectorPlexusDownload()
+        {
+            await DownloadDispatcher.GetInstance<VectorPlexusDownloader>().Prepare();
+            var ini = @"[General]
+                        directURL=https://vectorplexus.com/files/file/290-wabbajack-test-file";
+
+            var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
+
+            Assert.IsNotNull(state);
+
+            /*var url_state = DownloadDispatcher.ResolveArchive("https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1");
+            Assert.AreEqual("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt",
+                ((HTTPDownloader.State)url_state).Url);
+                */
+            var converted = state.ViaJSON();
+            Assert.IsTrue(await converted.Verify());
+            var filename = Guid.NewGuid().ToString();
+
+            Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
+
+            await converted.Download(new Archive { Name = "Vector Plexus Test.zip" }, filename);
+
+            Assert.AreEqual("eSIyd+KOG3s=", Utils.FileHash(filename));
+
+            Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
+        }
+
         [TestMethod]
         public async Task GameFileSourceDownload()
         {