created downloaders for dropbox, http, google, and nexus

Wabbajack.Common/Consts.cs

@@ -57,7 +57,7 @@ namespace Wabbajack.Common
             {
                 var platformType = Environment.Is64BitOperatingSystem ? "x64" : "x86";
                 var headerString =
-                    $"{AppName}/{Assembly.GetEntryAssembly().GetName().Version} ({Environment.OSVersion.VersionString}; {platformType}) {RuntimeInformation.FrameworkDescription}";
+                    $"{AppName}/{Assembly.GetEntryAssembly()?.GetName()?.Version ?? new Version(0, 1)} ({Environment.OSVersion.VersionString}; {platformType}) {RuntimeInformation.FrameworkDescription}";
                 return headerString;
             }
         }

Wabbajack.Test/DownloaderTests.cs

@@ -20,9 +20,7 @@ namespace Wabbajack.Test
             var ini = @"[General]
                         directURL=https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k";
 
-            var downloader = new MegaDownloader();
-            downloader.Init();
-            var state = (AbstractDownloadState)downloader.GetDownloaderState(ini.LoadIniString());
+            var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString());
 
             Assert.IsNotNull(state);
 
@@ -33,11 +31,107 @@ namespace Wabbajack.Test
             Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List<string>{"https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k" } }));
             Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>{ "blerg" }}));
 
-            converted.Download(new Archive() {Name = "MEGA Test.txt"}, filename);
+            converted.Download(new Archive {Name = "MEGA Test.txt"}, filename);
 
             Assert.AreEqual("Lb1iTsz3iyZeHGs3e94TVmOhf22sqtHLhqkCdXbjiyc=", Utils.FileSHA256(filename));
 
             Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
         }
+
+        [TestMethod]
+        public void DropboxTests()
+        {
+            var ini = @"[General]
+                        directURL=https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?dl=0";
+
+            var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString());
+
+            Assert.IsNotNull(state);
+
+            var converted = state.ViaJSON();
+            Assert.IsTrue(converted.Verify());
+            var filename = Guid.NewGuid().ToString();
+
+            Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?" } }));
+            Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "blerg" } }));
+
+            converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
+
+            Assert.AreEqual("Lb1iTsz3iyZeHGs3e94TVmOhf22sqtHLhqkCdXbjiyc=", Utils.FileSHA256(filename));
+
+            Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
+        }
+
+        [TestMethod]
+        public void GoogleDriveTests()
+        {
+            var ini = @"[General]
+                        directURL=https://drive.google.com/file/d/1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_/view?usp=sharing";
+
+            var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString());
+
+            Assert.IsNotNull(state);
+
+            var converted = state.ViaJSON();
+            Assert.IsTrue(converted.Verify());
+            var filename = Guid.NewGuid().ToString();
+
+            Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List<string> { "1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_" } }));
+            Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List<string>()}));
+
+            converted.Download(new Archive { Name = "MEGA Test.txt" }, filename);
+
+            Assert.AreEqual("Lb1iTsz3iyZeHGs3e94TVmOhf22sqtHLhqkCdXbjiyc=", Utils.FileSHA256(filename));
+
+            Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
+        }
+
+        [TestMethod]
+        public void HttpDownload()
+        {
+            var ini = @"[General]
+                        directURL=https://raw.githubusercontent.com/wabbajack-tools/opt-out-lists/master/ServerWhitelist.yml";
+
+            var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString());
+
+            Assert.IsNotNull(state);
+
+            var converted = state.ViaJSON();
+            Assert.IsTrue(converted.Verify());
+            var filename = Guid.NewGuid().ToString();
+
+            Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "https://raw.githubusercontent.com/" } }));
+            Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
+
+            converted.Download(new Archive { Name = "Github Test Test.txt" }, filename);
+
+            Assert.IsTrue(File.ReadAllText(filename).StartsWith("# Server whitelist for Wabbajack"));
+        }
+
+        [TestMethod]
+        public void NexusDownload()
+        {
+            var ini = @"[General]
+                        gameName=SkyrimSE
+                        modID = 12604
+                        fileID=35407";
+
+            var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString());
+
+            Assert.IsNotNull(state);
+
+            var converted = state.ViaJSON();
+            Assert.IsTrue(converted.Verify());
+            var filename = Guid.NewGuid().ToString();
+
+            Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> () }));
+
+            converted.Download(new Archive { Name = "SkyUI.7z" }, filename);
+
+            Assert.AreEqual(filename.FileSHA256(), "U3Xg6RBR9XrUY9/jQSu6WKu5dfhHmpaN2dTl0ylDFmI=");
+        }
     }
+
+
+
 }

Wabbajack/Compiler.cs

@@ -548,7 +548,7 @@ namespace Wabbajack
                         ModID = general.modID,
                         Version = general.version ?? "0.0.0.0"
                     };
-                    var info = new NexusApiClient().GetModInfo(nm);
+                    /*var info = new NexusApiClient().GetModInfo(nm);
                     nm.Author = info.author;
                     nm.UploadedBy = info.uploaded_by;
                     nm.UploaderProfile = info.uploaded_users_profile_url;
@@ -556,7 +556,7 @@ namespace Wabbajack
                     nm.SlideShowPic = info.picture_url;
                     nm.NexusURL = NexusApiUtils.GetModURL(info.game_name, info.mod_id);
                     nm.Summary = info.summary;
-                    nm.Adult = info.contains_adult_content;
+                    nm.Adult = info.contains_adult_content;*/
 
                     result = nm;
                 }

Wabbajack/Downloaders/DownloadDispatcher.cs

@@ -0,0 +1,45 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Security.Policy;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Wabbajack.Downloaders
+{
+    public static class DownloadDispatcher
+    {
+        private static List<IDownloader> _downloaders = new List<IDownloader>()
+        {
+            new MegaDownloader(),
+            new DropboxDownloader(),
+            new GoogleDriveDownloader(),
+            new HTTPDownloader(),
+            new NexusDownloader()
+        };
+
+        private static Dictionary<Type, IDownloader> _indexedDownloaders;
+
+        static DownloadDispatcher()
+        {
+            _indexedDownloaders = _downloaders.ToDictionary(d => d.GetType());
+        }
+
+        public static T GetInstance<T>()
+        {
+            return (T)_indexedDownloaders[typeof(T)];
+        }
+
+        public static AbstractDownloadState ResolveArchive(dynamic ini)
+        {
+            foreach (var d in _downloaders)
+            {
+                var result = d.GetDownloaderState(ini);
+                if (result != null) return result;
+            }
+
+            return null;
+        }
+
+    }
+}

Wabbajack/Downloaders/DropboxDownloader.cs

@@ -0,0 +1,38 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Web;
+
+namespace Wabbajack.Downloaders
+{
+    public class DropboxDownloader : IDownloader
+    {
+        public void Init()
+        {
+
+        }
+
+        public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
+        {
+            var urlstring = archive_ini?.General?.directURL;
+            if (urlstring == null) return null;
+            var uri = new UriBuilder((string)urlstring);
+            if (uri.Host != "www.dropbox.com") return null;
+            var query = HttpUtility.ParseQueryString(uri.Query);
+
+            if (query.GetValues("dl").Length > 0)
+                query.Remove("dl");
+
+            query.Set("dl", "1");
+
+            uri.Query = query.ToString();
+
+            return new HTTPDownloader.State()
+            {
+                Url = uri.ToString().Replace("dropbox.com:443/", "dropbox.com/")
+            };
+        }
+    }
+}

Wabbajack/Downloaders/GoogleDriveDownloader.cs

@@ -0,0 +1,63 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+using System.Security.Cryptography.X509Certificates;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.Validation;
+
+namespace Wabbajack.Downloaders
+{
+    public class GoogleDriveDownloader : IDownloader
+    {
+        public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
+        {
+            var url = archive_ini?.General?.directURL;
+            var regex = new Regex("((?<=id=)[a-zA-Z0-9_-]*)|(?<=\\/file\\/d\\/)[a-zA-Z0-9_-]*");
+            if (url != null && url.StartsWith("https://drive.google.com"))
+            {
+                var match = regex.Match(url);
+                return new State
+                {
+                    Id = match.ToString()
+                };
+            }
+
+            return null;
+        }
+
+        public class State : AbstractDownloadState
+        {
+            public string Id { get; set; }
+            public override bool IsWhitelisted(ServerWhitelist whitelist)
+            {
+                return whitelist.GoogleIDs.Contains(Id);
+            }
+
+            public override void Download(Archive a, string destination)
+            {
+                ToHttpState().Download(a, destination);
+            }
+
+            private HTTPDownloader.State ToHttpState()
+            {
+                var initial_url = $"https://drive.google.com/uc?id={Id}&export=download";
+                var client = new HttpClient();
+                var result = client.GetStringSync(initial_url);
+                var regex = new Regex("(?<=/uc\\?export=download&amp;confirm=).*(?=;id=)");
+                var confirm = regex.Match(result);
+                var url = $"https://drive.google.com/uc?export=download&confirm={confirm}&id={Id}";
+                var http_state = new HTTPDownloader.State {Url = url};
+                return http_state;
+            }
+
+            public override bool Verify()
+            {
+                return ToHttpState().Verify();
+            }
+        }
+    }
+}

Wabbajack/Downloaders/HTTPDownloader.cs

@@ -0,0 +1,124 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+using System.ServiceModel.Configuration;
+using System.Text;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Wabbajack.Common;
+using Wabbajack.Validation;
+using File = Alphaleonis.Win32.Filesystem.File;
+
+namespace Wabbajack.Downloaders
+{
+    public class HTTPDownloader : IDownloader
+    {
+
+        public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
+        {
+            var url = archive_ini?.General?.directURL;
+
+            if (url != null)
+            {
+                var tmp = new State
+                {
+                    Url = url
+                };
+                if (archive_ini?.General?.directURLHeaders != null)
+                {
+                    tmp.Headers = new List<string>();
+                    tmp.Headers.AddRange(archive_ini?.General.directURLHeaders.Split('|'));
+                }
+                return tmp;
+            }
+
+            return null;
+        }
+
+        public class State : AbstractDownloadState
+        {
+            public string Url { get; set; }
+
+            [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+            public List<string> Headers { get; set; }
+
+            public override bool IsWhitelisted(ServerWhitelist whitelist)
+            {
+                return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p));
+            }
+
+            public override void Download(Archive a, string destination)
+            {
+                DoDownload(a, destination, true);
+            }
+
+            public bool DoDownload(Archive a, string destination, bool download)
+            {
+                var client = new HttpClient();
+                client.DefaultRequestHeaders.Add("User-Agent", Consts.UserAgent);
+
+                if (Headers != null)
+                    foreach (var header in Headers)
+                    {
+                        var idx = header.IndexOf(':');
+                        var k = header.Substring(0, idx);
+                        var v = header.Substring(idx + 1);
+                        client.DefaultRequestHeaders.Add(k, v);
+                    }
+
+                long total_read = 0;
+                var buffer_size = 1024 * 32;
+
+                var response = client.GetSync(Url);
+                var stream = response.Content.ReadAsStreamAsync();
+                try
+                {
+                    stream.Wait();
+                }
+                catch (Exception ex)
+                {
+                }
+
+                ;
+                if (stream.IsFaulted)
+                {
+                    Utils.Log($"While downloading {Url} - {stream.Exception.ExceptionToString()}");
+                    return false;
+                }
+
+                if (!download)
+                    return true;
+
+                var header_var = "1";
+                if (response.Content.Headers.Contains("Content-Length"))
+                    header_var = response.Content.Headers.GetValues("Content-Length").FirstOrDefault();
+
+                var content_size = header_var != null ? long.Parse(header_var) : 1;
+
+
+                using (var webs = stream.Result)
+                using (var fs = File.OpenWrite(destination))
+                {
+                    var buffer = new byte[buffer_size];
+                    while (true)
+                    {
+                        var read = webs.Read(buffer, 0, buffer_size);
+                        if (read == 0) break;
+                        Utils.Status($"Downloading {a.Name}", (int)(total_read * 100 / content_size));
+
+                        fs.Write(buffer, 0, read);
+                        total_read += read;
+                    }
+                }
+
+                return true;
+            }
+
+            public override bool Verify()
+            {
+                return DoDownload(new Archive {Name = ""}, "", false);
+            }
+        }
+    }
+}

Wabbajack/Downloaders/IDownloader.cs

@@ -9,11 +9,6 @@ namespace Wabbajack.Downloaders
 {
     interface IDownloader
     {
-        /// <summary>
-        /// Setup the module. Run at the start of the application lifecycle
-        /// </summary>
-        void Init();
-
         AbstractDownloadState GetDownloaderState(dynamic archive_ini);
     }
 }

Wabbajack/Downloaders/MEGADownloader.cs

@@ -15,9 +15,6 @@ namespace Wabbajack.Downloaders
 {
     public class MegaDownloader : IDownloader
     {
-        public void Init()
-        {
-        }
 
         public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
         {
@@ -27,15 +24,8 @@ namespace Wabbajack.Downloaders
             return null;
         }
 
-        public class State : AbstractDownloadState
+        public class State : HTTPDownloader.State
         {
-            public string Url { get; set; }
-
-            public override bool IsWhitelisted(ServerWhitelist whitelist)
-            {
-                return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p));
-            }
-
             public override void Download(Archive a, string destination)
             {
                 var client = new MegaApiClient();

Wabbajack/Downloaders/NexusDownloader.cs

@@ -0,0 +1,101 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.NexusApi;
+using Wabbajack.Validation;
+
+namespace Wabbajack.Downloaders
+{
+    public class NexusDownloader : IDownloader
+    {
+        public AbstractDownloadState GetDownloaderState(dynamic archive_ini)
+        {
+            var general = archive_ini?.General;
+
+            if (general.modID != null && general.fileID != null && general.gameName != null)
+            {
+                var info = new NexusApiClient().GetModInfo(general.gameName, general.modID);
+                return new State
+                {
+                    GameName = general.gameName,
+                    FileID = general.fileID,
+                    ModID = general.modID,
+                    Version = general.version ?? "0.0.0.0",
+                    Author = info.author,
+                    UploadedBy = info.uploaded_by,
+                    UploaderProfile = info.uploaded_users_profile_url,
+                    ModName = info.name,
+                    SlideShowPic = info.picture_url,
+                    NexusURL = NexusApiUtils.GetModURL(info.game_name, info.mod_id),
+                    Summary = info.summary,
+                    Adult = info.contains_adult_content
+
+                };
+            }
+
+            return null;
+        }
+
+        public class State : AbstractDownloadState
+        {
+            public string Author;
+            public string FileID;
+            public string GameName;
+            public string ModID;
+            public string UploadedBy;
+            public string UploaderProfile;
+            public string Version;
+            public string SlideShowPic;
+            public string ModName;
+            public string NexusURL;
+            public string Summary;
+            public bool Adult;
+
+            public override bool IsWhitelisted(ServerWhitelist whitelist)
+            {
+                // Nexus files are always whitelisted
+                return true;
+            }
+
+            public override void Download(Archive a, string destination)
+            {
+                string url;
+                try
+                {
+                    url = new NexusApiClient().GetNexusDownloadLink(this, false);
+                }
+                catch (Exception ex)
+                {
+                    Utils.Log($"{a.Name} - Error Getting Nexus Download URL - {ex.Message}");
+                    return;
+                }
+
+                Utils.Log($"Downloading Nexus Archive - {a.Name} - {GameName} - {ModID} - {FileID}");
+
+                new HTTPDownloader.State
+                {
+                    Url = url
+                }.Download(a, destination);
+
+            }
+
+            public override bool Verify()
+            {
+                try
+                {
+                    new NexusApiClient().GetNexusDownloadLink(this, true);
+                    return true;
+                }
+                catch (Exception ex)
+                {
+                    Utils.Log($"{ModName} - {GameName} - {ModID} - {FileID} - Error Getting Nexus Download URL - {ex.Message}");
+                    return false;
+                }
+
+            }
+        }
+    }
+}

Wabbajack/Installer.cs

@@ -464,8 +464,10 @@ namespace Wabbajack
                 {
                     case NexusMod a:
                         string url;
+                        /*
                         try
                         {
+                            
                             url = new NexusApiClient().GetNexusDownloadLink(a, !download);
                             if (!download) return true;
                         }
@@ -477,6 +479,7 @@ namespace Wabbajack
 
                         Info($"Downloading Nexus Archive - {archive.Name} - {a.GameName} - {a.ModID} - {a.FileID}");
                         DownloadURLDirect(archive, url);
+                        */
                         return true;
                     case MEGAArchive a:
                         return DownloadMegaArchive(a, download);

Wabbajack/NexusApi/NexusApi.cs

@@ -11,6 +11,7 @@ using System.Reflection;
 using System.Security.Authentication;
 using System.Threading.Tasks;
 using Wabbajack.Common;
+using Wabbajack.Downloaders;
 using WebSocketSharp;
 using static Wabbajack.NexusApi.NexusApiUtils;
 
@@ -61,6 +62,12 @@ namespace Wabbajack.NexusApi
                     return File.ReadAllText(API_KEY_CACHE_FILE);
                 }
 
+                var env_key = Environment.GetEnvironmentVariable("NEXUSAPIKEY");
+                if (env_key != null)
+                {
+                    return env_key;
+                }
+
                 // open a web socket to receive the api key
                 var guid = Guid.NewGuid();
                 var _websocket = new WebSocket("wss://sso.nexusmods.com")
@@ -154,7 +161,7 @@ namespace Wabbajack.NexusApi
             headers.Add("apikey", _apiKey);
             headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
             headers.Add("Application-Name", Consts.AppName);
-            headers.Add("Application-Version", $"{Assembly.GetEntryAssembly().GetName().Version}");
+            headers.Add("Application-Version", $"{Assembly.GetEntryAssembly()?.GetName()?.Version ?? new Version(0, 1)}");
         }
 
         private T Get<T>(string url)
@@ -172,7 +179,7 @@ namespace Wabbajack.NexusApi
         }
 
 
-        public string GetNexusDownloadLink(NexusMod archive, bool cache = false)
+        public string GetNexusDownloadLink(NexusDownloader.State archive, bool cache = false)
         {
             if (cache && TryGetCachedLink(archive, out var result))
                 return result;
@@ -183,7 +190,7 @@ namespace Wabbajack.NexusApi
             return Get<List<DownloadLink>>(url).First().URI;
         }
 
-        private bool TryGetCachedLink(NexusMod archive, out string result)
+        private bool TryGetCachedLink(NexusDownloader.State archive, out string result)
         {
             if (!Directory.Exists(Consts.NexusCacheDirectory))
                 Directory.CreateDirectory(Consts.NexusCacheDirectory);
@@ -207,14 +214,14 @@ namespace Wabbajack.NexusApi
             return Get<NexusFileInfo>(url);
         }
 
-        public ModInfo GetModInfo(NexusMod archive)
+        public ModInfo GetModInfo(string gameName, string modId)
         {
             if (!Directory.Exists(Consts.NexusCacheDirectory))
                 Directory.CreateDirectory(Consts.NexusCacheDirectory);
 
             ModInfo result = null;
         TOP:
-            var path = Path.Combine(Consts.NexusCacheDirectory, $"mod-info-{archive.GameName}-{archive.ModID}.json");
+            var path = Path.Combine(Consts.NexusCacheDirectory, $"mod-info-{gameName}-{modId}.json");
             try
             {
                 if (File.Exists(path))
@@ -234,11 +241,11 @@ namespace Wabbajack.NexusApi
                 File.Delete(path);
             }
 
-            var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(archive.GameName)}/mods/{archive.ModID}.json";
+            var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(gameName)}/mods/{modId}.json";
             result = Get<ModInfo>(url);
 
-            result.game_name = archive.GameName;
-            result.mod_id = archive.ModID;
+            result.game_name = gameName;
+            result.mod_id = modId;
             result._internal_version = CACHED_VERSION_NUMBER;
             result.ToJSON(path);
             return result;

Wabbajack/Wabbajack.csproj

@@ -189,8 +189,13 @@
     </ApplicationDefinition>
     <Compile Include="Data.cs" />
     <Compile Include="Downloaders\AbstractDownloadState.cs" />
+    <Compile Include="Downloaders\GoogleDriveDownloader.cs" />
+    <Compile Include="Downloaders\HTTPDownloader.cs" />
+    <Compile Include="Downloaders\DownloadDispatcher.cs" />
+    <Compile Include="Downloaders\DropboxDownloader.cs" />
     <Compile Include="Downloaders\IDownloader.cs" />
     <Compile Include="Downloaders\MegaDownloader.cs" />
+    <Compile Include="Downloaders\NexusDownloader.cs" />
     <Compile Include="LambdaCommand.cs" />
     <Compile Include="UI\ModlistPropertiesWindow.xaml.cs">
       <DependentUpon>ModlistPropertiesWindow.xaml</DependentUpon>