diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 56b92c84..49826167 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -57,7 +57,7 @@ namespace Wabbajack.Common { var platformType = Environment.Is64BitOperatingSystem ? "x64" : "x86"; var headerString = - $"{AppName}/{Assembly.GetEntryAssembly().GetName().Version} ({Environment.OSVersion.VersionString}; {platformType}) {RuntimeInformation.FrameworkDescription}"; + $"{AppName}/{Assembly.GetEntryAssembly()?.GetName()?.Version ?? new Version(0, 1)} ({Environment.OSVersion.VersionString}; {platformType}) {RuntimeInformation.FrameworkDescription}"; return headerString; } } diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 3f5edd1b..c9353c69 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -20,9 +20,7 @@ namespace Wabbajack.Test var ini = @"[General] directURL=https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k"; - var downloader = new MegaDownloader(); - downloader.Init(); - var state = (AbstractDownloadState)downloader.GetDownloaderState(ini.LoadIniString()); + var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.IsNotNull(state); @@ -33,11 +31,107 @@ namespace Wabbajack.Test Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List{"https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k" } })); Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List{ "blerg" }})); - converted.Download(new Archive() {Name = "MEGA Test.txt"}, filename); + converted.Download(new Archive {Name = "MEGA Test.txt"}, filename); Assert.AreEqual("Lb1iTsz3iyZeHGs3e94TVmOhf22sqtHLhqkCdXbjiyc=", Utils.FileSHA256(filename)); Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!"); } + + [TestMethod] + public void DropboxTests() + { + var ini = @"[General] + directURL=https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?dl=0"; + + var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString()); + + Assert.IsNotNull(state); + + var converted = state.ViaJSON(); + Assert.IsTrue(converted.Verify()); + var filename = Guid.NewGuid().ToString(); + + Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List { "https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?" } })); + Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List { "blerg" } })); + + converted.Download(new Archive { Name = "MEGA Test.txt" }, filename); + + Assert.AreEqual("Lb1iTsz3iyZeHGs3e94TVmOhf22sqtHLhqkCdXbjiyc=", Utils.FileSHA256(filename)); + + Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!"); + } + + [TestMethod] + public void GoogleDriveTests() + { + var ini = @"[General] + directURL=https://drive.google.com/file/d/1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_/view?usp=sharing"; + + var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString()); + + Assert.IsNotNull(state); + + var converted = state.ViaJSON(); + Assert.IsTrue(converted.Verify()); + var filename = Guid.NewGuid().ToString(); + + Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List { "1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_" } })); + Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List()})); + + converted.Download(new Archive { Name = "MEGA Test.txt" }, filename); + + Assert.AreEqual("Lb1iTsz3iyZeHGs3e94TVmOhf22sqtHLhqkCdXbjiyc=", Utils.FileSHA256(filename)); + + Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!"); + } + + [TestMethod] + public void HttpDownload() + { + var ini = @"[General] + directURL=https://raw.githubusercontent.com/wabbajack-tools/opt-out-lists/master/ServerWhitelist.yml"; + + var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString()); + + Assert.IsNotNull(state); + + var converted = state.ViaJSON(); + Assert.IsTrue(converted.Verify()); + var filename = Guid.NewGuid().ToString(); + + Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List { "https://raw.githubusercontent.com/" } })); + Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); + + converted.Download(new Archive { Name = "Github Test Test.txt" }, filename); + + Assert.IsTrue(File.ReadAllText(filename).StartsWith("# Server whitelist for Wabbajack")); + } + + [TestMethod] + public void NexusDownload() + { + var ini = @"[General] + gameName=SkyrimSE + modID = 12604 + fileID=35407"; + + var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString()); + + Assert.IsNotNull(state); + + var converted = state.ViaJSON(); + Assert.IsTrue(converted.Verify()); + var filename = Guid.NewGuid().ToString(); + + Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List () })); + + converted.Download(new Archive { Name = "SkyUI.7z" }, filename); + + Assert.AreEqual(filename.FileSHA256(), "U3Xg6RBR9XrUY9/jQSu6WKu5dfhHmpaN2dTl0ylDFmI="); + } } + + + } diff --git a/Wabbajack/Compiler.cs b/Wabbajack/Compiler.cs index 779e0563..3c1a73e7 100644 --- a/Wabbajack/Compiler.cs +++ b/Wabbajack/Compiler.cs @@ -548,7 +548,7 @@ namespace Wabbajack ModID = general.modID, Version = general.version ?? "0.0.0.0" }; - var info = new NexusApiClient().GetModInfo(nm); + /*var info = new NexusApiClient().GetModInfo(nm); nm.Author = info.author; nm.UploadedBy = info.uploaded_by; nm.UploaderProfile = info.uploaded_users_profile_url; @@ -556,7 +556,7 @@ namespace Wabbajack nm.SlideShowPic = info.picture_url; nm.NexusURL = NexusApiUtils.GetModURL(info.game_name, info.mod_id); nm.Summary = info.summary; - nm.Adult = info.contains_adult_content; + nm.Adult = info.contains_adult_content;*/ result = nm; } diff --git a/Wabbajack/Downloaders/DownloadDispatcher.cs b/Wabbajack/Downloaders/DownloadDispatcher.cs new file mode 100644 index 00000000..6f916e81 --- /dev/null +++ b/Wabbajack/Downloaders/DownloadDispatcher.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Policy; +using System.Text; +using System.Threading.Tasks; + +namespace Wabbajack.Downloaders +{ + public static class DownloadDispatcher + { + private static List _downloaders = new List() + { + new MegaDownloader(), + new DropboxDownloader(), + new GoogleDriveDownloader(), + new HTTPDownloader(), + new NexusDownloader() + }; + + private static Dictionary _indexedDownloaders; + + static DownloadDispatcher() + { + _indexedDownloaders = _downloaders.ToDictionary(d => d.GetType()); + } + + public static T GetInstance() + { + return (T)_indexedDownloaders[typeof(T)]; + } + + public static AbstractDownloadState ResolveArchive(dynamic ini) + { + foreach (var d in _downloaders) + { + var result = d.GetDownloaderState(ini); + if (result != null) return result; + } + + return null; + } + + } +} diff --git a/Wabbajack/Downloaders/DropboxDownloader.cs b/Wabbajack/Downloaders/DropboxDownloader.cs new file mode 100644 index 00000000..3ccf48ad --- /dev/null +++ b/Wabbajack/Downloaders/DropboxDownloader.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Web; + +namespace Wabbajack.Downloaders +{ + public class DropboxDownloader : IDownloader + { + public void Init() + { + + } + + public AbstractDownloadState GetDownloaderState(dynamic archive_ini) + { + var urlstring = archive_ini?.General?.directURL; + if (urlstring == null) return null; + var uri = new UriBuilder((string)urlstring); + if (uri.Host != "www.dropbox.com") return null; + var query = HttpUtility.ParseQueryString(uri.Query); + + if (query.GetValues("dl").Length > 0) + query.Remove("dl"); + + query.Set("dl", "1"); + + uri.Query = query.ToString(); + + return new HTTPDownloader.State() + { + Url = uri.ToString().Replace("dropbox.com:443/", "dropbox.com/") + }; + } + } +} diff --git a/Wabbajack/Downloaders/GoogleDriveDownloader.cs b/Wabbajack/Downloaders/GoogleDriveDownloader.cs new file mode 100644 index 00000000..aec346a2 --- /dev/null +++ b/Wabbajack/Downloaders/GoogleDriveDownloader.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Validation; + +namespace Wabbajack.Downloaders +{ + public class GoogleDriveDownloader : IDownloader + { + public AbstractDownloadState GetDownloaderState(dynamic archive_ini) + { + var url = archive_ini?.General?.directURL; + var regex = new Regex("((?<=id=)[a-zA-Z0-9_-]*)|(?<=\\/file\\/d\\/)[a-zA-Z0-9_-]*"); + if (url != null && url.StartsWith("https://drive.google.com")) + { + var match = regex.Match(url); + return new State + { + Id = match.ToString() + }; + } + + return null; + } + + public class State : AbstractDownloadState + { + public string Id { get; set; } + public override bool IsWhitelisted(ServerWhitelist whitelist) + { + return whitelist.GoogleIDs.Contains(Id); + } + + public override void Download(Archive a, string destination) + { + ToHttpState().Download(a, destination); + } + + private HTTPDownloader.State ToHttpState() + { + var initial_url = $"https://drive.google.com/uc?id={Id}&export=download"; + var client = new HttpClient(); + var result = client.GetStringSync(initial_url); + var regex = new Regex("(?<=/uc\\?export=download&confirm=).*(?=;id=)"); + var confirm = regex.Match(result); + var url = $"https://drive.google.com/uc?export=download&confirm={confirm}&id={Id}"; + var http_state = new HTTPDownloader.State {Url = url}; + return http_state; + } + + public override bool Verify() + { + return ToHttpState().Verify(); + } + } + } +} diff --git a/Wabbajack/Downloaders/HTTPDownloader.cs b/Wabbajack/Downloaders/HTTPDownloader.cs new file mode 100644 index 00000000..8d50df3a --- /dev/null +++ b/Wabbajack/Downloaders/HTTPDownloader.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.ServiceModel.Configuration; +using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Wabbajack.Common; +using Wabbajack.Validation; +using File = Alphaleonis.Win32.Filesystem.File; + +namespace Wabbajack.Downloaders +{ + public class HTTPDownloader : IDownloader + { + + public AbstractDownloadState GetDownloaderState(dynamic archive_ini) + { + var url = archive_ini?.General?.directURL; + + if (url != null) + { + var tmp = new State + { + Url = url + }; + if (archive_ini?.General?.directURLHeaders != null) + { + tmp.Headers = new List(); + tmp.Headers.AddRange(archive_ini?.General.directURLHeaders.Split('|')); + } + return tmp; + } + + return null; + } + + public class State : AbstractDownloadState + { + public string Url { get; set; } + + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public List Headers { get; set; } + + public override bool IsWhitelisted(ServerWhitelist whitelist) + { + return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p)); + } + + public override void Download(Archive a, string destination) + { + DoDownload(a, destination, true); + } + + public bool DoDownload(Archive a, string destination, bool download) + { + var client = new HttpClient(); + client.DefaultRequestHeaders.Add("User-Agent", Consts.UserAgent); + + if (Headers != null) + foreach (var header in Headers) + { + var idx = header.IndexOf(':'); + var k = header.Substring(0, idx); + var v = header.Substring(idx + 1); + client.DefaultRequestHeaders.Add(k, v); + } + + long total_read = 0; + var buffer_size = 1024 * 32; + + var response = client.GetSync(Url); + var stream = response.Content.ReadAsStreamAsync(); + try + { + stream.Wait(); + } + catch (Exception ex) + { + } + + ; + if (stream.IsFaulted) + { + Utils.Log($"While downloading {Url} - {stream.Exception.ExceptionToString()}"); + return false; + } + + if (!download) + return true; + + var header_var = "1"; + if (response.Content.Headers.Contains("Content-Length")) + header_var = response.Content.Headers.GetValues("Content-Length").FirstOrDefault(); + + var content_size = header_var != null ? long.Parse(header_var) : 1; + + + using (var webs = stream.Result) + using (var fs = File.OpenWrite(destination)) + { + var buffer = new byte[buffer_size]; + while (true) + { + var read = webs.Read(buffer, 0, buffer_size); + if (read == 0) break; + Utils.Status($"Downloading {a.Name}", (int)(total_read * 100 / content_size)); + + fs.Write(buffer, 0, read); + total_read += read; + } + } + + return true; + } + + public override bool Verify() + { + return DoDownload(new Archive {Name = ""}, "", false); + } + } + } +} diff --git a/Wabbajack/Downloaders/IDownloader.cs b/Wabbajack/Downloaders/IDownloader.cs index dcacd032..e757e06a 100644 --- a/Wabbajack/Downloaders/IDownloader.cs +++ b/Wabbajack/Downloaders/IDownloader.cs @@ -9,11 +9,6 @@ namespace Wabbajack.Downloaders { interface IDownloader { - /// - /// Setup the module. Run at the start of the application lifecycle - /// - void Init(); - AbstractDownloadState GetDownloaderState(dynamic archive_ini); } } diff --git a/Wabbajack/Downloaders/MEGADownloader.cs b/Wabbajack/Downloaders/MEGADownloader.cs index f8ea5759..e5215eae 100644 --- a/Wabbajack/Downloaders/MEGADownloader.cs +++ b/Wabbajack/Downloaders/MEGADownloader.cs @@ -15,9 +15,6 @@ namespace Wabbajack.Downloaders { public class MegaDownloader : IDownloader { - public void Init() - { - } public AbstractDownloadState GetDownloaderState(dynamic archive_ini) { @@ -27,15 +24,8 @@ namespace Wabbajack.Downloaders return null; } - public class State : AbstractDownloadState + public class State : HTTPDownloader.State { - public string Url { get; set; } - - public override bool IsWhitelisted(ServerWhitelist whitelist) - { - return whitelist.AllowedPrefixes.Any(p => Url.StartsWith(p)); - } - public override void Download(Archive a, string destination) { var client = new MegaApiClient(); diff --git a/Wabbajack/Downloaders/NexusDownloader.cs b/Wabbajack/Downloaders/NexusDownloader.cs new file mode 100644 index 00000000..28c5ae71 --- /dev/null +++ b/Wabbajack/Downloaders/NexusDownloader.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.NexusApi; +using Wabbajack.Validation; + +namespace Wabbajack.Downloaders +{ + public class NexusDownloader : IDownloader + { + public AbstractDownloadState GetDownloaderState(dynamic archive_ini) + { + var general = archive_ini?.General; + + if (general.modID != null && general.fileID != null && general.gameName != null) + { + var info = new NexusApiClient().GetModInfo(general.gameName, general.modID); + return new State + { + GameName = general.gameName, + FileID = general.fileID, + ModID = general.modID, + Version = general.version ?? "0.0.0.0", + Author = info.author, + UploadedBy = info.uploaded_by, + UploaderProfile = info.uploaded_users_profile_url, + ModName = info.name, + SlideShowPic = info.picture_url, + NexusURL = NexusApiUtils.GetModURL(info.game_name, info.mod_id), + Summary = info.summary, + Adult = info.contains_adult_content + + }; + } + + return null; + } + + public class State : AbstractDownloadState + { + public string Author; + public string FileID; + public string GameName; + public string ModID; + public string UploadedBy; + public string UploaderProfile; + public string Version; + public string SlideShowPic; + public string ModName; + public string NexusURL; + public string Summary; + public bool Adult; + + public override bool IsWhitelisted(ServerWhitelist whitelist) + { + // Nexus files are always whitelisted + return true; + } + + public override void Download(Archive a, string destination) + { + string url; + try + { + url = new NexusApiClient().GetNexusDownloadLink(this, false); + } + catch (Exception ex) + { + Utils.Log($"{a.Name} - Error Getting Nexus Download URL - {ex.Message}"); + return; + } + + Utils.Log($"Downloading Nexus Archive - {a.Name} - {GameName} - {ModID} - {FileID}"); + + new HTTPDownloader.State + { + Url = url + }.Download(a, destination); + + } + + public override bool Verify() + { + try + { + new NexusApiClient().GetNexusDownloadLink(this, true); + return true; + } + catch (Exception ex) + { + Utils.Log($"{ModName} - {GameName} - {ModID} - {FileID} - Error Getting Nexus Download URL - {ex.Message}"); + return false; + } + + } + } + } +} diff --git a/Wabbajack/Installer.cs b/Wabbajack/Installer.cs index e25b6726..c4b585fa 100644 --- a/Wabbajack/Installer.cs +++ b/Wabbajack/Installer.cs @@ -464,8 +464,10 @@ namespace Wabbajack { case NexusMod a: string url; + /* try { + url = new NexusApiClient().GetNexusDownloadLink(a, !download); if (!download) return true; } @@ -477,6 +479,7 @@ namespace Wabbajack Info($"Downloading Nexus Archive - {archive.Name} - {a.GameName} - {a.ModID} - {a.FileID}"); DownloadURLDirect(archive, url); + */ return true; case MEGAArchive a: return DownloadMegaArchive(a, download); diff --git a/Wabbajack/NexusApi/NexusApi.cs b/Wabbajack/NexusApi/NexusApi.cs index 08713a1c..0862aa21 100644 --- a/Wabbajack/NexusApi/NexusApi.cs +++ b/Wabbajack/NexusApi/NexusApi.cs @@ -11,6 +11,7 @@ using System.Reflection; using System.Security.Authentication; using System.Threading.Tasks; using Wabbajack.Common; +using Wabbajack.Downloaders; using WebSocketSharp; using static Wabbajack.NexusApi.NexusApiUtils; @@ -61,6 +62,12 @@ namespace Wabbajack.NexusApi return File.ReadAllText(API_KEY_CACHE_FILE); } + var env_key = Environment.GetEnvironmentVariable("NEXUSAPIKEY"); + if (env_key != null) + { + return env_key; + } + // open a web socket to receive the api key var guid = Guid.NewGuid(); var _websocket = new WebSocket("wss://sso.nexusmods.com") @@ -154,7 +161,7 @@ namespace Wabbajack.NexusApi headers.Add("apikey", _apiKey); headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); headers.Add("Application-Name", Consts.AppName); - headers.Add("Application-Version", $"{Assembly.GetEntryAssembly().GetName().Version}"); + headers.Add("Application-Version", $"{Assembly.GetEntryAssembly()?.GetName()?.Version ?? new Version(0, 1)}"); } private T Get(string url) @@ -172,7 +179,7 @@ namespace Wabbajack.NexusApi } - public string GetNexusDownloadLink(NexusMod archive, bool cache = false) + public string GetNexusDownloadLink(NexusDownloader.State archive, bool cache = false) { if (cache && TryGetCachedLink(archive, out var result)) return result; @@ -183,7 +190,7 @@ namespace Wabbajack.NexusApi return Get>(url).First().URI; } - private bool TryGetCachedLink(NexusMod archive, out string result) + private bool TryGetCachedLink(NexusDownloader.State archive, out string result) { if (!Directory.Exists(Consts.NexusCacheDirectory)) Directory.CreateDirectory(Consts.NexusCacheDirectory); @@ -207,14 +214,14 @@ namespace Wabbajack.NexusApi return Get(url); } - public ModInfo GetModInfo(NexusMod archive) + public ModInfo GetModInfo(string gameName, string modId) { if (!Directory.Exists(Consts.NexusCacheDirectory)) Directory.CreateDirectory(Consts.NexusCacheDirectory); ModInfo result = null; TOP: - var path = Path.Combine(Consts.NexusCacheDirectory, $"mod-info-{archive.GameName}-{archive.ModID}.json"); + var path = Path.Combine(Consts.NexusCacheDirectory, $"mod-info-{gameName}-{modId}.json"); try { if (File.Exists(path)) @@ -234,11 +241,11 @@ namespace Wabbajack.NexusApi File.Delete(path); } - var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(archive.GameName)}/mods/{archive.ModID}.json"; + var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(gameName)}/mods/{modId}.json"; result = Get(url); - result.game_name = archive.GameName; - result.mod_id = archive.ModID; + result.game_name = gameName; + result.mod_id = modId; result._internal_version = CACHED_VERSION_NUMBER; result.ToJSON(path); return result; diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 21fb964b..df9b0de7 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -189,8 +189,13 @@ + + + + + ModlistPropertiesWindow.xaml