From 0441bbaf176f96ab58628a0582835ed0644ad7f9 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 14 Jun 2021 22:55:07 -0600 Subject: [PATCH] Downloading works, next up is metatdata --- .../Downloaders/AbstractDownloadState.cs | 2 +- .../AbstractIPS4OAuthDownloader.cs | 91 +++++- .../DTOs/IPS4OAuthFilesResponse.cs | 295 ++++++++++++++++++ .../Downloaders/DownloadDispatcher.cs | 2 +- .../Downloaders/VectorPlexusDownloader.cs | 90 +----- Wabbajack.Test/DownloaderTests.cs | 6 +- .../View Models/UserInterventionHandlers.cs | 4 +- 7 files changed, 387 insertions(+), 103 deletions(-) create mode 100644 Wabbajack.Lib/Downloaders/DTOs/IPS4OAuthFilesResponse.cs diff --git a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs index 2a7516a6..e72f4aa2 100644 --- a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs +++ b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs @@ -35,7 +35,7 @@ namespace Wabbajack.Lib.Downloaders typeof(MegaDownloader.State), typeof(ModDBDownloader.State), typeof(NexusDownloader.State), - typeof(VectorPlexusDownloader.State), + typeof(VectorPlexusOAuthDownloader.State), typeof(DeadlyStreamDownloader.State), typeof(TESAllianceDownloader.State), typeof(TESAllDownloader.State), diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs index 66acb700..23892bb8 100644 --- a/Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs @@ -5,13 +5,16 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Reactive; using System.Reactive.Linq; +using System.Threading; using System.Threading.Tasks; using System.Web; using Newtonsoft.Json; using ReactiveUI; using Wabbajack.Common; using Wabbajack.Common.Serialization.Json; +using Wabbajack.Lib.Downloaders.DTOs; using Wabbajack.Lib.Http; +using Wabbajack.Lib.Validation; namespace Wabbajack.Lib.Downloaders { @@ -19,7 +22,7 @@ namespace Wabbajack.Lib.Downloaders where TState : AbstractIPS4OAuthDownloader.State, new() where TDownloader : IDownloader { - public AbstractIPS4OAuthDownloader(string clientID, Uri authEndpoint, Uri tokenEndpoint, string encryptedKeyName) + public AbstractIPS4OAuthDownloader(string clientID, Uri authEndpoint, Uri tokenEndpoint, IEnumerable scopes, string encryptedKeyName) { ClientID = clientID; AuthorizationEndpoint = authEndpoint; @@ -27,7 +30,7 @@ namespace Wabbajack.Lib.Downloaders EncryptedKeyName = encryptedKeyName; TriggerLogin = ReactiveCommand.CreateFromTask( - execute: () => Utils.CatchAndLog(async () => await Utils.Log(new RequestOAuthLogin(ClientID, authEndpoint, tokenEndpoint, SiteName, new []{"profile"}, EncryptedKeyName)).Task), + execute: () => Utils.CatchAndLog(async () => await Utils.Log(new RequestOAuthLogin(ClientID, authEndpoint, tokenEndpoint, SiteName, scopes, EncryptedKeyName)).Task), canExecute: IsLoggedIn.Select(b => !b).ObserveOnGuiThread()); ClearLogin = ReactiveCommand.CreateFromTask( execute: () => Utils.CatchAndLog(async () => await Utils.DeleteEncryptedJson(EncryptedKeyName)), @@ -51,21 +54,33 @@ namespace Wabbajack.Lib.Downloaders private bool _isPrepared = false; public async Task GetDownloaderState(dynamic archiveINI, bool quickMode = false) { - if (archiveINI.General != null && archiveINI.General.directURL != null) + if (archiveINI.General.ips4Site == SiteName && archiveINI.General.ips4Mod != null && archiveINI.General.ips4File != null) { - var parsed = new Uri(archiveINI.General.directURL); - var fileID = parsed.AbsolutePath.Split("/").Last().Split("-").First(); - var modID = HttpUtility.ParseQueryString(parsed.Query).Get("r"); + if (!long.TryParse(archiveINI.General.ips4Mod, out long parsedMod)) + return null; + var state = new TState {IPS4Mod = parsedMod, IPS4File = archiveINI.General.ips4File}; + + if (!quickMode) + { + var downloads = await GetDownloads(state.IPS4Mod); + state.IPS4Url = downloads.Url ?? ""; + } + + return state; - if (!long.TryParse(fileID, out var fileIDParsed)) - return null; - if (modID != null && !long.TryParse(modID, out var modIDParsed)) - return null; } - throw new NotImplementedException(); + return null; } + public async Task GetDownloads(long modID) + { + var responseString = await (await GetAuthedClient())!.GetStringAsync(SiteURL+ $"api/downloads/files/{modID}") ; + return responseString.FromJsonString(); + } + + + public async Task Prepare() { @@ -95,8 +110,56 @@ namespace Wabbajack.Lib.Downloaders public abstract class State : AbstractDownloadState { - public long FileID { get; set; } - public long? ModID { get; set; } + public long IPS4Mod { get; set; } + public string IPS4File { get; set; } = ""; + + public string IPS4Url { get; set; } = ""; + + public override object[] PrimaryKey => new object[] {IPS4Mod, IPS4File}; + + public override bool IsWhitelisted(ServerWhitelist whitelist) + { + return true; + } + + public override async Task Download(Archive a, AbsolutePath destination) + { + var downloads = await TypedDownloader.GetDownloads(IPS4Mod); + var fileEntry = downloads.Files.First(f => f.Name == IPS4File); + if (a.Size != 0 && fileEntry.Size != a.Size) + throw new Exception( + $"File {IPS4File} on mod {IPS4Mod} on {TypedDownloader.SiteName} appears to be re-uploaded with the same name"); + + var state = new HTTPDownloader.State(fileEntry.Url!) {Client = TypedDownloader.AuthedClient!}; + if (a.Size == 0) a.Size = fileEntry.Size!.Value; + return await state.Download(a, destination); + } + + private static AbstractIPS4OAuthDownloader TypedDownloader => (AbstractIPS4OAuthDownloader)(object)DownloadDispatcher.GetInstance(); + + public override async Task Verify(Archive archive, CancellationToken? token = null) + { + var downloads = await DownloadDispatcher.GetInstance().GetDownloads(IPS4Mod); + var fileEntry = downloads.Files.FirstOrDefault(f => f.Name == IPS4File); + if (fileEntry == null) return false; + return archive.Size == 0 || fileEntry.Size == archive.Size; + } + + public override string? GetManifestURL(Archive a) + { + return IPS4Url; + } + + public override string[] GetMetaIni() + { + return new[] + { + "[General]", + $"ips4Site={TypedDownloader.SiteName}", + $"ips4Mod={IPS4Mod}", + $"ips4File={IPS4File}" + }; + } } } @@ -215,7 +278,5 @@ namespace Wabbajack.Lib.Downloaders Handled = true; _source.TrySetCanceled(); } - - } } diff --git a/Wabbajack.Lib/Downloaders/DTOs/IPS4OAuthFilesResponse.cs b/Wabbajack.Lib/Downloaders/DTOs/IPS4OAuthFilesResponse.cs new file mode 100644 index 00000000..7a80abc1 --- /dev/null +++ b/Wabbajack.Lib/Downloaders/DTOs/IPS4OAuthFilesResponse.cs @@ -0,0 +1,295 @@ +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace Wabbajack.Lib.Downloaders.DTOs +{ + public class IPS4OAuthFilesResponse + { + +// Root myDeserializedClass = JsonConvert.DeserializeObject(myJsonResponse); + public class Category + { + [JsonProperty("id")] + public int Id { get; set; } + + [JsonProperty("name")] + public string? Name { get; set; } + + [JsonProperty("url")] + public string? Url { get; set; } + + [JsonProperty("class")] + public string? Class { get; set; } + } + + public class PrimaryGroup + { + [JsonProperty("id")] + public int Id { get; set; } + + [JsonProperty("name")] + public string? Name { get; set; } + + [JsonProperty("formattedName")] + public string? FormattedName { get; set; } + } + + public class Author + { + [JsonProperty("id")] + public int Id { get; set; } + + [JsonProperty("name")] + public string? Name { get; set; } + + [JsonProperty("title")] + public string? Title { get; set; } + + [JsonProperty("formattedName")] + public string? FormattedName { get; set; } + + [JsonProperty("primaryGroup")] + public PrimaryGroup? PrimaryGroup { get; set; } + + [JsonProperty("joined")] + public DateTime Joined { get; set; } + + [JsonProperty("reputationPoints")] + public int ReputationPoints { get; set; } + + [JsonProperty("photoUrl")] + public string? PhotoUrl { get; set; } + + [JsonProperty("photoUrlIsDefault")] + public bool PhotoUrlIsDefault { get; set; } + + [JsonProperty("coverPhotoUrl")] + public string? CoverPhotoUrl { get; set; } + + [JsonProperty("profileUrl")] + public string? ProfileUrl { get; set; } + + [JsonProperty("posts")] + public int Posts { get; set; } + + [JsonProperty("lastActivity")] + public DateTime LastActivity { get; set; } + + [JsonProperty("lastVisit")] + public DateTime LastVisit { get; set; } + + [JsonProperty("lastPost")] + public DateTime LastPost { get; set; } + + [JsonProperty("profileViews")] + public int ProfileViews { get; set; } + + } + + public class File + { + [JsonProperty("name")] + public string? Name { get; set; } + + [JsonProperty("url")] + public string? Url { get; set; } + + [JsonProperty("size")] + public long? Size { get; set; } + } + + public class Screenshot + { + [JsonProperty("name")] + public string? Name { get; set; } + + [JsonProperty("url")] + public string? Url { get; set; } + + [JsonProperty("size")] + public int Size { get; set; } + } + + public class PrimaryScreenshot + { + [JsonProperty("name")] + public string? Name { get; set; } + + [JsonProperty("url")] + public string? Url { get; set; } + + [JsonProperty("size")] + public string? Size { get; set; } + } + + public class Forum + { + [JsonProperty("id")] + public int Id { get; set; } + + [JsonProperty("name")] + public string? Name { get; set; } + + [JsonProperty("topics")] + public int Topics { get; set; } + + [JsonProperty("url")] + public string? Url { get; set; } + } + + public class FirstPost + { + [JsonProperty("id")] + public int Id { get; set; } + + [JsonProperty("item_id")] + public int ItemId { get; set; } + + [JsonProperty("author")] + public Author? Author { get; set; } + + [JsonProperty("date")] + public DateTime Date { get; set; } + + [JsonProperty("content")] + public string? Content { get; set; } + + [JsonProperty("hidden")] + public bool Hidden { get; set; } + + [JsonProperty("url")] + public string? Url { get; set; } + } + + public class LastPost + { + [JsonProperty("id")] + public int Id { get; set; } + + [JsonProperty("item_id")] + public int ItemId { get; set; } + + [JsonProperty("author")] + public Author? Author { get; set; } + + [JsonProperty("date")] + public DateTime Date { get; set; } + + [JsonProperty("content")] + public string? Content { get; set; } + + [JsonProperty("hidden")] + public bool Hidden { get; set; } + + [JsonProperty("url")] + public string? Url { get; set; } + } + + public class Topic + { + [JsonProperty("id")] + public int Id { get; set; } + + [JsonProperty("title")] + public string? Title { get; set; } + + [JsonProperty("forum")] + public Forum? Forum { get; set; } + + [JsonProperty("posts")] + public int Posts { get; set; } + + [JsonProperty("views")] + public int Views { get; set; } + + [JsonProperty("locked")] + public bool Locked { get; set; } + + [JsonProperty("hidden")] + public bool Hidden { get; set; } + + [JsonProperty("pinned")] + public bool Pinned { get; set; } + + [JsonProperty("featured")] + public bool Featured { get; set; } + + [JsonProperty("archived")] + public bool Archived { get; set; } + + [JsonProperty("url")] + public string? Url { get; set; } + + [JsonProperty("rating")] + public double Rating { get; set; } + } + + public class Root + { + [JsonProperty("id")] + public int Id { get; set; } + + [JsonProperty("title")] + public string? Title { get; set; } + + [JsonProperty("category")] + public Category? Category { get; set; } + + [JsonProperty("author")] + public Author? Author { get; set; } + + [JsonProperty("date")] + public DateTime Date { get; set; } + + [JsonProperty("description")] + public string? Description { get; set; } + + [JsonProperty("version")] + public string? Version { get; set; } + + [JsonProperty("changelog")] + public string? Changelog { get; set; } + + [JsonProperty("files")] public List Files { get; set; } = new(); + + [JsonProperty("screenshots")] public List Screenshots { get; set; } = new(); + + [JsonProperty("primaryScreenshot")] + public PrimaryScreenshot PrimaryScreenshot { get; set; } = new(); + + [JsonProperty("downloads")] + public int Downloads { get; set; } + + [JsonProperty("comments")] + public int Comments { get; set; } + + [JsonProperty("reviews")] + public int Reviews { get; set; } + + [JsonProperty("views")] + public int Views { get; set; } + + [JsonProperty("tags")] public List Tags { get; set; } = new (); + + [JsonProperty("locked")] + public bool Locked { get; set; } + + [JsonProperty("hidden")] + public bool Hidden { get; set; } + + [JsonProperty("pinned")] + public bool Pinned { get; set; } + + [JsonProperty("featured")] + public bool Featured { get; set; } + + [JsonProperty("url")] + public string? Url { get; set; } + + [JsonProperty("topic")] public Topic Topic { get; set; } = new(); + } + + + } +} diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index 2c02acf3..92487b07 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -22,7 +22,7 @@ namespace Wabbajack.Lib.Downloaders new NexusDownloader(), new MediaFireDownloader(), new LoversLabDownloader(), - new VectorPlexusDownloader(), + new VectorPlexusOAuthDownloader(), new DeadlyStreamDownloader(), new TESAllianceDownloader(), new TESAllDownloader(), diff --git a/Wabbajack.Lib/Downloaders/VectorPlexusDownloader.cs b/Wabbajack.Lib/Downloaders/VectorPlexusDownloader.cs index 86cc79ad..cfa790d1 100644 --- a/Wabbajack.Lib/Downloaders/VectorPlexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/VectorPlexusDownloader.cs @@ -10,7 +10,7 @@ using Wabbajack.Lib.Validation; namespace Wabbajack.Lib.Downloaders { - public class VectorPlexusDownloader : AbstractIPS4OAuthDownloader + public class VectorPlexusOAuthDownloader : AbstractIPS4OAuthDownloader { #region INeedsDownload public override string SiteName => "Vector Plexus"; @@ -18,96 +18,22 @@ namespace Wabbajack.Lib.Downloaders public override Uri IconUri => new Uri("https://www.vectorplexus.com/favicon.ico"); #endregion - public VectorPlexusDownloader() : base("45c6d3c9867903a7daa6ded0a38cedf8", - new Uri("https://vectorplexus.com/oauth/authorize/"), new Uri("https://vectorplexus.com/oauth/token/"), + public VectorPlexusOAuthDownloader() : base("45c6d3c9867903a7daa6ded0a38cedf8", + new Uri("https://vectorplexus.com/oauth/authorize/"), + new Uri("https://vectorplexus.com/oauth/token/"), + new []{"profile", "get_downloads"}, "vector-plexus-oauth2") { } - public class State : AbstractIPS4OAuthDownloader.State + [JsonName("VectorPlexusOAuthDownloader+State")] + public class State : AbstractIPS4OAuthDownloader.State { - public override object[] PrimaryKey { get; } = Array.Empty(); - public override bool IsWhitelisted(ServerWhitelist whitelist) - { - throw new NotImplementedException(); - } - - public override Task Download(Archive a, AbsolutePath destination) - { - throw new NotImplementedException(); - } - - public override Task Verify(Archive archive, CancellationToken? token = null) - { - throw new NotImplementedException(); - } - public override IDownloader GetDownloader() { - throw new NotImplementedException(); - } - - public override string? GetManifestURL(Archive a) - { - throw new NotImplementedException(); - } - - public override string[] GetMetaIni() - { - throw new NotImplementedException(); + return DownloadDispatcher.GetInstance(); } } - - /* - [JsonName("VectorPlexusDownloader")] - public class State //: State - { - public override async Task LoadMetaData() - { - var html = await Downloader.AuthedClient.GetStringAsync(URL); - var doc = new HtmlDocument(); - doc.LoadHtml(html); - var node = doc.DocumentNode; - - Name = HttpUtility.HtmlDecode(node - .SelectNodes( - "//h1[@class='ipsType_pageTitle ipsContained_container']/span[@class='ipsType_break ipsContained']") - ?.First().InnerHtml); - - Author = HttpUtility.HtmlDecode(node - .SelectNodes( - "//div[@class='ipsBox_alt']/div[@class='ipsPhotoPanel ipsPhotoPanel_tiny ipsClearfix ipsSpacer_bottom']/div/p[@class='ipsType_reset ipsType_large ipsType_blendLinks']/a") - ?.First().InnerHtml); - - Version = HttpUtility.HtmlDecode(node - .SelectNodes("//section/h2[@class='ipsType_sectionHead']/span[@data-role='versionTitle']") - ? - .First().InnerHtml); - - var url = HttpUtility.HtmlDecode(node - .SelectNodes( - "//div[@class='ipsBox ipsSpacer_top ipsSpacer_double']/section/div[@class='ipsPad ipsAreaBackground']/div[@class='ipsCarousel ipsClearfix']/div[@class='ipsCarousel_inner']/ul[@class='cDownloadsCarousel ipsClearfix']/li[@class='ipsCarousel_item ipsAreaBackground_reset ipsPad_half']/span[@class='ipsThumb ipsThumb_medium ipsThumb_bg ipsCursor_pointer']") - ?.First().GetAttributeValue("data-fullurl", "none")); - - if (!string.IsNullOrWhiteSpace(url)) - { - ImageURL = new Uri(url); - return true; - } - - url = HttpUtility.HtmlDecode(node - .SelectNodes( - "//article[@class='ipsColumn ipsColumn_fluid']/div[@class='ipsPad']/section/div[@class='ipsType_richText ipsContained ipsType_break']/p/a/img[@class='ipsImage ipsImage_thumbnailed']") - ?.First().GetAttributeValue("src", "")); - if (!string.IsNullOrWhiteSpace(url)) - { - ImageURL = new Uri(url); - } - - return true; - } - } - */ } } diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 8d75f179..85aa31fd 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -400,9 +400,11 @@ namespace Wabbajack.Test [Fact] public async Task VectorPlexusDownload() { - await DownloadDispatcher.GetInstance().Prepare(); + await DownloadDispatcher.GetInstance().Prepare(); var ini = @"[General] - directURL=https://vectorplexus.com/files/file/290-wabbajack-test-file"; + ips4Site=Vector Plexus + ips4Mod=290 + ips4File=WABBAJACK_TEST_FILE.zip"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); diff --git a/Wabbajack/View Models/UserInterventionHandlers.cs b/Wabbajack/View Models/UserInterventionHandlers.cs index c0f49992..33e04e7e 100644 --- a/Wabbajack/View Models/UserInterventionHandlers.cs +++ b/Wabbajack/View Models/UserInterventionHandlers.cs @@ -113,9 +113,9 @@ namespace Wabbajack vm.Instructions = "Please log in and allow Wabbajack to access your account"; var wrapper = new CefSharpWrapper(vm.Browser); - var scopes = string.Join("&", oa.Scopes.Select(s => $"scope={s}")); + var scopes = string.Join(" ", oa.Scopes); var state = Guid.NewGuid().ToString(); - await wrapper.NavigateTo(new Uri(oa.AuthorizationEndpoint + $"?response_type=code&client_id={oa.ClientID}&state={state}&{scopes}")); + await wrapper.NavigateTo(new Uri(oa.AuthorizationEndpoint + $"?response_type=code&client_id={oa.ClientID}&state={state}&scope={scopes}")); Helpers.SchemeHandler = (browser, frame, _, request) => {