From 05399c4632f209b921038f4021576670137b046f Mon Sep 17 00:00:00 2001 From: erri120 Date: Sat, 4 Apr 2020 19:40:58 +0200 Subject: [PATCH 1/3] Attachments from IPS4 sites can now be downloaded --- .../Downloaders/AbstractIPS4Downloader.cs | 90 ++++++++++++------- 1 file changed, 57 insertions(+), 33 deletions(-) diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs index 4df8f011..2ab97779 100644 --- a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs @@ -5,7 +5,6 @@ using System.Net; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Web; -using HtmlAgilityPack; using Newtonsoft.Json; using Wabbajack.Common; using Wabbajack.Lib.Validation; @@ -29,7 +28,16 @@ namespace Wabbajack.Lib.Downloaders var absolute = true; if (url == null || url.Host != SiteURL.Host) return null; - + + if (url.PathAndQuery.StartsWith("/applications/core/interface/file/attachment")) + { + return new TState + { + IsAttachment = true, + FullURL = url.ToString() + }; + } + if (url.PathAndQuery.StartsWith("/index.php?")) { var id2 = HttpUtility.ParseQueryString(url.Query)["r"]; @@ -37,6 +45,7 @@ namespace Wabbajack.Lib.Downloaders var name = parsed[null].Split("/", StringSplitOptions.RemoveEmptyEntries).Last(); return new TState { + FullURL = url.AbsolutePath, FileID = id2, FileName = name }; @@ -56,6 +65,7 @@ namespace Wabbajack.Lib.Downloaders return new TState { + FullURL = url.AbsolutePath, FileID = id, FileName = file }; @@ -64,6 +74,8 @@ namespace Wabbajack.Lib.Downloaders public class State : AbstractDownloadState, IMetaState where TDownloader : IDownloader { + public string FullURL { get; set; } + public bool IsAttachment { get; set; } public string FileID { get; set; } public string FileName { get; set; } @@ -81,7 +93,9 @@ namespace Wabbajack.Lib.Downloaders get { return FileID == null - ? new object[] {Downloader.SiteURL, FileName} + ? IsAttachment + ? new object[] {Downloader.SiteURL, IsAttachment, FullURL} + : new object[] {Downloader.SiteURL, FileName} : new object[] {Downloader.SiteURL, FileName, FileID}; } } @@ -103,27 +117,32 @@ namespace Wabbajack.Lib.Downloaders private async Task ResolveDownloadStream() { - //var downloader = (AbstractNeedsLoginDownloader)(object)DownloadDispatcher.GetInstance(); - TOP: - var csrfurl = FileID == null - ? $"{Site}/files/file/{FileName}/?do=download" - : $"{Site}/files/file/{FileName}/?do=download&r={FileID}"; - var html = await Downloader.AuthedClient.GetStringAsync(csrfurl); + string url; + if (IsAttachment) + { + url = FullURL; + } + else + { + var csrfURL = FileID == null + ? $"{Site}/files/file/{FileName}/?do=download" + : $"{Site}/files/file/{FileName}/?do=download&r={FileID}"; + var html = await Downloader.AuthedClient.GetStringAsync(csrfURL); - var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])"); - var matches = pattern.Matches(html).Cast(); + var pattern = new Regex("(?<=csrfKey=).*(?=[&\"\'])|(?<=csrfKey: \").*(?=[&\"\'])"); + var matches = pattern.Matches(html).Cast(); var csrfKey = matches.Where(m => m.Length == 32).Select(m => m.ToString()).FirstOrDefault(); - if (csrfKey == null) - return null; + if (csrfKey == null) + return null; - var sep = Site.EndsWith("?") ? "&" : "?"; - var url = FileID == null - ? $"{Site}/files/file/{FileName}/{sep}do=download&confirm=1&t=1&csrfKey={csrfKey}" - : $"{Site}/files/file/{FileName}/{sep}do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}"; - + var sep = Site.EndsWith("?") ? "&" : "?"; + url = FileID == null + ? $"{Site}/files/file/{FileName}/{sep}do=download&confirm=1&t=1&csrfKey={csrfKey}" + : $"{Site}/files/file/{FileName}/{sep}do=download&r={FileID}&confirm=1&t=1&csrfKey={csrfKey}"; + } var streamResult = await Downloader.AuthedClient.GetAsync(url); if (streamResult.StatusCode != HttpStatusCode.OK) @@ -174,37 +193,42 @@ namespace Wabbajack.Lib.Downloaders public override string GetManifestURL(Archive a) { - return $"{Site}/files/file/{FileName}/?do=download&r={FileID}"; + return IsAttachment ? "" : $"{Site}/files/file/{FileName}/?do=download&r={FileID}"; } public override string[] GetMetaIni() { - if (FileID != null) - { - if (Site.EndsWith("?")) - { - return new[] - { - "[General]", $"directURL={Site}/files/file/{FileName}&do=download&r={FileID}&confirm=1&t=1" - }; - - } + if (IsAttachment) + return new[] {"[General]", $"directURL={FullURL}"}; + if (FileID == null) + return new[] {"[General]", $"directURL={Site}/files/file/{FileName}"}; + + if (Site.EndsWith("?")) + { return new[] { - "[General]", $"directURL={Site}/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1" + "[General]", $"directURL={Site}/files/file/{FileName}&do=download&r={FileID}&confirm=1&t=1" }; + } return new[] { - "[General]", - $"directURL={Site}/files/file/{FileName}" + "[General]", $"directURL={Site}/files/file/{FileName}/?do=download&r={FileID}&confirm=1&t=1" }; + } // from IMetaState - public string URL => $"{Site}/files/file/{FileName}"; + public string URL + { + get + { + return !IsAttachment ? $"{Site}/files/file/{FileName}" : ""; + } + } + public string Name { get; set; } public string Author { get; set; } public string Version { get; set; } From d58fcab92efceeb5c8f031f1f2f9b59a3f0ff92a Mon Sep 17 00:00:00 2001 From: erri120 Date: Sat, 4 Apr 2020 19:53:05 +0200 Subject: [PATCH 2/3] Fixed ManifestURL --- Wabbajack.Lib/ACompiler.cs | 3 +++ Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index f389ec69..86272a36 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -91,6 +91,9 @@ namespace Wabbajack.Lib { if (a.State is IMetaState metaState) { + if (string.IsNullOrWhiteSpace(metaState.URL)) + return; + var b = await metaState.LoadMetaData(); Utils.Log(b ? $"Getting meta data for {a.Name} was successful!" diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs index 2ab97779..d940b23a 100644 --- a/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs @@ -193,7 +193,7 @@ namespace Wabbajack.Lib.Downloaders public override string GetManifestURL(Archive a) { - return IsAttachment ? "" : $"{Site}/files/file/{FileName}/?do=download&r={FileID}"; + return IsAttachment ? FullURL : $"{Site}/files/file/{FileName}/?do=download&r={FileID}"; } public override string[] GetMetaIni() From 73e17931dc17f17ac33ba46c214a9306db5e79f0 Mon Sep 17 00:00:00 2001 From: erri120 Date: Sat, 4 Apr 2020 19:56:21 +0200 Subject: [PATCH 3/3] Fixed empty images in slideshow --- Wabbajack/View Models/SlideShow.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Wabbajack/View Models/SlideShow.cs b/Wabbajack/View Models/SlideShow.cs index 7f60f259..9902a08c 100644 --- a/Wabbajack/View Models/SlideShow.cs +++ b/Wabbajack/View Models/SlideShow.cs @@ -89,6 +89,7 @@ namespace Wabbajack return modList.SourceModList.Archives .Select(m => m.State) .OfType() + .Where(x => !string.IsNullOrEmpty(x.URL) && !string.IsNullOrEmpty(x.ImageURL)) .DistinctBy(x => x.URL) // Shuffle it .Shuffle(_random)