From 9779da19b205234824e1f50a45da417cc1db1fe1 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 27 Jun 2021 08:05:45 -0600 Subject: [PATCH] Support for attachments on ips4 sites --- .../CompilationSteps/MatchSimilarTextures.cs | 1 + .../AbstractIPS4OAuthDownloader.cs | 67 +++++++++++++++---- Wabbajack.Test/DownloaderTests.cs | 30 +++++++++ 3 files changed, 85 insertions(+), 13 deletions(-) diff --git a/Wabbajack.Lib/CompilationSteps/MatchSimilarTextures.cs b/Wabbajack.Lib/CompilationSteps/MatchSimilarTextures.cs index 71c174b1..69693d17 100644 --- a/Wabbajack.Lib/CompilationSteps/MatchSimilarTextures.cs +++ b/Wabbajack.Lib/CompilationSteps/MatchSimilarTextures.cs @@ -13,6 +13,7 @@ namespace Wabbajack.Lib.CompilationSteps { _byName = _compiler.IndexedFiles.SelectMany(kv => kv.Value) .Where(f => f.Name.FileName.Extension == DDS) + .Where(f => f.ImageState != null) .ToLookup(f => f.Name.FileName.FileNameWithoutExtension); } diff --git a/Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs b/Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs index fc9f2415..eb511971 100644 --- a/Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs +++ b/Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Net.Http.Headers; +using System.Printing; using System.Reactive; using System.Reactive.Linq; using System.Threading; @@ -74,6 +75,19 @@ namespace Wabbajack.Lib.Downloaders } + if (archiveINI.General.ips4Site == SiteName && archiveINI.General.ips4Attachment != null) + { + if (!long.TryParse(archiveINI.General.ips4Attachment, out long parsedMod)) + return null; + var state = new TState + { + IPS4Mod = parsedMod, IPS4File = archiveINI.General.ips4File, IsAttachment = true, + IPS4Url=$"{SiteURL}/applications/core/interface/file/attachment.php?id={parsedMod}" + }; + + return state; + } + return null; } @@ -122,10 +136,12 @@ namespace Wabbajack.Lib.Downloaders public abstract class State : AbstractDownloadState, IMetaState { public long IPS4Mod { get; set; } + + public bool IsAttachment { get; set; } = false; public string IPS4File { get; set; } = ""; public string IPS4Url { get; set; } = ""; - public override object[] PrimaryKey => new object[] {IPS4Mod, IPS4File}; + public override object[] PrimaryKey => new object[] {IPS4Mod, IPS4File ?? "", IsAttachment}; public override bool IsWhitelisted(ServerWhitelist whitelist) { @@ -134,25 +150,50 @@ namespace Wabbajack.Lib.Downloaders public override async Task Download(Archive a, AbsolutePath destination) { - var downloads = await TypedDownloader.GetDownloads(IPS4Mod); - var fileEntry = downloads.Files.First(f => f.Name == IPS4File); - if (a.Size != 0 && fileEntry.Size != a.Size) - throw new Exception( - $"File {IPS4File} on mod {IPS4Mod} on {TypedDownloader.SiteName} appears to be re-uploaded with the same name"); + if (IsAttachment) + { + var downloader = TypedDownloader; + using var driver = await WebAutomation.Driver.Create(); + await driver.NavigateToAndDownload( + new Uri($"{downloader.SiteURL}/applications/core/interface/file/attachment.php?id={IPS4Mod}"), destination); + return true; + } + else + { - var state = new HTTPDownloader.State(fileEntry.Url!); - if (a.Size == 0) a.Size = fileEntry.Size!.Value; - return await state.Download(a, destination); + var downloads = await TypedDownloader.GetDownloads(IPS4Mod); + var fileEntry = downloads.Files.First(f => f.Name == IPS4File); + if (a.Size != 0 && fileEntry.Size != a.Size) + throw new Exception( + $"File {IPS4File} on mod {IPS4Mod} on {TypedDownloader.SiteName} appears to be re-uploaded with the same name"); + + var state = new HTTPDownloader.State(fileEntry.Url!); + if (a.Size == 0) a.Size = fileEntry.Size!.Value; + return await state.Download(a, destination); + } } private static AbstractIPS4OAuthDownloader TypedDownloader => (AbstractIPS4OAuthDownloader)(object)DownloadDispatcher.GetInstance(); public override async Task Verify(Archive archive, CancellationToken? token = null) { - var downloads = await TypedDownloader.GetDownloads(IPS4Mod); - var fileEntry = downloads.Files.FirstOrDefault(f => f.Name == IPS4File); - if (fileEntry == null) return false; - return archive.Size == 0 || fileEntry.Size == archive.Size; + if (IsAttachment) + { + var downloader = TypedDownloader; + using var driver = await WebAutomation.Driver.Create(); + await using var tmp = new TempFile(); + var foundSize = await driver.NavigateToAndDownload( + new Uri($"{downloader.SiteURL}/applications/core/interface/file/attachment.php?id={IPS4Mod}"), tmp.Path); + return archive.Size == 0 || foundSize == archive.Size; + } + else + { + var downloads = await TypedDownloader.GetDownloads(IPS4Mod); + var fileEntry = downloads.Files.FirstOrDefault(f => f.Name == IPS4File); + if (fileEntry == null) return false; + return archive.Size == 0 || fileEntry.Size == archive.Size; + } + } public override string? GetManifestURL(Archive a) diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 9680e456..1c8654b4 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -356,6 +356,36 @@ namespace Wabbajack.Test Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); } + + [Fact] + public async Task LoversLabAttachmentDownload() + { + + + await DownloadDispatcher.GetInstance().Prepare(); + var ini = @"[General] + ips4Site=Lovers Lab + ips4Attachment=853295"; + + var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); + + Assert.NotNull(state); + + var converted = RoundTripState(state); + Assert.True(await converted.Verify(new Archive(state: null!) { Size = 1363396})); + + // Verify with different Size + Assert.False(await converted.Verify(new Archive(state: null!) { Size = 15})); + + + await using var filename = new TempFile(); + Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); + + await converted.Download(new Archive(state: null!) { Name = "LoversLab Test.txt" }, filename.Path); + + Assert.Equal(Hash.FromBase64("gLJDxGDaeQ0="), await filename.Path.FileHashAsync()); + + } [Fact] public async Task CanLoadOldLLMeta()