Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Switch LoverLab over to OAuth2

Browse Source
This commit is contained in:
Timothy Baldridge 2021-06-18 22:05:58 -06:00
parent da485f9df5
commit 6ba00d9d09
8 changed files with 94 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.Lib/Downloaders/AbstractDownloadState.cs
View File

@ -29,7 +29,7 @@ namespace Wabbajack.Lib.Downloaders
typeof(HTTPDownloader.State),
typeof(GameFileSourceDownloader.State),
typeof(GoogleDriveDownloader.State),
typeof(LoversLabDownloader.State),
typeof(LoversLabOAuthDownloader.State),
typeof(ManualDownloader.State),
typeof(MediaFireDownloader.State),
typeof(MegaDownloader.State),

34
Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs
View File

@ -8,6 +8,7 @@ using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using F23.StringSimilarity;
using Newtonsoft.Json;
using ReactiveUI;
using Wabbajack.Common;
@ -138,7 +139,7 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Verify(Archive archive, CancellationToken? token = null)
{
var downloads = await DownloadDispatcher.GetInstance<VectorPlexusOAuthDownloader>().GetDownloads(IPS4Mod);
var downloads = await TypedDownloader.GetDownloads(IPS4Mod);
var fileEntry = downloads.Files.FirstOrDefault(f => f.Name == IPS4File);
if (fileEntry == null) return false;
return archive.Size == 0 || fileEntry.Size == archive.Size;
@ -173,12 +174,41 @@ namespace Wabbajack.Lib.Downloaders
Name = data.Title;
Author = data.Author?.Name;
Version = data.Version;
ImageURL = data.PrimaryScreenshot.Url != null ? new Uri(data.PrimaryScreenshot.Url) : null;
ImageURL = data.PrimaryScreenshot?.Url != null ? new Uri(data.PrimaryScreenshot.Url) : null;
IsNSFW = true;
Description = "";
IPS4Url = data.Url!;
return true;
}
public async Task<List<Archive>> GetFilesInGroup()
{
var data = await TypedDownloader.GetDownloads(IPS4Mod);
return data.Files.Select(f => new Archive(new TState {IPS4Mod = IPS4Mod, IPS4File = f.Name!}) {Size = f.Size!.Value}).ToList();
}
public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
{
var files = await GetFilesInGroup();
var nl = new Levenshtein();
foreach (var newFile in files.OrderBy(f => nl.Distance(a.Name.ToLowerInvariant(), f.Name.ToLowerInvariant())))
{
var tmp = new TempFile();
await DownloadDispatcher.PrepareAll(new[] {newFile.State});
if (await newFile.State.Download(newFile, tmp.Path))
{
newFile.Size = tmp.Path.Size;
var tmpHash = await tmp.Path.FileHashAsync();
if (tmpHash == null) return default;
newFile.Hash = tmpHash.Value;
return (newFile, tmp);
}
await tmp.DisposeAsync();
}
return default;
}
}
}

6
Wabbajack.Lib/Downloaders/DTOs/IPS4OAuthFilesResponse.cs
View File

@ -74,13 +74,13 @@ namespace Wabbajack.Lib.Downloaders.DTOs
public int Posts { get; set; }
[JsonProperty("lastActivity")]
public DateTime LastActivity { get; set; }
public DateTime? LastActivity { get; set; }
[JsonProperty("lastVisit")]
public DateTime LastVisit { get; set; }
public DateTime? LastVisit { get; set; }
[JsonProperty("lastPost")]
public DateTime LastPost { get; set; }
public DateTime? LastPost { get; set; }
[JsonProperty("profileViews")]
public int ProfileViews { get; set; }

2
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders
new ModDBDownloader(),
new NexusDownloader(),
new MediaFireDownloader(),
new LoversLabDownloader(),
new LoversLabOAuthDownloader(),
new VectorPlexusOAuthDownloader(),
new DeadlyStreamDownloader(),
new TESAllianceDownloader(),

95
Wabbajack.Lib/Downloaders/LoversLabDownloader.cs
View File

@ -1,95 +0,0 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using HtmlAgilityPack;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib.WebAutomation;
namespace Wabbajack.Lib.Downloaders
{
public class LoversLabDownloader : AbstractIPS4Downloader<LoversLabDownloader, LoversLabDownloader.State>
{
#region INeedsDownload
public override string SiteName => "Lovers Lab";
public override Uri SiteURL => new Uri("https://www.loverslab.com");
public override Uri IconUri => new Uri("https://www.loverslab.com/favicon.ico");
#endregion
public LoversLabDownloader() : base(new Uri("https://www.loverslab.com/login"),
"loverslabcookies", "loverslab.com")
{
IsCloudFlareProtected = false;
}
protected override async Task WhileWaiting(IWebDriver browser)
{
try
{
await browser.EvaluateJavaScript(
"document.querySelectorAll(\".ll_adblock\").forEach(function (itm) { itm.innerHTML = \"\";});");
}
catch (Exception ex)
{
Utils.Error(ex);
}
}
[JsonName("LoversLabDownloader")]
public class State : State<LoversLabDownloader>
{
[JsonIgnore]
public override bool IsNSFW => true;
public override async Task<bool> LoadMetaData()
{
var cts = new CancellationTokenSource();
cts.CancelAfter(Consts.MaxVerifyTime);
var html = await GetStringAsync(URL, cts.Token);
var doc = new HtmlDocument();
doc.LoadHtml(html);
var node = doc.DocumentNode;
Name = HttpUtility.HtmlDecode(node
.SelectNodes(
"//h1[@class='ipsType_pageTitle ipsContained_container']/span[@class='ipsType_break ipsContained']")
?.First().InnerHtml);
Author = HttpUtility.HtmlDecode(node
.SelectNodes(
"//div[@class='ipsBox_alt']/div[@class='ipsPhotoPanel ipsPhotoPanel_tiny ipsClearfix ipsSpacer_bottom']/div/p[@class='ipsType_reset ipsType_large ipsType_blendLinks']/a")
?.First().InnerHtml);
Version = HttpUtility.HtmlDecode(node
.SelectNodes("//section/h2[@class='ipsType_sectionHead']/span[@data-role='versionTitle']")
?
.First().InnerHtml);
var url = HttpUtility.HtmlDecode(node
.SelectNodes(
"//div[@class='ipsBox ipsSpacer_top ipsSpacer_double']/section/div[@class='ipsPad ipsAreaBackground']/div[@class='ipsCarousel ipsClearfix']/div[@class='ipsCarousel_inner']/ul[@class='cDownloadsCarousel ipsClearfix']/li[@class='ipsCarousel_item ipsAreaBackground_reset ipsPad_half']/span[@class='ipsThumb ipsThumb_medium ipsThumb_bg ipsCursor_pointer']")
?.First().GetAttributeValue("data-fullurl", "none"));
if (!string.IsNullOrWhiteSpace(url))
{
ImageURL = new Uri(url);
return true;
}
url = HttpUtility.HtmlDecode(node
.SelectNodes(
"//article[@class='ipsColumn ipsColumn_fluid']/div[@class='ipsPad']/section/div[@class='ipsType_richText ipsContained ipsType_break']/p/a/img[@class='ipsImage ipsImage_thumbnailed']")
?.First().GetAttributeValue("src", ""));
if (!string.IsNullOrWhiteSpace(url))
{
ImageURL = new Uri(url);
}
return true;
}
}
}
}

39
Wabbajack.Lib/Downloaders/LoversLabOAuthDownloader.cs Normal file
View File

@ -0,0 +1,39 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using HtmlAgilityPack;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib.Validation;
namespace Wabbajack.Lib.Downloaders
{
public class LoversLabOAuthDownloader : AbstractIPS4OAuthDownloader<LoversLabOAuthDownloader, LoversLabOAuthDownloader.State>
{
#region INeedsDownload
public override string SiteName => "Lovers Lab";
public override Uri SiteURL => new("https://loverslab.com");
public override Uri IconUri => new("https://www.loverslab.com/favicon.ico");
#endregion
public LoversLabOAuthDownloader() : base("0b543a010bf1a8f0f4c5dae154fce7c3",
new Uri("https://loverslab.com/oauth/authorize/"),
new Uri("https://loverslab.com/oauth/token/"),
new []{"downloads"},
"lovers-lab-oauth2")
{
}
[JsonName("LoversLabOAuthDownloader+State")]
public class State : AbstractIPS4OAuthDownloader<LoversLabOAuthDownloader, LoversLabOAuthDownloader.State>.State
{
public override IDownloader GetDownloader()
{
return DownloadDispatcher.GetInstance<LoversLabOAuthDownloader>();
}
}
}
}

2
Wabbajack.Server/Services/ListValidator.cs
View File

@ -362,8 +362,6 @@ namespace Wabbajack.Server.Services
return (archive, ArchiveStatus.Valid);
case MediaFireDownloader.State _:
return (archive, ArchiveStatus.Valid);
case LoversLabDownloader.State _ :
return (archive, ArchiveStatus.Valid);
default:
{
if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash),

84
Wabbajack.Test/DownloaderTests.cs
View File

@ -285,78 +285,51 @@ namespace Wabbajack.Test
Assert.Equal(Hash.FromBase64("2lZt+1h6wxM="), await filename.Path.FileHashAsync());
}
[Fact]
public async Task CanFindOtherLLMods()
{
await DownloadDispatcher.GetInstance<LoversLabDownloader>().Prepare();
await DownloadDispatcher.GetInstance<LoversLabOAuthDownloader>().Prepare();
var ini = @"[General]
directURL=https://www.loverslab.com/files/file/1382-milk-mod-economy/?do=download&r=913360&confirm=1&t=1&csrfKey=7984faa4d27f6b638125daf38ae5f1191";
ips4Site=Lovers Lab
ips4Mod=11116
ips4File=ffooo.zip";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
var otherfiles = await ((LoversLabDownloader.State)state).GetFilesInGroup();
}
[Fact]
public async Task CanCancelLLValidation()
{
await using var filename = new TempFile();
if (!DownloadDispatcher.GetInstance<LoversLabDownloader>().IsCloudFlareProtected)
return;
await DownloadDispatcher.GetInstance<LoversLabDownloader>().Prepare();
var state = new LoversLabDownloader.State
{
FileName = "14424-books-of-dibella-se-alternate-start-plugin", FileID = "870820",
};
using var queue = new WorkQueue();
var tcs = new CancellationTokenSource();
tcs.CancelAfter(2);
await Assert.ThrowsAsync<TaskCanceledException>(async () =>
{
await Enumerable.Range(0, 2).PMap(queue,
async x =>
{
Assert.True(await state.Verify(new Archive(state: null!) {Size = 252269}, tcs.Token));
});
});
var otherfiles = await ((LoversLabOAuthDownloader.State)state).GetFilesInGroup();
}
[Fact]
public async Task CanGetLLMetadata()
{
await DownloadDispatcher.GetInstance<LoversLabDownloader>().Prepare();
await DownloadDispatcher.GetInstance<LoversLabOAuthDownloader>().Prepare();
var ini = @"[General]
directURL=https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1";
ips4Site=Lovers Lab
ips4Mod=11116
ips4File=WABBAJACK_TEST_FILE.zip";
var state = (LoversLabDownloader.State)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
var state = (LoversLabOAuthDownloader.State)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.True(await state.LoadMetaData());
Assert.Equal("halgari", state.Author);
}
[Fact]
public async Task LoversLabDownload()
{
await DownloadDispatcher.GetInstance<LoversLabDownloader>().Prepare();
await DownloadDispatcher.GetInstance<LoversLabOAuthDownloader>().Prepare();
var ini = @"[General]
directURL=https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1";
ips4Site=Lovers Lab
ips4Mod=11116
ips4File=WABBAJACK_TEST_FILE.zip";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
/*var url_state = DownloadDispatcher.ResolveArchive("https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1");
Assert.Equal("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt",
((HTTPDownloader.State)url_state).Url);
*/
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20}));
@ -373,30 +346,10 @@ namespace Wabbajack.Test
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
var files = await ((LoversLabDownloader.State)converted).GetFilesInGroup();
Assert.NotEmpty(files);
Assert.Equal("WABBAJACK_TEST_FILE.zip", files.First().Name);
Assert.True(files.All(f => !string.IsNullOrWhiteSpace(((LoversLabDownloader.State)f.State).FileID)));
((LoversLabDownloader.State)converted).FileID = "42";
var upgrade = await DownloadDispatcher.FindUpgrade(new Archive(converted) {Name = "WABBAJACK_TEST_FILE.zip"});
Assert.True(upgrade != default);
var newState = ((LoversLabDownloader.State)upgrade.Archive.State);
Assert.False(string.IsNullOrWhiteSpace(newState.FileID));
var roundTripped = newState.ViaJSON();
Assert.False(string.IsNullOrWhiteSpace(roundTripped.FileID));
}
[Fact]
public async Task VectorPlexusDownload()
{
@ -539,7 +492,6 @@ namespace Wabbajack.Test
{
// Test mode off for this test
Consts.TestMode = false;
await DownloadDispatcher.GetInstance<LoversLabDownloader>().Prepare();
var ini = $@"[General]
gameName={Game.SkyrimSpecialEdition.MetaData().MO2ArchiveName}
gameFile=Data/Update.esm";