wabbajack/Wabbajack.Test/DownloaderTests.cs
2020-04-10 13:32:35 -05:00

562 lines
24 KiB
C#

using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Threading;
using Wabbajack.Common;
using Wabbajack.Common.StatusFeed;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.LibCefHelpers;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using Xunit;
using Xunit.Abstractions;
using Directory = System.IO.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using Game = Wabbajack.Common.Game;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Test
{
public class DownloaderTests : XunitContextBase, IDisposable
{
private IDisposable _unsubMsgs;
private IDisposable _unsubErr;
public DownloaderTests(ITestOutputHelper helper) : base(helper)
{
Helpers.Init();
_unsubMsgs = Utils.LogMessages.OfType<IInfo>().Subscribe(onNext: msg => XunitContext.WriteLine(msg.ShortDescription));
_unsubErr = Utils.LogMessages.OfType<IUserIntervention>().Subscribe(msg =>
XunitContext.WriteLine("ERROR: User intervention required: " + msg.ShortDescription));
}
public override void Dispose()
{
base.Dispose();
_unsubErr.Dispose();
_unsubMsgs.Dispose();
}
[Fact]
public void TestAllPrepares()
{
DownloadDispatcher.Downloaders.Do(d => d.Prepare());
}
[Fact]
public async Task MegaDownload()
{
var ini = @"[General]
directURL=https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
var url_state = DownloadDispatcher.ResolveArchive(
"https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k");
Assert.Equal("https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k",
((MegaDownloader.State)url_state).Url);
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive(state: null!){Size = 20}));
using var filename = new TempFile();
Assert.True(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List<string>{"https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k" } }));
Assert.False(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>{ "blerg" }}));
await converted.Download(new Archive(state: null!) {Name = "MEGA Test.txt"}, filename.Path);
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
}
[Fact]
public async Task DropboxTests()
{
var ini = @"[General]
directURL=https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?dl=0";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
var url_state = DownloadDispatcher.ResolveArchive(
"https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?dl=0");
Assert.Equal("https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?dl=1",
((HTTPDownloader.State)url_state).Url);
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive(state: null!){Size = 20}));
using var filename = new TempFile();
Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?" } }));
Assert.False(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "blerg" } }));
await converted.Download(new Archive(state: null!) { Name = "MEGA Test.txt" }, filename.Path);
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
}
[Fact]
public async Task GoogleDriveTests()
{
var ini = @"[General]
directURL=https://drive.google.com/file/d/1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_/view?usp=sharing";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
var url_state = DownloadDispatcher.ResolveArchive(
"https://drive.google.com/file/d/1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_/view?usp=sharing");
Assert.Equal("1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_",
((GoogleDriveDownloader.State)url_state).Id);
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20}));
using var filename = new TempFile();
Assert.True(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List<string> { "1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_" } }));
Assert.False(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List<string>()}));
await converted.Download(new Archive(state: null!) { Name = "MEGA Test.txt" }, filename.Path);
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
}
[Fact]
public async Task HttpDownload()
{
var ini = @"[General]
directURL=http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
var url_state = DownloadDispatcher.ResolveArchive("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt");
Assert.Equal("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt",
((HTTPDownloader.State)url_state).Url);
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20}));
using var filename = new TempFile();
Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "http://build.wabbajack.org/" } }));
Assert.False(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
await converted.Download(new Archive(state: null!) { Name = "MEGA Test.txt" }, filename.Path);
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
}
[Fact]
public async Task ManualDownload()
{
var ini = @"[General]
manualURL=http://build.wabbajack.org/WABBAJACK_TEST_FILE.zip";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20}));
using var filename = new TempFile();
Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "http://build.wabbajack.org/" } }));
// Doesn't work well on the test server, so we're disabling for now
//await converted.Download(new Archive { Name = "WABBAJACK_TEST_FILE.zip", Size = 20, Hash = "eSIyd+KOG3s="}, filename.Path);
//Assert.Equal("eSIyd+KOG3s=", Utils.FileHash(filename.Path));
//Assert.Equal(File.ReadAllText(filename.Path), "Cheese for Everyone!");
}
/*
[Fact]
public async Task MediaFireDownload()
{
var ini = @"[General]
directURL=http://www.mediafire.com/file/agiqzm1xwebczpx/WABBAJACK_TEST_FILE.txt";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
var url_state = DownloadDispatcher.ResolveArchive(
"http://www.mediafire.com/file/agiqzm1xwebczpx/WABBAJACK_TEST_FILE.txt");
Assert.Equal("http://www.mediafire.com/file/agiqzm1xwebczpx/WABBAJACK_TEST_FILE.txt",
((MediaFireDownloader.State) url_state).Url);
var converted = RoundTripState(state);
Assert.True(await converted.Verify());
using var filename = new TempFile();
Assert.True(converted.IsWhitelisted(new ServerWhitelist
{AllowedPrefixes = new List<string> {"http://www.mediafire.com/file/agiqzm1xwebczpx/"}}));
Assert.False(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List<string>()}));
await converted.Download(new Archive {Name = "Media Fire Test.txt"}, filename.Path);
Assert.Equal(File.ReadAllText(filename.Path), "Cheese for Everyone!");
}*/
[Fact]
public async Task NexusDownload()
{
var ini = @"[General]
gameName=SkyrimSE
modID = 12604
fileID=35407";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20 }));
// Exercise the cache code
Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20 }));
using var filename = new TempFile();
Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
await converted.Download(new Archive(state: null!) { Name = "SkyUI.7z" }, filename.Path);
Assert.Equal(Hash.FromBase64("dF2yafV2Oks="), await filename.Path.FileHashAsync());
}
[Fact]
public async Task ModDbTests()
{
var ini = @"[General]
directURL=https://www.moddb.com/downloads/start/124908?referer=https%3A%2F%2Fwww.moddb.com%2Fmods%2Fautopause";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
var url_state = DownloadDispatcher.ResolveArchive(
"https://www.moddb.com/downloads/start/124908?referer=https%3A%2F%2Fwww.moddb.com%2Fmods%2Fautopause");
Assert.Equal("https://www.moddb.com/downloads/start/124908?referer=https%3A%2F%2Fwww.moddb.com%2Fmods%2Fautopause",
((ModDBDownloader.State)url_state).Url);
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20}));
using var filename = new TempFile();
Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
await converted.Download(new Archive(state: null!) { Name = "moddbtest.7z" }, filename.Path);
Assert.Equal(Hash.FromBase64("2lZt+1h6wxM="), await filename.Path.FileHashAsync());
}
[Fact]
public async Task LoversLabDownload()
{
await DownloadDispatcher.GetInstance<LoversLabDownloader>().Prepare();
var ini = @"[General]
directURL=https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
/*var url_state = DownloadDispatcher.ResolveArchive("https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1");
Assert.Equal("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt",
((HTTPDownloader.State)url_state).Url);
*/
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20}));
using var filename = new TempFile();
Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
await converted.Download(new Archive(state: null!) { Name = "LoversLab Test.txt" }, filename.Path);
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
}
[Fact]
public async Task VectorPlexusDownload()
{
await DownloadDispatcher.GetInstance<VectorPlexusDownloader>().Prepare();
var ini = @"[General]
directURL=https://vectorplexus.com/files/file/290-wabbajack-test-file";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
/*var url_state = DownloadDispatcher.ResolveArchive("https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1");
Assert.Equal("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt",
((HTTPDownloader.State)url_state).Url);
*/
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20}));
using var filename = new TempFile();
Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
await converted.Download(new Archive(state: null!) { Name = "Vector Plexus Test.zip" }, filename.Path);
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
}
[Fact]
public async Task TESAllianceDownload()
{
await DownloadDispatcher.GetInstance<TESAllianceDownloader>().Prepare();
const string ini = "[General]\n" +
"directURL=http://tesalliance.org/forums/index.php?/files/file/2035-wabbajack-test-file/";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20}));
using var filename = new TempFile();
Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
await converted.Download(new Archive(state: null!) { Name = "TESAlliance Test.zip" }, filename.Path);
Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync());
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
}
/* WAITING FOR APPROVAL BY MODERATOR
[Fact]
public async Task DeadlyStreamDownloader()
{
await DownloadDispatcher.GetInstance<DeadlyStreamDownloader>().Prepare();
const string ini = "[General]\n" +
"directURL=https://deadlystream.com/files/file/1550-wabbajack-test-file/";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive{Size = 20}));
using var filename = new TempFile();
Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
await converted.Download(new Archive { Name = "DeadlyStream Test.zip" }, filename.Path);
Assert.Equal("eSIyd+KOG3s=", filename.FileHash());
Assert.Equal(File.ReadAllText(filename.Path), "Cheese for Everyone!");
}*/
[Fact]
public async Task GameFileSourceDownload()
{
// Test mode off for this test
Consts.TestMode = false;
await DownloadDispatcher.GetInstance<LoversLabDownloader>().Prepare();
var ini = $@"[General]
gameName={Game.SkyrimSpecialEdition.MetaData().MO2ArchiveName}
gameFile=Data/Update.esm";
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20}));
using var filename = new TempFile();
Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
await converted.Download(new Archive(state: null!) { Name = "Update.esm" }, filename.Path);
Assert.Equal(Hash.FromBase64("/DLG/LjdGXI="), await Utils.FileHashAsync(filename.Path));
Assert.Equal(await filename.Path.ReadAllBytesAsync(), await Game.SkyrimSpecialEdition.MetaData().GameLocation().Combine("Data/Update.esm").ReadAllBytesAsync());
Consts.TestMode = true;
}
[Fact]
public async Task BethesdaNetDownload()
{
var downloader = DownloadDispatcher.GetInstance<BethesdaNetDownloader>();
Assert.True(await downloader.IsLoggedIn.FirstAsync());
var ini = $@"[General]
directURL=https://bethesda.net/en/mods/skyrim/mod-detail/4145641";
using var filename = new TempFile();
var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.NotNull(state);
var converted = state.ViaJSON();
Assert.True(await converted.Verify(new Archive(state: null!) { Name = "mod.ckm"}));
Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
await converted.Download(new Archive(state: null!) { Name = "mod.zip" }, filename.Path);
await using var fs = filename.Path.OpenRead();
using var archive = new ZipArchive(fs);
var entries = archive.Entries.Select(e => e.FullName).ToList();
Assert.Equal(entries, new List<string> {@"Data\TestCK.esp", @"Data\TestCK.ini"});
}
[Fact]
public async Task YoutubeDownloader()
{
var infered_ini = await DownloadDispatcher.Infer(new Uri("https://www.youtube.com/watch?v=4ceowgHn8BE"));
Assert.IsAssignableFrom<YouTubeDownloader.State>(infered_ini);
Assert.Equal(15, ((YouTubeDownloader.State)infered_ini).Tracks.Count);
var ini = string.Join("\n", infered_ini.GetMetaIni());
var state = (YouTubeDownloader.State)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.Equal(15, state.Tracks.Count);
Assert.NotNull(state);
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive(state: null!) { Name = "yt_test.zip"}));
Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
using var tempFile = new TempFile();
await converted.Download(new Archive(state: null!) { Name = "yt_test.zip"}, tempFile.Path);
Assert.Equal(Hash.FromBase64("kD36zbA2X9Q="), await tempFile.Path.FileHashAsync());
}
/// <summary>
/// Tests that files from different sources don't overwrite eachother when downloaded by AInstaller
/// </summary>
/// <returns></returns>
[Fact]
public async Task DownloadRenamingTests()
{
// Test mode off for this test
Consts.TestMode = false;
var inia = $@"[General]
gameName={Game.SkyrimSpecialEdition.MetaData().MO2ArchiveName}
gameFile=Data/Update.esm";
var statea = (GameFileSourceDownloader.State)await DownloadDispatcher.ResolveArchive(inia.LoadIniString());
var inib = $@"[General]
gameName={Game.SkyrimSpecialEdition.MetaData().MO2ArchiveName}
gameFile=Data/Skyrim.esm";
var stateb = (GameFileSourceDownloader.State)await DownloadDispatcher.ResolveArchive(inib.LoadIniString());
var archivesa = new List<Archive>()
{
new Archive(statea) {Hash = statea.Hash, Name = "Download.esm" }
};
var archivesb = new List<Archive>()
{
new Archive(stateb) {Hash = stateb.Hash, Name = "Download.esm" }
};
var folder = ((RelativePath)"DownloadTests").RelativeToEntryPoint();
await folder.DeleteDirectory();
folder.CreateDirectory();
var inst = new TestInstaller(default, null, default, folder, null);
await inst.DownloadMissingArchives(archivesa, true);
await inst.DownloadMissingArchives(archivesb, true);
Assert.Equal(new[]
{
(RelativePath)@"Download.esm",
(RelativePath)@"Download.esm.xxHash",
(RelativePath)@"Download_ed33cbb256e5328361da8d9227df9cab1bb43a79a87dca2f223b2e2762ccaad1_.esm",
(RelativePath)@"Download_ed33cbb256e5328361da8d9227df9cab1bb43a79a87dca2f223b2e2762ccaad1_.esm.xxHash"
}.OrderBy(a => a).ToArray(),
folder.EnumerateFiles().Select(f => f.FileName).OrderBy(a => a).ToArray());
Consts.TestMode = true;
}
private T RoundTripState<T>(T state)
{
return state.ToJson().FromJsonString<T>();
}
[Fact]
public async Task TestUpgrading()
{
await using var folder = new TempFolder();
var dest = folder.Dir.Combine("Cori.7z");
var archive = new Archive(
new NexusDownloader.State
{
Game = Game.SkyrimSpecialEdition,
ModID = 24808,
FileID = 123501
})
{
Name = "Cori.7z",
Hash = Hash.FromBase64("gCRVrvzDNH0="),
};
Assert.True(await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, dest));
Assert.Equal(Hash.FromBase64("gCRVrvzDNH0="), await dest.FileHashCachedAsync());
}
class TestInstaller : AInstaller
{
public TestInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters parameters)
: base(archive, modList, outputFolder, downloadFolder, parameters, steps: 1)
{
Queue.SetActiveThreadsObservable(Observable.Return(1));
}
protected override Task<bool> _Begin(CancellationToken cancel)
{
throw new NotImplementedException();
}
public override ModManager ModManager { get => ModManager.MO2; }
}
}
}