using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; using System.Threading.Tasks; using System.Reactive.Linq; using System.Reactive.Subjects; using System.Threading; using Wabbajack.Common; using Wabbajack.Common.StatusFeed; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.Http; using Wabbajack.Lib.LibCefHelpers; using Wabbajack.Lib.NexusApi; using Wabbajack.Lib.Validation; using Wabbajack.Lib.WebAutomation; using Xunit; using Xunit.Abstractions; using Directory = System.IO.Directory; using File = Alphaleonis.Win32.Filesystem.File; using Game = Wabbajack.Common.Game; using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Test { public class DownloaderTests : XunitContextBase, IDisposable { private IDisposable _unsubMsgs; private IDisposable _unsubErr; public DownloaderTests(ITestOutputHelper helper) : base(helper) { Helpers.Init(); _unsubMsgs = Utils.LogMessages.OfType().Subscribe(onNext: msg => XunitContext.WriteLine(msg.ShortDescription)); _unsubErr = Utils.LogMessages.OfType().Subscribe(msg => XunitContext.WriteLine("ERROR: User intervention required: " + msg.ShortDescription)); } public override void Dispose() { base.Dispose(); _unsubErr.Dispose(); _unsubMsgs.Dispose(); } /* [Fact] public async Task TestAllPrepares() { foreach (var downloader in DownloadDispatcher.Downloaders) await downloader.Prepare(); }*/ [Fact] public async Task MegaDownload() { var ini = @"[General] directURL=https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); var url_state = DownloadDispatcher.ResolveArchive( "https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k"); Assert.Equal("https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k", ((MegaDownloader.State)url_state).Url); var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!){Size = 20})); await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List{"https://mega.nz/#!CsMSFaaJ!-uziC4mbJPRy2e4pPk8Gjb3oDT_38Be9fzZ6Ld4NL-k" } })); Assert.False(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List{ "blerg" }})); await converted.Download(new Archive(state: null!) {Name = "MEGA Test.txt"}, filename.Path); Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); } [Fact] public async Task DropboxTests() { var ini = @"[General] directURL=https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?dl=0"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); var url_state = DownloadDispatcher.ResolveArchive( "https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?dl=0"); Assert.Equal("https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?dl=1", ((HTTPDownloader.State)url_state).Url); var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!){Size = 20})); await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List { "https://www.dropbox.com/s/5hov3m2pboppoc2/WABBAJACK_TEST_FILE.txt?" } })); Assert.False(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List { "blerg" } })); await converted.Download(new Archive(state: null!) { Name = "MEGA Test.txt" }, filename.Path); Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); } [Fact] public async Task GoogleDriveTests() { var ini = @"[General] directURL=https://drive.google.com/file/d/1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_/view?usp=sharing"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); var url_state = DownloadDispatcher.ResolveArchive( "https://drive.google.com/file/d/1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_/view?usp=sharing"); Assert.Equal("1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_", ((GoogleDriveDownloader.State)url_state).Id); var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List { "1grLRTrpHxlg7VPxATTFNfq2OkU_Plvh_" } })); Assert.False(converted.IsWhitelisted(new ServerWhitelist { GoogleIDs = new List()})); await converted.Download(new Archive(state: null!) { Name = "MEGA Test.txt" }, filename.Path); Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); } [Fact] public async Task HttpDownload() { var ini = @"[General] directURL=http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); var url_state = DownloadDispatcher.ResolveArchive("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt"); Assert.Equal("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt", ((HTTPDownloader.State)url_state).Url); var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List { "http://build.wabbajack.org/" } })); Assert.False(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); await converted.Download(new Archive(state: null!) { Name = "MEGA Test.txt" }, filename.Path); Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); } [Fact] public async Task ManualDownload() { var ini = @"[General] manualURL=http://build.wabbajack.org/WABBAJACK_TEST_FILE.zip"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List { "http://build.wabbajack.org/" } })); // Doesn't work well on the test server, so we're disabling for now //await converted.Download(new Archive { Name = "WABBAJACK_TEST_FILE.zip", Size = 20, Hash = "eSIyd+KOG3s="}, filename.Path); //Assert.Equal("eSIyd+KOG3s=", Utils.FileHash(filename.Path)); //Assert.Equal(File.ReadAllText(filename.Path), "Cheese for Everyone!"); } [Fact] public async Task MediaFireDownload() { var ini = @"[General] directURL=http://www.mediafire.com/file/agiqzm1xwebczpx/WABBAJACK_TEST_FILE.txt"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); var url_state = DownloadDispatcher.ResolveArchive( "http://www.mediafire.com/file/agiqzm1xwebczpx/WABBAJACK_TEST_FILE.txt"); Assert.Equal("http://www.mediafire.com/file/agiqzm1xwebczpx/WABBAJACK_TEST_FILE.txt", ((MediaFireDownloader.State) url_state).Url); var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20 })); await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List {"http://www.mediafire.com/file/agiqzm1xwebczpx/"}})); Assert.False(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List()})); await converted.Download(new Archive(state: null!) { Name = "Media Fire Test.zip" }, filename.Path); Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); } [Fact] public async Task NexusDownload() { var ini = @"[General] gameName=SkyrimSE modID = 12604 fileID=35407"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20 })); // Exercise the cache code Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20 })); await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); await converted.Download(new Archive(state: null!) { Name = "SkyUI.7z" }, filename.Path); Assert.Equal(Hash.FromBase64("dF2yafV2Oks="), await filename.Path.FileHashAsync()); } [Fact] public async Task ModDbTests() { var ini = @"[General] directURL=https://www.moddb.com/downloads/start/124908?referer=https%3A%2F%2Fwww.moddb.com%2Fmods%2Fautopause"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); var url_state = DownloadDispatcher.ResolveArchive( "https://www.moddb.com/downloads/start/124908?referer=https%3A%2F%2Fwww.moddb.com%2Fmods%2Fautopause"); Assert.Equal("https://www.moddb.com/downloads/start/124908?referer=https%3A%2F%2Fwww.moddb.com%2Fmods%2Fautopause", ((ModDBDownloader.State)url_state).Url); var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); await converted.Download(new Archive(state: null!) { Name = "moddbtest.7z" }, filename.Path); Assert.Equal(Hash.FromBase64("2lZt+1h6wxM="), await filename.Path.FileHashAsync()); } [Fact] public async Task CanFindOtherLLMods() { await DownloadDispatcher.GetInstance().Prepare(); var ini = @"[General] directURL=https://www.loverslab.com/files/file/1382-milk-mod-economy/?do=download&r=913360&confirm=1&t=1&csrfKey=7984faa4d27f6b638125daf38ae5f1191"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); var otherfiles = await ((LoversLabDownloader.State)state).GetFilesInGroup(); } [Fact] public async Task CanCancelLLValidation() { await using var filename = new TempFile(); await DownloadDispatcher.GetInstance().Prepare(); var state = new LoversLabDownloader.State { FileName = "14424-books-of-dibella-se-alternate-start-plugin", FileID = "870820", }; using var queue = new WorkQueue(); var tcs = new CancellationTokenSource(); tcs.CancelAfter(2); await Assert.ThrowsAsync(async () => { await Enumerable.Range(0, 2).PMap(queue, async x => { Assert.True(await state.Verify(new Archive(state: null!) {Size = 252269}, tcs.Token)); }); }); } [Fact] public async Task CanGetLLMetadata() { await DownloadDispatcher.GetInstance().Prepare(); var ini = @"[General] directURL=https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1"; var state = (LoversLabDownloader.State)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.True(await state.LoadMetaData()); Assert.Equal("halgari", state.Author); } [Fact] public async Task LoversLabDownload() { await DownloadDispatcher.GetInstance().Prepare(); var ini = @"[General] directURL=https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); /*var url_state = DownloadDispatcher.ResolveArchive("https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1"); Assert.Equal("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt", ((HTTPDownloader.State)url_state).Url); */ var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); // Verify with different Size Assert.False(await converted.Verify(new Archive(state: null!) { Size = 15})); await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); await converted.Download(new Archive(state: null!) { Name = "LoversLab Test.txt" }, filename.Path); Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); var files = await ((LoversLabDownloader.State)converted).GetFilesInGroup(); Assert.NotEmpty(files); Assert.Equal("WABBAJACK_TEST_FILE.zip", files.First().Name); Assert.True(files.All(f => !string.IsNullOrWhiteSpace(((LoversLabDownloader.State)f.State).FileID))); ((LoversLabDownloader.State)converted).FileID = "42"; var upgrade = await DownloadDispatcher.FindUpgrade(new Archive(converted) {Name = "WABBAJACK_TEST_FILE.zip"}); Assert.True(upgrade != default); var newState = ((LoversLabDownloader.State)upgrade.Archive.State); Assert.False(string.IsNullOrWhiteSpace(newState.FileID)); var roundTripped = newState.ViaJSON(); Assert.False(string.IsNullOrWhiteSpace(roundTripped.FileID)); } /* [Fact] public async Task VectorPlexusDownload() { await DownloadDispatcher.GetInstance().Prepare(); var ini = @"[General] directURL=https://vectorplexus.com/files/file/290-wabbajack-test-file"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); await converted.Download(new Archive(state: null!) { Name = "Vector Plexus Test.zip" }, filename.Path); Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); }*/ [Fact] public async Task YandexDownloader() { await DownloadDispatcher.GetInstance().Prepare(); var ini = @"[General] directURL=https://yadi.sk/d/jqwQT4ByYtC9Tw"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() {"https://yadi.sk"} })); await converted.Download(new Archive(state: null!) { Name = "WABBAJACK_TEST_FILE.txt" }, filename.Path); Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); } [Fact] public async Task TESAllianceDownload() { await DownloadDispatcher.GetInstance().Prepare(); const string ini = "[General]\n" + "directURL=http://tesalliance.org/forums/index.php?/files/file/2035-wabbajack-test-file/"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); await converted.Download(new Archive(state: null!) { Name = "TESAlliance Test.zip" }, filename.Path); Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); } [Fact] public async Task TESAllDownloader() { await DownloadDispatcher.GetInstance().Prepare(); const string ini = "[General]\n" + "directURL=https://tesall.ru/files/download/594545"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); await converted.Download(new Archive(state: null!) { Name = "TESAll Test.zip" }, filename.Path); Assert.Equal(Hash.FromBase64("eSIyd+KOG3s="), await filename.Path.FileHashAsync()); Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync()); } /* WAITING FOR APPROVAL BY MODERATOR [Fact] public async Task DeadlyStreamDownloader() { await DownloadDispatcher.GetInstance().Prepare(); const string ini = "[General]\n" + "directURL=https://deadlystream.com/files/file/1550-wabbajack-test-file/"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive{Size = 20})); using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); await converted.Download(new Archive { Name = "DeadlyStream Test.zip" }, filename.Path); Assert.Equal("eSIyd+KOG3s=", filename.FileHash()); Assert.Equal(File.ReadAllText(filename.Path), "Cheese for Everyone!"); }*/ [Fact] public async Task GameFileSourceDownload() { // Test mode off for this test Consts.TestMode = false; await DownloadDispatcher.GetInstance().Prepare(); var ini = $@"[General] gameName={Game.SkyrimSpecialEdition.MetaData().MO2ArchiveName} gameFile=Data/Update.esm"; var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.NotNull(state); var converted = RoundTripState(state); Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20})); await using var filename = new TempFile(); Assert.True(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); await converted.Download(new Archive(state: null!) { Name = "Update.esm" }, filename.Path); Assert.Equal(Hash.FromBase64("/DLG/LjdGXI="), await filename.Path.FileHashAsync()); Assert.Equal(await filename.Path.ReadAllBytesAsync(), await Game.SkyrimSpecialEdition.MetaData().GameLocation().Combine("Data/Update.esm").ReadAllBytesAsync()); Consts.TestMode = true; } /// /// Tests that files from different sources don't overwrite eachother when downloaded by AInstaller /// /// [Fact] public async Task DownloadRenamingTests() { // Test mode off for this test Consts.TestMode = false; var inia = $@"[General] gameName={Game.SkyrimSpecialEdition.MetaData().MO2ArchiveName} gameFile=Data/Update.esm"; var statea = (GameFileSourceDownloader.State)await DownloadDispatcher.ResolveArchive(inia.LoadIniString()); var inib = $@"[General] gameName={Game.SkyrimSpecialEdition.MetaData().MO2ArchiveName} gameFile=Data/Skyrim.esm"; var stateb = (GameFileSourceDownloader.State)await DownloadDispatcher.ResolveArchive(inib.LoadIniString()); var archivesa = new List() { new Archive(statea) {Hash = statea.Hash, Name = "Download.esm" } }; var archivesb = new List() { new Archive(stateb) {Hash = stateb.Hash, Name = "Download.esm" } }; var folder = ((RelativePath)"DownloadTests").RelativeToEntryPoint(); await folder.DeleteDirectory(); folder.CreateDirectory(); var inst = new TestInstaller(default, new ModList {GameType = Game.SkyrimSpecialEdition}, default, folder, null); await inst.DownloadMissingArchives(archivesa, true); await inst.DownloadMissingArchives(archivesb, true); Assert.Equal(new[] { (RelativePath)@"Download.esm", (RelativePath)@"Download_ed33cbb256e5328361da8d9227df9cab1bb43a79a87dca2f223b2e2762ccaad1_.esm", }.OrderBy(a => a).ToArray(), folder.EnumerateFiles().Select(f => f.FileName).OrderBy(a => a).ToArray()); Consts.TestMode = true; } private T RoundTripState(T state) { return state.ToJson().FromJsonString(); } /* TODO : Disabled for now [Fact] public async Task TestUpgrading() { await using var folder = await TempFolder.Create(); var dest = folder.Dir.Combine("Cori.7z"); var archive = new Archive( new NexusDownloader.State { Game = Game.SkyrimSpecialEdition, ModID = 24808, FileID = 123501 }) { Name = "Cori.7z", Hash = Hash.FromBase64("gCRVrvzDNH0="), }; Utils.Log($"Getting Hash for {(long)archive.Hash}"); Assert.True(await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, dest)); Assert.Equal(Hash.FromBase64("gCRVrvzDNH0="), await dest.FileHashCachedAsync()); }*/ class TestInstaller : AInstaller { public TestInstaller(AbsolutePath archive, ModList modList, AbsolutePath outputFolder, AbsolutePath downloadFolder, SystemParameters parameters) : base(archive, modList, outputFolder, downloadFolder, parameters, steps: 1, modList.GameType) { DesiredThreads.OnNext(1); } protected override Task _Begin(CancellationToken cancel) { throw new NotImplementedException(); } public override ModManager ModManager { get => ModManager.MO2; } } } }