Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Wabbajack.Lib downloaders enable nullable

Browse Source
This commit is contained in:
Justin Swanson 2020-04-09 15:20:34 -05:00
parent a7ad4f1f16
commit a29eb93caf
29 changed files with 234 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.BuildServer.Test/ModListValidationTests.cs
View File

@ -130,7 +130,7 @@ namespace Wabbajack.BuildServer.Test
Hash = await test_archive_path.FileHashAsync(),
Name = "test_archive",
Size = test_archive_path.Size,
State = new HTTPDownloader.State {Url = MakeURL("test_archive.txt")}
State = new HTTPDownloader.State(MakeURL("test_archive.txt"))
}
}
};

10
Wabbajack.BuildServer.Test/ModlistUpdater.cs
View File

@ -45,10 +45,7 @@ namespace Wabbajack.BuildServer.Test
Archive = new Archive
{
Name = "Oldfile",
State = new HTTPDownloader.State
{
Url = MakeURL("old_file_data.random"),
}
State = new HTTPDownloader.State(MakeURL("old_file_data.random"))
}
}
});
@ -60,10 +57,7 @@ namespace Wabbajack.BuildServer.Test
Archive = new Archive
{
Name = "Newfile",
State = new HTTPDownloader.State
{
Url = MakeURL("new_file_data.random"),
}
State = new HTTPDownloader.State(MakeURL("new_file_data.random"))
}
}
});

6
Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs
View File

@ -22,10 +22,9 @@ namespace Wabbajack.BuildServer.Models.Jobs
var states = GameRegistry.Games.Values
.Where(game => game.GameLocation() != null && game.MainExecutable != null)
.SelectMany(game => game.GameLocation().EnumerateFiles()
.Select(file => new GameFileSourceDownloader.State
.Select(file => new GameFileSourceDownloader.State(game.InstalledVersion)
{
Game = game.Game,
GameVersion = game.InstalledVersion,
GameFile = file.RelativeTo(game.GameLocation()),
}))
.ToList();
@ -59,11 +58,10 @@ namespace Wabbajack.BuildServer.Models.Jobs
.ToList();
foreach (var job in jobs)
await sql.EnqueueJob(job);
await sql.EnqueueJob(job);
return JobResult.Success();
}
}
}
}

5
Wabbajack.BuildServer/Models/Jobs/IndexDynDOLOD.cs
View File

@ -41,10 +41,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
Archive = new Archive
{
Name = Guid.NewGuid() + ".7z",
State = new MegaDownloader.State
{
Url = url.ToString()
}
State = new MegaDownloader.State(url.ToString())
}
}
})

5
Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs
View File

@ -61,10 +61,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
Name = file.MungedName,
Size = file.Size,
Hash = file.Hash,
State = new HTTPDownloader.State
{
Url = file.Uri
}
State = new HTTPDownloader.State(file.Uri)
}
}
});

1
Wabbajack.Lib/Downloaders/AFKModsDownloader.cs
View File

@ -1,4 +1,5 @@
using System;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{

17
Wabbajack.Lib/Downloaders/AbstractDownloadState.cs
View File

@ -5,25 +5,25 @@ using System.Threading.Tasks;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib.Validation;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
public interface IMetaState
{
Uri URL { get; }
string Name { get; set; }
string Author { get; set; }
string Version { get; set; }
string ImageURL { get; set; }
string? Name { get; set; }
string? Author { get; set; }
string? Version { get; set; }
string? ImageURL { get; set; }
bool IsNSFW { get; set; }
string Description { get; set; }
string? Description { get; set; }
Task<bool> LoadMetaData();
}
public abstract class AbstractDownloadState
{
public static List<Type> KnownSubTypes = new List<Type>
{
typeof(HTTPDownloader.State),
@ -48,7 +48,7 @@ namespace Wabbajack.Lib.Downloaders
static AbstractDownloadState()
{
NameToType = KnownSubTypes.ToDictionary(t => t.FullName.Substring(t.Namespace.Length + 1), t => t);
NameToType = KnownSubTypes.ToDictionary(t => t.FullName!.Substring(t.Namespace!.Length + 1), t => t);
TypeToName = NameToType.ToDictionary(k => k.Value, k => k.Key);
}
@ -68,7 +68,6 @@ namespace Wabbajack.Lib.Downloaders
}
}
/// <summary>
/// Returns true if this file is allowed to be downloaded via whitelist
/// </summary>
@ -96,7 +95,7 @@ namespace Wabbajack.Lib.Downloaders
public abstract IDownloader GetDownloader();
public abstract string GetManifestURL(Archive a);
public abstract string? GetManifestURL(Archive a);
public abstract string[] GetMetaIni();
}
}

41
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -8,6 +8,7 @@ using System.Web;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib.Validation;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
@ -18,10 +19,12 @@ namespace Wabbajack.Lib.Downloaders
where TState : AbstractIPS4Downloader<TDownloader, TState>.State<TDownloader>, new()
where TDownloader : IDownloader
{
public override string SiteName { get; }
public override Uri SiteURL { get; }
protected AbstractIPS4Downloader(Uri loginUri, string encryptedKeyName, string cookieDomain)
: base(loginUri, encryptedKeyName, cookieDomain, "ips4_member_id")
{
}
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
Uri url = DownloaderUtils.GetDirectURL(archiveINI);
@ -70,23 +73,23 @@ namespace Wabbajack.Lib.Downloaders
};
}
public class State<TStateDownloader> : AbstractDownloadState, IMetaState where TStateDownloader : IDownloader
public class State<TStateDownloader> : AbstractDownloadState, IMetaState
where TStateDownloader : IDownloader
{
public string FullURL { get; set; }
public string FullURL { get; set; } = string.Empty;
public bool IsAttachment { get; set; }
public string FileID { get; set; }
public string FileName { get; set; }
public string FileID { get; set; } = string.Empty;
public string FileName { get; set; } = string.Empty;
// from IMetaState
public Uri URL => new Uri($"{Site}/files/file/{FileName}");
public string Name { get; set; }
public string Author { get; set; }
public string Version { get; set; }
public string ImageURL { get; set; }
public string? Name { get; set; }
public string? Author { get; set; }
public string? Version { get; set; }
public string? ImageURL { get; set; }
public virtual bool IsNSFW { get; set; }
public string Description { get; set; }
public string? Description { get; set; }
private static bool IsHTTPS => Downloader.SiteURL.AbsolutePath.StartsWith("https://");
private static string URLPrefix => IsHTTPS ? "https://" : "http://";
@ -119,6 +122,7 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Download(Archive a, AbsolutePath destination)
{
await using var stream = await ResolveDownloadStream();
if (stream == null) return false;
await using (var file = destination.Create())
{
await stream.CopyToAsync(file);
@ -126,7 +130,7 @@ namespace Wabbajack.Lib.Downloaders
return true;
}
private async Task<Stream> ResolveDownloadStream()
private async Task<Stream?> ResolveDownloadStream()
{
TOP:
string url;
@ -236,12 +240,5 @@ namespace Wabbajack.Lib.Downloaders
return false;
}
}
protected AbstractIPS4Downloader(Uri loginUri, string encryptedKeyName, string cookieDomain) :
base(loginUri, encryptedKeyName, cookieDomain, "ips4_member_id")
{
}
}
}

2
Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs
View File

@ -138,6 +138,4 @@ namespace Wabbajack.Lib.Downloaders
}
}
}
}

98
Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs
View File

@ -24,12 +24,22 @@ using Wabbajack.Lib.Validation;
using File = Alphaleonis.Win32.Filesystem.File;
using Game = Wabbajack.Common.Game;
using Path = Alphaleonis.Win32.Filesystem.Path;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
public class BethesdaNetDownloader : IUrlDownloader, INeedsLogin
{
public const string DataName = "bethesda-net-data";
public ReactiveCommand<Unit, Unit> TriggerLogin { get; }
public ReactiveCommand<Unit, Unit> ClearLogin { get; }
public IObservable<bool> IsLoggedIn => Utils.HaveEncryptedJsonObservable(DataName);
public string SiteName => "Bethesda.NET";
public IObservable<string> MetaInfo => Observable.Return(""); //"Wabbajack will start the game, then exit once you enter the Mods page";
public Uri SiteURL => new Uri("https://bethesda.net");
public Uri? IconUri { get; }
public BethesdaNetDownloader()
{
TriggerLogin = ReactiveCommand.CreateFromTask(() => Utils.CatchAndLog(RequestLoginAndCache), IsLoggedIn.Select(b => !b).ObserveOn(RxApp.MainThreadScheduler));
@ -38,23 +48,23 @@ namespace Wabbajack.Lib.Downloaders
private static async Task RequestLoginAndCache()
{
var result = await Utils.Log(new RequestBethesdaNetLogin()).Task;
await Utils.Log(new RequestBethesdaNetLogin()).Task;
}
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var url = (Uri)DownloaderUtils.GetDirectURL(archiveINI);
return StateFromUrl(url);
}
internal static AbstractDownloadState StateFromUrl(Uri url)
internal static AbstractDownloadState? StateFromUrl(Uri url)
{
if (url != null && url.Host == "bethesda.net" && url.AbsolutePath.StartsWith("/en/mods/"))
{
var split = url.AbsolutePath.Split('/');
var game = split[3];
var modId = split[5];
return new State {GameName = game, ContentId = modId};
return new State(gameName: game, contentId: modId);
}
return null;
}
@ -65,9 +75,10 @@ namespace Wabbajack.Lib.Downloaders
await Utils.Log(new RequestBethesdaNetLogin()).Task;
}
public static async Task<BethesdaNetData> Login(Game game)
public static async Task<BethesdaNetData?> Login(Game game)
{
var metadata = game.MetaData();
if (metadata.MainExecutable == null) throw new NotImplementedException();
var gamePath = metadata.GameLocation().Combine(metadata.MainExecutable);
var info = new ProcessStartInfo
{
@ -102,28 +113,25 @@ namespace Wabbajack.Lib.Downloaders
}
}
public AbstractDownloadState GetDownloaderState(string url)
public AbstractDownloadState? GetDownloaderState(string url)
{
return StateFromUrl(new Uri(url));
}
public ReactiveCommand<Unit, Unit> TriggerLogin { get; }
public ReactiveCommand<Unit, Unit> ClearLogin { get; }
public IObservable<bool> IsLoggedIn => Utils.HaveEncryptedJsonObservable(DataName);
public string SiteName => "Bethesda.NET";
public IObservable<string> MetaInfo => Observable.Return(""); //"Wabbajack will start the game, then exit once you enter the Mods page";
public Uri SiteURL => new Uri("https://bethesda.net");
public Uri IconUri { get; }
[JsonName("BethesdaNetDownloader")]
public class State : AbstractDownloadState
{
public string GameName { get; set; }
public string ContentId { get; set; }
public string GameName { get; }
public string ContentId { get; }
[JsonIgnore]
public override object[] PrimaryKey => new object[] {GameName, ContentId};
public override object[] PrimaryKey => new object[] { GameName, ContentId };
public State(string gameName, string contentId)
{
GameName = gameName;
ContentId = contentId;
}
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
@ -142,8 +150,8 @@ namespace Wabbajack.Lib.Downloaders
using var got = await client.GetAsync(
$"https://content.cdp.bethesda.net/{collected.CDPProductId}/{collected.CDPPropertiesId}/{chunk.sha}");
var data = await got.Content.ReadAsByteArrayAsync();
if (collected.AESKey != null)
AESCTRDecrypt(collected.AESKey, collected.AESIV, data);
if (collected.AESKey != null)
AESCTRDecrypt(collected.AESKey, collected.AESIV!, data);
if (chunk.uncompressed_size == chunk.chunk_size)
await file.WriteAsync(data, 0, data.Length);
@ -197,7 +205,7 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Verify(Archive archive)
{
var info = await ResolveDownloadInfo();
await ResolveDownloadInfo();
return true;
}
@ -221,7 +229,7 @@ namespace Wabbajack.Lib.Downloaders
client.Headers.Add(("x-cdp-app", "UGC SDK"));
client.Headers.Add(("x-cdp-app-ver", "0.9.11314/debug"));
client.Headers.Add(("x-cdp-lib-ver", "0.9.11314/debug"));
client.Headers.Add(("x-cdp-platform","Win/32"));
client.Headers.Add(("x-cdp-platform", "Win/32"));
posted = await client.PostAsync("https://api.bethesda.net/cdp-user/auth",
new StringContent("{\"access_token\": \"" + info.AccessToken + "\"}", Encoding.UTF8,
@ -232,10 +240,10 @@ namespace Wabbajack.Lib.Downloaders
var got = await client.GetAsync($"https://api.bethesda.net/mods/ugc-workshop/content/get?content_id={ContentId}");
JObject data = JObject.Parse(await got.Content.ReadAsStringAsync());
var content = data["platform"]["response"]["content"];
var content = data["platform"]!["response"]!["content"]!;
info.CDPBranchId = (int)content["cdp_branch_id"];
info.CDPProductId = (int)content["cdp_product_id"];
info.CDPBranchId = (int)content["cdp_branch_id"]!;
info.CDPProductId = (int)content["cdp_product_id"]!;
client.Headers.Add(("Authorization", $"Token {info.CDPToken}"));
client.Headers.Add(("Accept", "application/json"));
@ -245,7 +253,7 @@ namespace Wabbajack.Lib.Downloaders
$"https://api.bethesda.net/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/tree/.json");
var tree = (await got.Content.ReadAsStringAsync()).FromJsonString<CDPTree>();
got.Dispose();
got = await client.PostAsync($"https://api.bethesda.net/mods/ugc-content/add-subscription", new StringContent($"{{\"content_id\": \"{ContentId}\"}}", Encoding.UTF8, "application/json"));
@ -254,14 +262,14 @@ namespace Wabbajack.Lib.Downloaders
$"https://api.bethesda.net/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/depots/.json");
var props_obj = JObject.Parse(await got.Content.ReadAsStringAsync()).Properties().First();
info.CDPPropertiesId = (int)props_obj.Value["properties_id"];
info.CDPPropertiesId = (int)props_obj.Value["properties_id"]!;
info.AESKey = props_obj.Value["ex_info_A"].Select(e => (byte)e).ToArray();
info.AESIV = props_obj.Value["ex_info_B"].Select(e => (byte)e).Take(16).ToArray();
return (client, tree, info);
}
static int AESCTRDecrypt(byte[] Key, byte[] IV, byte[] Data)
{
IBufferedCipher cipher = CipherUtilities.GetCipher("AES/CTR/NoPadding");
@ -282,27 +290,26 @@ namespace Wabbajack.Lib.Downloaders
public override string[] GetMetaIni()
{
return new[] {"[General]", $"directURL=https://bethesda.net/en/mods/{GameName}/mod-detail/{ContentId}"};
return new[] { "[General]", $"directURL=https://bethesda.net/en/mods/{GameName}/mod-detail/{ContentId}" };
}
private class BeamLoginResponse
{
public string access_token { get; set; }
public string access_token { get; set; } = string.Empty;
}
private class CDPLoginResponse
{
public string token { get; set; }
public string token { get; set; } = string.Empty;
}
private class CollectedBNetInfo
{
public byte[] AESKey { get; set; }
public byte[] AESIV { get; set; }
public string AccessToken { get; set; }
public string CDPToken { get; set; }
public byte[] AESKey { get; set; } = null!;
public byte[] AESIV { get; set; } = null!;
public string AccessToken { get; set; } = string.Empty;
public string CDPToken { get; set; } = string.Empty;
public int CDPBranchId { get; set; }
public int CDPProductId { get; set; }
public int CDPPropertiesId { get; set; }
@ -310,24 +317,24 @@ namespace Wabbajack.Lib.Downloaders
public class CDPTree
{
public List<Depot> depot_list { get; set; }
public List<Depot> depot_list { get; set; } = null!;
public class Depot
{
public List<CDPFile> file_list { get; set; }
public List<CDPFile> file_list { get; set; } = null!;
public class CDPFile
{
public int chunk_count { get; set; }
public List<Chunk> chunk_list { get; set; }
public List<Chunk> chunk_list { get; set; } = null!;
public string name { get; set; }
public string? name { get; set; }
public class Chunk
{
public int chunk_size { get; set; }
public int index { get; set; }
public string sha { get; set; }
public string sha { get; set; } = string.Empty;
public int uncompressed_size { get; set; }
}
}
@ -344,7 +351,7 @@ namespace Wabbajack.Lib.Downloaders
public class RequestBethesdaNetLogin : AUserIntervention
{
public override string ShortDescription => "Logging into Bethesda.NET";
public override string ExtendedDescription { get; }
public override string ExtendedDescription { get; } = string.Empty;
private readonly TaskCompletionSource<BethesdaNetData> _source = new TaskCompletionSource<BethesdaNetData>();
public Task<BethesdaNetData> Task => _source.Task;
@ -365,8 +372,7 @@ namespace Wabbajack.Lib.Downloaders
public class BethesdaNetData
{
public string body { get; set; }
public string body { get; set; } = string.Empty;
public Dictionary<string, string> headers = new Dictionary<string, string>();
}
}

1
Wabbajack.Lib/Downloaders/DeadlyStreamDownloader.cs
View File

@ -1,4 +1,5 @@
using System;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{

5
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -115,10 +115,7 @@ namespace Wabbajack.Lib.Downloaders
var patchState = new Archive
{
Name = patchName,
State = new HTTPDownloader.State
{
Url = $"https://wabbajackcdn.b-cdn.net/updates/{patchName}"
}
State = new HTTPDownloader.State($"https://wabbajackcdn.b-cdn.net/updates/{patchName}")
};
var patchResult = await Download(patchState, patchPath);

10
Wabbajack.Lib/Downloaders/DropboxDownloader.cs
View File

@ -2,18 +2,19 @@
using System.Threading.Tasks;
using System.Web;
using Wabbajack.Common;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
public class DropboxDownloader : IDownloader, IUrlDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var urlstring = archiveINI?.General?.directURL;
return GetDownloaderState(urlstring);
}
public AbstractDownloadState GetDownloaderState(string url)
public AbstractDownloadState? GetDownloaderState(string url)
{
try
{
@ -29,10 +30,7 @@ namespace Wabbajack.Lib.Downloaders
uri.Query = query.ToString();
return new HTTPDownloader.State()
{
Url = uri.ToString().Replace("dropbox.com:443/", "dropbox.com/")
};
return new HTTPDownloader.State(uri.ToString().Replace("dropbox.com:443/", "dropbox.com/"));
}
catch (Exception)
{

25
Wabbajack.Lib/Downloaders/GameFileSourceDownloader.cs
View File

@ -4,17 +4,18 @@ using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib.Validation;
using Game = Wabbajack.Common.Game;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
public class GameFileSourceDownloader : IDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var gameName = (string)archiveINI?.General?.gameName;
var gameFile = (string)archiveINI?.General?.gameFile;
var gameName = (string?)archiveINI?.General?.gameName;
var gameFile = (string?)archiveINI?.General?.gameFile;
if (gameFile == null || gameFile == null)
if (gameName == null || gameFile == null)
return null;
var game = GameRegistry.GetByFuzzyName(gameName);
@ -23,18 +24,17 @@ namespace Wabbajack.Lib.Downloaders
var path = game.TryGetGameLocation();
var filePath = path?.Combine(gameFile);
if (!filePath?.Exists ?? false)
if (!(filePath?.Exists ?? false))
return null;
var fp = filePath.Value;
var hash = await fp.FileHashCachedAsync();
return new State
return new State(game.InstalledVersion)
{
Game = game.Game,
GameFile = (RelativePath)gameFile,
Hash = hash,
GameVersion = game.InstalledVersion
Hash = hash
};
}
@ -48,7 +48,12 @@ namespace Wabbajack.Lib.Downloaders
public Game Game { get; set; }
public RelativePath GameFile { get; set; }
public Hash Hash { get; set; }
public string GameVersion { get; set; }
public string GameVersion { get; }
public State(string gameVersion)
{
GameVersion = gameVersion;
}
[JsonIgnore]
internal AbsolutePath SourcePath => Game.MetaData().GameLocation().Combine(GameFile);
@ -81,7 +86,7 @@ namespace Wabbajack.Lib.Downloaders
return DownloadDispatcher.GetInstance<GameFileSourceDownloader>();
}
public override string GetManifestURL(Archive a)
public override string? GetManifestURL(Archive a)
{
return null;
}

21
Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs
View File

@ -5,27 +5,25 @@ using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib.Exceptions;
using Wabbajack.Lib.Validation;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
public class GoogleDriveDownloader : IDownloader, IUrlDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var url = archiveINI?.General?.directURL;
return GetDownloaderState(url);
}
public AbstractDownloadState GetDownloaderState(string url)
public AbstractDownloadState? GetDownloaderState(string url)
{
if (url != null && url.StartsWith("https://drive.google.com"))
{
var regex = new Regex("((?<=id=)[a-zA-Z0-9_-]*)|(?<=\\/file\\/d\\/)[a-zA-Z0-9_-]*");
var match = regex.Match(url);
return new State
{
Id = match.ToString()
};
return new State(match.ToString());
}
return null;
@ -38,10 +36,15 @@ namespace Wabbajack.Lib.Downloaders
[JsonName("GoogleDriveDownloader")]
public class State : AbstractDownloadState
{
public string Id { get; set; }
public string Id { get; }
[JsonIgnore]
public override object[] PrimaryKey { get => new object[] {Id}; }
public override object[] PrimaryKey => new object[] { Id };
public State(string id)
{
Id = id;
}
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
@ -64,7 +67,7 @@ namespace Wabbajack.Lib.Downloaders
var regex = new Regex("(?<=/uc\\?export=download&amp;confirm=).*(?=;id=)");
var confirm = regex.Match(await response.Content.ReadAsStringAsync());
var url = $"https://drive.google.com/uc?export=download&confirm={confirm}&id={Id}";
var httpState = new HTTPDownloader.State {Url = url, Client = client};
var httpState = new HTTPDownloader.State(url) { Client = client };
return httpState;
}

48
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -10,34 +10,30 @@ using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib.Exceptions;
using Wabbajack.Lib.Validation;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
public class HTTPDownloader : IDownloader, IUrlDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var url = archiveINI?.General?.directURL;
return GetDownloaderState(url, archiveINI);
}
public AbstractDownloadState GetDownloaderState(string uri)
public AbstractDownloadState? GetDownloaderState(string uri)
{
return GetDownloaderState(uri, null);
}
public AbstractDownloadState GetDownloaderState(string url, dynamic archiveINI)
public AbstractDownloadState? GetDownloaderState(string url, dynamic? archiveINI)
{
if (url != null)
{
var tmp = new State
{
Url = url
};
var tmp = new State(url);
if (archiveINI?.General?.directURLHeaders != null)
{
tmp.Headers = new List<string>();
tmp.Headers.AddRange(archiveINI?.General.directURLHeaders.Split('|'));
}
return tmp;
@ -53,15 +49,20 @@ namespace Wabbajack.Lib.Downloaders
[JsonName("HttpDownloader")]
public class State : AbstractDownloadState
{
public string Url { get; set; }
public string Url { get; }
public List<string> Headers { get; set; }
public List<string> Headers { get; } = new List<string>();
[JsonIgnore]
public Common.Http.Client Client { get; set; }
public Common.Http.Client? Client { get; set; }
[JsonIgnore]
public override object[] PrimaryKey { get => new object[] {Url};}
public override object[] PrimaryKey => new object[] { Url };
public State(string url)
{
Url = url;
}
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
@ -85,19 +86,18 @@ namespace Wabbajack.Lib.Downloaders
var client = Client ?? new Common.Http.Client();
client.Headers.Add(("User-Agent", Consts.UserAgent));
if (Headers != null)
foreach (var header in Headers)
{
var idx = header.IndexOf(':');
var k = header.Substring(0, idx);
var v = header.Substring(idx + 1);
client.Headers.Add((k, v));
}
foreach (var header in Headers)
{
var idx = header.IndexOf(':');
var k = header.Substring(0, idx);
var v = header.Substring(idx + 1);
client.Headers.Add((k, v));
}
long totalRead = 0;
var bufferSize = 1024 * 32;
Utils.Status($"Starting Download {a?.Name ?? Url}", Percent.Zero);
Utils.Status($"Starting Download {a.Name ?? Url}", Percent.Zero);
var response = await client.GetAsync(Url);
TOP:
@ -177,7 +177,7 @@ TOP:
if (read == 0) break;
Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(totalRead, contentSize));
fs.Write(buffer, 0, read);
fs!.Write(buffer, 0, read);
totalRead += read;
}
}
@ -203,7 +203,7 @@ TOP:
public override string[] GetMetaIni()
{
if (Headers != null)
if (Headers.Count > 0)
return new [] {"[General]",
$"directURL={Url}",
$"directURLHeaders={string.Join("|", Headers)}"};

4
Wabbajack.Lib/Downloaders/IDownloader.cs
View File

@ -1,15 +1,15 @@
using System.Threading.Tasks;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
public interface IDownloader
{
Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode = false);
Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode = false);
/// <summary>
/// Called before any downloads are inacted by the installer;
/// </summary>
Task Prepare();
}
}

3
Wabbajack.Lib/Downloaders/INeedsLogin.cs
View File

@ -2,6 +2,7 @@
using System.Reactive;
using System.Security;
using ReactiveUI;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
@ -13,7 +14,7 @@ namespace Wabbajack.Lib.Downloaders
string SiteName { get; }
IObservable<string> MetaInfo { get; }
Uri SiteURL { get; }
Uri IconUri { get; }
Uri? IconUri { get; }
}
public struct LoginReturnMessage

1
Wabbajack.Lib/Downloaders/LoversLabDownloader.cs
View File

@ -7,6 +7,7 @@ using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib.WebAutomation;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{

14
Wabbajack.Lib/Downloaders/MEGADownloader.cs
View File

@ -8,6 +8,7 @@ using Newtonsoft.Json;
using ReactiveUI;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
@ -23,8 +24,6 @@ namespace Wabbajack.Lib.Downloaders
try
{
authInfos = MegaApiClient.GenerateAuthInfos(username, password.ToNormalString());
username = null;
password = null;
}
catch (ApiException e)
{
@ -72,16 +71,16 @@ namespace Wabbajack.Lib.Downloaders
IsLoggedIn.ObserveOnGuiThread());
}
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var url = archiveINI?.General?.directURL;
return GetDownloaderState(url);
}
public AbstractDownloadState GetDownloaderState(string url)
public AbstractDownloadState? GetDownloaderState(string url)
{
if (url != null && url.StartsWith(Consts.MegaPrefix))
return new State { Url = url};
return new State(url);
return null;
}
@ -92,6 +91,11 @@ namespace Wabbajack.Lib.Downloaders
[JsonName("MegaDownloader")]
public class State : HTTPDownloader.State
{
public State(string url)
: base(url)
{
}
private static MegaApiClient MegaApiClient => DownloadDispatcher.GetInstance<MegaDownloader>().MegaApiClient;
private void MegaLogin()

26
Wabbajack.Lib/Downloaders/ManualDownloader.cs
View File

@ -6,6 +6,7 @@ using Wabbajack.Common;
using Wabbajack.Common.IO;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib.Validation;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
@ -18,8 +19,8 @@ namespace Wabbajack.Lib.Downloaders
class FileEvent
{
public string FullPath { get; set; }
public string Name { get; set; }
public string FullPath { get; set; } = string.Empty;
public string Name { get; set; } = string.Empty;
public long Size { get; set; }
}
@ -57,10 +58,10 @@ namespace Wabbajack.Lib.Downloaders
}
}
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var url = archiveINI?.General?.manualURL;
return url != null ? new State { Url = url} : null;
return url != null ? new State(url) : null;
}
public async Task Prepare()
@ -70,10 +71,15 @@ namespace Wabbajack.Lib.Downloaders
[JsonName("ManualDownloader")]
public class State : AbstractDownloadState
{
public string Url { get; set; }
public string Url { get; }
[JsonIgnore]
public override object[] PrimaryKey { get => new object[] {Url}; }
public override object[] PrimaryKey => new object[] { Url };
public State(string url)
{
Url = url;
}
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
@ -83,7 +89,7 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Download(Archive a, AbsolutePath destination)
{
var (uri, client) = await Utils.Log(await ManuallyDownloadFile.Create(this)).Task;
var state = new HTTPDownloader.State {Url = uri.ToString(), Client = client};
var state = new HTTPDownloader.State(uri.ToString()) { Client = client };
return await state.Download(a, destination);
}
@ -104,10 +110,10 @@ namespace Wabbajack.Lib.Downloaders
public override string[] GetMetaIni()
{
return new [] {
return new []
{
"[General]",
$"manualURL={Url}"
$"manualURL={Url}",
};
}
}

30
Wabbajack.Lib/Downloaders/MediaFireDownloader.cs
View File

@ -5,27 +5,30 @@ using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.Validation;
using Wabbajack.Lib.WebAutomation;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
public class MediaFireDownloader : IUrlDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
Uri url = DownloaderUtils.GetDirectURL(archiveINI);
if (url == null || url.Host != "www.mediafire.com") return null;
return new State
{
Url = url.ToString()
};
return new State(url.ToString());
}
public class State : AbstractDownloadState
{
public string Url { get; set; }
public string Url { get; }
public override object[] PrimaryKey { get => new object[] {Url};}
public override object[] PrimaryKey => new object[] { Url };
public State(string url)
{
Url = url;
}
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
@ -35,6 +38,7 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Download(Archive a, AbsolutePath destination)
{
var result = await Resolve();
if (result == null) return false;
return await result.Download(a, destination);
}
@ -43,7 +47,7 @@ namespace Wabbajack.Lib.Downloaders
return await Resolve() != null;
}
private async Task<HTTPDownloader.State> Resolve()
private async Task<HTTPDownloader.State?> Resolve()
{
using (var d = await Driver.Create())
{
@ -52,10 +56,9 @@ namespace Wabbajack.Lib.Downloaders
await Task.Delay(1000);
var newURL = await d.GetAttr("a.input", "href");
if (newURL == null || !newURL.StartsWith("http")) return null;
return new HTTPDownloader.State()
return new HTTPDownloader.State(newURL)
{
Client = new Common.Http.Client(),
Url = newURL
};
}
}
@ -84,15 +87,12 @@ namespace Wabbajack.Lib.Downloaders
{
}
public AbstractDownloadState GetDownloaderState(string u)
public AbstractDownloadState? GetDownloaderState(string u)
{
var url = new Uri(u);
if (url.Host != "www.mediafire.com") return null;
return new State
{
Url = url.ToString()
};
return new State(url.ToString());
}
}
}

21
Wabbajack.Lib/Downloaders/ModDBDownloader.cs
View File

@ -6,25 +6,23 @@ using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
using Wabbajack.Lib.Validation;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
public class ModDBDownloader : IDownloader, IUrlDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var url = archiveINI?.General?.directURL;
return GetDownloaderState(url);
}
public AbstractDownloadState GetDownloaderState(string url)
public AbstractDownloadState? GetDownloaderState(string url)
{
if (url != null && url.StartsWith("https://www.moddb.com/downloads/start"))
{
return new State
{
Url = url
};
return new State(url);
}
return null;
@ -37,10 +35,15 @@ namespace Wabbajack.Lib.Downloaders
[JsonName("ModDBDownloader")]
public class State : AbstractDownloadState
{
public string Url { get; set; }
public string Url { get; }
[JsonIgnore]
public override object[] PrimaryKey { get => new object[]{Url}; }
public override object[] PrimaryKey => new object[] { Url };
public State(string url)
{
Url = url;
}
public override bool IsWhitelisted(ServerWhitelist whitelist)
{
@ -56,7 +59,7 @@ namespace Wabbajack.Lib.Downloaders
{
try
{
await new HTTPDownloader.State {Url = url}.Download(a, destination);
await new HTTPDownloader.State(url).Download(a, destination);
return true;
}
catch (Exception)

24
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -12,6 +12,7 @@ using Wabbajack.Common.StatusFeed.Errors;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using Game = Wabbajack.Common.Game;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
@ -19,8 +20,8 @@ namespace Wabbajack.Lib.Downloaders
{
private bool _prepared;
private AsyncLock _lock = new AsyncLock();
private UserStatus _status;
private NexusApiClient _client;
private UserStatus? _status;
private NexusApiClient? _client;
public IObservable<bool> IsLoggedIn => Utils.HaveEncryptedJsonObservable("nexusapikey");
@ -50,9 +51,9 @@ namespace Wabbajack.Lib.Downloaders
canExecute: IsLoggedIn.ObserveOnGuiThread());
}
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var general = archiveINI?.General;
var general = archiveINI.General;
if (general.modID != null && general.fileID != null && general.gameName != null)
{
@ -135,17 +136,17 @@ namespace Wabbajack.Lib.Downloaders
[JsonIgnore]
public Uri URL => new Uri($"http://nexusmods.com/{Game.MetaData().NexusName}/mods/{ModID}");
public string Name { get; set; }
public string? Name { get; set; }
public string Author { get; set; }
public string? Author { get; set; }
public string Version { get; set; }
public string? Version { get; set; }
public string ImageURL { get; set; }
public string? ImageURL { get; set; }
public bool IsNSFW { get; set; }
public string Description { get; set; }
public string? Description { get; set; }
[JsonProperty("GameName")]
[JsonConverter(typeof(Utils.GameConverter))]
@ -184,10 +185,7 @@ namespace Wabbajack.Lib.Downloaders
Utils.Log($"Downloading Nexus Archive - {a.Name} - {Game} - {ModID} - {FileID}");
return await new HTTPDownloader.State
{
Url = url
}.Download(a, destination);
return await new HTTPDownloader.State(url).Download(a, destination);
}
public override async Task<bool> Verify(Archive a)

1
Wabbajack.Lib/Downloaders/TESAllianceDownloader.cs
View File

@ -1,6 +1,7 @@
using System;
using Wabbajack.Common;
using Wabbajack.Common.Serialization.Json;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{

1
Wabbajack.Lib/Downloaders/VectorPlexusDownloader.cs
View File

@ -1,5 +1,6 @@
using System;
using Wabbajack.Common.Serialization.Json;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{

22
Wabbajack.Lib/Downloaders/YouTubeDownloader.cs
View File

@ -15,15 +15,16 @@ using YoutubeExplode.Exceptions;
using YoutubeExplode.Models.MediaStreams;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
#nullable enable
namespace Wabbajack.Lib.Downloaders
{
public class YouTubeDownloader : IDownloader
{
public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI, bool quickMode)
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode)
{
var directURL = (Uri)DownloaderUtils.GetDirectURL(archiveINI);
var state = (State)UriToState(directURL);
var state = UriToState(directURL) as State;
if (state == null) return state;
var idx = 0;
@ -44,7 +45,7 @@ namespace Wabbajack.Lib.Downloaders
return state;
}
internal static AbstractDownloadState UriToState(Uri directURL)
internal static AbstractDownloadState? UriToState(Uri directURL)
{
if (directURL == null || !directURL.Host.EndsWith("youtube.com"))
{
@ -52,7 +53,7 @@ namespace Wabbajack.Lib.Downloaders
}
var key = HttpUtility.ParseQueryString(directURL.Query)["v"];
return key != null ? new State {Key = key} : null;
return key != null ? new State(key) : null;
}
public async Task Prepare()
@ -62,13 +63,18 @@ namespace Wabbajack.Lib.Downloaders
[JsonName("YouTubeDownloader")]
public class State : AbstractDownloadState
{
public string Key { get; set; }
public string Key { get; }
public List<Track> Tracks { get; set; } = new List<Track>();
[JsonIgnore]
public override object[] PrimaryKey => new object[] {Key};
public State(string key)
{
Key = key;
}
[JsonName("YouTubeTrack")]
public class Track
{
@ -78,8 +84,8 @@ namespace Wabbajack.Lib.Downloaders
WAV
}
public FormatEnum Format { get; set; }
public string Name { get; set; }
public string Name { get; set; } = string.Empty;
public TimeSpan Start { get; set; }
@ -164,7 +170,7 @@ namespace Wabbajack.Lib.Downloaders
p.Start();
ChildProcessTracker.AddProcess(p);
var output = await p.StandardError.ReadToEndAsync();
await p.StandardError.ReadToEndAsync();
try
{

8
Wabbajack.Test/ContentRightsManagementTests.cs
View File

@ -73,7 +73,7 @@ namespace Wabbajack.Test
modlist.GameType = Game.Skyrim;
modlist.Archives[0] = new Archive()
{
State = new HTTPDownloader.State() { Url = "https://somebadplace.com" },
State = new HTTPDownloader.State("https://somebadplace.com"),
Hash = Hash.FromLong(42)
};
var errors = await validate.Validate(modlist);
@ -83,7 +83,7 @@ namespace Wabbajack.Test
modlist.GameType = Game.Skyrim;
modlist.Archives[0] = new Archive
{
State = new HTTPDownloader.State { Url = "https://somegoodplace.com/baz.7z" },
State = new HTTPDownloader.State("https://somegoodplace.com/baz.7z"),
Hash = Hash.FromLong(42)
};
errors = await validate.Validate(modlist);
@ -94,7 +94,7 @@ namespace Wabbajack.Test
modlist.GameType = Game.Skyrim;
modlist.Archives[0] = new Archive
{
State = new GoogleDriveDownloader.State { Id = "bleg"},
State = new GoogleDriveDownloader.State("bleg"),
Hash = Hash.FromLong(42)
};
errors = await validate.Validate(modlist);
@ -104,7 +104,7 @@ namespace Wabbajack.Test
modlist.GameType = Game.Skyrim;
modlist.Archives[0] = new Archive
{
State = new GoogleDriveDownloader.State { Id = "googleDEADBEEF" },
State = new GoogleDriveDownloader.State("googleDEADBEEF"),
Hash = Hash.FromLong(42)
};
errors = await validate.Validate(modlist);

8
Wabbajack.Test/RestartingDownloadsTests.cs
View File

@ -72,8 +72,8 @@ namespace Wabbajack.Test
{
using var testFile = new TempFile();
using var server = new CrappyRandomServer();
var state = new HTTPDownloader.State {Url = $"http://localhost:{server.Port}/foo"};
var state = new HTTPDownloader.State($"http://localhost:{server.Port}/foo");
await state.Download(testFile.Path);
Assert.Equal(server.Data, await testFile.Path.ReadAllBytesAsync());
@ -87,8 +87,4 @@ namespace Wabbajack.Test
_unsubIntevention?.Dispose();
}
}
}