From bf19ad959a1450ef2d4e8dfd5ca8fd0ed65c53db Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 8 Feb 2020 16:53:11 -0700 Subject: [PATCH] Command line options for downloading files from the CLI, fixes for Bethesda.NET integration. --- Wabbajack.CLI/OptionsDefinition.cs | 2 +- Wabbajack.CLI/Program.cs | 1 + Wabbajack.CLI/Verbs/DownloadUrl.cs | 50 +++++++++++++++++++ Wabbajack.Common/Extensions/RxExt.cs | 2 +- .../Downloaders/BethesdaNetDownloader.cs | 17 ++++--- .../Downloaders/DownloadDispatcher.cs | 11 ++++ .../UrlDownloaders/BethesdaNetInferencer.cs | 12 +++++ .../UrlDownloaders/IUrlInferencer.cs | 9 ++++ 8 files changed, 96 insertions(+), 8 deletions(-) create mode 100644 Wabbajack.CLI/Verbs/DownloadUrl.cs create mode 100644 Wabbajack.Lib/Downloaders/UrlDownloaders/BethesdaNetInferencer.cs create mode 100644 Wabbajack.Lib/Downloaders/UrlDownloaders/IUrlInferencer.cs diff --git a/Wabbajack.CLI/OptionsDefinition.cs b/Wabbajack.CLI/OptionsDefinition.cs index 5cf11f0d..dd176155 100644 --- a/Wabbajack.CLI/OptionsDefinition.cs +++ b/Wabbajack.CLI/OptionsDefinition.cs @@ -6,7 +6,7 @@ namespace Wabbajack.CLI public class OptionsDefinition { public static Type[] AllOptions = { - typeof(OptionsDefinition), typeof(Encrypt), typeof(Decrypt), typeof(Validate) + typeof(OptionsDefinition), typeof(Encrypt), typeof(Decrypt), typeof(Validate), typeof(DownloadUrl) }; } } diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index 9d0c3236..c1d043ae 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -12,6 +12,7 @@ namespace Wabbajack.CLI (Encrypt opts) => Encrypt.Run(opts), (Decrypt opts) => Decrypt.Run(opts), (Validate opts) => Validate.Run(opts), + (DownloadUrl opts) => DownloadUrl.Run(opts), errs => 1); } } diff --git a/Wabbajack.CLI/Verbs/DownloadUrl.cs b/Wabbajack.CLI/Verbs/DownloadUrl.cs new file mode 100644 index 00000000..173dcc22 --- /dev/null +++ b/Wabbajack.CLI/Verbs/DownloadUrl.cs @@ -0,0 +1,50 @@ +using System; +using System.Reactive.Disposables; +using System.Reactive.Linq; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("download-url", HelpText = "Infer a download state from a URL and download it")] + public class DownloadUrl + { + [Option('u', "url", Required = true, HelpText = "Url to download")] + public Uri Url { get; set; } + + [Option('o', "output", Required = true, HelpText = "Output file name")] + public string Output { get; set; } + + public static int Run(DownloadUrl opts) + { + var state = DownloadDispatcher.Infer(opts.Url); + if (state == null) + { + Console.WriteLine($"Could not find download source for URL {opts.Url}"); + return 1; + } + + DownloadDispatcher.PrepareAll(new []{state}); + + using var queue = new WorkQueue(); + queue.Status + .Where(s => s.ProgressPercent != Percent.Zero) + .Debounce(TimeSpan.FromSeconds(1)) + .Subscribe(s => Console.WriteLine($"Downloading {s.ProgressPercent}")); + + new[] {state} + .PMap(queue, async s => + { + await s.Download(new Archive {Name = Path.GetFileName(opts.Output)}, opts.Output); + }).Wait(); + + File.WriteAllLines(opts.Output + ".meta", state.GetMetaIni()); + return 0; + } + + } +} diff --git a/Wabbajack.Common/Extensions/RxExt.cs b/Wabbajack.Common/Extensions/RxExt.cs index e0ad02ce..ddc3f1fe 100644 --- a/Wabbajack.Common/Extensions/RxExt.cs +++ b/Wabbajack.Common/Extensions/RxExt.cs @@ -88,7 +88,7 @@ namespace Wabbajack /// Inspiration: /// http://reactivex.io/documentation/operators/debounce.html /// https://stackoverflow.com/questions/20034476/how-can-i-use-reactive-extensions-to-throttle-events-using-a-max-window-size - public static IObservable Debounce(this IObservable source, TimeSpan interval, IScheduler scheduler) + public static IObservable Debounce(this IObservable source, TimeSpan interval, IScheduler scheduler = null) { scheduler = scheduler ?? Scheduler.Default; return Observable.Create(o => diff --git a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs index 59696f10..2c9dac4f 100644 --- a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs +++ b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs @@ -45,7 +45,7 @@ namespace Wabbajack.Lib.Downloaders return StateFromUrl(url); } - private static AbstractDownloadState StateFromUrl(Uri url) + internal static AbstractDownloadState StateFromUrl(Uri url) { if (url != null && url.Host == "bethesda.net" && url.AbsolutePath.StartsWith("/en/mods/")) { @@ -137,7 +137,8 @@ namespace Wabbajack.Lib.Downloaders var got = await client.GetAsync( $"https://content.cdp.bethesda.net/{collected.CDPProductId}/{collected.CDPPropertiesId}/{chunk.sha}"); var data = await got.Content.ReadAsByteArrayAsync(); - AESCTRDecrypt(collected.AESKey, collected.AESIV, data); + if (collected.AESKey != null) + AESCTRDecrypt(collected.AESKey, collected.AESIV, data); if (chunk.uncompressed_size == chunk.chunk_size) await file.WriteAsync(data, 0, data.Length); @@ -233,20 +234,24 @@ namespace Wabbajack.Lib.Downloaders info.CDPProductId = (int)content["cdp_product_id"]; client.DefaultRequestHeaders.Add("Authorization", $"Token {info.CDPToken}"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); got = await client.GetAsync( $"/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/tree/.json"); var tree = (await got.Content.ReadAsStringAsync()).FromJSONString(); + + got = await client.PostAsync($"/mods/ugc-content/add-subscription", new StringContent($"{{\"content_id\": \"{ContentId}\"}}", Encoding.UTF8, "application/json")); got = await client.GetAsync( $"/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/depots/.json"); var props_obj = JObject.Parse(await got.Content.ReadAsStringAsync()).Properties().First(); info.CDPPropertiesId = (int)props_obj.Value["properties_id"]; + info.AESKey = props_obj.Value["ex_info_A"].Select(e => (byte)e).ToArray(); info.AESIV = props_obj.Value["ex_info_B"].Select(e => (byte)e).Take(16).ToArray(); - + return (client, tree, info); } @@ -260,17 +265,17 @@ namespace Wabbajack.Lib.Downloaders public override IDownloader GetDownloader() { - throw new NotImplementedException(); + return DownloadDispatcher.GetInstance(); } public override string GetManifestURL(Archive a) { - throw new NotImplementedException(); + return $"https://bethesda.net/en/mods/{GameName}/mod-detail/{ContentId}"; } public override string[] GetMetaIni() { - throw new NotImplementedException(); + return new[] {"[General]", $"directURL=https://bethesda.net/en/mods/{GameName}/mod-detail/{ContentId}"}; } diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index 27b9728e..75b4be49 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Wabbajack.Common; +using Wabbajack.Lib.Downloaders.UrlDownloaders; namespace Wabbajack.Lib.Downloaders { @@ -27,6 +28,11 @@ namespace Wabbajack.Lib.Downloaders new ManualDownloader(), }; + public static readonly List Inferencers = new List() + { + new BethesdaNetInferencer() + }; + private static readonly Dictionary IndexedDownloaders; static DownloadDispatcher() @@ -34,6 +40,11 @@ namespace Wabbajack.Lib.Downloaders IndexedDownloaders = Downloaders.ToDictionary(d => d.GetType()); } + public static AbstractDownloadState Infer(Uri uri) + { + return Inferencers.Select(infer => infer.Infer(uri)).FirstOrDefault(result => result != null); + } + public static T GetInstance() where T : IDownloader { var inst = (T)IndexedDownloaders[typeof(T)]; diff --git a/Wabbajack.Lib/Downloaders/UrlDownloaders/BethesdaNetInferencer.cs b/Wabbajack.Lib/Downloaders/UrlDownloaders/BethesdaNetInferencer.cs new file mode 100644 index 00000000..273f415d --- /dev/null +++ b/Wabbajack.Lib/Downloaders/UrlDownloaders/BethesdaNetInferencer.cs @@ -0,0 +1,12 @@ +using System; + +namespace Wabbajack.Lib.Downloaders.UrlDownloaders +{ + public class BethesdaNetInferencer : IUrlInferencer + { + public AbstractDownloadState Infer(Uri uri) + { + return BethesdaNetDownloader.StateFromUrl(uri); + } + } +} diff --git a/Wabbajack.Lib/Downloaders/UrlDownloaders/IUrlInferencer.cs b/Wabbajack.Lib/Downloaders/UrlDownloaders/IUrlInferencer.cs new file mode 100644 index 00000000..b9ad04f7 --- /dev/null +++ b/Wabbajack.Lib/Downloaders/UrlDownloaders/IUrlInferencer.cs @@ -0,0 +1,9 @@ +using System; + +namespace Wabbajack.Lib.Downloaders.UrlDownloaders +{ + public interface IUrlInferencer + { + AbstractDownloadState Infer(Uri uri); + } +}