Merge pull request #494 from wabbajack-tools/beth-net-cli-download

Command line options for downloading files

Wabbajack.CLI/OptionsDefinition.cs

@@ -6,7 +6,7 @@ namespace Wabbajack.CLI
     public class OptionsDefinition
     {
         public static Type[] AllOptions = {
-            typeof(OptionsDefinition), typeof(Encrypt), typeof(Decrypt), typeof(Validate)
+            typeof(OptionsDefinition), typeof(Encrypt), typeof(Decrypt), typeof(Validate), typeof(DownloadUrl)
         };
     }
 }

Wabbajack.CLI/Program.cs

@@ -12,6 +12,7 @@ namespace Wabbajack.CLI
                     (Encrypt opts) => Encrypt.Run(opts),
                     (Decrypt opts) => Decrypt.Run(opts),
                     (Validate opts) => Validate.Run(opts),
+                    (DownloadUrl opts) => DownloadUrl.Run(opts),
                     errs => 1);
         }
     }

Wabbajack.CLI/Verbs/DownloadUrl.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Reactive.Disposables;
+using System.Reactive.Linq;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using CommandLine;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+
+namespace Wabbajack.CLI.Verbs
+{
+    [Verb("download-url", HelpText = "Infer a download state from a URL and download it")]
+    public class DownloadUrl
+    {
+        [Option('u', "url", Required = true, HelpText = "Url to download")]
+        public Uri Url { get; set; }
+
+        [Option('o', "output", Required = true, HelpText = "Output file name")]
+        public string Output { get; set; }
+
+        public static int Run(DownloadUrl opts)
+        {
+            var state = DownloadDispatcher.Infer(opts.Url);
+            if (state == null)
+            {
+                Console.WriteLine($"Could not find download source for URL {opts.Url}");
+                return 1;
+            }
+            
+            DownloadDispatcher.PrepareAll(new []{state});
+
+            using var queue = new WorkQueue();
+            queue.Status
+                .Where(s => s.ProgressPercent != Percent.Zero)
+                .Debounce(TimeSpan.FromSeconds(1))
+                .Subscribe(s => Console.WriteLine($"Downloading {s.ProgressPercent}"));
+
+                new[] {state}
+                .PMap(queue, async s =>
+                {
+                    await s.Download(new Archive {Name = Path.GetFileName(opts.Output)}, opts.Output);
+                }).Wait();
+
+            File.WriteAllLines(opts.Output + ".meta", state.GetMetaIni());
+            return 0;
+        }
+
+    }
+}

Wabbajack.Common/Extensions/RxExt.cs

@@ -88,7 +88,7 @@ namespace Wabbajack
         /// Inspiration:
         /// http://reactivex.io/documentation/operators/debounce.html
         /// https://stackoverflow.com/questions/20034476/how-can-i-use-reactive-extensions-to-throttle-events-using-a-max-window-size
-        public static IObservable<T> Debounce<T>(this IObservable<T> source, TimeSpan interval, IScheduler scheduler)
+        public static IObservable<T> Debounce<T>(this IObservable<T> source, TimeSpan interval, IScheduler scheduler = null)
         {
             scheduler = scheduler ?? Scheduler.Default;
             return Observable.Create<T>(o =>

Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs

@@ -45,7 +45,7 @@ namespace Wabbajack.Lib.Downloaders
             return StateFromUrl(url);
         }
 
-        private static AbstractDownloadState StateFromUrl(Uri url)
+        internal static AbstractDownloadState StateFromUrl(Uri url)
         {
             if (url != null && url.Host == "bethesda.net" && url.AbsolutePath.StartsWith("/en/mods/"))
             {
@@ -137,6 +137,7 @@ namespace Wabbajack.Lib.Downloaders
                     var got = await client.GetAsync(
                         $"https://content.cdp.bethesda.net/{collected.CDPProductId}/{collected.CDPPropertiesId}/{chunk.sha}");
                     var data = await got.Content.ReadAsByteArrayAsync();
+                    if (collected.AESKey != null) 
                         AESCTRDecrypt(collected.AESKey, collected.AESIV, data);
 
                     if (chunk.uncompressed_size == chunk.chunk_size)
@@ -233,17 +234,21 @@ namespace Wabbajack.Lib.Downloaders
                 info.CDPProductId = (int)content["cdp_product_id"];
 
                 client.DefaultRequestHeaders.Add("Authorization", $"Token {info.CDPToken}");
+                client.DefaultRequestHeaders.Add("Accept", "application/json");
 
                 got = await client.GetAsync(
                     $"/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/tree/.json");
 
                 var tree = (await got.Content.ReadAsStringAsync()).FromJSONString<CDPTree>();
                 
+                got = await client.PostAsync($"/mods/ugc-content/add-subscription", new StringContent($"{{\"content_id\": \"{ContentId}\"}}", Encoding.UTF8, "application/json"));
+
                 got = await client.GetAsync(
                     $"/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/depots/.json");
 
                 var props_obj = JObject.Parse(await got.Content.ReadAsStringAsync()).Properties().First();
                 info.CDPPropertiesId = (int)props_obj.Value["properties_id"];
+                
                 info.AESKey = props_obj.Value["ex_info_A"].Select(e => (byte)e).ToArray();
                 info.AESIV = props_obj.Value["ex_info_B"].Select(e => (byte)e).Take(16).ToArray();
 
@@ -260,17 +265,17 @@ namespace Wabbajack.Lib.Downloaders
 
             public override IDownloader GetDownloader()
             {
-                throw new NotImplementedException();
+                return DownloadDispatcher.GetInstance<BethesdaNetDownloader>();
             }
 
             public override string GetManifestURL(Archive a)
             {
-                throw new NotImplementedException();
+                return $"https://bethesda.net/en/mods/{GameName}/mod-detail/{ContentId}";
             }
 
             public override string[] GetMetaIni()
             {
-                throw new NotImplementedException();
+                return new[] {"[General]", $"directURL=https://bethesda.net/en/mods/{GameName}/mod-detail/{ContentId}"};
             }
 
 

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Wabbajack.Common;
+using Wabbajack.Lib.Downloaders.UrlDownloaders;
 
 namespace Wabbajack.Lib.Downloaders
 {
@@ -27,6 +28,11 @@ namespace Wabbajack.Lib.Downloaders
             new ManualDownloader(),
         };
 
+        public static readonly List<IUrlInferencer> Inferencers = new List<IUrlInferencer>()
+        {
+            new BethesdaNetInferencer()
+        };
+
         private static readonly Dictionary<Type, IDownloader> IndexedDownloaders;
 
         static DownloadDispatcher()
@@ -34,6 +40,11 @@ namespace Wabbajack.Lib.Downloaders
             IndexedDownloaders = Downloaders.ToDictionary(d => d.GetType());
         }
 
+        public static AbstractDownloadState Infer(Uri uri)
+        {
+            return Inferencers.Select(infer => infer.Infer(uri)).FirstOrDefault(result => result != null);
+        }
+
         public static T GetInstance<T>() where T : IDownloader
         {
             var inst = (T)IndexedDownloaders[typeof(T)];

Wabbajack.Lib/Downloaders/UrlDownloaders/BethesdaNetInferencer.cs

@@ -0,0 +1,12 @@
+using System;
+
+namespace Wabbajack.Lib.Downloaders.UrlDownloaders
+{
+    public class BethesdaNetInferencer : IUrlInferencer
+    {
+        public AbstractDownloadState Infer(Uri uri)
+        {
+            return BethesdaNetDownloader.StateFromUrl(uri);
+        }
+    }
+}

Wabbajack.Lib/Downloaders/UrlDownloaders/IUrlInferencer.cs

@@ -0,0 +1,9 @@
+using System;
+
+namespace Wabbajack.Lib.Downloaders.UrlDownloaders
+{
+    public interface IUrlInferencer
+    {
+        AbstractDownloadState Infer(Uri uri);
+    }
+}