Youtube Downloader (#596)

* Can download audio tracks from Youtube, re-encoding to XWM in the process

Wabbajack.CLI/Verbs/DownloadUrl.cs

@@ -21,7 +21,7 @@ namespace Wabbajack.CLI.Verbs
 
         protected override async Task<int> Run()
         {
-            var state = DownloadDispatcher.Infer(Url);
+            var state = await DownloadDispatcher.Infer(Url);
             if (state == null)
                 return CLIUtils.Exit($"Could not find download source for URL {Url}", 1);
 

Wabbajack.Common/DynamicIniData.cs

@@ -34,6 +34,18 @@ namespace Wabbajack.Common
             result = new SectionData(_value[binder.Name]);
             return true;
         }
+        
+        public override bool TryGetIndex(GetIndexBinder binder, object[] indexes, out object result)
+        {
+            if (indexes.Length > 1)
+            {
+                result = null;
+                return false;
+            }
+
+            result = new SectionData(_value[(string) indexes[0]]);
+            return true;
+        }
     }
 
     public class SectionData : DynamicObject

Wabbajack.Common/Http/ClientFactory.cs

@@ -6,7 +6,7 @@ namespace Wabbajack.Common.Http
     public static class ClientFactory
     {
         private static SysHttp.SocketsHttpHandler _socketsHandler { get; }
-        internal static SysHttp.HttpClient Client { get; }
+        public static SysHttp.HttpClient Client { get; }
         internal static CookieContainer Cookies { get; }
 
         static ClientFactory()

Wabbajack.Lib/CerasConfig.cs

@@ -30,7 +30,7 @@ namespace Wabbajack.Lib
                     typeof(PropertyFile), typeof(SteamMeta), typeof(SteamWorkshopDownloader), typeof(SteamWorkshopDownloader.State),
                     typeof(LoversLabDownloader.State), typeof(GameFileSourceDownloader.State), typeof(VectorPlexusDownloader.State),
                     typeof(DeadlyStreamDownloader.State), typeof(AFKModsDownloader.State), typeof(TESAllianceDownloader.State),
-                    typeof(TES3ArchiveState), typeof(TES3FileState), typeof(BethesdaNetDownloader.State)
+                    typeof(TES3ArchiveState), typeof(TES3FileState), typeof(BethesdaNetDownloader.State), typeof(YouTubeDownloader)
                 },
             };
             Config.VersionTolerance.Mode = VersionToleranceMode.Standard;

Wabbajack.Lib/Downloaders/AbstractDownloadState.cs

@@ -26,7 +26,8 @@ namespace Wabbajack.Lib.Downloaders
             typeof(DeadlyStreamDownloader.State),
             typeof(AFKModsDownloader.State),
             typeof(TESAllianceDownloader.State),
-            typeof(BethesdaNetDownloader.State)
+            typeof(BethesdaNetDownloader.State),
+            typeof(YouTubeDownloader.State)
         };
         public static Dictionary<string, Type> NameToType { get; set; }
         public static Dictionary<Type, string> TypeToName { get; set; }

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -25,13 +25,15 @@ namespace Wabbajack.Lib.Downloaders
             new BethesdaNetDownloader(),
             new AFKModsDownloader(),
             new TESAllianceDownloader(),
+            new YouTubeDownloader(),
             new HTTPDownloader(),
             new ManualDownloader(),
         };
 
         public static readonly List<IUrlInferencer> Inferencers = new List<IUrlInferencer>()
         {
-            new BethesdaNetInferencer()
+            new BethesdaNetInferencer(),
+            new YoutubeInferencer()
         };
 
         private static readonly Dictionary<Type, IDownloader> IndexedDownloaders;
@@ -41,9 +43,16 @@ namespace Wabbajack.Lib.Downloaders
             IndexedDownloaders = Downloaders.ToDictionary(d => d.GetType());
         }
 
-        public static AbstractDownloadState Infer(Uri uri)
+        public static async Task<AbstractDownloadState> Infer(Uri uri)
         {
-            return Inferencers.Select(infer => infer.Infer(uri)).FirstOrDefault(result => result != null);
+            foreach (var inf in Inferencers)
+            {
+                var state = await inf.Infer(uri);
+                if (state != null)
+                    return state;
+            }
+            return null;
+
         }
 
         public static T GetInstance<T>() where T : IDownloader

Wabbajack.Lib/Downloaders/UrlDownloaders/BethesdaNetInferencer.cs

@@ -1,10 +1,11 @@
 using System;
+using System.Threading.Tasks;
 
 namespace Wabbajack.Lib.Downloaders.UrlDownloaders
 {
     public class BethesdaNetInferencer : IUrlInferencer
     {
-        public AbstractDownloadState Infer(Uri uri)
+        public async Task<AbstractDownloadState> Infer(Uri uri)
         {
             return BethesdaNetDownloader.StateFromUrl(uri);
         }

Wabbajack.Lib/Downloaders/UrlDownloaders/IUrlInferencer.cs

@@ -1,9 +1,10 @@
 using System;
+using System.Threading.Tasks;
 
 namespace Wabbajack.Lib.Downloaders.UrlDownloaders
 {
     public interface IUrlInferencer
     {
-        AbstractDownloadState Infer(Uri uri);
+        Task<AbstractDownloadState> Infer(Uri uri);
     }
 }

Wabbajack.Lib/Downloaders/UrlDownloaders/YoutubeInferencer.cs

@@ -0,0 +1,81 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using YoutubeExplode;
+
+namespace Wabbajack.Lib.Downloaders.UrlDownloaders
+{
+    public class YoutubeInferencer : IUrlInferencer
+    {
+        
+        public async Task<AbstractDownloadState> Infer(Uri uri)
+        {
+            var state = (YouTubeDownloader.State)YouTubeDownloader.UriToState(uri);
+            if (state == null) return null;
+            
+            var client = new YoutubeClient(Common.Http.ClientFactory.Client);
+            var video = await client.GetVideoAsync(state.Key);
+
+            var desc = video.Description;
+
+            var lines = desc.Split('\n', StringSplitOptions.RemoveEmptyEntries)
+                .Select(line => line.Trim())
+                .Select(line =>
+                {
+                    var segments = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
+                    if (segments.Length == 0) return (TimeSpan.Zero, null);
+                    
+                    if (TryParseEx(segments.First(), out var s1))
+                        return (s1, string.Join(" ", segments.Skip(1)));
+                    if (TryParseEx(Enumerable.Last(segments), out var s2))
+                        return (s2, string.Join(" ", Utils.ButLast(segments)));
+                    return (TimeSpan.Zero, null);
+                })
+                .Where(t => t.Item2 != null)
+                .ToList();
+
+            var tracks = lines.Select((line, idx) => new YouTubeDownloader.State.Track
+            {
+                Name = Sanitize(line.Item2),
+                Start = line.Item1,
+                End = idx < lines.Count - 1 ? lines[idx + 1].Item1 : video.Duration,
+                Format = YouTubeDownloader.State.Track.FormatEnum.XWM
+            }).ToList();
+
+            state.Tracks = tracks;
+            
+            return state;
+        }
+
+        private string Sanitize(string input)
+        {
+            return input.Replace(":", "_").Replace("'", "").Replace("\"", "");
+        }
+
+        private static bool TryParseEx(string s, out TimeSpan span)
+        {
+            var ints = s.Split(':').Select(segment => int.TryParse(segment, out int v) ? v : -1).ToArray();
+            if (ints.Any(i => i == -1))
+            {
+                span = TimeSpan.Zero;
+                return false;
+            }
+
+            switch (ints.Length)
+            {
+                case 2:
+                    span = new TimeSpan(0, ints[0], ints[1]);
+                    break;
+                case 3:
+                    span = new TimeSpan(ints[0], ints[1], ints[2]);
+                    break;
+                default:
+                    span = TimeSpan.Zero;
+                    return false;
+            }
+
+            return true;
+        }
+    }
+}

Wabbajack.Lib/Downloaders/YouTubeDownloader.cs

@@ -0,0 +1,261 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Web;
+using Wabbajack.Common;
+using Wabbajack.Lib.Validation;
+using YoutubeExplode;
+using YoutubeExplode.Exceptions;
+using YoutubeExplode.Models.MediaStreams;
+using File = Alphaleonis.Win32.Filesystem.File;
+using Path = Alphaleonis.Win32.Filesystem.Path;
+
+namespace Wabbajack.Lib.Downloaders
+{
+    public class YouTubeDownloader : IDownloader
+    {
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
+        {
+            var directURL = (Uri)DownloaderUtils.GetDirectURL(archiveINI);
+            var state = (State)UriToState(directURL);
+            if (state == null) return state;
+
+            var idx = 0;
+            while (true)
+            {
+                var section = archiveINI[$"track/{idx}"];
+                if (section.name == null) break;
+                
+                var track = new State.Track();
+                track.Name = section.name;
+                track.Start = TimeSpan.Parse(section.start);
+                track.End = TimeSpan.Parse(section.end);
+                track.Format = Enum.Parse<State.Track.FormatEnum>(section.format);
+                state.Tracks.Add(track);
+                idx += 1;
+            }
+
+            return state;
+        }
+
+        internal static AbstractDownloadState UriToState(Uri directURL)
+        {
+            if (directURL == null || !directURL.Host.EndsWith("youtube.com"))
+            {
+                return null;
+            }
+
+            var key = HttpUtility.ParseQueryString(directURL.Query)["v"];
+            return key != null ? new State {Key = key} : null;
+        }
+
+        public async Task Prepare()
+        {
+        }
+
+        public class State : AbstractDownloadState
+        {
+            public string Key { get; set; }
+            public List<Track> Tracks { get; set; } = new List<Track>();
+            public override object[] PrimaryKey => new object[] {Key};
+
+            public class Track
+            {
+                public enum FormatEnum
+                {
+                    XWM,
+                    WAV
+                }
+                public FormatEnum Format { get; set; }
+                public string Name { get; set; }
+                public TimeSpan Start { get; set; }
+                public TimeSpan End { get; set; }
+            }
+            
+            public override bool IsWhitelisted(ServerWhitelist whitelist)
+            {
+                return true;
+            }
+
+            public override async Task<bool> Download(Archive a, string destination)
+            {
+                try
+                {
+                    using var queue = new WorkQueue();
+                    using var folder = new TempFolder();
+                    Directory.CreateDirectory(Path.Combine(folder.Dir.FullName, "tracks"));
+                    var client = new YoutubeClient(Common.Http.ClientFactory.Client);
+                    var meta = await client.GetVideoAsync(Key);
+                    var video = await client.GetVideoMediaStreamInfosAsync(Key);
+                    var all = video.GetAll();
+                    var stream = video.GetAll().OfType<AudioStreamInfo>().Where(f => f.AudioEncoding == AudioEncoding.Aac).OrderByDescending(a => a.Bitrate)
+                        .ToArray().First();
+
+                    var initialDownload = Path.Combine(folder.Dir.FullName, "initial_download");
+
+                    var trackFolder = Path.Combine(folder.Dir.FullName, "tracks");
+
+                    await using (var fs = File.Create(initialDownload))
+                    {
+                        await client.DownloadMediaStreamAsync(stream, fs, new Progress($"Downloading {a.Name}"),
+                            CancellationToken.None);
+                    }
+
+                    File.Copy(initialDownload, @$"c:\tmp\{Path.GetFileName(destination)}.dest_stream");
+                    
+                    await Tracks.PMap(queue, async track =>
+                    {
+                        Utils.Status($"Extracting track {track.Name}");
+                        await ExtractTrack(initialDownload, trackFolder, track);
+                    });
+
+                    await using var dest = File.Create(destination);
+                    using var ar = new ZipArchive(dest, ZipArchiveMode.Create);
+                    foreach (var track in Directory.EnumerateFiles(trackFolder).OrderBy(e => e))
+                    {
+                        Utils.Status($"Adding {Path.GetFileName(track)} to archive");
+                        var entry = ar.CreateEntry(Path.Combine("Data", "tracks", track.RelativeTo(trackFolder)), CompressionLevel.NoCompression);
+                        entry.LastWriteTime = meta.UploadDate;
+                        await using var es = entry.Open();
+                        await using var ins = File.OpenRead(track);
+                        await ins.CopyToAsync(es);
+                    }
+                        
+                    return true;
+                }
+                catch (VideoUnavailableException ex)
+                {
+                    return false;
+                }
+
+            }
+
+            private const string FFMpegPath = "Downloaders/Converters/ffmpeg.exe";
+            private const string xWMAEncodePath = "Downloaders/Converters/xWMAEncode.exe";
+            private async Task ExtractTrack(string source, string dest_folder, Track track)
+            {
+                var info = new ProcessStartInfo
+                {
+                    FileName = FFMpegPath,
+                    Arguments =
+                        $"-threads 1 -i \"{source}\" -ss {track.Start} -t {track.End - track.Start} \"{dest_folder}\\{track.Name}.wav\"",
+                    RedirectStandardError = true,
+                    RedirectStandardInput = true,
+                    RedirectStandardOutput = true,
+                    UseShellExecute = false,
+                    CreateNoWindow = true
+                };
+                
+                var p = new Process {StartInfo = info};
+                p.Start();
+                ChildProcessTracker.AddProcess(p);
+
+                var output = await p.StandardError.ReadToEndAsync();
+
+                try
+                {
+                    p.PriorityClass = ProcessPriorityClass.BelowNormal;
+                }
+                catch (Exception e)
+                {
+                    Utils.Error(e, "Error while setting process priority level for ffmpeg.exe");
+                }
+                p.WaitForExit();
+
+                if (track.Format == Track.FormatEnum.WAV) return;
+                
+                info = new ProcessStartInfo
+                {
+                    FileName = xWMAEncodePath,
+                    Arguments =
+                        $"-b 192000 \"{dest_folder}\\{track.Name}.wav\" \"{dest_folder}\\{track.Name}.xwm\"",
+                    RedirectStandardError = true,
+                    RedirectStandardInput = true,
+                    RedirectStandardOutput = true,
+                    UseShellExecute = false,
+                    CreateNoWindow = true
+                };
+                
+                p = new Process {StartInfo = info};
+
+                p.Start();
+                ChildProcessTracker.AddProcess(p);
+
+                var output2 = await p.StandardError.ReadToEndAsync();
+
+                try
+                {
+                    p.PriorityClass = ProcessPriorityClass.BelowNormal;
+                }
+                catch (Exception e)
+                {
+                    Utils.Error(e, "Error while setting process priority level for ffmpeg.exe");
+                }
+                p.WaitForExit();
+                
+                if (File.Exists($"{dest_folder}\\{track.Name}.wav"))
+                    File.Delete($"{dest_folder}\\{track.Name}.wav");
+                
+            }
+
+            private class Progress : IProgress<double>
+            {
+                private string _prefix;
+
+                public Progress(string prefix)
+                {
+                    _prefix = prefix;
+                }
+                public void Report(double value)
+                {
+                    Utils.Status(_prefix, Percent.FactoryPutInRange(value));
+                }
+            }
+
+            public override async Task<bool> Verify(Archive archive)
+            {
+                try
+                {
+                    var client = new YoutubeClient(Common.Http.ClientFactory.Client);
+                    var video = await client.GetVideoAsync(Key);
+                    return true;
+                }
+                catch (VideoUnavailableException ex)
+                {
+                    return false;
+                }
+                
+            }
+
+            public override IDownloader GetDownloader()
+            {
+                return DownloadDispatcher.GetInstance<YouTubeDownloader>();
+            }
+
+            public override string GetManifestURL(Archive a)
+            {
+                return $"https://www.youtube.com/watch?v={Key}";
+            }
+
+            public override string[] GetMetaIni()
+            {
+                IEnumerable<string> start = new List<string> {"[General]", $"directURL=https://www.youtube.com/watch?v={Key}"};
+                start = start.Concat(Tracks.SelectMany((track, idx) =>
+                {
+                    return new[]
+                    {
+                        $"\n[track/{idx}]", $"name={track.Name}", $"start={track.Start}", $"end={track.End}",
+                        $"format={track.Format}"
+                    };
+
+                }));
+                return start.ToArray();
+            }
+        }
+    }
+}

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -69,6 +69,9 @@
       <PackageReference Include="YamlDotNet.NetCore">
         <Version>1.0.0</Version>
       </PackageReference>
+      <PackageReference Include="YoutubeExplode">
+        <Version>4.7.13</Version>
+      </PackageReference>
     </ItemGroup>
     <ItemGroup>
       <ProjectReference Include="..\Compression.BSA\Compression.BSA.csproj" />
@@ -80,8 +83,15 @@
       <None Update="Downloaders\BethesdaNet\bethnetlogin.exe">
         <CopyToOutputDirectory>Always</CopyToOutputDirectory>
       </None>
+      <None Update="Downloaders\Converters\ffmpeg.exe">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </None>
+      <None Update="Downloaders\Converters\xWMAEncode.exe">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </None>
     </ItemGroup>
     <ItemGroup>
       <Folder Include="Downloaders\BethesdaNet" />
+      <Folder Include="Downloaders\Converters" />
     </ItemGroup>
 </Project>

Wabbajack.Test/DownloaderTests.cs

@@ -444,6 +444,33 @@ namespace Wabbajack.Test
             CollectionAssert.AreEqual(entries, new List<string> {@"Data\TestCK.esp", @"Data\TestCK.ini"});
         }
         
+        [TestMethod]
+        public async Task YoutubeDownloader()
+        {
+
+            var infered_ini = await DownloadDispatcher.Infer(new Uri("https://www.youtube.com/watch?v=4ceowgHn8BE"));
+            Assert.IsInstanceOfType(infered_ini, typeof(YouTubeDownloader.State));
+            Assert.AreEqual(15, ((YouTubeDownloader.State)infered_ini).Tracks.Count);
+            
+            var ini = string.Join("\n", infered_ini.GetMetaIni());
+            
+            var state = (YouTubeDownloader.State)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
+            Assert.AreEqual(15, state.Tracks.Count);
+            Assert.IsNotNull(state);
+
+            
+            
+            var converted = state.ViaJSON();
+            Assert.IsTrue(await converted.Verify(new Archive {Name = "yt_test.zip"}));
+
+            Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
+
+            using var tempFile = new TempFile();
+            await converted.Download(new Archive {Name = "yt_test.zip"}, tempFile.File.FullName);
+            File.Copy(tempFile.File.FullName, "c:\\tmp\\" + Path.GetFileName(tempFile.File.FullName) + ".zip");
+            Assert.AreEqual("kD36zbA2X9Q=", await tempFile.File.FullName.FileHashAsync());
+        }
+        
         
         /// <summary>
         /// Tests that files from different sources don't overwrite eachother when downloaded by AInstaller