From c225c468fdf7036f6a2b9175e3e5479a0e4240fa Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 27 May 2020 06:05:55 -0600 Subject: [PATCH] Several fixes for the Youtube Downloader inferencing code. --- Wabbajack.CLI/Program.cs | 5 ++++ Wabbajack.CLI/Wabbajack.CLI.csproj | 2 ++ Wabbajack.Common/ProcessHelper.cs | 2 +- .../UrlDownloaders/YoutubeInferencer.cs | 29 ++++++++++++++----- .../Downloaders/YouTubeDownloader.cs | 20 ++++++------- Wabbajack.Test/DownloaderTests.cs | 2 +- 6 files changed, 41 insertions(+), 19 deletions(-) diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index eea8793d..cd0d445a 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -1,5 +1,9 @@ using CommandLine; using Wabbajack.CLI.Verbs; +using Wabbajack.Common; +using Console = System.Console; +using System.Reactive.Linq; +using System; namespace Wabbajack.CLI { @@ -7,6 +11,7 @@ namespace Wabbajack.CLI { private static int Main(string[] args) { + Utils.LogMessages.Subscribe(Console.WriteLine); return Parser.Default.ParseArguments(args, OptionsDefinition.AllOptions) .MapResult( (Encrypt opts) => opts.Execute(), diff --git a/Wabbajack.CLI/Wabbajack.CLI.csproj b/Wabbajack.CLI/Wabbajack.CLI.csproj index 3337797b..a1381447 100644 --- a/Wabbajack.CLI/Wabbajack.CLI.csproj +++ b/Wabbajack.CLI/Wabbajack.CLI.csproj @@ -20,6 +20,8 @@ + + diff --git a/Wabbajack.Common/ProcessHelper.cs b/Wabbajack.Common/ProcessHelper.cs index 8c81852d..a333b576 100644 --- a/Wabbajack.Common/ProcessHelper.cs +++ b/Wabbajack.Common/ProcessHelper.cs @@ -105,7 +105,7 @@ namespace Wabbajack.Common Output.OnCompleted(); if (result != 0 && ThrowOnNonZeroExitCode) - throw new Exception($"Error executing {Path} - Exit Code {result} - Check the log for more information"); + throw new Exception($"Error executing {Path} - Exit Code {result} - Check the log for more information - {string.Join(" ", args.Select(a => a!.ToString()))}"); return result; } diff --git a/Wabbajack.Lib/Downloaders/UrlDownloaders/YoutubeInferencer.cs b/Wabbajack.Lib/Downloaders/UrlDownloaders/YoutubeInferencer.cs index 4e2c8175..a1c19239 100644 --- a/Wabbajack.Lib/Downloaders/UrlDownloaders/YoutubeInferencer.cs +++ b/Wabbajack.Lib/Downloaders/UrlDownloaders/YoutubeInferencer.cs @@ -1,6 +1,9 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using DynamicData; +using Org.BouncyCastle.Utilities.Collections; using Wabbajack.Common; using YoutubeExplode; @@ -17,21 +20,28 @@ namespace Wabbajack.Lib.Downloaders.UrlDownloaders var video = await client.Videos.GetAsync(state.Key); var desc = video.Description; + + var replaceChars = new HashSet() {'_', '(', ')', '-'}; var lines = desc.Split('\n', StringSplitOptions.RemoveEmptyEntries) .Select(line => line.Trim()) .Select(line => { - var segments = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); - if (segments.Length == 0) return (TimeSpan.Zero, string.Empty); - if (TryParseEx(segments.First(), out var s1)) - return (s1, string.Join(" ", segments.Skip(1))); - if (TryParseEx(Enumerable.Last(segments), out var s2)) - return (s2, string.Join(" ", Utils.ButLast(segments))); + var segments = replaceChars.Aggregate(line, (acc, c) => acc.Replace(c, ' ')) + .Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (segments.Length == 0) return (TimeSpan.Zero, string.Empty); + + foreach (var segment in segments) + { + if (TryParseEx(segment, out var si)) + { + return (si, string.Join(" ", segments.Where(s => !s.Contains(":")))); + } + } return (TimeSpan.Zero, string.Empty); }) - .Where(t => t.Item2 != null) + .Where(t => t.Item2 != string.Empty) .ToList(); var tracks = lines.Select((line, idx) => new YouTubeDownloader.State.Track @@ -42,6 +52,11 @@ namespace Wabbajack.Lib.Downloaders.UrlDownloaders Format = YouTubeDownloader.State.Track.FormatEnum.XWM }).ToList(); + foreach (var track in tracks) + { + Utils.Log($"Inferred Track {track.Name} {track.Format} {track.Start}-{track.End}"); + } + state.Tracks = tracks; return state; diff --git a/Wabbajack.Lib/Downloaders/YouTubeDownloader.cs b/Wabbajack.Lib/Downloaders/YouTubeDownloader.cs index 75ee4072..0462b04b 100644 --- a/Wabbajack.Lib/Downloaders/YouTubeDownloader.cs +++ b/Wabbajack.Lib/Downloaders/YouTubeDownloader.cs @@ -120,8 +120,6 @@ namespace Wabbajack.Lib.Downloaders CancellationToken.None); } - await initialDownload.CopyToAsync(destination.WithExtension(new Extension(".dest_stream"))); - await Tracks.PMap(queue, async track => { Utils.Status($"Extracting track {track.Name}"); @@ -133,7 +131,7 @@ namespace Wabbajack.Lib.Downloaders foreach (var track in trackFolder.EnumerateFiles().OrderBy(e => e)) { Utils.Status($"Adding {track.FileName} to archive"); - var entry = ar.CreateEntry(Path.Combine("Data", "tracks", (string)track.RelativeTo(trackFolder)), CompressionLevel.NoCompression); + var entry = ar.CreateEntry(Path.Combine("Data", "Music", (string)track.RelativeTo(trackFolder)), CompressionLevel.NoCompression); entry.LastWriteTime = meta.UploadDate; await using var es = entry.Open(); await using var ins = await track.OpenRead(); @@ -155,10 +153,12 @@ namespace Wabbajack.Lib.Downloaders private Extension XWMExtension = new Extension(".xwm"); private async Task ExtractTrack(AbsolutePath source, AbsolutePath destFolder, Track track) { + Utils.Log($"Extracting {track.Name}"); + var wavFile = track.Name.RelativeTo(destFolder).WithExtension(WAVExtension); var process = new ProcessHelper { Path = FFMpegPath, - Arguments = new object[] {"-threads", 1, "-i", source, "-ss", track.Start, "-t", track.End - track.Start, track.Name.RelativeTo(destFolder).WithExtension(WAVExtension)}, + Arguments = new object[] {"-hide_banner", "-loglevel", "panic", "-threads", 1, "-i", source, "-ss", track.Start, "-t", track.End - track.Start, wavFile}, ThrowOnNonZeroExitCode = true }; @@ -169,27 +169,27 @@ namespace Wabbajack.Lib.Downloaders }); await process.Start(); + ffmpegLogs.Dispose(); if (track.Format == Track.FormatEnum.WAV) return; process = new ProcessHelper() { Path = xWMAEncodePath, - Arguments = new object[] {"-b", 192000, track.Name.RelativeTo(destFolder).WithExtension(WAVExtension), track.Name.RelativeTo(destFolder).WithExtension(XWMExtension)}, + Arguments = new object[] {"-b", 192000, wavFile, wavFile.ReplaceExtension(XWMExtension)}, ThrowOnNonZeroExitCode = true }; var xwmLogs = process.Output.Where(arg => arg.Type == ProcessHelper.StreamType.Output) .ForEachAsync(val => { - Utils.Status($"Encoding {track.Name} - {val.Line}"); + Utils.Log($"Encoding {track.Name} - {val.Line}"); }); await process.Start(); - - if (File.Exists($"{destFolder}\\{track.Name}.wav")) - File.Delete($"{destFolder}\\{track.Name}.wav"); - + xwmLogs.Dispose(); + + await wavFile.DeleteAsync(); } private class Progress : IProgress diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 99f34864..ee91102d 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -465,7 +465,7 @@ namespace Wabbajack.Test using var tempFile = new TempFile(); await converted.Download(new Archive(state: null!) { Name = "yt_test.zip"}, tempFile.Path); - Assert.Equal(Hash.FromBase64("kD36zbA2X9Q="), await tempFile.Path.FileHashAsync()); + Assert.Equal(Hash.FromBase64("pD7UoVNY4o8="), await tempFile.Path.FileHashAsync()); }