Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Several fixes for the Youtube Downloader inferencing code.

Browse Source
This commit is contained in:
Timothy Baldridge 2020-05-27 06:05:55 -06:00
parent 90d7d3b15c
commit c225c468fd
6 changed files with 41 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Wabbajack.CLI/Program.cs
View File

@ -1,5 +1,9 @@
using CommandLine;
using Wabbajack.CLI.Verbs;
using Wabbajack.Common;
using Console = System.Console;
using System.Reactive.Linq;
using System;
namespace Wabbajack.CLI
{
@ -7,6 +11,7 @@ namespace Wabbajack.CLI
{
private static int Main(string[] args)
{
Utils.LogMessages.Subscribe(Console.WriteLine);
return Parser.Default.ParseArguments(args, OptionsDefinition.AllOptions)
.MapResult(
(Encrypt opts) => opts.Execute(),

2
Wabbajack.CLI/Wabbajack.CLI.csproj
View File

@ -20,6 +20,8 @@
<PackageReference Include="CommandLineParser" Version="2.8.0" />
<PackageReference Include="F23.StringSimilarity" Version="3.1.0" />
<PackageReference Include="Markdig" Version="0.20.0" />
<PackageReference Include="System.Reactive" Version="4.4.1" />
<PackageReference Include="System.Reactive.Linq" Version="4.4.1" />
</ItemGroup>
<ItemGroup>

2
Wabbajack.Common/ProcessHelper.cs
View File

@ -105,7 +105,7 @@ namespace Wabbajack.Common
Output.OnCompleted();
if (result != 0 && ThrowOnNonZeroExitCode)
throw new Exception($"Error executing {Path} - Exit Code {result} - Check the log for more information");
throw new Exception($"Error executing {Path} - Exit Code {result} - Check the log for more information - {string.Join(" ", args.Select(a => a!.ToString()))}");
return result;
}

29
Wabbajack.Lib/Downloaders/UrlDownloaders/YoutubeInferencer.cs
View File

@ -1,6 +1,9 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using DynamicData;
using Org.BouncyCastle.Utilities.Collections;
using Wabbajack.Common;
using YoutubeExplode;
@ -17,21 +20,28 @@ namespace Wabbajack.Lib.Downloaders.UrlDownloaders
var video = await client.Videos.GetAsync(state.Key);
var desc = video.Description;
var replaceChars = new HashSet<char>() {'_', '(', ')', '-'};
var lines = desc.Split('\n', StringSplitOptions.RemoveEmptyEntries)
.Select(line => line.Trim())
.Select(line =>
{
var segments = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (segments.Length == 0) return (TimeSpan.Zero, string.Empty);
if (TryParseEx(segments.First(), out var s1))
return (s1, string.Join(" ", segments.Skip(1)));
if (TryParseEx(Enumerable.Last(segments), out var s2))
return (s2, string.Join(" ", Utils.ButLast(segments)));
var segments = replaceChars.Aggregate(line, (acc, c) => acc.Replace(c, ' '))
.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (segments.Length == 0) return (TimeSpan.Zero, string.Empty);
foreach (var segment in segments)
{
if (TryParseEx(segment, out var si))
{
return (si, string.Join(" ", segments.Where(s => !s.Contains(":"))));
}
}
return (TimeSpan.Zero, string.Empty);
})
.Where(t => t.Item2 != null)
.Where(t => t.Item2 != string.Empty)
.ToList();
var tracks = lines.Select((line, idx) => new YouTubeDownloader.State.Track
@ -42,6 +52,11 @@ namespace Wabbajack.Lib.Downloaders.UrlDownloaders
Format = YouTubeDownloader.State.Track.FormatEnum.XWM
}).ToList();
foreach (var track in tracks)
{
Utils.Log($"Inferred Track {track.Name} {track.Format} {track.Start}-{track.End}");
}
state.Tracks = tracks;
return state;

20
Wabbajack.Lib/Downloaders/YouTubeDownloader.cs
View File

@ -120,8 +120,6 @@ namespace Wabbajack.Lib.Downloaders
CancellationToken.None);
}
await initialDownload.CopyToAsync(destination.WithExtension(new Extension(".dest_stream")));
await Tracks.PMap(queue, async track =>
{
Utils.Status($"Extracting track {track.Name}");
@ -133,7 +131,7 @@ namespace Wabbajack.Lib.Downloaders
foreach (var track in trackFolder.EnumerateFiles().OrderBy(e => e))
{
Utils.Status($"Adding {track.FileName} to archive");
var entry = ar.CreateEntry(Path.Combine("Data", "tracks", (string)track.RelativeTo(trackFolder)), CompressionLevel.NoCompression);
var entry = ar.CreateEntry(Path.Combine("Data", "Music", (string)track.RelativeTo(trackFolder)), CompressionLevel.NoCompression);
entry.LastWriteTime = meta.UploadDate;
await using var es = entry.Open();
await using var ins = await track.OpenRead();
@ -155,10 +153,12 @@ namespace Wabbajack.Lib.Downloaders
private Extension XWMExtension = new Extension(".xwm");
private async Task ExtractTrack(AbsolutePath source, AbsolutePath destFolder, Track track)
{
Utils.Log($"Extracting {track.Name}");
var wavFile = track.Name.RelativeTo(destFolder).WithExtension(WAVExtension);
var process = new ProcessHelper
{
Path = FFMpegPath,
Arguments = new object[] {"-threads", 1, "-i", source, "-ss", track.Start, "-t", track.End - track.Start, track.Name.RelativeTo(destFolder).WithExtension(WAVExtension)},
Arguments = new object[] {"-hide_banner", "-loglevel", "panic", "-threads", 1, "-i", source, "-ss", track.Start, "-t", track.End - track.Start, wavFile},
ThrowOnNonZeroExitCode = true
};
@ -169,27 +169,27 @@ namespace Wabbajack.Lib.Downloaders
});
await process.Start();
ffmpegLogs.Dispose();
if (track.Format == Track.FormatEnum.WAV) return;
process = new ProcessHelper()
{
Path = xWMAEncodePath,
Arguments = new object[] {"-b", 192000, track.Name.RelativeTo(destFolder).WithExtension(WAVExtension), track.Name.RelativeTo(destFolder).WithExtension(XWMExtension)},
Arguments = new object[] {"-b", 192000, wavFile, wavFile.ReplaceExtension(XWMExtension)},
ThrowOnNonZeroExitCode = true
};
var xwmLogs = process.Output.Where(arg => arg.Type == ProcessHelper.StreamType.Output)
.ForEachAsync(val =>
{
Utils.Status($"Encoding {track.Name} - {val.Line}");
Utils.Log($"Encoding {track.Name} - {val.Line}");
});
await process.Start();
if (File.Exists($"{destFolder}\\{track.Name}.wav"))
File.Delete($"{destFolder}\\{track.Name}.wav");
xwmLogs.Dispose();
await wavFile.DeleteAsync();
}
private class Progress : IProgress<double>

2
Wabbajack.Test/DownloaderTests.cs
View File

@ -465,7 +465,7 @@ namespace Wabbajack.Test
using var tempFile = new TempFile();
await converted.Download(new Archive(state: null!) { Name = "yt_test.zip"}, tempFile.Path);
Assert.Equal(Hash.FromBase64("kD36zbA2X9Q="), await tempFile.Path.FileHashAsync());
Assert.Equal(Hash.FromBase64("pD7UoVNY4o8="), await tempFile.Path.FileHashAsync());
}