From 61c841f05326d3bf8a7cccd09b470ddbf23d955f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 28 Jan 2020 21:17:24 -0700 Subject: [PATCH] Basic workings of BNet downloadings. Can download and convert a mod into a .zip --- Wabbajack.Common/Utils.cs | 14 +++ .../Downloaders/AbstractDownloadState.cs | 3 +- .../Downloaders/BethesdaNetDownloader.cs | 111 ++++++++++++------ .../Downloaders/DownloadDispatcher.cs | 1 + Wabbajack.Lib/Wabbajack.Lib.csproj | 8 +- Wabbajack.Test/DownloaderTests.cs | 15 ++- .../View Models/UserInterventionHandlers.cs | 4 + 7 files changed, 107 insertions(+), 49 deletions(-) diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index bcfd5f59..6235d73a 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -1143,6 +1143,20 @@ namespace Wabbajack.Common { return path.ToLower().TrimEnd('\\').StartsWith(parent.ToLower().TrimEnd('\\') + "\\"); } + + public static async Task CopyToLimitAsync(this Stream frm, Stream tw, long limit) + { + var buff = new byte[1024]; + while (limit > 0) + { + var to_read = Math.Min(buff.Length, limit); + var read = await frm.ReadAsync(buff, 0, (int)to_read); + await tw.WriteAsync(buff, 0, read); + limit -= read; + } + + tw.Flush(); + } public class NexusErrorResponse { diff --git a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs index bd46e4a9..3e9d9187 100644 --- a/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs +++ b/Wabbajack.Lib/Downloaders/AbstractDownloadState.cs @@ -25,7 +25,8 @@ namespace Wabbajack.Lib.Downloaders typeof(VectorPlexusDownloader.State), typeof(DeadlyStreamDownloader.State), typeof(AFKModsDownloader.State), - typeof(TESAllianceDownloader.State) + typeof(TESAllianceDownloader.State), + typeof(BethesdaNetDownloader.State) }; public static Dictionary NameToType { get; set; } public static Dictionary TypeToName { get; set; } diff --git a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs index e59ea1ca..7beb2a93 100644 --- a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs +++ b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; using System.IO; +using System.IO.Compression; using System.Linq; using System.Net.Http; using System.Reactive; @@ -26,21 +27,16 @@ namespace Wabbajack.Lib.Downloaders { public class BethesdaNetDownloader : IUrlDownloader, INeedsLogin { - public const string DataName = "BethesdaNetData"; + public const string DataName = "bethesda-net-data"; public BethesdaNetDownloader() { - TriggerLogin = ReactiveCommand.Create(async () => await Utils.Log(new RequestBethesdaNetLogin()).Task, IsLoggedIn.Select(b => !b).ObserveOn(RxApp.MainThreadScheduler)); + TriggerLogin = ReactiveCommand.CreateFromTask(() => Utils.CatchAndLog(RequestLoginAndCache), IsLoggedIn.Select(b => !b).ObserveOn(RxApp.MainThreadScheduler)); ClearLogin = ReactiveCommand.Create(() => Utils.DeleteEncryptedJson(DataName), IsLoggedIn.ObserveOn(RxApp.MainThreadScheduler)); + } - if (File.Exists("bethnetlogin.exe")) return; - - using (var os = File.OpenWrite("bethnetlogin.exe")) - using (var i = Assembly.GetExecutingAssembly().GetManifestResourceStream("Wabbajack.Lib.Downloaders.BethesdaNet.bethnetlogin.exe")) - { - i.CopyTo(os); - } - - + private static async Task RequestLoginAndCache() + { + var result = await Utils.Log(new RequestBethesdaNetLogin()).Task; } public async Task GetDownloaderState(dynamic archiveINI) @@ -66,14 +62,16 @@ namespace Wabbajack.Lib.Downloaders public static async Task Login() { var game = Path.Combine(Game.SkyrimSpecialEdition.MetaData().GameLocation(), "SkyrimSE.exe"); - var info = new ProcessStartInfo(); - info.FileName = "bethnetlogin.exe"; - info.Arguments = $"\"{game}\" SkyrimSE.exe"; - info.RedirectStandardError = true; - info.RedirectStandardInput = true; - info.RedirectStandardOutput = true; - info.UseShellExecute = false; - info.CreateNoWindow = true; + var info = new ProcessStartInfo + { + FileName = @"Downloaders\BethesdaNet\bethnetlogin.exe", + Arguments = $"\"{game}\" SkyrimSE.exe", + RedirectStandardError = true, + RedirectStandardInput = true, + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true + }; var process = Process.Start(info); ChildProcessTracker.AddProcess(process); string last_line = ""; @@ -100,7 +98,7 @@ namespace Wabbajack.Lib.Downloaders public ReactiveCommand ClearLogin { get; } public IObservable IsLoggedIn => Utils.HaveEncryptedJsonObservable(DataName); public string SiteName => "Bethesda.NET"; - public IObservable MetaInfo => "Wabbajack will start the game, then exit once you enter the Mods page"; + public IObservable MetaInfo => Observable.Return(""); //"Wabbajack will start the game, then exit once you enter the Mods page"; public Uri SiteURL => new Uri("https://bethesda.net"); public Uri IconUri { get; } @@ -108,7 +106,7 @@ namespace Wabbajack.Lib.Downloaders { public string GameName { get; set; } public string ContentId { get; set; } - public override object[] PrimaryKey { get; } + public override object[] PrimaryKey => new object[] {GameName, ContentId}; public override bool IsWhitelisted(ServerWhitelist whitelist) { @@ -118,32 +116,67 @@ namespace Wabbajack.Lib.Downloaders public override async Task Download(Archive a, string destination) { var (client, info, collected) = await ResolveDownloadInfo(); - using (var file = File.OpenWrite(destination)) + using var tf = new TempFile(); + await using var file = tf.File.Create(); + var max_chunks = info.depot_list[0].file_list[0].chunk_count; + foreach (var chunk in info.depot_list[0].file_list[0].chunk_list.OrderBy(c => c.index)) { + Utils.Status($"Downloading {a.Name}", chunk.index * 100 / max_chunks); + var got = await client.GetAsync( + $"https://content.cdp.bethesda.net/{collected.CDPProductId}/{collected.CDPPropertiesId}/{chunk.sha}"); + var data = await got.Content.ReadAsByteArrayAsync(); + AESCTRDecrypt(collected.AESKey, collected.AESIV, data); - var max_chunks = info.depot_list[0].file_list[0].chunk_count; - foreach (var chunk in info.depot_list[0].file_list[0].chunk_list.OrderBy(c => c.index)) + if (chunk.uncompressed_size == chunk.chunk_size) + await file.WriteAsync(data, 0, data.Length); + else { - Utils.Status($"Downloading {a.Name}", chunk.index * 100 / max_chunks); - var got = await client.GetAsync( - $"https://content.cdp.bethesda.net/{collected.CDPProductId}/{collected.CDPPropertiesId}/{chunk.sha}"); - var data = await got.Content.ReadAsByteArrayAsync(); - AESCTRDecrypt(collected.AESKey, collected.AESIV, data); - - if (chunk.uncompressed_size == chunk.chunk_size) - await file.WriteAsync(data, 0, data.Length); - else - { - using (var ms = new MemoryStream(data)) - using (var zlibStream = new ICSharpCode.SharpZipLib.Zip.Compression.Streams.InflaterInputStream(ms)) - await zlibStream.CopyToAsync(file); - } + using (var ms = new MemoryStream(data)) + using (var zlibStream = + new ICSharpCode.SharpZipLib.Zip.Compression.Streams.InflaterInputStream(ms)) + await zlibStream.CopyToAsync(file); } } + file.Close(); + await ConvertCKMToZip(file.Name, destination); return true; } + + private const uint CKM_Magic = 0x52415442; // BTAR + private async Task ConvertCKMToZip(string src, string dest) + { + using var reader = new BinaryReader(File.OpenRead(src)); + var magic = reader.ReadUInt32(); + if (magic != CKM_Magic) + throw new InvalidDataException("Invalid magic format in CKM parsing"); + + ushort majorVersion = reader.ReadUInt16(); + ushort minorVersion = reader.ReadUInt16(); + if (majorVersion != 1) + throw new InvalidDataException("Archive major version is unknown. Should be 1."); + + if (minorVersion < 2 || minorVersion > 4) + throw new InvalidDataException("Archive minor version is unknown. Should be 2, 3, or 4."); + + await using var fos = File.Create(dest); + using var archive = new ZipArchive(fos, ZipArchiveMode.Create); + while (reader.PeekChar() != -1) + { + ushort nameLength = reader.ReadUInt16(); + string name = Encoding.UTF8.GetString(reader.ReadBytes(nameLength)); + ulong dataLength = reader.ReadUInt64(); + + if (dataLength > int.MaxValue) + throw new Exception(); + + var entry = archive.CreateEntry(name, CompressionLevel.NoCompression); + await using var output = entry.Open(); + await reader.BaseStream.CopyToLimitAsync(output, (long)dataLength); + } + } + public override async Task Verify(Archive archive) { var info = await ResolveDownloadInfo(); @@ -286,7 +319,7 @@ namespace Wabbajack.Lib.Downloaders public class RequestBethesdaNetLogin : AUserIntervention { - public override string ShortDescription => "Getting LoversLab information"; + public override string ShortDescription => "Logging into Bethesda.NET"; public override string ExtendedDescription { get; } private readonly TaskCompletionSource _source = new TaskCompletionSource(); diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index b908fe40..27b9728e 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -20,6 +20,7 @@ namespace Wabbajack.Lib.Downloaders new LoversLabDownloader(), new VectorPlexusDownloader(), new DeadlyStreamDownloader(), + new BethesdaNetDownloader(), new AFKModsDownloader(), new TESAllianceDownloader(), new HTTPDownloader(), diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 478bf737..befc0e94 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -5,9 +5,6 @@ AnyCPU;x64 - - 1.8.1 - 75.1.143 @@ -68,6 +65,9 @@ 1.0.1 + + 1.0.0 + 1.0.0 @@ -83,7 +83,7 @@ - PreserveNewest + Always diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index 6de88937..3b7911e4 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.IO.Compression; +using System.Linq; using System.Threading.Tasks; using System.Reactive.Linq; using Alphaleonis.Win32.Filesystem; @@ -420,11 +422,9 @@ namespace Wabbajack.Test Assert.IsTrue(await downloader.IsLoggedIn.FirstAsync()); var ini = $@"[General] - directURL=https://bethesda.net/en/mods/fallout4/mod-detail/4145641"; - /*var ini = $@"[General] - directURL=https://bethesda.net/en/mods/fallout4/mod-detail/3411824";*/ - + directURL=https://bethesda.net/en/mods/skyrim/mod-detail/4145641"; + var filename = Guid.NewGuid().ToString(); var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); Assert.IsNotNull(state); @@ -433,7 +433,12 @@ namespace Wabbajack.Test Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); - await converted.Download(new Archive { Name = "mod.ckm" }, "c:\\tmp\\mod.ckm"); + await converted.Download(new Archive { Name = "mod.zip" }, filename); + + await using var fs = File.OpenRead(filename); + using var archive = new ZipArchive(fs); + var entries = archive.Entries.Select(e => e.FullName).ToList(); + CollectionAssert.AreEqual(entries, new List {@"Data\TestCK.esp", @"Data\TestCK.ini"}); } diff --git a/Wabbajack/View Models/UserInterventionHandlers.cs b/Wabbajack/View Models/UserInterventionHandlers.cs index 88c5e731..bc61899c 100644 --- a/Wabbajack/View Models/UserInterventionHandlers.cs +++ b/Wabbajack/View Models/UserInterventionHandlers.cs @@ -66,6 +66,10 @@ namespace Wabbajack c.Resume(key); }); break; + case RequestBethesdaNetLogin c: + var data = await BethesdaNetDownloader.Login(); + c.Resume(data); + break; case AbstractNeedsLoginDownloader.RequestSiteLogin c: await WrapBrowserJob(msg, async (vm, cancel) => {