diff --git a/Wabbajack.Lib/Downloaders/BethesdaNet/bethnetlogin.exe b/Wabbajack.Lib/Downloaders/BethesdaNet/bethnetlogin.exe new file mode 100644 index 00000000..a87eb7ed Binary files /dev/null and b/Wabbajack.Lib/Downloaders/BethesdaNet/bethnetlogin.exe differ diff --git a/Wabbajack.Lib/Downloaders/BethesdaNet/bethnetlogin.py b/Wabbajack.Lib/Downloaders/BethesdaNet/bethnetlogin.py new file mode 100644 index 00000000..356d55c6 --- /dev/null +++ b/Wabbajack.Lib/Downloaders/BethesdaNet/bethnetlogin.py @@ -0,0 +1,103 @@ +""" +Simple process that uses Fria to inspect the HTTP headers, and sent data during a Skyrim/Fallout login to Bethesda.NET. That data is output to the screen and can later be used to +allow Wabbajack to log into Bethesda.NET. +""" + +import frida +import sys +from subprocess import Popen, PIPE +import psutil, time, json + +known_headers = {} +shutdown = False + +def on_message(message, data): + msg_type, msg_data = message["payload"] + if msg_type == "header": + header, value = msg_data.split(": "); + if header not in known_headers: + known_headers[header] = value; + if msg_type == "data": + try: + data = json.loads(msg_data) + if "scheme" in data and "language" in data and "payload" in data: + shutdown_and_print(data) + except: + return + +def main(target_process): + session = frida.attach(target_process) + + script = session.create_script(""" + + // Find base address of current imported jvm.dll by main process fledge.exe + var reqHeaders = Module.getExportByName('winhttp.dll', 'WinHttpAddRequestHeaders'); + + Interceptor.attach(reqHeaders, { // Intercept calls to our SetAesDecrypt function + + // When function is called, print out its parameters + onEnter: function (args) { + send(['header', args[1].readUtf16String(args[2].toInt32())]); + + }, + // When function is finished + onLeave: function (retval) { + } + }); + + var reqHeaders = Module.getExportByName('winhttp.dll', 'WinHttpWriteData'); + console.log("WinHttpAddRequestHeaders: " + reqHeaders); + + Interceptor.attach(reqHeaders, { // Intercept calls to our SetAesDecrypt function + + // When function is called, print out its parameters + onEnter: function (args) { + send(['data', args[1].readUtf8String(args[2].toInt32())]); + + }, + // When function is finished + onLeave: function (retval) { + } + }); + + +""") + script.on('message', on_message) + script.load() + + while not shutdown: + time.sleep(0.5); + + session.detach() + +def wait_for_game(name): + while True: + time.sleep(1); + for proc in psutil.process_iter(): + if proc.name() == name: + return proc.pid; + +def shutdown_and_print(data): + global shutdown + output = {"body": json.dumps(data), "headers": known_headers} + + print(json.dumps(output)) + + for proc in psutil.process_iter(): + if proc.pid == pid: + proc.kill(); + break + + shutdown = True; + + + +if __name__ == '__main__': + start = """C:\Steam\steamapps\common\Skyrim Special Edition\SkyrimSE.exe""" + wait_for = "SkyrimSE.exe" + if len(sys.argv) == 3: + start = sys.argv[1]; + wait_for = sys.argv[2] + target_process = Popen([start]) + pid = wait_for_game(wait_for); + main(pid) \ No newline at end of file diff --git a/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs new file mode 100644 index 00000000..e59ea1ca --- /dev/null +++ b/Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs @@ -0,0 +1,315 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Reactive; +using System.Reactive.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Input; +using Newtonsoft.Json.Linq; +using Org.BouncyCastle.Crypto; +using Org.BouncyCastle.Crypto.Parameters; +using Org.BouncyCastle.Security; +using ReactiveUI; +using Wabbajack.Common; +using Wabbajack.Lib.Validation; +using File = Alphaleonis.Win32.Filesystem.File; +using Game = Wabbajack.Common.Game; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.Lib.Downloaders +{ + public class BethesdaNetDownloader : IUrlDownloader, INeedsLogin + { + public const string DataName = "BethesdaNetData"; + public BethesdaNetDownloader() + { + TriggerLogin = ReactiveCommand.Create(async () => await Utils.Log(new RequestBethesdaNetLogin()).Task, IsLoggedIn.Select(b => !b).ObserveOn(RxApp.MainThreadScheduler)); + ClearLogin = ReactiveCommand.Create(() => Utils.DeleteEncryptedJson(DataName), IsLoggedIn.ObserveOn(RxApp.MainThreadScheduler)); + + if (File.Exists("bethnetlogin.exe")) return; + + using (var os = File.OpenWrite("bethnetlogin.exe")) + using (var i = Assembly.GetExecutingAssembly().GetManifestResourceStream("Wabbajack.Lib.Downloaders.BethesdaNet.bethnetlogin.exe")) + { + i.CopyTo(os); + } + + + } + + public async Task GetDownloaderState(dynamic archiveINI) + { + var url = (Uri)DownloaderUtils.GetDirectURL(archiveINI); + if (url != null && url.Host == "bethesda.net" && url.AbsolutePath.StartsWith("/en/mods/")) + { + var split = url.AbsolutePath.Split('/'); + var game = split[3]; + var modId = split[5]; + return new State {GameName = game, ContentId = modId}; + } + + return null; + } + + public async Task Prepare() + { + if (Utils.HaveEncryptedJson(DataName)) return; + await Utils.Log(new RequestBethesdaNetLogin()).Task; + } + + public static async Task Login() + { + var game = Path.Combine(Game.SkyrimSpecialEdition.MetaData().GameLocation(), "SkyrimSE.exe"); + var info = new ProcessStartInfo(); + info.FileName = "bethnetlogin.exe"; + info.Arguments = $"\"{game}\" SkyrimSE.exe"; + info.RedirectStandardError = true; + info.RedirectStandardInput = true; + info.RedirectStandardOutput = true; + info.UseShellExecute = false; + info.CreateNoWindow = true; + var process = Process.Start(info); + ChildProcessTracker.AddProcess(process); + string last_line = ""; + + while (true) + { + var line = await process.StandardOutput.ReadLineAsync(); + if (line == null) break; + last_line = line; + } + + var result = last_line.FromJSONString(); + result.ToEcryptedJson(DataName); + return result; + } + + public AbstractDownloadState GetDownloaderState(string url) + { + throw new NotImplementedException(); + } + + public event PropertyChangedEventHandler PropertyChanged; + public ReactiveCommand TriggerLogin { get; } + public ReactiveCommand ClearLogin { get; } + public IObservable IsLoggedIn => Utils.HaveEncryptedJsonObservable(DataName); + public string SiteName => "Bethesda.NET"; + public IObservable MetaInfo => "Wabbajack will start the game, then exit once you enter the Mods page"; + public Uri SiteURL => new Uri("https://bethesda.net"); + public Uri IconUri { get; } + + public class State : AbstractDownloadState + { + public string GameName { get; set; } + public string ContentId { get; set; } + public override object[] PrimaryKey { get; } + + public override bool IsWhitelisted(ServerWhitelist whitelist) + { + return true; + } + + public override async Task Download(Archive a, string destination) + { + var (client, info, collected) = await ResolveDownloadInfo(); + using (var file = File.OpenWrite(destination)) + { + + var max_chunks = info.depot_list[0].file_list[0].chunk_count; + foreach (var chunk in info.depot_list[0].file_list[0].chunk_list.OrderBy(c => c.index)) + { + Utils.Status($"Downloading {a.Name}", chunk.index * 100 / max_chunks); + var got = await client.GetAsync( + $"https://content.cdp.bethesda.net/{collected.CDPProductId}/{collected.CDPPropertiesId}/{chunk.sha}"); + var data = await got.Content.ReadAsByteArrayAsync(); + AESCTRDecrypt(collected.AESKey, collected.AESIV, data); + + if (chunk.uncompressed_size == chunk.chunk_size) + await file.WriteAsync(data, 0, data.Length); + else + { + using (var ms = new MemoryStream(data)) + using (var zlibStream = new ICSharpCode.SharpZipLib.Zip.Compression.Streams.InflaterInputStream(ms)) + await zlibStream.CopyToAsync(file); + } + } + } + + return true; + } + + public override async Task Verify(Archive archive) + { + var info = await ResolveDownloadInfo(); + return true; + } + + private async Task<(HttpClient, CDPTree, CollectedBNetInfo)> ResolveDownloadInfo() + { + var info = new CollectedBNetInfo(); + + var login_info = Utils.FromEncryptedJson(DataName); + + var client = new HttpClient(); + + client.BaseAddress = new Uri("https://api.bethesda.net"); + client.DefaultRequestHeaders.Add("User-Agent", "bnet"); + foreach (var header in login_info.headers.Where(h => h.Key.ToLower().StartsWith("x-"))) + client.DefaultRequestHeaders.Add(header.Key, header.Value); + + var posted = await client.PostAsync("/beam/accounts/external_login", + new StringContent(login_info.body, Encoding.UTF8, "application/json")); + + info.AccessToken = (await posted.Content.ReadAsStringAsync()).FromJSONString().access_token; + + client.DefaultRequestHeaders.Add("x-cdp-app", "UGC SDK"); + client.DefaultRequestHeaders.Add("x-cdp-app-ver", "0.9.11314/debug"); + client.DefaultRequestHeaders.Add("x-cdp-lib-ver", "0.9.11314/debug"); + client.DefaultRequestHeaders.Add("x-cdp-platform","Win/32"); + + posted = await client.PostAsync("cdp-user/auth", + new StringContent("{\"access_token\": \"" + info.AccessToken + "\"}", Encoding.UTF8, + "application/json")); + info.CDPToken = (await posted.Content.ReadAsStringAsync()).FromJSONString().token; + + client.DefaultRequestHeaders.Add("X-Access-Token", info.AccessToken); + var got = await client.GetAsync($"mods/ugc-workshop/content/get?content_id={ContentId}"); + JObject data = JObject.Parse(await got.Content.ReadAsStringAsync()); + + var content = data["platform"]["response"]["content"]; + + info.CDPBranchId = (int)content["cdp_branch_id"]; + info.CDPProductId = (int)content["cdp_product_id"]; + + client.DefaultRequestHeaders.Add("Authorization", $"Token {info.CDPToken}"); + + got = await client.GetAsync( + $"/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/tree/.json"); + + var tree = (await got.Content.ReadAsStringAsync()).FromJSONString(); + + got = await client.GetAsync( + $"/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/depots/.json"); + + var props_obj = JObject.Parse(await got.Content.ReadAsStringAsync()).Properties().First(); + info.CDPPropertiesId = (int)props_obj.Value["properties_id"]; + info.AESKey = props_obj.Value["ex_info_A"].Select(e => (byte)e).ToArray(); + info.AESIV = props_obj.Value["ex_info_B"].Select(e => (byte)e).Take(16).ToArray(); + + return (client, tree, info); + } + + static int AESCTRDecrypt(byte[] Key, byte[] IV, byte[] Data) + { + IBufferedCipher cipher = CipherUtilities.GetCipher("AES/CTR/NoPadding"); + cipher.Init(false, new ParametersWithIV(ParameterUtilities.CreateKeyParameter("AES", Key), IV)); + + return cipher.DoFinal(Data, Data, 0); + } + + public override IDownloader GetDownloader() + { + throw new NotImplementedException(); + } + + public override string GetReportEntry(Archive a) + { + throw new NotImplementedException(); + } + + public override string[] GetMetaIni() + { + throw new NotImplementedException(); + } + + + private class BeamLoginResponse + { + public string access_token { get; set; } + + } + + private class CDPLoginResponse + { + public string token { get; set; } + } + + private class CollectedBNetInfo + { + public byte[] AESKey { get; set; } + public byte[] AESIV { get; set; } + public string AccessToken { get; set; } + public string CDPToken { get; set; } + public int CDPBranchId { get; set; } + public int CDPProductId { get; set; } + public int CDPPropertiesId { get; set; } + } + + public class CDPTree + { + public List depot_list { get; set; } + + public class Depot + { + public List file_list { get; set; } + + public class CDPFile + { + public int chunk_count { get; set; } + public List chunk_list { get; set; } + + public string name { get; set; } + + public class Chunk + { + public int chunk_size { get; set; } + public int index { get; set; } + public string sha { get; set; } + public int uncompressed_size { get; set; } + } + } + } + } + + } + } + + internal class DownloadInfo + { + } + + public class RequestBethesdaNetLogin : AUserIntervention + { + public override string ShortDescription => "Getting LoversLab information"; + public override string ExtendedDescription { get; } + + private readonly TaskCompletionSource _source = new TaskCompletionSource(); + public Task Task => _source.Task; + + public void Resume(BethesdaNetData data) + { + Handled = true; + _source.SetResult(data); + } + + public override void Cancel() + { + Handled = true; + _source.SetCanceled(); + } + + } + + public class BethesdaNetData + { + public string body { get; set; } + public Dictionary headers = new Dictionary(); + } + +} diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 1572743e..478bf737 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -5,6 +5,9 @@ AnyCPU;x64 + + 1.8.1 + 75.1.143 @@ -79,5 +82,11 @@ + + PreserveNewest + + + + \ No newline at end of file diff --git a/Wabbajack.Test/DownloaderTests.cs b/Wabbajack.Test/DownloaderTests.cs index d3a4636b..6de88937 100644 --- a/Wabbajack.Test/DownloaderTests.cs +++ b/Wabbajack.Test/DownloaderTests.cs @@ -411,6 +411,31 @@ namespace Wabbajack.Test CollectionAssert.AreEqual(File.ReadAllBytes(Path.Combine(Game.SkyrimSpecialEdition.MetaData().GameLocation(), "Data/Update.esm")), File.ReadAllBytes(filename)); Consts.TestMode = true; } + + [TestMethod] + public async Task BethesdaNetDownload() + { + + var downloader = DownloadDispatcher.GetInstance(); + Assert.IsTrue(await downloader.IsLoggedIn.FirstAsync()); + + var ini = $@"[General] + directURL=https://bethesda.net/en/mods/fallout4/mod-detail/4145641"; + /*var ini = $@"[General] + directURL=https://bethesda.net/en/mods/fallout4/mod-detail/3411824";*/ + + + var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString()); + Assert.IsNotNull(state); + + var converted = state.ViaJSON(); + Assert.IsTrue(await converted.Verify(new Archive {Name = "mod.ckm"})); + + Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List() })); + + await converted.Download(new Archive { Name = "mod.ckm" }, "c:\\tmp\\mod.ckm"); + } + }