Merge pull request #442 from wabbajack-tools/bethesda-net-redux-again-finally

Bethesda net redux again finally

Wabbajack.Common/Utils.cs

@@ -1143,6 +1143,20 @@ namespace Wabbajack.Common
         {
             return path.ToLower().TrimEnd('\\').StartsWith(parent.ToLower().TrimEnd('\\') + "\\");
         }
+        
+        public static async Task CopyToLimitAsync(this Stream frm, Stream tw, long limit)
+        {
+            var buff = new byte[1024];
+            while (limit > 0)
+            {
+                var to_read = Math.Min(buff.Length, limit);
+                var read = await frm.ReadAsync(buff, 0, (int)to_read);
+                await tw.WriteAsync(buff, 0, read);
+                limit -= read;
+            }
+
+            tw.Flush();
+        }
 
         public class NexusErrorResponse
         {

Wabbajack.Lib/Downloaders/AbstractDownloadState.cs

@@ -25,7 +25,8 @@ namespace Wabbajack.Lib.Downloaders
             typeof(VectorPlexusDownloader.State),
             typeof(DeadlyStreamDownloader.State),
             typeof(AFKModsDownloader.State),
-            typeof(TESAllianceDownloader.State)
+            typeof(TESAllianceDownloader.State),
+            typeof(BethesdaNetDownloader.State)
         };
         public static Dictionary<string, Type> NameToType { get; set; }
         public static Dictionary<Type, string> TypeToName { get; set; }

Wabbajack.Lib/Downloaders/BethesdaNet/bethnetlogin.py

@@ -0,0 +1,103 @@
+"""
+Simple process that uses Fria to inspect the HTTP headers, and sent data during a Skyrim/Fallout login to Bethesda.NET. That data is output to the screen and can later be used to 
+allow Wabbajack to log into Bethesda.NET.
+"""
+
+import frida
+import sys
+from subprocess import Popen, PIPE
+import psutil, time, json
+
+known_headers = {}
+shutdown = False
+
+def on_message(message, data):
+    msg_type, msg_data = message["payload"]
+    if msg_type == "header":
+        header, value = msg_data.split(": ");
+        if header not in known_headers:
+            known_headers[header] = value;
+    if msg_type == "data":
+        try:
+            data = json.loads(msg_data)
+            if "scheme" in data and "language" in data and "payload" in data:
+                shutdown_and_print(data)
+        except:
+            return
+
+def main(target_process):
+    session = frida.attach(target_process)
+
+    script = session.create_script("""
+
+    // Find base address of current imported jvm.dll by main process fledge.exe
+    var reqHeaders = Module.getExportByName('winhttp.dll', 'WinHttpAddRequestHeaders');
+
+    Interceptor.attach(reqHeaders, { // Intercept calls to our SetAesDecrypt function
+
+        // When function is called, print out its parameters
+        onEnter: function (args) {
+            send(['header', args[1].readUtf16String(args[2].toInt32())]);
+
+        },
+        // When function is finished
+        onLeave: function (retval) {
+        }
+    });
+    
+    var reqHeaders = Module.getExportByName('winhttp.dll', 'WinHttpWriteData');
+    console.log("WinHttpAddRequestHeaders: " + reqHeaders);
+
+    Interceptor.attach(reqHeaders, { // Intercept calls to our SetAesDecrypt function
+
+        // When function is called, print out its parameters
+        onEnter: function (args) {
+            send(['data', args[1].readUtf8String(args[2].toInt32())]);
+
+        },
+        // When function is finished
+        onLeave: function (retval) {
+        }
+    });
+    
+    
+""")
+    script.on('message', on_message)
+    script.load()
+    
+    while not shutdown:
+        time.sleep(0.5);
+        
+    session.detach()
+    
+def wait_for_game(name):
+    while True:
+        time.sleep(1);
+        for proc in psutil.process_iter():
+            if proc.name() == name:
+                return proc.pid;
+                
+def shutdown_and_print(data):
+    global shutdown
+    output = {"body": json.dumps(data), "headers": known_headers}
+    
+    print(json.dumps(output))
+    
+    for proc in psutil.process_iter():
+        if proc.pid == pid:
+            proc.kill();
+            break
+            
+    shutdown = True;
+    
+    
+
+if __name__ == '__main__':
+    start = """C:\Steam\steamapps\common\Skyrim Special Edition\SkyrimSE.exe"""
+    wait_for = "SkyrimSE.exe"
+    if len(sys.argv) == 3:
+        start = sys.argv[1];
+        wait_for = sys.argv[2]
+    target_process = Popen([start])
+    pid = wait_for_game(wait_for);
+    main(pid)

Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs

@@ -0,0 +1,352 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Net.Http;
+using System.Reactive;
+using System.Reactive.Linq;
+using System.Reflection;
+using System.Text;
+using System.Threading.Tasks;
+using System.Windows.Input;
+using Newtonsoft.Json.Linq;
+using Org.BouncyCastle.Crypto;
+using Org.BouncyCastle.Crypto.Parameters;
+using Org.BouncyCastle.Security;
+using ReactiveUI;
+using Wabbajack.Common;
+using Wabbajack.Lib.Validation;
+using File = Alphaleonis.Win32.Filesystem.File;
+using Game = Wabbajack.Common.Game;
+using Path = Alphaleonis.Win32.Filesystem.Path;
+
+namespace Wabbajack.Lib.Downloaders
+{
+    public class BethesdaNetDownloader : IUrlDownloader, INeedsLogin
+    {
+        public const string DataName = "bethesda-net-data";
+        public BethesdaNetDownloader()
+        {
+            TriggerLogin = ReactiveCommand.CreateFromTask(() => Utils.CatchAndLog(RequestLoginAndCache), IsLoggedIn.Select(b => !b).ObserveOn(RxApp.MainThreadScheduler));
+            ClearLogin = ReactiveCommand.Create(() => Utils.DeleteEncryptedJson(DataName), IsLoggedIn.ObserveOn(RxApp.MainThreadScheduler));
+        }
+
+        private static async Task RequestLoginAndCache()
+        {
+            var result = await Utils.Log(new RequestBethesdaNetLogin()).Task;
+        }
+
+        public async Task<AbstractDownloadState> GetDownloaderState(dynamic archiveINI)
+        {
+            var url = (Uri)DownloaderUtils.GetDirectURL(archiveINI);
+            return StateFromUrl(url);
+        }
+
+        private static AbstractDownloadState StateFromUrl(Uri url)
+        {
+            if (url != null && url.Host == "bethesda.net" && url.AbsolutePath.StartsWith("/en/mods/"))
+            {
+                var split = url.AbsolutePath.Split('/');
+                var game = split[3];
+                var modId = split[5];
+                return new State {GameName = game, ContentId = modId};
+            }
+            return null;
+        }
+
+        public async Task Prepare()
+        {
+            if (Utils.HaveEncryptedJson(DataName)) return;
+            await Utils.Log(new RequestBethesdaNetLogin()).Task;
+        }
+
+        public static async Task<BethesdaNetData> Login()
+        {
+            var game = Path.Combine(Game.SkyrimSpecialEdition.MetaData().GameLocation(), "SkyrimSE.exe");
+            var info = new ProcessStartInfo
+            {
+                FileName = @"Downloaders\BethesdaNet\bethnetlogin.exe",
+                Arguments = $"\"{game}\" SkyrimSE.exe",
+                RedirectStandardError = true,
+                RedirectStandardInput = true,
+                RedirectStandardOutput = true,
+                UseShellExecute = false,
+                CreateNoWindow = true
+            };
+            var process = Process.Start(info);
+            ChildProcessTracker.AddProcess(process);
+            string last_line = "";
+
+            while (true)
+            {
+                var line = await process.StandardOutput.ReadLineAsync();
+                if (line == null) break;
+                last_line = line;
+            }
+
+            var result =  last_line.FromJSONString<BethesdaNetData>();
+            result.ToEcryptedJson(DataName);
+            return result;
+        }
+
+        public AbstractDownloadState GetDownloaderState(string url)
+        {
+            return StateFromUrl(new Uri(url));
+        }
+
+        public event PropertyChangedEventHandler PropertyChanged;
+        public ReactiveCommand<Unit, Unit> TriggerLogin { get; }
+        public ReactiveCommand<Unit, Unit> ClearLogin { get; }
+        public IObservable<bool> IsLoggedIn => Utils.HaveEncryptedJsonObservable(DataName);
+        public string SiteName => "Bethesda.NET";
+        public IObservable<string> MetaInfo => Observable.Return(""); //"Wabbajack will start the game, then exit once you enter the Mods page";
+        public Uri SiteURL => new Uri("https://bethesda.net");
+        public Uri IconUri { get; }
+
+        public class State : AbstractDownloadState
+        {
+            public string GameName { get; set; }
+            public string ContentId { get; set; }
+            public override object[] PrimaryKey => new object[] {GameName, ContentId};
+
+            public override bool IsWhitelisted(ServerWhitelist whitelist)
+            {
+                return true;
+            }
+
+            public override async Task<bool> Download(Archive a, string destination)
+            {
+                var (client, info, collected) = await ResolveDownloadInfo();
+                using var tf = new TempFile();
+                await using var file = tf.File.Create();
+                var max_chunks = info.depot_list[0].file_list[0].chunk_count;
+                foreach (var chunk in info.depot_list[0].file_list[0].chunk_list.OrderBy(c => c.index))
+                {
+                    Utils.Status($"Downloading {a.Name}", chunk.index * 100 / max_chunks);
+                    var got = await client.GetAsync(
+                        $"https://content.cdp.bethesda.net/{collected.CDPProductId}/{collected.CDPPropertiesId}/{chunk.sha}");
+                    var data = await got.Content.ReadAsByteArrayAsync();
+                    AESCTRDecrypt(collected.AESKey, collected.AESIV, data);
+
+                    if (chunk.uncompressed_size == chunk.chunk_size)
+                        await file.WriteAsync(data, 0, data.Length);
+                    else
+                    {
+                        using (var ms = new MemoryStream(data))
+                        using (var zlibStream =
+                            new ICSharpCode.SharpZipLib.Zip.Compression.Streams.InflaterInputStream(ms))
+                            await zlibStream.CopyToAsync(file);
+                    }
+                }
+                file.Close();
+                await ConvertCKMToZip(file.Name, destination);
+
+                return true;
+            }
+
+
+            private const uint CKM_Magic = 0x52415442; // BTAR
+            private async Task ConvertCKMToZip(string src, string dest)
+            {
+                using var reader = new BinaryReader(File.OpenRead(src));
+                var magic = reader.ReadUInt32();
+                if (magic != CKM_Magic)
+                    throw new InvalidDataException("Invalid magic format in CKM parsing");
+
+                ushort majorVersion = reader.ReadUInt16();
+                ushort minorVersion = reader.ReadUInt16();
+                if (majorVersion != 1)
+                    throw new InvalidDataException("Archive major version is unknown. Should be 1.");
+
+                if (minorVersion < 2 || minorVersion > 4)
+                    throw new InvalidDataException("Archive minor version is unknown. Should be 2, 3, or 4.");
+
+                await using var fos = File.Create(dest);
+                using var archive = new ZipArchive(fos, ZipArchiveMode.Create);
+                while (reader.PeekChar() != -1)
+                {
+                    ushort nameLength = reader.ReadUInt16();
+                    string name = Encoding.UTF8.GetString(reader.ReadBytes(nameLength));
+                    ulong dataLength = reader.ReadUInt64();
+
+                    if (dataLength > int.MaxValue)
+                        throw new Exception();
+
+                    var entry = archive.CreateEntry(name, CompressionLevel.NoCompression);
+                    await using var output = entry.Open();
+                    await reader.BaseStream.CopyToLimitAsync(output, (long)dataLength);
+                }
+            }
+
+            public override async Task<bool> Verify(Archive archive)
+            {
+                var info = await ResolveDownloadInfo();
+                return true;
+            }
+
+            private async Task<(HttpClient, CDPTree, CollectedBNetInfo)> ResolveDownloadInfo()
+            {
+                var info = new CollectedBNetInfo();
+
+                var login_info = Utils.FromEncryptedJson<BethesdaNetData>(DataName);
+
+                var client = new HttpClient();
+
+                client.BaseAddress = new Uri("https://api.bethesda.net");
+                client.DefaultRequestHeaders.Add("User-Agent", "bnet");
+                foreach (var header in login_info.headers.Where(h => h.Key.ToLower().StartsWith("x-")))
+                    client.DefaultRequestHeaders.Add(header.Key, header.Value);
+
+                var posted = await client.PostAsync("/beam/accounts/external_login",
+                    new StringContent(login_info.body, Encoding.UTF8, "application/json"));
+
+                info.AccessToken = (await posted.Content.ReadAsStringAsync()).FromJSONString<BeamLoginResponse>().access_token;
+
+                client.DefaultRequestHeaders.Add("x-cdp-app", "UGC SDK");
+                client.DefaultRequestHeaders.Add("x-cdp-app-ver", "0.9.11314/debug");
+                client.DefaultRequestHeaders.Add("x-cdp-lib-ver", "0.9.11314/debug");
+                client.DefaultRequestHeaders.Add("x-cdp-platform","Win/32");
+
+                posted = await client.PostAsync("cdp-user/auth",
+                    new StringContent("{\"access_token\": \"" + info.AccessToken + "\"}", Encoding.UTF8,
+                        "application/json"));
+                info.CDPToken = (await posted.Content.ReadAsStringAsync()).FromJSONString<CDPLoginResponse>().token;
+
+                client.DefaultRequestHeaders.Add("X-Access-Token", info.AccessToken);
+                var got = await client.GetAsync($"mods/ugc-workshop/content/get?content_id={ContentId}");
+                JObject data = JObject.Parse(await got.Content.ReadAsStringAsync());
+
+                var content = data["platform"]["response"]["content"];
+
+                info.CDPBranchId = (int)content["cdp_branch_id"];
+                info.CDPProductId = (int)content["cdp_product_id"];
+
+                client.DefaultRequestHeaders.Add("Authorization", $"Token {info.CDPToken}");
+
+                got = await client.GetAsync(
+                    $"/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/tree/.json");
+
+                var tree = (await got.Content.ReadAsStringAsync()).FromJSONString<CDPTree>();
+
+                got = await client.GetAsync(
+                    $"/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/depots/.json");
+
+                var props_obj = JObject.Parse(await got.Content.ReadAsStringAsync()).Properties().First();
+                info.CDPPropertiesId = (int)props_obj.Value["properties_id"];
+                info.AESKey = props_obj.Value["ex_info_A"].Select(e => (byte)e).ToArray();
+                info.AESIV = props_obj.Value["ex_info_B"].Select(e => (byte)e).Take(16).ToArray();
+                
+                return (client, tree, info);
+            }
+            
+            static int AESCTRDecrypt(byte[] Key, byte[] IV, byte[] Data)
+            {
+                IBufferedCipher cipher = CipherUtilities.GetCipher("AES/CTR/NoPadding");
+                cipher.Init(false, new ParametersWithIV(ParameterUtilities.CreateKeyParameter("AES", Key), IV));
+
+                return cipher.DoFinal(Data, Data, 0);
+            }
+
+            public override IDownloader GetDownloader()
+            {
+                throw new NotImplementedException();
+            }
+
+            public override string GetReportEntry(Archive a)
+            {
+                throw new NotImplementedException();
+            }
+
+            public override string[] GetMetaIni()
+            {
+                throw new NotImplementedException();
+            }
+
+
+            private class BeamLoginResponse
+            {
+                public string access_token { get; set; }
+
+            }
+
+            private class CDPLoginResponse
+            {
+                public string token { get; set; }
+            }
+
+            private class CollectedBNetInfo
+            {
+                public byte[] AESKey { get; set; }
+                public byte[] AESIV { get; set; }
+                public string AccessToken { get; set; }
+                public string CDPToken { get; set; }
+                public int CDPBranchId { get; set; }
+                public int CDPProductId { get; set; }
+                public int CDPPropertiesId { get; set; }
+            }
+
+            public class CDPTree
+            {
+                public List<Depot> depot_list { get; set; }
+
+                public class Depot
+                {
+                    public List<CDPFile> file_list { get; set; }
+
+                    public class CDPFile
+                    {
+                        public int chunk_count { get; set; }
+                        public List<Chunk> chunk_list { get; set; }
+
+                        public string name { get; set; }
+
+                        public class Chunk
+                        {
+                            public int chunk_size { get; set; }
+                            public int index { get; set; }
+                            public string sha { get; set; }
+                            public int uncompressed_size { get; set; }
+                        }
+                    }
+                }
+            }
+
+        }
+    }
+
+    internal class DownloadInfo
+    {
+    }
+
+    public class RequestBethesdaNetLogin : AUserIntervention
+    {
+        public override string ShortDescription => "Logging into Bethesda.NET";
+        public override string ExtendedDescription { get; }
+
+        private readonly TaskCompletionSource<BethesdaNetData> _source = new TaskCompletionSource<BethesdaNetData>();
+        public Task<BethesdaNetData> Task => _source.Task;
+
+        public void Resume(BethesdaNetData data)
+        {
+            Handled = true;
+            _source.SetResult(data);
+        }
+
+        public override void Cancel()
+        {
+            Handled = true;
+            _source.SetCanceled();
+        }
+
+    }
+
+    public class BethesdaNetData
+    {
+        public string body { get; set; }
+        public Dictionary<string, string> headers = new Dictionary<string, string>();
+    }
+
+}

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -20,6 +20,7 @@ namespace Wabbajack.Lib.Downloaders
             new LoversLabDownloader(),
             new VectorPlexusDownloader(),
             new DeadlyStreamDownloader(),
+            new BethesdaNetDownloader(),
             new AFKModsDownloader(),
             new TESAllianceDownloader(),
             new HTTPDownloader(),

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -65,6 +65,9 @@
       <PackageReference Include="WebSocketSharp-netstandard">
         <Version>1.0.1</Version>
       </PackageReference>
+      <PackageReference Include="XC.BouncyCastle.Crypto">
+        <Version>1.0.0</Version>
+      </PackageReference>
       <PackageReference Include="YamlDotNet.NetCore">
         <Version>1.0.0</Version>
       </PackageReference>
@@ -79,5 +82,11 @@
       <EmbeddedResource Include="LibCefHelpers\cefsharp.7z" />
       <None Remove="css-min.css" />
       <EmbeddedResource Include="css-min.css" />
+      <None Update="Downloaders\BethesdaNet\bethnetlogin.exe">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </None>
+    </ItemGroup>
+    <ItemGroup>
+      <Folder Include="Downloaders\BethesdaNet" />
     </ItemGroup>
 </Project>

Wabbajack.Test/DownloaderTests.cs

@@ -1,5 +1,7 @@
 using System;
 using System.Collections.Generic;
+using System.IO.Compression;
+using System.Linq;
 using System.Threading.Tasks;
 using System.Reactive.Linq;
 using Alphaleonis.Win32.Filesystem;
@@ -411,6 +413,34 @@ namespace Wabbajack.Test
             CollectionAssert.AreEqual(File.ReadAllBytes(Path.Combine(Game.SkyrimSpecialEdition.MetaData().GameLocation(), "Data/Update.esm")), File.ReadAllBytes(filename));
             Consts.TestMode = true;
         }
+        
+        [TestMethod]
+        public async Task BethesdaNetDownload()
+        {
+
+            var downloader = DownloadDispatcher.GetInstance<BethesdaNetDownloader>();
+            Assert.IsTrue(await downloader.IsLoggedIn.FirstAsync());
+
+            var ini = $@"[General]
+                              directURL=https://bethesda.net/en/mods/skyrim/mod-detail/4145641";
+
+            var filename = Guid.NewGuid().ToString();
+            var state = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(ini.LoadIniString());
+            Assert.IsNotNull(state);
+
+            var converted = state.ViaJSON();
+            Assert.IsTrue(await converted.Verify(new Archive {Name = "mod.ckm"}));
+
+            Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
+
+            await converted.Download(new Archive { Name = "mod.zip" }, filename);
+
+            await using var fs = File.OpenRead(filename);
+            using var archive = new ZipArchive(fs);
+            var entries = archive.Entries.Select(e => e.FullName).ToList();
+            CollectionAssert.AreEqual(entries, new List<string> {@"Data\TestCK.esp", @"Data\TestCK.ini"});
+        }
+
 
     }
 

Wabbajack/View Models/UserInterventionHandlers.cs

@@ -66,6 +66,10 @@ namespace Wabbajack
                         c.Resume(key);
                     });
                     break;
+                case RequestBethesdaNetLogin c:
+                    var data = await BethesdaNetDownloader.Login();
+                    c.Resume(data);
+                    break;
                 case AbstractNeedsLoginDownloader.RequestSiteLogin c:
                     await WrapBrowserJob(msg, async (vm, cancel) =>
                     {