From 30b29d890df508a0866b176a573c5cc24ac38f5f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 11 Feb 2022 16:40:38 -0700 Subject: [PATCH] Can download Bethesda.NET CC files --- Wabbajack.CLI/Verbs/DownloadUrl.cs | 5 +- Wabbajack.DTOs/DownloadStates/Bethesda.cs | 4 +- .../BethesdaDownloader.cs | 48 +++++++++++++++---- .../Wabbajack.Downloaders.Bethesda.csproj | 5 ++ Wabbajack.Networking.BethesdaNet/Client.cs | 4 +- 5 files changed, 53 insertions(+), 13 deletions(-) diff --git a/Wabbajack.CLI/Verbs/DownloadUrl.cs b/Wabbajack.CLI/Verbs/DownloadUrl.cs index af6fc344..b800e967 100644 --- a/Wabbajack.CLI/Verbs/DownloadUrl.cs +++ b/Wabbajack.CLI/Verbs/DownloadUrl.cs @@ -4,9 +4,11 @@ using System.CommandLine.Invocation; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Wabbajack.Common; using Wabbajack.Downloaders; using Wabbajack.DTOs; using Wabbajack.Paths; +using Wabbajack.Paths.IO; namespace Wabbajack.CLI.Verbs; @@ -42,7 +44,8 @@ public class DownloadUrl : IVerb } var archive = new Archive() {State = parsed, Name = output.FileName.ToString()}; - await _dispatcher.Download(archive, output, CancellationToken.None); + var hash = await _dispatcher.Download(archive, output, CancellationToken.None); ; + Console.WriteLine($"Download complete: {output.Size().ToFileSizeString()} {hash} {hash.ToHex()} {(long)hash}"); return 0; } } \ No newline at end of file diff --git a/Wabbajack.DTOs/DownloadStates/Bethesda.cs b/Wabbajack.DTOs/DownloadStates/Bethesda.cs index 792baabd..c0af7fd8 100644 --- a/Wabbajack.DTOs/DownloadStates/Bethesda.cs +++ b/Wabbajack.DTOs/DownloadStates/Bethesda.cs @@ -6,11 +6,11 @@ namespace Wabbajack.DTOs.DownloadStates; public class Bethesda : ADownloadState { public override string TypeName { get; } = "Bethesda"; - public override object[] PrimaryKey => new object[] {Game, IsCCMod, ProductId, BranchID, ContentId}; + public override object[] PrimaryKey => new object[] {Game, IsCCMod, ProductId, BranchId, ContentId}; public Game Game { get; set; } public bool IsCCMod { get; set; } public string ContentId { get; set; } public long ProductId { get; set; } - public long BranchID { get; set; } + public long BranchId { get; set; } public string Name { get; set; } } \ No newline at end of file diff --git a/Wabbajack.Downloaders.Bethesda/BethesdaDownloader.cs b/Wabbajack.Downloaders.Bethesda/BethesdaDownloader.cs index 91b1d5bc..82099a55 100644 --- a/Wabbajack.Downloaders.Bethesda/BethesdaDownloader.cs +++ b/Wabbajack.Downloaders.Bethesda/BethesdaDownloader.cs @@ -1,3 +1,7 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using CS_AES_CTR; +using ICSharpCode.SharpZipLib.Zip.Compression.Streams; using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Downloaders.Interfaces; @@ -38,28 +42,56 @@ public class BethesdaDownloader : ADownloader, IUr var chunks = tree!.DepotList.First().FileList.First().ChunkList; + using var os = destination.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.Read); + + var hasher = new xxHashAlgorithm(0); + Hash finalHash = default; + + var aesKey = depot.ExInfoA.ToArray(); + var aesIV = depot.ExInfoB.Take(16).ToArray(); + await chunks.PMapAll(async chunk => { - var data = await GetChunk(state, chunk, token); + var data = await GetChunk(state, chunk, depot.PropertiesId, token); var reported = job.Report(data.Length, token); -// Decrypt and Decompress + var aesCtr = new AES_CTR(aesKey, aesIV, false); + data = aesCtr.DecryptBytes(data); + if (chunk.UncompressedSize != chunk.ChunkSize) + { + var inflater = new InflaterInputStream(new MemoryStream(data)); + data = await inflater.ReadAllAsync(); + } + await reported; return data; + }) + .Do(async data => + { + if (data.Length < tree.DepotList.First().BytesPerChunk) + { + hasher.HashBytes(data); + } + else + { + finalHash = Hash.FromULong(hasher.FinalizeHashValueInternal(data)); + } + + await os.WriteAsync(data, token); }); - return default; + return finalHash; } - private async Task GetChunk(DTOs.DownloadStates.Bethesda state, Chunk chunk, + private async Task GetChunk(DTOs.DownloadStates.Bethesda state, Chunk chunk, long propertiesId, CancellationToken token) { - var uri = new Uri($"https://content.cdp.bethesda.net/{state.ProductId}/{state.BranchID}/{chunk.Sha}"); + var uri = new Uri($"https://content.cdp.bethesda.net/{state.ProductId}/{propertiesId}/{chunk.Sha}"); var msg = new HttpRequestMessage(HttpMethod.Get, uri); msg.Headers.Add("User-Agent", "bnet"); using var job = await _limiter.Begin("Getting chunk", chunk.ChunkSize, token); - using var response = await _httpClient.GetAsync(uri, token); + using var response = await _httpClient.SendAsync(msg, token); if (!response.IsSuccessStatusCode) throw new HttpException(response); await job.Report(chunk.ChunkSize, token); @@ -128,7 +160,7 @@ public class BethesdaDownloader : ADownloader, IUr Game = game.Game, IsCCMod = isCCMod, ProductId = productId, - BranchID = branchId, + BranchId = branchId, ContentId = path[3] }; } @@ -136,7 +168,7 @@ public class BethesdaDownloader : ADownloader, IUr public Uri UnParse(IDownloadState state) { var cstate = (DTOs.DownloadStates.Bethesda) state; - return new Uri($"bethesda://{cstate.Game}/{(cstate.IsCCMod ? "cc" : "mod")}/{cstate.ProductId}/{cstate.BranchID}/{cstate.ContentId}"); + return new Uri($"bethesda://{cstate.Game}/{(cstate.IsCCMod ? "cc" : "mod")}/{cstate.ProductId}/{cstate.BranchId}/{cstate.ContentId}"); } public ValueTask GetChunkedSeekableStream(Archive archive, CancellationToken token) diff --git a/Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj b/Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj index 3affa506..8126cfb1 100644 --- a/Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj +++ b/Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj @@ -12,4 +12,9 @@ + + + + + diff --git a/Wabbajack.Networking.BethesdaNet/Client.cs b/Wabbajack.Networking.BethesdaNet/Client.cs index 80120b30..0f4132c2 100644 --- a/Wabbajack.Networking.BethesdaNet/Client.cs +++ b/Wabbajack.Networking.BethesdaNet/Client.cs @@ -135,7 +135,7 @@ public class Client ContentId = c.ContentId, IsCCMod = c.CcMod, ProductId = c.CdpProductId, - BranchID = c.CdpBranchId + BranchId = c.CdpBranchId })); } @@ -165,7 +165,7 @@ public class Client { await EnsureAuthed(token); var msg = MakeMessage(HttpMethod.Get, - new Uri($"https://api.bethesda.net/cdp-user/projects/{state.ProductId}/branches/{state.BranchID}/{type}/.json")); + new Uri($"https://api.bethesda.net/cdp-user/projects/{state.ProductId}/branches/{state.BranchId}/{type}/.json")); msg.Headers.Add("x-src-fp", FingerprintKey); msg.Headers.Add("x-cdp-app", "UGC SDK"); msg.Headers.Add("x-cdp-app-ver", "0.9.11314/debug");