From b923cac804e06a66f39d903f739e3ede76763c0d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 11 Feb 2022 07:03:16 -0700 Subject: [PATCH] Start to implement downloading --- .../BethesdaDownloader.cs | 35 +++- .../Wabbajack.Downloaders.Bethesda.csproj | 1 + Wabbajack.Networking.BethesdaNet/Client.cs | 19 +- Wabbajack.Networking.BethesdaNet/DTOs/Tree.cs | 194 ++++++++++++++++++ 4 files changed, 243 insertions(+), 6 deletions(-) create mode 100644 Wabbajack.Networking.BethesdaNet/DTOs/Tree.cs diff --git a/Wabbajack.Downloaders.Bethesda/BethesdaDownloader.cs b/Wabbajack.Downloaders.Bethesda/BethesdaDownloader.cs index b482995e..91b1d5bc 100644 --- a/Wabbajack.Downloaders.Bethesda/BethesdaDownloader.cs +++ b/Wabbajack.Downloaders.Bethesda/BethesdaDownloader.cs @@ -1,10 +1,13 @@ using Microsoft.Extensions.Logging; +using Wabbajack.Common; using Wabbajack.Downloaders.Interfaces; using Wabbajack.DTOs; using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.Validation; using Wabbajack.Hashing.xxHash64; using Wabbajack.Networking.BethesdaNet; +using Wabbajack.Networking.BethesdaNet.DTOs; +using Wabbajack.Networking.Http; using Wabbajack.Paths; using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; @@ -31,10 +34,38 @@ public class BethesdaDownloader : ADownloader, IUr { var depot = await _client.GetDepots(state, token); + var tree = await _client.GetTree(state, token); + var chunks = tree!.DepotList.First().FileList.First().ChunkList; + + await chunks.PMapAll(async chunk => + { + var data = await GetChunk(state, chunk, token); + var reported = job.Report(data.Length, token); + +// Decrypt and Decompress + + await reported; + return data; + }); + return default; } + private async Task GetChunk(DTOs.DownloadStates.Bethesda state, Chunk chunk, + CancellationToken token) + { + var uri = new Uri($"https://content.cdp.bethesda.net/{state.ProductId}/{state.BranchID}/{chunk.Sha}"); + var msg = new HttpRequestMessage(HttpMethod.Get, uri); + msg.Headers.Add("User-Agent", "bnet"); + using var job = await _limiter.Begin("Getting chunk", chunk.ChunkSize, token); + using var response = await _httpClient.GetAsync(uri, token); + if (!response.IsSuccessStatusCode) + throw new HttpException(response); + await job.Report(chunk.ChunkSize, token); + return await response.Content.ReadAsByteArrayAsync(token); + } + public override async Task Prepare() { await _client.CDPAuth(CancellationToken.None); @@ -59,8 +90,8 @@ public class BethesdaDownloader : ADownloader, IUr public override async Task Verify(Archive archive, DTOs.DownloadStates.Bethesda state, IJob job, CancellationToken token) { - await _client.GetDepots(state, token); - throw new NotImplementedException(); + var depot = await _client.GetDepots(state, token); + return depot != null; } public override IEnumerable MetaIni(Archive a, DTOs.DownloadStates.Bethesda state) diff --git a/Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj b/Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj index a3799d4d..3affa506 100644 --- a/Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj +++ b/Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj @@ -7,6 +7,7 @@ + diff --git a/Wabbajack.Networking.BethesdaNet/Client.cs b/Wabbajack.Networking.BethesdaNet/Client.cs index ed90e823..80120b30 100644 --- a/Wabbajack.Networking.BethesdaNet/Client.cs +++ b/Wabbajack.Networking.BethesdaNet/Client.cs @@ -152,9 +152,20 @@ public class Client } public async Task GetDepots(Bethesda state, CancellationToken token) + { + return (await MakeCdpRequest>(state, "depots", token))?.Values.First(); + } + + public async Task GetTree(Bethesda state, CancellationToken token) + { + return await MakeCdpRequest(state, "tree", token); + } + + private async Task MakeCdpRequest(Bethesda state, string type, CancellationToken token) { await EnsureAuthed(token); - var msg = MakeMessage(HttpMethod.Get, new Uri($"https://api.bethesda.net/cdp-user/projects/{state.ProductId}/branches/{state.BranchID}/depots/.json")); + var msg = MakeMessage(HttpMethod.Get, + new Uri($"https://api.bethesda.net/cdp-user/projects/{state.ProductId}/branches/{state.BranchID}/{type}/.json")); msg.Headers.Add("x-src-fp", FingerprintKey); msg.Headers.Add("x-cdp-app", "UGC SDK"); msg.Headers.Add("x-cdp-app-ver", "0.9.11314/debug"); @@ -165,8 +176,8 @@ public class Client using var request = await _httpClient.SendAsync(msg, token); if (!request.IsSuccessStatusCode) throw new HttpException(request); - - var response = await request.Content.ReadFromJsonAsync>(_jsonOptions, token); - return response!.Values.First(); + + var response = await request.Content.ReadFromJsonAsync(_jsonOptions, token); + return response; } } \ No newline at end of file diff --git a/Wabbajack.Networking.BethesdaNet/DTOs/Tree.cs b/Wabbajack.Networking.BethesdaNet/DTOs/Tree.cs new file mode 100644 index 00000000..8c864a50 --- /dev/null +++ b/Wabbajack.Networking.BethesdaNet/DTOs/Tree.cs @@ -0,0 +1,194 @@ +using System.Text.Json.Serialization; + +namespace Wabbajack.Networking.BethesdaNet.DTOs; + + +public class BuildHistory +{ + [JsonPropertyName("id")] + public int Id { get; set; } + + [JsonPropertyName("description")] + public string Description { get; set; } +} + +public class BuildFields +{ + [JsonPropertyName("id")] + public int Id { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("create_date")] + public string CreateDate { get; set; } + + [JsonPropertyName("description")] + public string Description { get; set; } + + [JsonPropertyName("build_type")] + public int BuildType { get; set; } + + [JsonPropertyName("locked")] + public bool Locked { get; set; } + + [JsonPropertyName("storage_key")] + public string StorageKey { get; set; } + + [JsonPropertyName("major")] + public bool Major { get; set; } +} + +public class Chunk +{ + [JsonPropertyName("index")] + public int Index { get; set; } + + [JsonPropertyName("chunk_size")] + public int ChunkSize { get; set; } + + [JsonPropertyName("uncompressed_size")] + public int UncompressedSize { get; set; } + + [JsonPropertyName("sha")] + public string Sha { get; set; } +} + +public class FileList +{ + [JsonPropertyName("file_id")] + public int FileId { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("sha")] + public string Sha { get; set; } + + [JsonPropertyName("file_size")] + public int FileSize { get; set; } + + [JsonPropertyName("compressed_size")] + public int CompressedSize { get; set; } + + [JsonPropertyName("chunk_count")] + public int ChunkCount { get; set; } + + [JsonPropertyName("modifiable")] + public bool Modifiable { get; set; } + + [JsonPropertyName("chunk_list")] + public Chunk[] ChunkList { get; set; } +} + +public class DepotList +{ + [JsonPropertyName("id")] + public int Id { get; set; } + + [JsonPropertyName("properties_id")] + public int PropertiesId { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("build")] + public int Build { get; set; } + + [JsonPropertyName("bytes_per_chunk")] + public int BytesPerChunk { get; set; } + + [JsonPropertyName("size_on_disk")] + public int SizeOnDisk { get; set; } + + [JsonPropertyName("download_size")] + public int DownloadSize { get; set; } + + [JsonPropertyName("depot_type")] + public int DepotType { get; set; } + + [JsonPropertyName("deployment_order")] + public int DeploymentOrder { get; set; } + + [JsonPropertyName("compression_type")] + public int CompressionType { get; set; } + + [JsonPropertyName("encryption_type")] + public int EncryptionType { get; set; } + + [JsonPropertyName("language")] + public int Language { get; set; } + + [JsonPropertyName("region")] + public int Region { get; set; } + + [JsonPropertyName("default_region")] + public bool DefaultRegion { get; set; } + + [JsonPropertyName("default_language")] + public bool DefaultLanguage { get; set; } + + [JsonPropertyName("platform")] + public int Platform { get; set; } + + [JsonPropertyName("architecture")] + public int Architecture { get; set; } + + [JsonPropertyName("is_dlc")] + public bool IsDlc { get; set; } + + [JsonPropertyName("file_list")] + public FileList[] FileList { get; set; } +} + +public class Tree +{ + [JsonPropertyName("id")] + public int Id { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("entitlement_id")] + public int EntitlementId { get; set; } + + [JsonPropertyName("branch_type")] + public int BranchType { get; set; } + + [JsonPropertyName("project")] + public int Project { get; set; } + + [JsonPropertyName("build")] + public int Build { get; set; } + + [JsonPropertyName("available")] + public bool Available { get; set; } + + [JsonPropertyName("preload")] + public bool Preload { get; set; } + + [JsonPropertyName("preload_ondeck")] + public bool PreloadOndeck { get; set; } + + [JsonPropertyName("diff_type")] + public int DiffType { get; set; } + + [JsonPropertyName("build_history_length")] + public int BuildHistoryLength { get; set; } + + [JsonPropertyName("promote_ondeck_after_diff")] + public bool PromoteOndeckAfterDiff { get; set; } + + [JsonPropertyName("storage_url")] + public string StorageUrl { get; set; } + + [JsonPropertyName("build_history")] + public List BuildHistory { get; set; } + + [JsonPropertyName("build_fields")] + public BuildFields BuildFields { get; set; } + + [JsonPropertyName("depot_list")] + public List DepotList { get; set; } +} +