Start to implement downloading

Wabbajack.Downloaders.Bethesda/BethesdaDownloader.cs

@@ -1,10 +1,13 @@
 using Microsoft.Extensions.Logging;
+using Wabbajack.Common;
 using Wabbajack.Downloaders.Interfaces;
 using Wabbajack.DTOs;
 using Wabbajack.DTOs.DownloadStates;
 using Wabbajack.DTOs.Validation;
 using Wabbajack.Hashing.xxHash64;
 using Wabbajack.Networking.BethesdaNet;
+using Wabbajack.Networking.BethesdaNet.DTOs;
+using Wabbajack.Networking.Http;
 using Wabbajack.Paths;
 using Wabbajack.Paths.IO;
 using Wabbajack.RateLimiter;
@@ -31,10 +34,38 @@ public class BethesdaDownloader : ADownloader<DTOs.DownloadStates.Bethesda>, IUr
     {
 
         var depot = await _client.GetDepots(state, token);
+        var tree = await _client.GetTree(state, token);
 
+        var chunks = tree!.DepotList.First().FileList.First().ChunkList;
+
+        await chunks.PMapAll(async chunk =>
+        {
+            var data = await GetChunk(state, chunk, token);
+            var reported = job.Report(data.Length, token);
+            
+// Decrypt and Decompress
+
+            await reported;
+            return data;
+        });
+        
         return default;
     }
 
+    private async Task<byte[]> GetChunk(DTOs.DownloadStates.Bethesda state, Chunk chunk,
+        CancellationToken token)
+    {
+        var uri = new Uri($"https://content.cdp.bethesda.net/{state.ProductId}/{state.BranchID}/{chunk.Sha}");
+        var msg = new HttpRequestMessage(HttpMethod.Get, uri);
+        msg.Headers.Add("User-Agent", "bnet");
+        using var job = await _limiter.Begin("Getting chunk", chunk.ChunkSize, token);
+        using var response = await _httpClient.GetAsync(uri, token);
+        if (!response.IsSuccessStatusCode)
+            throw new HttpException(response);
+        await job.Report(chunk.ChunkSize, token);
+        return await response.Content.ReadAsByteArrayAsync(token);
+    }
+
     public override async Task<bool> Prepare()
     {
         await _client.CDPAuth(CancellationToken.None);
@@ -59,8 +90,8 @@ public class BethesdaDownloader : ADownloader<DTOs.DownloadStates.Bethesda>, IUr
 
     public override async Task<bool> Verify(Archive archive, DTOs.DownloadStates.Bethesda state, IJob job, CancellationToken token)
     {
-        await _client.GetDepots(state, token);
-        throw new NotImplementedException();
+        var depot = await _client.GetDepots(state, token);
+        return depot != null;
     }
 
     public override IEnumerable<string> MetaIni(Archive a, DTOs.DownloadStates.Bethesda state)

Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj

@@ -7,6 +7,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+      <ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
       <ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />
       <ProjectReference Include="..\Wabbajack.Networking.BethesdaNet\Wabbajack.Networking.BethesdaNet.csproj" />
     </ItemGroup>

Wabbajack.Networking.BethesdaNet/Client.cs

@@ -152,9 +152,20 @@ public class Client
     }
 
     public async Task<Depot?> GetDepots(Bethesda state, CancellationToken token)
+    {
+        return (await MakeCdpRequest<Dictionary<string, Depot>>(state, "depots", token))?.Values.First();
+    }
+
+    public async Task<Tree?> GetTree(Bethesda state, CancellationToken token)
+    {
+        return await MakeCdpRequest<Tree>(state, "tree", token);
+    }
+
+    private async Task<T?> MakeCdpRequest<T>(Bethesda state, string type, CancellationToken token)
     {
         await EnsureAuthed(token);
-        var msg = MakeMessage(HttpMethod.Get, new Uri($"https://api.bethesda.net/cdp-user/projects/{state.ProductId}/branches/{state.BranchID}/depots/.json"));
+        var msg = MakeMessage(HttpMethod.Get,
+            new Uri($"https://api.bethesda.net/cdp-user/projects/{state.ProductId}/branches/{state.BranchID}/{type}/.json"));
         msg.Headers.Add("x-src-fp", FingerprintKey);
         msg.Headers.Add("x-cdp-app", "UGC SDK");
         msg.Headers.Add("x-cdp-app-ver", "0.9.11314/debug");
@@ -165,8 +176,8 @@ public class Client
         using var request = await _httpClient.SendAsync(msg, token);
         if (!request.IsSuccessStatusCode)
             throw new HttpException(request);
-
-        var response = await request.Content.ReadFromJsonAsync<Dictionary<string, Depot>>(_jsonOptions, token);
-        return response!.Values.First();
+        
+        var response = await request.Content.ReadFromJsonAsync<T>(_jsonOptions, token);
+        return response;
     }
 }

Wabbajack.Networking.BethesdaNet/DTOs/Tree.cs

@@ -0,0 +1,194 @@
+using System.Text.Json.Serialization;
+
+namespace Wabbajack.Networking.BethesdaNet.DTOs;
+
+
+public class BuildHistory
+{
+    [JsonPropertyName("id")]
+    public int Id { get; set; }
+
+    [JsonPropertyName("description")]
+    public string Description { get; set; }
+}
+
+public class BuildFields
+{
+    [JsonPropertyName("id")]
+    public int Id { get; set; }
+
+    [JsonPropertyName("name")]
+    public string Name { get; set; }
+
+    [JsonPropertyName("create_date")]
+    public string CreateDate { get; set; }
+
+    [JsonPropertyName("description")]
+    public string Description { get; set; }
+
+    [JsonPropertyName("build_type")]
+    public int BuildType { get; set; }
+
+    [JsonPropertyName("locked")]
+    public bool Locked { get; set; }
+
+    [JsonPropertyName("storage_key")]
+    public string StorageKey { get; set; }
+
+    [JsonPropertyName("major")]
+    public bool Major { get; set; }
+}
+
+public class Chunk
+{
+    [JsonPropertyName("index")]
+    public int Index { get; set; }
+
+    [JsonPropertyName("chunk_size")]
+    public int ChunkSize { get; set; }
+
+    [JsonPropertyName("uncompressed_size")]
+    public int UncompressedSize { get; set; }
+
+    [JsonPropertyName("sha")]
+    public string Sha { get; set; }
+}
+
+public class FileList
+{
+    [JsonPropertyName("file_id")]
+    public int FileId { get; set; }
+
+    [JsonPropertyName("name")]
+    public string Name { get; set; }
+
+    [JsonPropertyName("sha")]
+    public string Sha { get; set; }
+
+    [JsonPropertyName("file_size")]
+    public int FileSize { get; set; }
+
+    [JsonPropertyName("compressed_size")]
+    public int CompressedSize { get; set; }
+
+    [JsonPropertyName("chunk_count")]
+    public int ChunkCount { get; set; }
+
+    [JsonPropertyName("modifiable")]
+    public bool Modifiable { get; set; }
+
+    [JsonPropertyName("chunk_list")]
+    public Chunk[] ChunkList { get; set; }
+}
+
+public class DepotList
+{
+    [JsonPropertyName("id")]
+    public int Id { get; set; }
+
+    [JsonPropertyName("properties_id")]
+    public int PropertiesId { get; set; }
+
+    [JsonPropertyName("name")]
+    public string Name { get; set; }
+
+    [JsonPropertyName("build")]
+    public int Build { get; set; }
+
+    [JsonPropertyName("bytes_per_chunk")]
+    public int BytesPerChunk { get; set; }
+
+    [JsonPropertyName("size_on_disk")]
+    public int SizeOnDisk { get; set; }
+
+    [JsonPropertyName("download_size")]
+    public int DownloadSize { get; set; }
+
+    [JsonPropertyName("depot_type")]
+    public int DepotType { get; set; }
+
+    [JsonPropertyName("deployment_order")]
+    public int DeploymentOrder { get; set; }
+
+    [JsonPropertyName("compression_type")]
+    public int CompressionType { get; set; }
+
+    [JsonPropertyName("encryption_type")]
+    public int EncryptionType { get; set; }
+
+    [JsonPropertyName("language")]
+    public int Language { get; set; }
+
+    [JsonPropertyName("region")]
+    public int Region { get; set; }
+
+    [JsonPropertyName("default_region")]
+    public bool DefaultRegion { get; set; }
+
+    [JsonPropertyName("default_language")]
+    public bool DefaultLanguage { get; set; }
+
+    [JsonPropertyName("platform")]
+    public int Platform { get; set; }
+
+    [JsonPropertyName("architecture")]
+    public int Architecture { get; set; }
+
+    [JsonPropertyName("is_dlc")]
+    public bool IsDlc { get; set; }
+
+    [JsonPropertyName("file_list")]
+    public FileList[] FileList { get; set; }
+}
+
+public class Tree
+{
+    [JsonPropertyName("id")]
+    public int Id { get; set; }
+
+    [JsonPropertyName("name")]
+    public string Name { get; set; }
+
+    [JsonPropertyName("entitlement_id")]
+    public int EntitlementId { get; set; }
+
+    [JsonPropertyName("branch_type")]
+    public int BranchType { get; set; }
+
+    [JsonPropertyName("project")]
+    public int Project { get; set; }
+
+    [JsonPropertyName("build")]
+    public int Build { get; set; }
+    
+    [JsonPropertyName("available")]
+    public bool Available { get; set; }
+
+    [JsonPropertyName("preload")]
+    public bool Preload { get; set; }
+
+    [JsonPropertyName("preload_ondeck")]
+    public bool PreloadOndeck { get; set; }
+    
+    [JsonPropertyName("diff_type")]
+    public int DiffType { get; set; }
+
+    [JsonPropertyName("build_history_length")]
+    public int BuildHistoryLength { get; set; }
+    
+    [JsonPropertyName("promote_ondeck_after_diff")]
+    public bool PromoteOndeckAfterDiff { get; set; }
+
+    [JsonPropertyName("storage_url")]
+    public string StorageUrl { get; set; }
+    
+    [JsonPropertyName("build_history")]
+    public List<BuildHistory> BuildHistory { get; set; }
+
+    [JsonPropertyName("build_fields")]
+    public BuildFields BuildFields { get; set; }
+
+    [JsonPropertyName("depot_list")]
+    public List<DepotList> DepotList { get; set; }
+}
+