Can download Bethesda.NET CC files

Wabbajack.CLI/Verbs/DownloadUrl.cs

@@ -4,9 +4,11 @@ using System.CommandLine.Invocation;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
+using Wabbajack.Common;
 using Wabbajack.Downloaders;
 using Wabbajack.DTOs;
 using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
 
 namespace Wabbajack.CLI.Verbs;
 
@@ -42,7 +44,8 @@ public class DownloadUrl : IVerb
         }
 
         var archive = new Archive() {State = parsed, Name = output.FileName.ToString()};
-        await _dispatcher.Download(archive, output, CancellationToken.None);
+        var hash = await _dispatcher.Download(archive, output, CancellationToken.None); ;
+        Console.WriteLine($"Download complete: {output.Size().ToFileSizeString()} {hash} {hash.ToHex()} {(long)hash}");
         return 0;
     }
 }

Wabbajack.DTOs/DownloadStates/Bethesda.cs

@@ -6,11 +6,11 @@ namespace Wabbajack.DTOs.DownloadStates;
 public class Bethesda : ADownloadState
 {
     public override string TypeName { get; } = "Bethesda";
-    public override object[] PrimaryKey => new object[] {Game, IsCCMod, ProductId, BranchID, ContentId};
+    public override object[] PrimaryKey => new object[] {Game, IsCCMod, ProductId, BranchId, ContentId};
     public Game Game { get; set; }
     public bool IsCCMod { get; set; }
     public string ContentId { get; set; }
     public long ProductId { get; set; }
-    public long BranchID { get; set; }
+    public long BranchId { get; set; }
     public string Name { get; set; }
 }

Wabbajack.Downloaders.Bethesda/BethesdaDownloader.cs

@@ -1,3 +1,7 @@
+using System.IO.Compression;
+using System.Security.Cryptography;
+using CS_AES_CTR;
+using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
 using Microsoft.Extensions.Logging;
 using Wabbajack.Common;
 using Wabbajack.Downloaders.Interfaces;
@@ -38,28 +42,56 @@ public class BethesdaDownloader : ADownloader<DTOs.DownloadStates.Bethesda>, IUr
 
         var chunks = tree!.DepotList.First().FileList.First().ChunkList;
 
+        using var os = destination.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.Read);
+        
+        var hasher = new xxHashAlgorithm(0);
+        Hash finalHash = default;
+
+        var aesKey = depot.ExInfoA.ToArray();
+        var aesIV = depot.ExInfoB.Take(16).ToArray();
+
         await chunks.PMapAll(async chunk =>
         {
-            var data = await GetChunk(state, chunk, token);
+            var data = await GetChunk(state, chunk, depot.PropertiesId, token);
             var reported = job.Report(data.Length, token);
             
-// Decrypt and Decompress
+            var aesCtr = new AES_CTR(aesKey, aesIV, false);
+            data = aesCtr.DecryptBytes(data);
+
+            if (chunk.UncompressedSize != chunk.ChunkSize)
+            {
+                var inflater = new InflaterInputStream(new MemoryStream(data));
+                data = await inflater.ReadAllAsync();
+            }
             
             await reported;
             return data;
-        });
+        })
-        
+        .Do(async data =>
-        return default;
+        {
+            if (data.Length < tree.DepotList.First().BytesPerChunk)
+            {
+                hasher.HashBytes(data);
+            }
+            else
+            {
+                finalHash = Hash.FromULong(hasher.FinalizeHashValueInternal(data));
             }
             
-    private async Task<byte[]> GetChunk(DTOs.DownloadStates.Bethesda state, Chunk chunk,
+            await os.WriteAsync(data, token);
+        });
+        
+        return finalHash;
+    }
+
+    private async Task<byte[]> GetChunk(DTOs.DownloadStates.Bethesda state, Chunk chunk, long propertiesId,
         CancellationToken token)
     {
-        var uri = new Uri($"https://content.cdp.bethesda.net/{state.ProductId}/{state.BranchID}/{chunk.Sha}");
+        var uri = new Uri($"https://content.cdp.bethesda.net/{state.ProductId}/{propertiesId}/{chunk.Sha}");
         var msg = new HttpRequestMessage(HttpMethod.Get, uri);
         msg.Headers.Add("User-Agent", "bnet");
         using var job = await _limiter.Begin("Getting chunk", chunk.ChunkSize, token);
-        using var response = await _httpClient.GetAsync(uri, token);
+        using var response = await _httpClient.SendAsync(msg, token);
         if (!response.IsSuccessStatusCode)
             throw new HttpException(response);
         await job.Report(chunk.ChunkSize, token);
@@ -128,7 +160,7 @@ public class BethesdaDownloader : ADownloader<DTOs.DownloadStates.Bethesda>, IUr
             Game = game.Game,
             IsCCMod = isCCMod,
             ProductId = productId,
-            BranchID = branchId,
+            BranchId = branchId,
             ContentId = path[3]
         };
     }
@@ -136,7 +168,7 @@ public class BethesdaDownloader : ADownloader<DTOs.DownloadStates.Bethesda>, IUr
     public Uri UnParse(IDownloadState state)
     {
         var cstate = (DTOs.DownloadStates.Bethesda) state;
-        return new Uri($"bethesda://{cstate.Game}/{(cstate.IsCCMod ? "cc" : "mod")}/{cstate.ProductId}/{cstate.BranchID}/{cstate.ContentId}");
+        return new Uri($"bethesda://{cstate.Game}/{(cstate.IsCCMod ? "cc" : "mod")}/{cstate.ProductId}/{cstate.BranchId}/{cstate.ContentId}");
     }
 
     public ValueTask<Stream> GetChunkedSeekableStream(Archive archive, CancellationToken token)

Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj

@@ -12,4 +12,9 @@
       <ProjectReference Include="..\Wabbajack.Networking.BethesdaNet\Wabbajack.Networking.BethesdaNet.csproj" />
     </ItemGroup>
 
+    <ItemGroup>
+      <PackageReference Include="LibAES-CTR" Version="0.9.6" />
+      <PackageReference Include="SharpZipLib" Version="1.3.3" />
+    </ItemGroup>
+
 </Project>

Wabbajack.Networking.BethesdaNet/Client.cs

@@ -135,7 +135,7 @@ public class Client
                 ContentId = c.ContentId,
                 IsCCMod = c.CcMod,
                 ProductId = c.CdpProductId,
-                BranchID = c.CdpBranchId
+                BranchId = c.CdpBranchId
             }));
     }
 
@@ -165,7 +165,7 @@ public class Client
     {
         await EnsureAuthed(token);
         var msg = MakeMessage(HttpMethod.Get,
-            new Uri($"https://api.bethesda.net/cdp-user/projects/{state.ProductId}/branches/{state.BranchID}/{type}/.json"));
+            new Uri($"https://api.bethesda.net/cdp-user/projects/{state.ProductId}/branches/{state.BranchId}/{type}/.json"));
         msg.Headers.Add("x-src-fp", FingerprintKey);
         msg.Headers.Add("x-cdp-app", "UGC SDK");
         msg.Headers.Add("x-cdp-app-ver", "0.9.11314/debug");