Implement bethesda downloader, and depot handling

Wabbajack.CLI/Verbs/ListCreationClubContent.cs

@@ -7,7 +7,10 @@ using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Wabbajack.Common;
+using Wabbajack.Downloaders.Bethesda;
 using Wabbajack.DTOs;
+using Wabbajack.DTOs.DownloadStates;
+using Wabbajack.DTOs.Logins.BethesdaNet;
 using Wabbajack.Networking.BethesdaNet;
 using Wabbajack.Paths;
 
@@ -17,11 +20,13 @@ public class ListCreationClubContent : IVerb
 {
     private readonly ILogger<ListCreationClubContent> _logger;
     private readonly Client _client;
+    private readonly BethesdaDownloader _downloader;
 
-    public ListCreationClubContent(ILogger<ListCreationClubContent> logger, Client wjClient)
+    public ListCreationClubContent(ILogger<ListCreationClubContent> logger, Client wjClient, Wabbajack.Downloaders.Bethesda.BethesdaDownloader downloader)
     {
         _logger = logger;
         _client = wjClient;
+        _downloader = downloader;
     }
     public Command MakeCommand()
     {
@@ -39,12 +44,12 @@ public class ListCreationClubContent : IVerb
         var falloutContent = (await _client.ListContent(Game.Fallout4, token))
             .Select(f => (Game.Fallout4, f));
 
-        foreach (var (game, content) in skyrimContent.Concat(falloutContent).OrderBy(f => f.f.Name))
+        foreach (var (game, content) in skyrimContent.Concat(falloutContent).OrderBy(f => f.f.Content.Name))
         {
             Console.WriteLine($"Game: {game}");
-            Console.WriteLine($"Name: {content.Name}");
+            Console.WriteLine($"Name: {content.Content.Name}");
-            Console.WriteLine($"Download Size: {content.DepotSize.ToFileSizeString()}");
+            Console.WriteLine($"Download Size: {content.Content.DepotSize.ToFileSizeString()}");
-            Console.WriteLine($"Uri: bethesda://{game}/cc/{content.ContentId}");
+            Console.WriteLine($"Uri: {_downloader.UnParse(content.State)}");
             Console.WriteLine("-----------------------------------");
         }
 

Wabbajack.CLI/Wabbajack.CLI.csproj

@@ -22,6 +22,7 @@
     </ItemGroup>
 
     <ItemGroup>
+        <ProjectReference Include="..\Wabbajack.Downloaders.Bethesda\Wabbajack.Downloaders.Bethesda.csproj" />
         <ProjectReference Include="..\Wabbajack.Downloaders.Dispatcher\Wabbajack.Downloaders.Dispatcher.csproj" />
         <ProjectReference Include="..\Wabbajack.Hashing.xxHash64\Wabbajack.Hashing.xxHash64.csproj" />
         <ProjectReference Include="..\Wabbajack.Networking.Discord\Wabbajack.Networking.Discord.csproj" />

Wabbajack.DTOs/DownloadStates/Bethesda.cs

@@ -0,0 +1,16 @@
+using Wabbajack.DTOs.JsonConverters;
+
+namespace Wabbajack.DTOs.DownloadStates;
+
+[JsonName("Bethesda")]
+public class Bethesda : ADownloadState
+{
+    public override string TypeName { get; } = "Bethesda";
+    public override object[] PrimaryKey => new object[] {Game, IsCCMod, ProductId, BranchID, ContentId};
+    public Game Game { get; set; }
+    public bool IsCCMod { get; set; }
+    public string ContentId { get; set; }
+    public long ProductId { get; set; }
+    public long BranchID { get; set; }
+    public string Name { get; set; }
+}

Wabbajack.Downloaders.Bethesda/BethesdaDownloader.cs

@@ -0,0 +1,115 @@
+using Microsoft.Extensions.Logging;
+using Wabbajack.Downloaders.Interfaces;
+using Wabbajack.DTOs;
+using Wabbajack.DTOs.DownloadStates;
+using Wabbajack.DTOs.Validation;
+using Wabbajack.Hashing.xxHash64;
+using Wabbajack.Networking.BethesdaNet;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+using Wabbajack.RateLimiter;
+
+namespace Wabbajack.Downloaders.Bethesda;
+
+public class BethesdaDownloader : ADownloader<DTOs.DownloadStates.Bethesda>, IUrlDownloader, IChunkedSeekableStreamDownloader
+{
+    private readonly Client _client;
+    private readonly IResource<HttpClient> _limiter;
+    private readonly HttpClient _httpClient;
+    private readonly ILogger<BethesdaDownloader> _logger;
+
+    public BethesdaDownloader(ILogger<BethesdaDownloader> logger, Client client, HttpClient httpClient, IResource<HttpClient> limiter)
+    {
+        _logger = logger;
+        _client = client;
+        _limiter = limiter;
+        _httpClient = httpClient;
+    }
+    
+    
+    public override async Task<Hash> Download(Archive archive, DTOs.DownloadStates.Bethesda state, AbsolutePath destination, IJob job, CancellationToken token)
+    {
+
+        var depot = await _client.GetDepots(state, token);
+
+        return default;
+    }
+
+    public override async Task<bool> Prepare()
+    {
+        await _client.CDPAuth(CancellationToken.None);
+        return true;
+    }
+
+    public override bool IsAllowed(ServerAllowList allowList, IDownloadState state)
+    {
+        return true;
+    }
+
+    public override IDownloadState? Resolve(IReadOnlyDictionary<string, string> iniData)
+    {
+        if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"], UriKind.Absolute, out var uri))
+        {
+            return Parse(uri);
+        }
+        return null;
+    }
+
+    public override Priority Priority => Priority.Normal;
+
+    public override async Task<bool> Verify(Archive archive, DTOs.DownloadStates.Bethesda state, IJob job, CancellationToken token)
+    {
+        await _client.GetDepots(state, token);
+        throw new NotImplementedException();
+    }
+
+    public override IEnumerable<string> MetaIni(Archive a, DTOs.DownloadStates.Bethesda state)
+    {
+        return new[] {$"directURL={UnParse(state)}"};
+    }
+
+    public IDownloadState? Parse(Uri uri)
+    {
+        if (uri.Scheme != "bethesda") return null;
+        var path = uri.PathAndQuery.Split("/", StringSplitOptions.RemoveEmptyEntries);
+        if (path.Length != 4) return null;
+        var game = GameRegistry.TryGetByFuzzyName(uri.Host);
+        if (game == null) return null;
+
+        if (!long.TryParse(path[1], out var productId)) return null;
+        if (!long.TryParse(path[2], out var branchId)) return null;
+        
+        bool isCCMod = false;
+        switch (path[0])
+        {
+            case "cc":
+                isCCMod = true;
+                break;
+            case "mod":
+                isCCMod = false;
+                break;
+            default:
+                return null;
+        }
+
+        return new DTOs.DownloadStates.Bethesda
+        {
+            Game = game.Game,
+            IsCCMod = isCCMod,
+            ProductId = productId,
+            BranchID = branchId,
+            ContentId = path[3]
+        };
+    }
+
+    public Uri UnParse(IDownloadState state)
+    {
+        var cstate = (DTOs.DownloadStates.Bethesda) state;
+        return new Uri($"bethesda://{cstate.Game}/{(cstate.IsCCMod ? "cc" : "mod")}/{cstate.ProductId}/{cstate.BranchID}/{cstate.ContentId}");
+    }
+
+    public ValueTask<Stream> GetChunkedSeekableStream(Archive archive, CancellationToken token)
+    {
+        throw new NotImplementedException();
+    }
+}

Wabbajack.Downloaders.Bethesda/ServiceExtensions.cs

@@ -0,0 +1,13 @@
+using Microsoft.Extensions.DependencyInjection;
+using Wabbajack.Downloaders.Interfaces;
+using Wabbajack.DTOs;
+
+namespace Wabbajack.Downloaders.Bethesda;
+
+public static class ServiceExtensions
+{
+    public static IServiceCollection AddBethesdaDownloader(this IServiceCollection services)
+    {
+        return services.AddAllSingleton<IDownloader, IDownloader<DTOs.DownloadStates.Bethesda>, IUrlDownloader, BethesdaDownloader>();
+    }
+}

Wabbajack.Downloaders.Bethesda/Wabbajack.Downloaders.Bethesda.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net6.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />
+      <ProjectReference Include="..\Wabbajack.Networking.BethesdaNet\Wabbajack.Networking.BethesdaNet.csproj" />
+    </ItemGroup>
+
+</Project>

Wabbajack.Downloaders.Dispatcher.Test/DownloaderTests.cs

@@ -180,6 +180,34 @@ public class DownloaderTests
                             "https://authored-files.wabbajack.org/Tonal%20Architect_WJ_TEST_FILES.zip_9cb97a01-3354-4077-9e4a-7e808d47794fFFOOO")
                     }
                 }
+            },
+            // Bethesda
+            new object[]
+            {
+                new Archive
+                {
+                    Hash = default,
+                    State = new DTOs.DownloadStates.Bethesda
+                    {
+                        Game = Game.SkyrimSpecialEdition,
+                        IsCCMod = true,
+                        ProductId = 4,
+                        BranchID = 90898,
+                        ContentId = "4059054"
+                    }
+                },
+                new Archive
+                {
+                    Hash = default,
+                    State = new DTOs.DownloadStates.Bethesda
+                    {
+                        Game = Game.SkyrimSpecialEdition,
+                        IsCCMod = true,
+                        ProductId = 6,
+                        BranchID = 9898,
+                        ContentId = "059054"
+                    }
+                },
             }
         };
 

Wabbajack.Downloaders.Dispatcher/ServiceExtensions.cs

@@ -1,4 +1,5 @@
 using Microsoft.Extensions.DependencyInjection;
+using Wabbajack.Downloaders.Bethesda;
 using Wabbajack.Downloaders.Http;
 using Wabbajack.Downloaders.IPS4OAuth2Downloader;
 using Wabbajack.Downloaders.MediaFire;
@@ -24,6 +25,7 @@ public static class ServiceExtensions
             .AddIPS4OAuth2Downloaders()
             .AddWabbajackCDNDownloader()
             .AddGameFileDownloader()
+            .AddBethesdaDownloader()
             .AddWabbajackClient()
             .AddSingleton<DownloadDispatcher>();
     }

Wabbajack.Downloaders.Dispatcher/Wabbajack.Downloaders.Dispatcher.csproj

@@ -8,6 +8,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <ProjectReference Include="..\Wabbajack.Downloaders.Bethesda\Wabbajack.Downloaders.Bethesda.csproj" />
         <ProjectReference Include="..\Wabbajack.Downloaders.GameFile\Wabbajack.Downloaders.GameFile.csproj" />
         <ProjectReference Include="..\Wabbajack.Downloaders.GoogleDrive\Wabbajack.Downloaders.GoogleDrive.csproj" />
         <ProjectReference Include="..\Wabbajack.Downloaders.Http\Wabbajack.Downloaders.Http.csproj" />

Wabbajack.Downloaders.Http/Wabbajack.Downloaders.Http.csproj

@@ -11,6 +11,7 @@
         <ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
         <ProjectReference Include="..\Wabbajack.Downloaders.Interfaces\Wabbajack.Downloaders.Interfaces.csproj" />
         <ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
+        <ProjectReference Include="..\Wabbajack.Networking.BethesdaNet\Wabbajack.Networking.BethesdaNet.csproj" />
         <ProjectReference Include="..\Wabbajack.Networking.Http.Interfaces\Wabbajack.Networking.Http.Interfaces.csproj" />
         <ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
     </ItemGroup>

Wabbajack.Networking.BethesdaNet/Client.cs

@@ -2,8 +2,10 @@ using System.Net.Http.Json;
 using System.Text;
 using System.Text.Encodings.Web;
 using System.Text.Json;
+using System.Text.Json.Serialization;
 using Microsoft.Extensions.Logging;
 using Wabbajack.DTOs;
+using Wabbajack.DTOs.DownloadStates;
 using Wabbajack.DTOs.Logins;
 using Wabbajack.DTOs.Logins.BethesdaNet;
 using Wabbajack.Networking.BethesdaNet.DTOs;
@@ -37,7 +39,8 @@ public class Client
         _httpClient = client;
         _jsonOptions = new JsonSerializerOptions
         {
-            Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
+            Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+            NumberHandling = JsonNumberHandling.AllowReadingFromString,
         };
         SetFingerprint();
     }
@@ -106,7 +109,7 @@ public class Client
         FingerprintKey = string.Concat(Array.ConvertAll(keyBytes, x => x.ToString("X2")));
     }
 
-    public async Task<IEnumerable<Content>> ListContent(Game game, CancellationToken token)
+    public async Task<IEnumerable<(Content Content, Bethesda State)>> ListContent(Game game, CancellationToken token)
     {
         var gameKey = game switch
         {
@@ -125,7 +128,15 @@ public class Client
         if (!request.IsSuccessStatusCode)
             throw new HttpException(request);
         var response = await request.Content.ReadFromJsonAsync<ListSubscribeResponse>(_jsonOptions, token);
-        return response!.Platform.Response.Content;
+        return response!.Platform.Response.Content
+            .Select(c => (c, new Bethesda
+            {
+                Game = game,
+                ContentId = c.ContentId,
+                IsCCMod = c.CcMod,
+                ProductId = c.CdpProductId,
+                BranchID = c.CdpBranchId
+            }));
     }
 
     private async Task EnsureAuthed(CancellationToken token)
@@ -133,4 +144,29 @@ public class Client
         if (_entitlementData == null)
             await CDPAuth(token);
     }
+
+    private int ProductId(Game game)
+    {
+        if (game == Game.SkyrimSpecialEdition) return 4;
+        return 0;
+    }
+
+    public async Task<Depot?> GetDepots(Bethesda state, CancellationToken token)
+    {
+        await EnsureAuthed(token);
+        var msg = MakeMessage(HttpMethod.Get, new Uri($"https://api.bethesda.net/cdp-user/projects/{state.ProductId}/branches/{state.BranchID}/depots/.json"));
+        msg.Headers.Add("x-src-fp", FingerprintKey);
+        msg.Headers.Add("x-cdp-app", "UGC SDK");
+        msg.Headers.Add("x-cdp-app-ver", "0.9.11314/debug");
+        msg.Headers.Add("x-cdp-lib-ver", "0.9.11314/debug");
+        msg.Headers.Add("x-cdp-platform", "Win/32");
+        msg.Headers.Add("Authorization", $"Token {_entitlementData!.Token}");
+
+        using var request = await _httpClient.SendAsync(msg, token);
+        if (!request.IsSuccessStatusCode)
+            throw new HttpException(request);
+
+        var response = await request.Content.ReadFromJsonAsync<Dictionary<string, Depot>>(_jsonOptions, token);
+        return response!.Values.First();
+    }
 }

Wabbajack.Networking.BethesdaNet/DTOs/Depot.cs

@@ -0,0 +1,66 @@
+using System.Text.Json.Serialization;
+
+namespace Wabbajack.Networking.BethesdaNet.DTOs;
+
+public class Depot
+{
+    [JsonPropertyName("id")]
+    public int Id { get; set; }
+
+    [JsonPropertyName("properties_id")]
+    public int PropertiesId { get; set; }
+
+    [JsonPropertyName("name")]
+    public string Name { get; set; }
+
+    [JsonPropertyName("build")]
+    public int Build { get; set; }
+
+    [JsonPropertyName("bytes_per_chunk")]
+    public int BytesPerChunk { get; set; }
+
+    [JsonPropertyName("size_on_disk")]
+    public int SizeOnDisk { get; set; }
+
+    [JsonPropertyName("download_size")]
+    public int DownloadSize { get; set; }
+
+    [JsonPropertyName("depot_type")]
+    public int DepotType { get; set; }
+
+    [JsonPropertyName("deployment_order")]
+    public int DeploymentOrder { get; set; }
+
+    [JsonPropertyName("compression_type")]
+    public int CompressionType { get; set; }
+
+    [JsonPropertyName("encryption_type")]
+    public int EncryptionType { get; set; }
+
+    [JsonPropertyName("language")]
+    public int Language { get; set; }
+
+    [JsonPropertyName("region")]
+    public int Region { get; set; }
+
+    [JsonPropertyName("default_region")]
+    public bool DefaultRegion { get; set; }
+
+    [JsonPropertyName("default_language")]
+    public bool DefaultLanguage { get; set; }
+
+    [JsonPropertyName("platform")]
+    public int Platform { get; set; }
+
+    [JsonPropertyName("architecture")]
+    public int Architecture { get; set; }
+
+    [JsonPropertyName("ex_info_A")]
+    public List<byte> ExInfoA { get; set; }
+
+    [JsonPropertyName("ex_info_B")]
+    public List<byte> ExInfoB { get; set; }
+
+    [JsonPropertyName("is_dlc")]
+    public bool IsDlc { get; set; }
+}

Wabbajack.Networking.BethesdaNet/DTOs/ListSubscribeResponse.cs

@@ -68,7 +68,7 @@ public class Content
     public int RatingCount { get; set; }
 
     [JsonPropertyName("cdp_branch_id")]
-    public int CdpBranchId { get; set; }
+    public long CdpBranchId { get; set; }
 
     [JsonPropertyName("type")]
     public string Type { get; set; }
@@ -107,7 +107,7 @@ public class Content
     public string ContentId { get; set; }
 
     [JsonPropertyName("cdp_product_id")]
-    public int CdpProductId { get; set; }
+    public long CdpProductId { get; set; }
 }
 
 public class Response

Wabbajack.sln

@@ -135,6 +135,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Compression.Zip.T
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Networking.BethesdaNet", "Wabbajack.Networking.BethesdaNet\Wabbajack.Networking.BethesdaNet.csproj", "{A3813D73-9A8E-4CE7-861A-C59043DFFC14}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Downloaders.Bethesda", "Wabbajack.Downloaders.Bethesda\Wabbajack.Downloaders.Bethesda.csproj", "{B10BB6D6-B3FC-4A76-8A07-6A0A0ADDE198}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -369,6 +371,10 @@ Global
 		{A3813D73-9A8E-4CE7-861A-C59043DFFC14}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{A3813D73-9A8E-4CE7-861A-C59043DFFC14}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{A3813D73-9A8E-4CE7-861A-C59043DFFC14}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B10BB6D6-B3FC-4A76-8A07-6A0A0ADDE198}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B10BB6D6-B3FC-4A76-8A07-6A0A0ADDE198}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B10BB6D6-B3FC-4A76-8A07-6A0A0ADDE198}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B10BB6D6-B3FC-4A76-8A07-6A0A0ADDE198}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -415,6 +421,7 @@ Global
 		{64AD7E26-5643-4969-A61C-E0A90FA25FCB} = {F677890D-5109-43BC-97C7-C4CD47C8EE0C}
 		{6D7EA87E-6ABE-4BA3-B93A-BE5E71A4DE7C} = {F677890D-5109-43BC-97C7-C4CD47C8EE0C}
 		{A3813D73-9A8E-4CE7-861A-C59043DFFC14} = {F01F8595-5FD7-4506-8469-F4A5522DACC1}
+		{B10BB6D6-B3FC-4A76-8A07-6A0A0ADDE198} = {98B731EE-4FC0-4482-A069-BCBA25497871}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {0AA30275-0F38-4A7D-B645-F5505178DDE8}