Merge pull request #1803 from wabbajack-tools/download-from-steam

Download from steam

.github/workflows/tests.yaml

@@ -7,7 +7,7 @@ on:
     branches: [ main ]
 
 env:
-  VERSION: 3.0.0.0-beta1
+  VERSION: 3.0.0.0-beta2
 
 jobs:
   build:

Wabbajack.CLI/Program.cs

@@ -11,6 +11,7 @@ using Octokit;
 using Wabbajack.CLI.TypeConverters;
 using Wabbajack.CLI.Verbs;
 using Wabbajack.DTOs.GitHub;
+using Wabbajack.DTOs.Interventions;
 using Wabbajack.Networking.Http;
 using Wabbajack.Networking.Http.Interfaces;
 using Wabbajack.Networking.WabbajackClientApi;
@@ -63,6 +64,11 @@ internal class Program
                 services.AddSingleton<IVerb, GenerateMetricsReports>();
                 services.AddSingleton<IVerb, ForceHeal>();
                 services.AddSingleton<IVerb, MirrorFile>();
+                services.AddSingleton<IVerb, SteamLogin>();
+                services.AddSingleton<IVerb, SteamAppDumpInfo>();
+                services.AddSingleton<IVerb, SteamDownloadFile>();
+
+                services.AddSingleton<IUserInterventionHandler, UserInterventionHandler>();
             }).Build();
 
         var service = host.Services.GetService<CommandLineBuilder>();

Wabbajack.CLI/UserInterventionHandler.cs

@@ -0,0 +1,27 @@
+using System;
+using Wabbajack.DTOs.Interventions;
+using Wabbajack.Networking.Steam.UserInterventions;
+
+namespace Wabbajack.CLI;
+
+public class UserInterventionHandler : IUserInterventionHandler
+{
+    public void Raise(IUserIntervention intervention)
+    {
+        if (intervention is GetAuthCode gac)
+        {
+            switch (gac.Type)
+            {
+                case GetAuthCode.AuthType.EmailCode:
+                    Console.WriteLine("Please enter the Steam code that was just emailed to you");
+                    break;
+                case GetAuthCode.AuthType.TwoFactorAuth:
+                    Console.WriteLine("Please enter your 2FA code for Steam");
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException();
+            }
+            gac.Finish(Console.ReadLine()!.Trim());
+        }
+    }
+}

Wabbajack.CLI/Verbs/SteamDownloadFile.cs

@@ -0,0 +1,95 @@
+using System.CommandLine;
+using System.CommandLine.Invocation;
+using System.Linq;
+using System.Threading.Tasks;
+using FluentFTP.Helpers;
+using Microsoft.Extensions.Logging;
+using SteamKit2;
+using Wabbajack.DTOs;
+using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Networking.Http.Interfaces;
+using Wabbajack.Networking.Steam;
+using Wabbajack.Paths;
+
+namespace Wabbajack.CLI.Verbs;
+
+public class SteamDownloadFile : IVerb
+{
+    private readonly ILogger<SteamDownloadFile> _logger;
+    private readonly Client _client;
+    private readonly ITokenProvider<SteamLoginState> _token;
+    private readonly DepotDownloader _downloader;
+    private readonly DTOSerializer _dtos;
+    private readonly Wabbajack.Networking.WabbajackClientApi.Client _wjClient;
+
+    public SteamDownloadFile(ILogger<SteamDownloadFile> logger, Client steamClient, ITokenProvider<SteamLoginState> token, 
+        DepotDownloader downloader, DTOSerializer dtos, Wabbajack.Networking.WabbajackClientApi.Client wjClient)
+    {
+        _logger = logger;
+        _client = steamClient;
+        _token = token;
+        _downloader = downloader;
+        _dtos = dtos;
+        _wjClient = wjClient;
+    }
+    public Command MakeCommand()
+    {
+        var command = new Command("steam-download-file");
+        command.Description = "Dumps information to the console about the given app";
+        
+        command.Add(new Option<string>(new[] {"-g", "-game", "-gameName"}, "Wabbajack game name"));
+
+        command.Add(new Option<string>(new[] {"-v", "-version"}, "Version of the game to download for"));
+        command.Add(new Option<string>(new[] {"-f", "-file"}, "File to download (relative path)"));
+        command.Add(new Option<string>(new[] {"-o", "-output"}, "Output location"));
+        command.Handler = CommandHandler.Create(Run);
+        return command;
+    }
+
+    private async Task<int> Run(string gameName, string version, string file, AbsolutePath output)
+    {
+        if (!GameRegistry.TryGetByFuzzyName(gameName, out var game))
+            _logger.LogError("Can't find definition for {Game}", gameName);
+
+        await _client.Login();
+        
+        var definition = await _wjClient.GetGameArchives(game.Game, version);
+        var manifests = await _wjClient.GetSteamManifests(game.Game, version);
+        
+        _logger.LogInformation("Found {Count} manifests, looking for file", manifests.Length);
+
+        SteamManifest? steamManifest = null;
+        DepotManifest? depotManifest = null;
+        DepotManifest.FileData? fileData = null;
+
+        var appId = (uint) game.SteamIDs.First();
+        
+        foreach (var manifest in manifests)
+        {
+            steamManifest = manifest;
+            depotManifest = await _client.GetAppManifest(appId, manifest.Depot, manifest.Manifest);
+            fileData = depotManifest.Files!.FirstOrDefault(f => f.FileName == file);
+            if (fileData != default)
+            {
+                break;
+            }
+        }
+
+        if (fileData == default)
+        {
+            _logger.LogError("Cannot find {File} in any manifests", file);
+            return 1;
+        }
+        
+        _logger.LogInformation("File has is {Size} and {ChunkCount} chunks", fileData.TotalSize.FileSizeToString(), fileData.Chunks.Count);
+
+        await _client.Download(appId, depotManifest!.DepotID, steamManifest!.Manifest,  fileData, output);
+
+        _logger.LogInformation("File downloaded");
+
+        return 0;
+
+
+
+    }
+}

Wabbajack.CLI/Verbs/SteamDumpAppInfo.cs

@@ -0,0 +1,71 @@
+using System;
+using System.CommandLine;
+using System.CommandLine.Invocation;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+using SteamKit2;
+using Wabbajack.DTOs;
+using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Networking.Http.Interfaces;
+using Wabbajack.Networking.Steam;
+using JsonSerializer = System.Text.Json.JsonSerializer;
+
+namespace Wabbajack.CLI.Verbs;
+
+public class SteamAppDumpInfo : IVerb
+{
+    private readonly ILogger<SteamAppDumpInfo> _logger;
+    private readonly Client _client;
+    private readonly ITokenProvider<SteamLoginState> _token;
+    private readonly DepotDownloader _downloader;
+    private readonly DTOSerializer _dtos;
+
+    public SteamAppDumpInfo(ILogger<SteamAppDumpInfo> logger, Client steamClient, ITokenProvider<SteamLoginState> token, 
+        DepotDownloader downloader, DTOSerializer dtos)
+    {
+        _logger = logger;
+        _client = steamClient;
+        _token = token;
+        _downloader = downloader;
+        _dtos = dtos;
+    }
+    public Command MakeCommand()
+    {
+        var command = new Command("steam-app-dump-info");
+        command.Description = "Dumps information to the console about the given app";
+        
+        command.Add(new Option<string>(new[] {"-g", "-game", "-gameName"}, "Wabbajack game name"));
+        command.Handler = CommandHandler.Create(Run);
+        return command;
+    }
+
+    public async Task<int> Run(string gameName)
+    {
+        if (!GameRegistry.TryGetByFuzzyName(gameName, out var game))
+        {
+            _logger.LogError("Can't find game {GameName} in game registry", gameName);
+            return 1;
+        }
+
+        await _client.Login();
+        var appId = (uint) game.SteamIDs.First();
+
+        if (!await _downloader.AccountHasAccess(appId))
+        {
+            _logger.LogError("Your account does not have access to this Steam App");
+            return 1;
+        }
+
+        var appData = await _downloader.GetAppInfo((uint)game.SteamIDs.First());
+
+        Console.WriteLine("App Depots: ");
+        
+        Console.WriteLine(_dtos.Serialize(appData, true));
+
+        return 0;
+    }
+
+    
+}

Wabbajack.CLI/Verbs/SteamLogin.cs

@@ -0,0 +1,58 @@
+using System;
+using System.CommandLine;
+using System.CommandLine.Invocation;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.Networking.Http.Interfaces;
+using Wabbajack.Networking.Steam;
+using Wabbajack.Paths;
+
+namespace Wabbajack.CLI.Verbs;
+
+public class SteamLogin : IVerb
+{
+    private readonly ILogger<SteamLogin> _logger;
+    private readonly Client _client;
+    private readonly ITokenProvider<SteamLoginState> _token;
+
+    public SteamLogin(ILogger<SteamLogin> logger, Client steamClient, ITokenProvider<SteamLoginState> token)
+    {
+        _logger = logger;
+        _client = steamClient;
+        _token = token;
+    }
+    public Command MakeCommand()
+    {
+        var command = new Command("steam-login");
+        command.Description = "Logs into Steam via interactive prompts";
+        
+        command.Add(new Option<string>(new[] {"-u", "-user"}, "Username for login"));
+        command.Handler = CommandHandler.Create(Run);
+        return command;
+    }
+
+    public async Task<int> Run(string user)
+    {
+        var token = await _token.Get();
+        
+        if (token == null || token.User != user || string.IsNullOrWhiteSpace(token.Password))
+        {
+            Console.WriteLine("Please enter password");
+            var password = Console.ReadLine() ?? "";
+            
+            await _token.SetToken(new SteamLoginState
+            {
+                User = user,
+                Password = password.Trim()
+            });
+        }
+        
+        _logger.LogInformation("Attempting login");
+        await _client.Login();
+
+        await Task.Delay(10000);
+
+        return 0;
+    }
+    
+}

Wabbajack.DTOs/Game/SteamManifest.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace Wabbajack.DTOs;
+
+public class SteamManifest
+{
+    [JsonPropertyName("Depot")]
+    public uint Depot { get; set; }
+    
+    [JsonPropertyName("Manifest")]
+    public ulong Manifest { get; set; }
+}

Wabbajack.DTOs/Interventions/AUserIntervention.cs

@@ -0,0 +1,38 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Wabbajack.DTOs.Interventions;
+
+public class AUserIntervention<T> : IUserIntervention
+{
+    private readonly TaskCompletionSource<T> _tcs;
+    private readonly CancellationTokenSource _ct;
+
+    protected AUserIntervention()
+    {
+        _tcs = new TaskCompletionSource<T>();
+        _ct = new CancellationTokenSource();
+    }
+    public void Cancel()
+    {
+        _ct.Cancel();
+        _tcs.SetCanceled(Token);
+    }
+    
+    public bool Handled => _tcs.Task.IsCompleted;
+    public CancellationToken Token => _ct.Token;
+    public Task<T> Task => _tcs.Task;
+
+    public void Finish(T value)
+    {
+        _tcs.SetResult(value);
+        _ct.Cancel();
+    }
+    
+    public void SetException(Exception exception)
+    {
+        _ct.Cancel();
+        _tcs.SetException(exception);
+    }
+}

Wabbajack.DTOs/Interventions/IUserIntervention.cs

@@ -25,4 +25,4 @@ public interface IUserIntervention
     public CancellationToken Token { get; }
 
     void SetException(Exception exception);
-}
+}

Wabbajack.Networking.Steam.Test/ClientTests.cs

@@ -0,0 +1,36 @@
+using System.Threading.Tasks;
+using Wabbajack.DTOs.Interventions;
+using Wabbajack.Networking.Http.Interfaces;
+using Xunit;
+
+namespace Wabbajack.Networking.Steam.Test;
+
+public class ClientTests
+{
+    private readonly ITokenProvider<SteamLoginState> _token;
+    private readonly Client _steamClient;
+    private readonly IUserInterventionHandler _userInterventionHandler;
+
+    public ClientTests(ITokenProvider<SteamLoginState> token, Client client, IUserInterventionHandler userInterventionHandler)
+    {
+        _token = token;
+        _steamClient = client;
+        _userInterventionHandler = userInterventionHandler;
+    }
+    
+    [Fact]
+    public async Task CanGetLogin()
+    {
+        var token = await _token.Get();
+        Assert.NotNull(token);
+        Assert.NotEmpty(token!.User);
+    }
+
+    [Fact]
+    public async Task CanLogin()
+    {
+        await _steamClient.Connect();
+        await _steamClient.Login();
+
+    }
+}

Wabbajack.Networking.Steam.Test/Startup.cs

@@ -0,0 +1,20 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Wabbajack.Services.OSIntegrated;
+using Xunit.DependencyInjection;
+using Xunit.DependencyInjection.Logging;
+
+namespace Wabbajack.Networking.Steam.Test;
+
+public class Startup
+{
+    public void ConfigureServices(IServiceCollection service)
+    {
+        service.AddOSIntegrated();
+    }
+
+    public void Configure(ILoggerFactory loggerFactory, ITestOutputHelperAccessor accessor)
+    {
+        loggerFactory.AddProvider(new XunitTestOutputLoggerProvider(accessor, delegate { return true; }));
+    }
+}

Wabbajack.Networking.Steam.Test/Wabbajack.Networking.Steam.Test.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net6.0</TargetFramework>
+        <Nullable>enable</Nullable>
+
+        <IsPackable>false</IsPackable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0" />
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
+        <PackageReference Include="xunit" Version="2.4.1" />
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+            <PrivateAssets>all</PrivateAssets>
+        </PackageReference>
+        <PackageReference Include="coverlet.collector" Version="3.1.0">
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+            <PrivateAssets>all</PrivateAssets>
+        </PackageReference>
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Wabbajack.Services.OSIntegrated\Wabbajack.Services.OSIntegrated.csproj" />
+    </ItemGroup>
+
+</Project>

Wabbajack.Networking.Steam/Client.cs

@@ -0,0 +1,442 @@
+using System.Collections.Concurrent;
+using System.IO.Compression;
+using System.Security;
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualBasic.CompilerServices;
+using SteamKit2;
+using SteamKit2.CDN;
+using SteamKit2.Internal;
+using Wabbajack.DTOs.Interventions;
+using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Hashing.xxHash64;
+using Wabbajack.Networking.Http.Interfaces;
+using Wabbajack.Networking.Steam.DTOs;
+using Wabbajack.Networking.Steam.UserInterventions;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+
+namespace Wabbajack.Networking.Steam;
+
+public class Client : IDisposable
+{
+    private readonly ILogger<Client> _logger;
+    private readonly HttpClient _httpClient;
+    private readonly SteamClient _client;
+    private readonly SteamUser _steamUser;
+    private readonly CallbackManager _manager;
+    private readonly ITokenProvider<SteamLoginState> _token;
+    private TaskCompletionSource _loginTask;
+    private TaskCompletionSource _connectTask;
+    private readonly CancellationTokenSource _cancellationSource;
+
+    private string? _twoFactorCode;
+    private string? _authCode;
+    private readonly IUserInterventionHandler _interventionHandler;
+    private bool _isConnected;
+    private bool _isLoggedIn;
+    private bool _haveSigFile;
+
+    public TaskCompletionSource _licenseRequest = new();
+    private readonly SteamApps _steamApps;
+    private readonly DTOSerializer _dtos;
+    private IReadOnlyCollection<Server> _cdnServers = Array.Empty<Server>();
+
+    public SteamApps.LicenseListCallback.License[] Licenses { get; private set; }
+
+    public ConcurrentDictionary<uint, ulong> PackageTokens { get; } = new();
+
+    public ConcurrentDictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo?> PackageInfos { get; } = new();
+
+    public ConcurrentDictionary<uint, (SteamApps.PICSProductInfoCallback.PICSProductInfo ProductInfo, AppInfo AppInfo)> AppInfo { get; } =
+        new();
+
+    public ConcurrentDictionary<uint, ulong> AppTokens { get; } = new();
+    
+    
+    public ConcurrentDictionary<uint, byte[]> DepotKeys { get; } = new();
+
+
+    public Client(ILogger<Client> logger, HttpClient client, ITokenProvider<SteamLoginState> token,
+        IUserInterventionHandler interventionHandler, DTOSerializer dtos)
+    {
+        _logger = logger;
+        _httpClient = client;
+        _dtos = dtos;
+        _interventionHandler = interventionHandler;
+        _client = new SteamClient(SteamConfiguration.Create(c =>
+        {
+            c.WithProtocolTypes(ProtocolTypes.WebSocket);
+            c.WithUniverse(EUniverse.Public);
+        }));
+        
+
+        _cancellationSource = new CancellationTokenSource();
+        
+        _token = token;
+
+        _manager = new CallbackManager(_client);
+
+        _steamUser = _client.GetHandler<SteamUser>()!;
+        _steamApps = _client.GetHandler<SteamApps>()!;
+        
+        _manager.Subscribe<SteamClient.ConnectedCallback>( OnConnected );
+        _manager.Subscribe<SteamClient.DisconnectedCallback>( OnDisconnected );
+
+        _manager.Subscribe<SteamUser.LoggedOnCallback>( OnLoggedOn );
+        _manager.Subscribe<SteamUser.LoggedOffCallback>( OnLoggedOff );
+
+        _manager.Subscribe<SteamApps.LicenseListCallback>(OnLicenseList);
+        
+        _manager.Subscribe<SteamUser.UpdateMachineAuthCallback>( OnUpdateMachineAuthCallback );
+
+        _isConnected = false;
+        _isLoggedIn = false;
+        _haveSigFile = false;
+
+        new Thread(() =>
+        {
+            while (!_cancellationSource.IsCancellationRequested)
+            {
+                _manager.RunWaitCallbacks(TimeSpan.FromMilliseconds(250));
+            }
+        })
+        {
+            Name = "Steam Client callback runner",
+            IsBackground = true
+        }
+        .Start();
+
+    }
+
+    private void OnLicenseList(SteamApps.LicenseListCallback obj)
+    {       
+        if (obj.Result != EResult.OK)
+        {
+            _licenseRequest.TrySetException(new SteamException("While getting licenses", obj.Result, EResult.Invalid));
+        }
+        _logger.LogInformation("Got {LicenseCount} licenses from Steam", obj.LicenseList.Count);
+        Licenses = obj.LicenseList.ToArray();
+        _licenseRequest.TrySetResult();
+    }
+
+    private void OnUpdateMachineAuthCallback(SteamUser.UpdateMachineAuthCallback callback)
+    {
+        Task.Run(async () =>
+        {
+            int fileSize;
+            byte[] sentryHash;
+            
+            var token = await _token.Get();
+
+            var ms = new MemoryStream();
+            
+            if (token?.SentryFile != null)
+                await ms.WriteAsync(token.SentryFile);
+            
+            ms.Seek(callback.Offset, SeekOrigin.Begin);
+            ms.Write(callback.Data, 0, callback.BytesToWrite);
+            fileSize = (int) ms.Length;
+
+            token!.SentryFile = ms.ToArray();
+            sentryHash = CryptoHelper.SHAHash(token.SentryFile);
+
+            await _token.SetToken(token);
+            
+
+            _steamUser.SendMachineAuthResponse(new SteamUser.MachineAuthDetails
+            {
+                JobID = callback.JobID,
+                FileName = callback.FileName,
+
+                BytesWritten = callback.BytesToWrite,
+                FileSize = token.SentryFile.Length,
+                Offset = callback.Offset,
+                Result = EResult.OK,
+                LastError = 0,
+                OneTimePassword = callback.OneTimePassword,
+                SentryFileHash = sentryHash
+            });
+
+            _haveSigFile = true;
+            _loginTask.TrySetResult();
+        });
+    }
+
+    private void OnLoggedOff(SteamUser.LoggedOffCallback obj)
+    {
+        _isLoggedIn = false;
+    }
+
+    private void OnLoggedOn(SteamUser.LoggedOnCallback callback)
+    {
+        Task.Run(async () =>
+        {
+            var isSteamGuard = callback.Result == EResult.AccountLogonDenied;
+            var is2FA = callback.Result == EResult.AccountLoginDeniedNeedTwoFactor;
+
+            if (isSteamGuard || is2FA)
+            {
+                _logger.LogInformation("Account is SteamGuard protected");
+                if (is2FA)
+                {
+                    var intervention = new GetAuthCode(GetAuthCode.AuthType.TwoFactorAuth);
+                    _interventionHandler.Raise(intervention);
+                    _twoFactorCode = await intervention.Task;
+                }
+                else
+                {
+                    var intervention = new GetAuthCode(GetAuthCode.AuthType.EmailCode);
+                    _interventionHandler.Raise(intervention);
+                    _authCode = await intervention.Task;
+                }
+                
+                var tcs = Login(_loginTask);
+                return;
+            }
+
+            if (callback.Result != EResult.OK)
+            {
+                _loginTask.SetException(new SteamException("Unable to log in", callback.Result, callback.ExtendedResult));
+                return;
+            }
+
+            _isLoggedIn = true;
+            _logger.LogInformation("Logged into Steam");
+            if (_haveSigFile) 
+                _loginTask.SetResult();
+        });
+    }
+
+    private void OnDisconnected(SteamClient.DisconnectedCallback obj)
+    {
+        _isConnected = false;
+        _logger.LogInformation("Logged out");
+    }
+
+    private void OnConnected(SteamClient.ConnectedCallback obj)
+    {
+        Task.Run(async () =>
+        {
+            var state = (await _token.Get())!;
+            _logger.LogInformation("Connected to Steam, logging in as {User}", state.User);
+
+            byte[]? sentryHash = null;
+            
+            
+            if (state.SentryFile != null)
+            {
+                _logger.LogInformation("Existing login keys found, reusing");
+                sentryHash = CryptoHelper.SHAHash(state.SentryFile);
+                _haveSigFile = true;
+            }
+            else
+            {
+                _haveSigFile = false;
+            }
+            
+
+            _isConnected = true;
+            
+            _steamUser.LogOn(new SteamUser.LogOnDetails
+            {
+                Username = state.User,
+                Password = state.Password,
+                AuthCode = _authCode,
+                TwoFactorCode = _twoFactorCode,
+                SentryFileHash = sentryHash,
+                RequestSteam2Ticket = true
+            });
+        });
+    }
+    
+    public Task Connect()
+    {
+        _connectTask = new TaskCompletionSource();
+
+        _client.Connect();
+        return _connectTask.Task;
+    }
+
+    public void Dispose()
+    {
+        _httpClient.Dispose();
+        _cancellationSource.Cancel();
+        _cancellationSource.Dispose();
+    }
+
+    public async Task Login(TaskCompletionSource? tcs = null)
+    {
+        _loginTask = tcs ?? new TaskCompletionSource();
+        _logger.LogInformation("Attempting login");
+        _client.Connect();
+
+        await _loginTask.Task; 
+        await _licenseRequest.Task;
+    }
+
+    public async Task<Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo?>> GetPackageInfos(IEnumerable<uint> packageIds)
+    {
+        var packages = packageIds.Where(id => !PackageInfos.ContainsKey(id)).ToList();
+
+        if (packages.Count > 0)
+        {
+            var packageRequests = new List<SteamApps.PICSRequest>();
+
+            foreach (var package in packages)
+            {
+                var request = new SteamApps.PICSRequest(package);
+                if (PackageTokens.TryGetValue(package, out var token))
+                {
+                    request.AccessToken = token;
+                }
+
+                packageRequests.Add(request);
+            }
+
+            _logger.LogInformation("Requesting {Count} package infos", packageRequests.Count);
+
+            var results = await _steamApps.PICSGetProductInfo(new List<SteamApps.PICSRequest>(), packageRequests);
+
+            if (results.Failed)
+                throw new SteamException("Exception getting product info", EResult.Invalid, EResult.Invalid);
+
+            foreach (var packageInfo in results.Results!)
+            {
+                foreach (var package in packageInfo.Packages.Select(v => v.Value))
+                {
+                    PackageInfos[package.ID] = package;
+                }
+
+                foreach (var package in packageInfo.UnknownPackages)
+                {
+                    PackageInfos[package] = null;
+                }
+            }
+        }
+
+        return packages.Distinct().ToDictionary(p => p, p => PackageInfos[p]);
+
+    }
+
+    public async Task<AppInfo> GetAppInfo(uint appId)
+    {
+        if (AppInfo.TryGetValue(appId, out var info))
+            return info.AppInfo;
+
+        var result = await _steamApps.PICSGetAccessTokens(new List<uint> {appId}, new List<uint>());
+        
+        if (result.AppTokensDenied.Contains(appId))
+            throw new SteamException($"Cannot get app token for {appId}", EResult.Invalid, EResult.Invalid);
+
+        foreach (var token in result.AppTokens)
+        {
+            AppTokens[token.Key] = token.Value;
+        }
+
+        var request = new SteamApps.PICSRequest(appId);
+        if (AppTokens.ContainsKey(appId))
+        {
+            request.AccessToken = AppTokens[appId];
+        }
+
+        var appResult = await _steamApps.PICSGetProductInfo(new List<SteamApps.PICSRequest> {request},
+            new List<SteamApps.PICSRequest>());
+
+        if (appResult.Failed)
+            throw new SteamException($"Error getting app info for {appId}", EResult.Invalid, EResult.Invalid);
+
+        foreach (var (_, value) in appResult.Results!.SelectMany(v => v.Apps))
+        {
+            var translated = KeyValueTranslator.Translate<AppInfo>(value.KeyValues, _dtos);
+            AppInfo[value.ID] = (value, translated);
+        }
+
+        return AppInfo[appId].AppInfo;
+    }
+
+    public async Task<IReadOnlyCollection<Server>> LoadCDNServers()
+    {
+        if (_cdnServers.Count > 0) return _cdnServers;
+        _logger.LogInformation("Loading CDN servers");
+        _cdnServers = await ContentServerDirectoryService.LoadAsync(_client.Configuration);
+        _logger.LogInformation("{Count} servers found", _cdnServers.Count);
+        
+        return _cdnServers;
+    }
+
+    public async Task<DepotManifest> GetAppManifest(uint appId, uint depotId, ulong manifestId)
+    {
+        await LoadCDNServers();
+        var client = _cdnServers.First();
+
+        var uri = new UriBuilder()
+        {
+            Host = client.Host,
+            Port = client.Port,
+            Scheme = client.Protocol.ToString(),
+            Path = $"depot/{depotId}/manifest/{manifestId}/5"
+        }.Uri;
+
+        var rawData = await _httpClient.GetByteArrayAsync(uri);
+
+        using var zip = new ZipArchive(new MemoryStream(rawData));
+        var firstEntry = zip.Entries.First();
+        var data = new MemoryStream();
+        await using var entryStream = firstEntry.Open();
+        await entryStream.CopyToAsync(data);
+        var manifest = DepotManifest.Deserialize(data.ToArray());
+
+        if (manifest.FilenamesEncrypted)
+            manifest.DecryptFilenames(await GetDepotKey(depotId, appId));
+
+        return manifest;
+    }
+    
+    
+    public async ValueTask<byte[]> GetDepotKey(uint depotId, uint appId)
+    {
+        if (DepotKeys.ContainsKey(depotId))
+            return DepotKeys[depotId];
+        
+        _logger.LogInformation("Requesting Depot Key for {DepotId}", depotId);
+
+        var result = await _steamApps.GetDepotDecryptionKey(depotId, appId);
+        if (result.Result != EResult.OK)
+            throw new SteamException($"Error getting Depot Key for {depotId} {appId}", result.Result, EResult.Invalid);
+
+        DepotKeys[depotId] = result.DepotKey;
+        return result.DepotKey;
+    }
+
+    public async Task Download(uint appId, uint depotId, ulong manifest, DepotManifest.FileData fileData, AbsolutePath output)
+    {
+        await LoadCDNServers();
+        var client = _cdnServers.First();
+        var depotKey = await GetDepotKey(depotId, appId);
+
+        await using var os = output.Open(FileMode.Create, FileAccess.Write, FileShare.Read);
+        
+        foreach (var chunk in fileData.Chunks.OrderBy(c => c.Offset))
+        {
+
+            var chunkId = chunk.ChunkID!.ToHex();
+
+
+            var uri = new UriBuilder()
+            {
+                Host = client.Host,
+                Port = client.Port,
+                Scheme = client.Protocol.ToString(),
+                Path = $"depot/{depotId}/chunk/{chunkId}"
+            }.Uri;
+
+            var data = await _httpClient.GetByteArrayAsync(uri);
+            var chunkData = new DepotChunk(chunk, data);
+            chunkData.Process(depotKey);
+
+            await os.WriteAsync(chunkData.Data);
+
+        }
+        
+    }
+}

Wabbajack.Networking.Steam/DTOs/AppInfo.cs

@@ -0,0 +1,43 @@
+using System.Text.Json.Serialization;
+using Wabbajack.DTOs.JsonConverters;
+
+namespace Wabbajack.Networking.Steam.DTOs;
+
+
+public class AppInfo
+{
+
+    [JsonPropertyName("depots")] 
+    public Dictionary<string, Depot> Depots { get; set; } = new();
+
+}
+
+public class Depot
+{
+    [JsonPropertyName("name")]
+    public string Name { get; set; }
+    
+    [JsonPropertyName("config")]
+    public DepotConfig Config { get; set; }
+    
+    [JsonPropertyName("maxsize")]
+    public ulong MaxSize { get; set; }
+    
+    [JsonPropertyName("depotfromapp")]
+    public uint DepotFromApp { get; set; }
+    
+    [JsonPropertyName("sharedinstall")]
+    public uint SharedInstall { get; set; }
+
+    [JsonPropertyName("manifests")] 
+    public Dictionary<string, string> Manifests { get; set; } = new();
+}
+
+public class DepotConfig
+{
+    [JsonPropertyName("oslist")]
+    public string OSList { get; set; }
+
+    [JsonPropertyName("language")]
+    public string Language { get; set; }
+}

Wabbajack.Networking.Steam/DepotDownloader.cs

@@ -0,0 +1,39 @@
+using Microsoft.Extensions.Logging;
+using SteamKit2;
+using Wabbajack.Networking.Steam.DTOs;
+
+namespace Wabbajack.Networking.Steam;
+
+public class DepotDownloader
+{
+    private readonly ILogger<DepotDownloader> _logger;
+    private readonly Client _steamClient;
+
+    public DepotDownloader(ILogger<DepotDownloader> logger, Client client)
+    {
+        _logger = logger;
+        _steamClient = client;
+    }
+
+    public async Task<bool> AccountHasAccess(uint depotId)
+    {
+        var packages = _steamClient.Licenses.Select(l => l.PackageID);
+        var infos = await _steamClient.GetPackageInfos(packages);
+
+        foreach (var info in infos.Where(i => i.Value != null))
+        {
+            if (info.Value!.KeyValues["appids"].Children.Any(child => child.AsUnsignedInteger() == depotId))
+                return true;
+            
+            if (info.Value!.KeyValues["depotids"].Children.Any(child => child.AsUnsignedInteger() == depotId))
+                return true;
+
+        }
+        return false;
+    }
+
+    public async Task<AppInfo> GetAppInfo(uint appId)
+    {
+        return await _steamClient.GetAppInfo(appId);
+    }
+}

Wabbajack.Networking.Steam/KeyValueTranslator.cs

@@ -0,0 +1,47 @@
+using System.Text;
+using System.Text.Json;
+using SteamKit2;
+using Wabbajack.DTOs.JsonConverters;
+
+namespace Wabbajack.Networking.Steam;
+
+public class KeyValueTranslator
+{
+    public static void Translate(Utf8JsonWriter wtr, List<KeyValue> kvs)
+    {
+        foreach (var kv in kvs)
+        {
+            wtr.WritePropertyName(kv.Name!);
+            if (kv.Value == null)
+            {
+                wtr.WriteStartObject();
+                Translate(wtr, kv.Children);
+                wtr.WriteEndObject();
+            }
+            else
+            {
+                wtr.WriteStringValue(kv.Value);
+            }
+        }
+    }
+    
+
+    public static T Translate<T>(KeyValue kv, DTOSerializer dtos)
+    {
+        var ms = new MemoryStream();
+        var wtr = new Utf8JsonWriter(ms, new JsonWriterOptions()
+        {
+            Indented = true,
+        });
+
+        wtr.WriteStartObject();
+        Translate(wtr, kv.Children);
+        wtr.WriteEndObject();
+        wtr.Flush();
+
+        var str = Encoding.UTF8.GetString(ms.ToArray());
+        
+        
+        return JsonSerializer.Deserialize<T>(str, dtos.Options)!;
+    }
+}

Wabbajack.Networking.Steam/ServiceExtensions.cs

@@ -0,0 +1,13 @@
+using Microsoft.Extensions.DependencyInjection;
+using Wabbajack.Networking.Steam;
+
+namespace Wabbajack.Networking.NexusApi;
+
+public static class ServiceExtensions
+{
+    public static void AddSteam(this IServiceCollection services)
+    {
+        services.AddSingleton<Client>();
+        services.AddSingleton<DepotDownloader>();
+    }
+}

Wabbajack.Networking.Steam/SteamException.cs

@@ -0,0 +1,16 @@
+using SteamKit2;
+
+namespace Wabbajack.Networking.Steam;
+
+public class SteamException : Exception
+{
+    public EResult Result { get; }
+    public EResult ExtendedResult { get; }
+    
+    public SteamException(string message, EResult result, EResult extendedResult) : base($"{message} {result} / {extendedResult}")
+    {
+        Result = result;
+        ExtendedResult = extendedResult;
+    }
+    
+}

Wabbajack.Networking.Steam/SteamLoginState.cs

@@ -0,0 +1,8 @@
+namespace Wabbajack.Networking.Steam;
+
+public class SteamLoginState
+{
+    public byte[]? SentryFile { get; set; }
+    public string User { get; set; }
+    public string Password { get; set; }
+}

Wabbajack.Networking.Steam/UserInterventions/GetAuthCode.cs

@@ -0,0 +1,14 @@
+using Wabbajack.DTOs.Interventions;
+
+namespace Wabbajack.Networking.Steam.UserInterventions;
+
+public class GetAuthCode : AUserIntervention<string>
+{
+    public enum AuthType
+    {
+        TwoFactorAuth,
+        EmailCode
+    }
+    public GetAuthCode(AuthType type) => Type = type;
+    public AuthType Type { get; }
+}

Wabbajack.Networking.Steam/UserInterventions/GetUsernameAndPassword.cs

@@ -0,0 +1,19 @@
+using Wabbajack.DTOs.Interventions;
+
+namespace Wabbajack.Networking.Steam.UserInterventions;
+
+public class GetUsernameAndPassword : IUserIntervention
+{
+    public void Cancel()
+    {
+        throw new NotImplementedException();
+    }
+
+    public bool Handled { get; }
+    public CancellationToken Token { get; }
+    
+    public void SetException(Exception exception)
+    {
+        throw new NotImplementedException();
+    }
+}

Wabbajack.Networking.Steam/Wabbajack.Networking.Steam.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net6.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0" />
+      <PackageReference Include="SteamKit2" Version="2.4.0" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
+      <ProjectReference Include="..\Wabbajack.Networking.Http.Interfaces\Wabbajack.Networking.Http.Interfaces.csproj" />
+      <ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />
+    </ItemGroup>
+
+</Project>

Wabbajack.Networking.WabbajackClientApi/Client.cs

@@ -340,4 +340,10 @@ public class Client
     {
         return (await _client.GetFromJsonAsync<ForcedRemoval[]>("https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/configs/forced_removal.json", _dtos.Options, token))!;
     }
+
+    public async Task<SteamManifest[]> GetSteamManifests(Game game, string version)
+    {
+        var url = $"https://raw.githubusercontent.com/wabbajack-tools/indexed-game-files/master/{game}/{version}_steam_manifests.json";
+        return await _client.GetFromJsonAsync<SteamManifest[]>(url, _dtos.Options) ?? Array.Empty<SteamManifest>();
+    }
 }

Wabbajack.Services.OSIntegrated/LoggingRateLimiterReporter.cs

@@ -41,13 +41,21 @@ public class LoggingRateLimiterReporter : IDisposable
         var report = NextReport();
         var sb = new StringBuilder();
         sb.Append($"[#{_reportNumber}] ");
+        
+        var found = false;
         foreach (var (prev, next, limiter) in _prevReport.Zip(report, _limiters))
         {
             var throughput = next.Transferred - prev.Transferred;
-            sb.Append($"{limiter.Name}: [{next.Running}/{next.Pending}] {throughput.ToFileSizeString()}/sec ");
+            if (throughput > 0)
+            {
+                found = true;
+                sb.Append($"{limiter.Name}: [{next.Running}/{next.Pending}] {throughput.ToFileSizeString()}/sec ");
+            }
         }
 
-        _logger.LogInformation(sb.ToString());
+        if (found) 
+            _logger.LogInformation(sb.ToString());
+        
         _prevReport = report;
     }
 }

Wabbajack.Services.OSIntegrated/ServiceExtensions.cs

@@ -15,13 +15,14 @@ using Wabbajack.Networking.Discord;
 using Wabbajack.Networking.Http;
 using Wabbajack.Networking.Http.Interfaces;
 using Wabbajack.Networking.NexusApi;
-using Wabbajack.Networking.WabbajackClientApi;
+using Wabbajack.Networking.Steam;
 using Wabbajack.Paths;
 using Wabbajack.Paths.IO;
 using Wabbajack.RateLimiter;
 using Wabbajack.Services.OSIntegrated.Services;
 using Wabbajack.Services.OSIntegrated.TokenProviders;
 using Wabbajack.VFS;
+using Client = Wabbajack.Networking.WabbajackClientApi.Client;
 
 namespace Wabbajack.Services.OSIntegrated;
 
@@ -106,6 +107,8 @@ public static class ServiceExtensions
         service.AddSingleton<HttpClient>();
         service.AddAllSingleton<IHttpDownloader, SingleThreadedDownloader>();
 
+        service.AddSteam();
+            
         service.AddSingleton<Client>();
         service.AddSingleton<WriteOnlyClient>();
 
@@ -118,6 +121,10 @@ public static class ServiceExtensions
             .AddAllSingleton<ITokenProvider<VectorPlexusLoginState>, EncryptedJsonTokenProvider<VectorPlexusLoginState>,
                 VectorPlexusTokenProvider>();
 
+        service
+            .AddAllSingleton<ITokenProvider<SteamLoginState>, EncryptedJsonTokenProvider<SteamLoginState>,
+                SteamTokenProvider>();
+
         service.AddAllSingleton<ITokenProvider<WabbajackApiState>, WabbajackApiTokenProvider>();
 
         service

Wabbajack.Services.OSIntegrated/TokenProviders/SteamTokenProvider.cs

@@ -0,0 +1,13 @@
+using Microsoft.Extensions.Logging;
+using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Networking.Steam;
+
+namespace Wabbajack.Services.OSIntegrated.TokenProviders;
+
+public class SteamTokenProvider : EncryptedJsonTokenProvider<SteamLoginState>
+{
+    public SteamTokenProvider(ILogger<SteamTokenProvider> logger, DTOSerializer dtos) : base(logger, dtos,
+        "steam-login")
+    {
+    }
+}

Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj

@@ -21,6 +21,7 @@
         <ProjectReference Include="..\Wabbajack.Downloaders.Dispatcher\Wabbajack.Downloaders.Dispatcher.csproj" />
         <ProjectReference Include="..\Wabbajack.Installer\Wabbajack.Installer.csproj" />
         <ProjectReference Include="..\Wabbajack.Networking.Discord\Wabbajack.Networking.Discord.csproj" />
+        <ProjectReference Include="..\Wabbajack.Networking.Steam\Wabbajack.Networking.Steam.csproj" />
         <ProjectReference Include="..\Wabbajack.VFS\Wabbajack.VFS.csproj" />
     </ItemGroup>
 

Wabbajack.sln

@@ -116,6 +116,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Launcher", "Wabba
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.App.Wpf", "Wabbajack.App.Wpf\Wabbajack.App.Wpf.csproj", "{372B2DD2-EAA3-4E18-98A7-B9838C7B41F4}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Networking.Steam", "Wabbajack.Networking.Steam\Wabbajack.Networking.Steam.csproj", "{AB9A5C22-10CC-4EE0-A808-FB1DC9E24247}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Networking.Steam.Test", "Wabbajack.Networking.Steam.Test\Wabbajack.Networking.Steam.Test.csproj", "{D6351587-CAF6-4CB6-A2BD-5368E69F297C}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -322,6 +326,14 @@ Global
 		{372B2DD2-EAA3-4E18-98A7-B9838C7B41F4}.Debug|Any CPU.Build.0 = Debug|x64
 		{372B2DD2-EAA3-4E18-98A7-B9838C7B41F4}.Release|Any CPU.ActiveCfg = Release|x64
 		{372B2DD2-EAA3-4E18-98A7-B9838C7B41F4}.Release|Any CPU.Build.0 = Release|x64
+		{AB9A5C22-10CC-4EE0-A808-FB1DC9E24247}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AB9A5C22-10CC-4EE0-A808-FB1DC9E24247}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AB9A5C22-10CC-4EE0-A808-FB1DC9E24247}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AB9A5C22-10CC-4EE0-A808-FB1DC9E24247}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D6351587-CAF6-4CB6-A2BD-5368E69F297C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D6351587-CAF6-4CB6-A2BD-5368E69F297C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D6351587-CAF6-4CB6-A2BD-5368E69F297C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D6351587-CAF6-4CB6-A2BD-5368E69F297C}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{4057B668-8595-44FE-9805-007B284A838F} = {98B731EE-4FC0-4482-A069-BCBA25497871}
@@ -358,5 +370,7 @@ Global
 		{29AC8A68-D5EC-43F5-B2CC-72A75545E418} = {98B731EE-4FC0-4482-A069-BCBA25497871}
 		{DEB4B073-4EAA-49FD-9D43-F0F8CB930E7A} = {F01F8595-5FD7-4506-8469-F4A5522DACC1}
 		{4F252332-CA77-41DE-95A8-9DF38A81D675} = {98B731EE-4FC0-4482-A069-BCBA25497871}
+		{AB9A5C22-10CC-4EE0-A808-FB1DC9E24247} = {F01F8595-5FD7-4506-8469-F4A5522DACC1}
+		{D6351587-CAF6-4CB6-A2BD-5368E69F297C} = {F01F8595-5FD7-4506-8469-F4A5522DACC1}
 	EndGlobalSection
 EndGlobal