diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 54eeddce..e8bb4ac9 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -7,7 +7,7 @@ on: branches: [ main ] env: - VERSION: 3.0.0.0-beta1 + VERSION: 3.0.0.0-beta2 jobs: build: diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index 3f28c809..27e22063 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -11,6 +11,7 @@ using Octokit; using Wabbajack.CLI.TypeConverters; using Wabbajack.CLI.Verbs; using Wabbajack.DTOs.GitHub; +using Wabbajack.DTOs.Interventions; using Wabbajack.Networking.Http; using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Networking.WabbajackClientApi; @@ -63,6 +64,11 @@ internal class Program services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + services.AddSingleton(); }).Build(); var service = host.Services.GetService(); diff --git a/Wabbajack.CLI/UserInterventionHandler.cs b/Wabbajack.CLI/UserInterventionHandler.cs new file mode 100644 index 00000000..28313f21 --- /dev/null +++ b/Wabbajack.CLI/UserInterventionHandler.cs @@ -0,0 +1,27 @@ +using System; +using Wabbajack.DTOs.Interventions; +using Wabbajack.Networking.Steam.UserInterventions; + +namespace Wabbajack.CLI; + +public class UserInterventionHandler : IUserInterventionHandler +{ + public void Raise(IUserIntervention intervention) + { + if (intervention is GetAuthCode gac) + { + switch (gac.Type) + { + case GetAuthCode.AuthType.EmailCode: + Console.WriteLine("Please enter the Steam code that was just emailed to you"); + break; + case GetAuthCode.AuthType.TwoFactorAuth: + Console.WriteLine("Please enter your 2FA code for Steam"); + break; + default: + throw new ArgumentOutOfRangeException(); + } + gac.Finish(Console.ReadLine()!.Trim()); + } + } +} \ No newline at end of file diff --git a/Wabbajack.CLI/Verbs/SteamDownloadFile.cs b/Wabbajack.CLI/Verbs/SteamDownloadFile.cs new file mode 100644 index 00000000..36e8e6fd --- /dev/null +++ b/Wabbajack.CLI/Verbs/SteamDownloadFile.cs @@ -0,0 +1,95 @@ +using System.CommandLine; +using System.CommandLine.Invocation; +using System.Linq; +using System.Threading.Tasks; +using FluentFTP.Helpers; +using Microsoft.Extensions.Logging; +using SteamKit2; +using Wabbajack.DTOs; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Networking.Http.Interfaces; +using Wabbajack.Networking.Steam; +using Wabbajack.Paths; + +namespace Wabbajack.CLI.Verbs; + +public class SteamDownloadFile : IVerb +{ + private readonly ILogger _logger; + private readonly Client _client; + private readonly ITokenProvider _token; + private readonly DepotDownloader _downloader; + private readonly DTOSerializer _dtos; + private readonly Wabbajack.Networking.WabbajackClientApi.Client _wjClient; + + public SteamDownloadFile(ILogger logger, Client steamClient, ITokenProvider token, + DepotDownloader downloader, DTOSerializer dtos, Wabbajack.Networking.WabbajackClientApi.Client wjClient) + { + _logger = logger; + _client = steamClient; + _token = token; + _downloader = downloader; + _dtos = dtos; + _wjClient = wjClient; + } + public Command MakeCommand() + { + var command = new Command("steam-download-file"); + command.Description = "Dumps information to the console about the given app"; + + command.Add(new Option(new[] {"-g", "-game", "-gameName"}, "Wabbajack game name")); + + command.Add(new Option(new[] {"-v", "-version"}, "Version of the game to download for")); + command.Add(new Option(new[] {"-f", "-file"}, "File to download (relative path)")); + command.Add(new Option(new[] {"-o", "-output"}, "Output location")); + command.Handler = CommandHandler.Create(Run); + return command; + } + + private async Task Run(string gameName, string version, string file, AbsolutePath output) + { + if (!GameRegistry.TryGetByFuzzyName(gameName, out var game)) + _logger.LogError("Can't find definition for {Game}", gameName); + + await _client.Login(); + + var definition = await _wjClient.GetGameArchives(game.Game, version); + var manifests = await _wjClient.GetSteamManifests(game.Game, version); + + _logger.LogInformation("Found {Count} manifests, looking for file", manifests.Length); + + SteamManifest? steamManifest = null; + DepotManifest? depotManifest = null; + DepotManifest.FileData? fileData = null; + + var appId = (uint) game.SteamIDs.First(); + + foreach (var manifest in manifests) + { + steamManifest = manifest; + depotManifest = await _client.GetAppManifest(appId, manifest.Depot, manifest.Manifest); + fileData = depotManifest.Files!.FirstOrDefault(f => f.FileName == file); + if (fileData != default) + { + break; + } + } + + if (fileData == default) + { + _logger.LogError("Cannot find {File} in any manifests", file); + return 1; + } + + _logger.LogInformation("File has is {Size} and {ChunkCount} chunks", fileData.TotalSize.FileSizeToString(), fileData.Chunks.Count); + + await _client.Download(appId, depotManifest!.DepotID, steamManifest!.Manifest, fileData, output); + + _logger.LogInformation("File downloaded"); + + return 0; + + + + } +} \ No newline at end of file diff --git a/Wabbajack.CLI/Verbs/SteamDumpAppInfo.cs b/Wabbajack.CLI/Verbs/SteamDumpAppInfo.cs new file mode 100644 index 00000000..d2c95f9f --- /dev/null +++ b/Wabbajack.CLI/Verbs/SteamDumpAppInfo.cs @@ -0,0 +1,71 @@ +using System; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using SteamKit2; +using Wabbajack.DTOs; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Networking.Http.Interfaces; +using Wabbajack.Networking.Steam; +using JsonSerializer = System.Text.Json.JsonSerializer; + +namespace Wabbajack.CLI.Verbs; + +public class SteamAppDumpInfo : IVerb +{ + private readonly ILogger _logger; + private readonly Client _client; + private readonly ITokenProvider _token; + private readonly DepotDownloader _downloader; + private readonly DTOSerializer _dtos; + + public SteamAppDumpInfo(ILogger logger, Client steamClient, ITokenProvider token, + DepotDownloader downloader, DTOSerializer dtos) + { + _logger = logger; + _client = steamClient; + _token = token; + _downloader = downloader; + _dtos = dtos; + } + public Command MakeCommand() + { + var command = new Command("steam-app-dump-info"); + command.Description = "Dumps information to the console about the given app"; + + command.Add(new Option(new[] {"-g", "-game", "-gameName"}, "Wabbajack game name")); + command.Handler = CommandHandler.Create(Run); + return command; + } + + public async Task Run(string gameName) + { + if (!GameRegistry.TryGetByFuzzyName(gameName, out var game)) + { + _logger.LogError("Can't find game {GameName} in game registry", gameName); + return 1; + } + + await _client.Login(); + var appId = (uint) game.SteamIDs.First(); + + if (!await _downloader.AccountHasAccess(appId)) + { + _logger.LogError("Your account does not have access to this Steam App"); + return 1; + } + + var appData = await _downloader.GetAppInfo((uint)game.SteamIDs.First()); + + Console.WriteLine("App Depots: "); + + Console.WriteLine(_dtos.Serialize(appData, true)); + + return 0; + } + + +} \ No newline at end of file diff --git a/Wabbajack.CLI/Verbs/SteamLogin.cs b/Wabbajack.CLI/Verbs/SteamLogin.cs new file mode 100644 index 00000000..5d99c89a --- /dev/null +++ b/Wabbajack.CLI/Verbs/SteamLogin.cs @@ -0,0 +1,58 @@ +using System; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.Networking.Http.Interfaces; +using Wabbajack.Networking.Steam; +using Wabbajack.Paths; + +namespace Wabbajack.CLI.Verbs; + +public class SteamLogin : IVerb +{ + private readonly ILogger _logger; + private readonly Client _client; + private readonly ITokenProvider _token; + + public SteamLogin(ILogger logger, Client steamClient, ITokenProvider token) + { + _logger = logger; + _client = steamClient; + _token = token; + } + public Command MakeCommand() + { + var command = new Command("steam-login"); + command.Description = "Logs into Steam via interactive prompts"; + + command.Add(new Option(new[] {"-u", "-user"}, "Username for login")); + command.Handler = CommandHandler.Create(Run); + return command; + } + + public async Task Run(string user) + { + var token = await _token.Get(); + + if (token == null || token.User != user || string.IsNullOrWhiteSpace(token.Password)) + { + Console.WriteLine("Please enter password"); + var password = Console.ReadLine() ?? ""; + + await _token.SetToken(new SteamLoginState + { + User = user, + Password = password.Trim() + }); + } + + _logger.LogInformation("Attempting login"); + await _client.Login(); + + await Task.Delay(10000); + + return 0; + } + +} \ No newline at end of file diff --git a/Wabbajack.DTOs/Game/SteamManifest.cs b/Wabbajack.DTOs/Game/SteamManifest.cs new file mode 100644 index 00000000..836121f8 --- /dev/null +++ b/Wabbajack.DTOs/Game/SteamManifest.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace Wabbajack.DTOs; + +public class SteamManifest +{ + [JsonPropertyName("Depot")] + public uint Depot { get; set; } + + [JsonPropertyName("Manifest")] + public ulong Manifest { get; set; } +} \ No newline at end of file diff --git a/Wabbajack.DTOs/Interventions/AUserIntervention.cs b/Wabbajack.DTOs/Interventions/AUserIntervention.cs new file mode 100644 index 00000000..e9a5a2b0 --- /dev/null +++ b/Wabbajack.DTOs/Interventions/AUserIntervention.cs @@ -0,0 +1,38 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Wabbajack.DTOs.Interventions; + +public class AUserIntervention : IUserIntervention +{ + private readonly TaskCompletionSource _tcs; + private readonly CancellationTokenSource _ct; + + protected AUserIntervention() + { + _tcs = new TaskCompletionSource(); + _ct = new CancellationTokenSource(); + } + public void Cancel() + { + _ct.Cancel(); + _tcs.SetCanceled(Token); + } + + public bool Handled => _tcs.Task.IsCompleted; + public CancellationToken Token => _ct.Token; + public Task Task => _tcs.Task; + + public void Finish(T value) + { + _tcs.SetResult(value); + _ct.Cancel(); + } + + public void SetException(Exception exception) + { + _ct.Cancel(); + _tcs.SetException(exception); + } +} \ No newline at end of file diff --git a/Wabbajack.DTOs/Interventions/IUserIntervention.cs b/Wabbajack.DTOs/Interventions/IUserIntervention.cs index 8326336c..89c24cd1 100644 --- a/Wabbajack.DTOs/Interventions/IUserIntervention.cs +++ b/Wabbajack.DTOs/Interventions/IUserIntervention.cs @@ -25,4 +25,4 @@ public interface IUserIntervention public CancellationToken Token { get; } void SetException(Exception exception); -} \ No newline at end of file +} diff --git a/Wabbajack.Networking.Steam.Test/ClientTests.cs b/Wabbajack.Networking.Steam.Test/ClientTests.cs new file mode 100644 index 00000000..e28b6f32 --- /dev/null +++ b/Wabbajack.Networking.Steam.Test/ClientTests.cs @@ -0,0 +1,36 @@ +using System.Threading.Tasks; +using Wabbajack.DTOs.Interventions; +using Wabbajack.Networking.Http.Interfaces; +using Xunit; + +namespace Wabbajack.Networking.Steam.Test; + +public class ClientTests +{ + private readonly ITokenProvider _token; + private readonly Client _steamClient; + private readonly IUserInterventionHandler _userInterventionHandler; + + public ClientTests(ITokenProvider token, Client client, IUserInterventionHandler userInterventionHandler) + { + _token = token; + _steamClient = client; + _userInterventionHandler = userInterventionHandler; + } + + [Fact] + public async Task CanGetLogin() + { + var token = await _token.Get(); + Assert.NotNull(token); + Assert.NotEmpty(token!.User); + } + + [Fact] + public async Task CanLogin() + { + await _steamClient.Connect(); + await _steamClient.Login(); + + } +} \ No newline at end of file diff --git a/Wabbajack.Networking.Steam.Test/Startup.cs b/Wabbajack.Networking.Steam.Test/Startup.cs new file mode 100644 index 00000000..3074aae9 --- /dev/null +++ b/Wabbajack.Networking.Steam.Test/Startup.cs @@ -0,0 +1,20 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Wabbajack.Services.OSIntegrated; +using Xunit.DependencyInjection; +using Xunit.DependencyInjection.Logging; + +namespace Wabbajack.Networking.Steam.Test; + +public class Startup +{ + public void ConfigureServices(IServiceCollection service) + { + service.AddOSIntegrated(); + } + + public void Configure(ILoggerFactory loggerFactory, ITestOutputHelperAccessor accessor) + { + loggerFactory.AddProvider(new XunitTestOutputLoggerProvider(accessor, delegate { return true; })); + } +} \ No newline at end of file diff --git a/Wabbajack.Networking.Steam.Test/Wabbajack.Networking.Steam.Test.csproj b/Wabbajack.Networking.Steam.Test/Wabbajack.Networking.Steam.Test.csproj new file mode 100644 index 00000000..01103a7d --- /dev/null +++ b/Wabbajack.Networking.Steam.Test/Wabbajack.Networking.Steam.Test.csproj @@ -0,0 +1,28 @@ + + + + net6.0 + enable + + false + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/Wabbajack.Networking.Steam/Client.cs b/Wabbajack.Networking.Steam/Client.cs new file mode 100644 index 00000000..47dd8047 --- /dev/null +++ b/Wabbajack.Networking.Steam/Client.cs @@ -0,0 +1,442 @@ +using System.Collections.Concurrent; +using System.IO.Compression; +using System.Security; +using Microsoft.Extensions.Logging; +using Microsoft.VisualBasic.CompilerServices; +using SteamKit2; +using SteamKit2.CDN; +using SteamKit2.Internal; +using Wabbajack.DTOs.Interventions; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Networking.Http.Interfaces; +using Wabbajack.Networking.Steam.DTOs; +using Wabbajack.Networking.Steam.UserInterventions; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; + +namespace Wabbajack.Networking.Steam; + +public class Client : IDisposable +{ + private readonly ILogger _logger; + private readonly HttpClient _httpClient; + private readonly SteamClient _client; + private readonly SteamUser _steamUser; + private readonly CallbackManager _manager; + private readonly ITokenProvider _token; + private TaskCompletionSource _loginTask; + private TaskCompletionSource _connectTask; + private readonly CancellationTokenSource _cancellationSource; + + private string? _twoFactorCode; + private string? _authCode; + private readonly IUserInterventionHandler _interventionHandler; + private bool _isConnected; + private bool _isLoggedIn; + private bool _haveSigFile; + + public TaskCompletionSource _licenseRequest = new(); + private readonly SteamApps _steamApps; + private readonly DTOSerializer _dtos; + private IReadOnlyCollection _cdnServers = Array.Empty(); + + public SteamApps.LicenseListCallback.License[] Licenses { get; private set; } + + public ConcurrentDictionary PackageTokens { get; } = new(); + + public ConcurrentDictionary PackageInfos { get; } = new(); + + public ConcurrentDictionary AppInfo { get; } = + new(); + + public ConcurrentDictionary AppTokens { get; } = new(); + + + public ConcurrentDictionary DepotKeys { get; } = new(); + + + public Client(ILogger logger, HttpClient client, ITokenProvider token, + IUserInterventionHandler interventionHandler, DTOSerializer dtos) + { + _logger = logger; + _httpClient = client; + _dtos = dtos; + _interventionHandler = interventionHandler; + _client = new SteamClient(SteamConfiguration.Create(c => + { + c.WithProtocolTypes(ProtocolTypes.WebSocket); + c.WithUniverse(EUniverse.Public); + })); + + + _cancellationSource = new CancellationTokenSource(); + + _token = token; + + _manager = new CallbackManager(_client); + + _steamUser = _client.GetHandler()!; + _steamApps = _client.GetHandler()!; + + _manager.Subscribe( OnConnected ); + _manager.Subscribe( OnDisconnected ); + + _manager.Subscribe( OnLoggedOn ); + _manager.Subscribe( OnLoggedOff ); + + _manager.Subscribe(OnLicenseList); + + _manager.Subscribe( OnUpdateMachineAuthCallback ); + + _isConnected = false; + _isLoggedIn = false; + _haveSigFile = false; + + new Thread(() => + { + while (!_cancellationSource.IsCancellationRequested) + { + _manager.RunWaitCallbacks(TimeSpan.FromMilliseconds(250)); + } + }) + { + Name = "Steam Client callback runner", + IsBackground = true + } + .Start(); + + } + + private void OnLicenseList(SteamApps.LicenseListCallback obj) + { + if (obj.Result != EResult.OK) + { + _licenseRequest.TrySetException(new SteamException("While getting licenses", obj.Result, EResult.Invalid)); + } + _logger.LogInformation("Got {LicenseCount} licenses from Steam", obj.LicenseList.Count); + Licenses = obj.LicenseList.ToArray(); + _licenseRequest.TrySetResult(); + } + + private void OnUpdateMachineAuthCallback(SteamUser.UpdateMachineAuthCallback callback) + { + Task.Run(async () => + { + int fileSize; + byte[] sentryHash; + + var token = await _token.Get(); + + var ms = new MemoryStream(); + + if (token?.SentryFile != null) + await ms.WriteAsync(token.SentryFile); + + ms.Seek(callback.Offset, SeekOrigin.Begin); + ms.Write(callback.Data, 0, callback.BytesToWrite); + fileSize = (int) ms.Length; + + token!.SentryFile = ms.ToArray(); + sentryHash = CryptoHelper.SHAHash(token.SentryFile); + + await _token.SetToken(token); + + + _steamUser.SendMachineAuthResponse(new SteamUser.MachineAuthDetails + { + JobID = callback.JobID, + FileName = callback.FileName, + + BytesWritten = callback.BytesToWrite, + FileSize = token.SentryFile.Length, + Offset = callback.Offset, + Result = EResult.OK, + LastError = 0, + OneTimePassword = callback.OneTimePassword, + SentryFileHash = sentryHash + }); + + _haveSigFile = true; + _loginTask.TrySetResult(); + }); + } + + private void OnLoggedOff(SteamUser.LoggedOffCallback obj) + { + _isLoggedIn = false; + } + + private void OnLoggedOn(SteamUser.LoggedOnCallback callback) + { + Task.Run(async () => + { + var isSteamGuard = callback.Result == EResult.AccountLogonDenied; + var is2FA = callback.Result == EResult.AccountLoginDeniedNeedTwoFactor; + + if (isSteamGuard || is2FA) + { + _logger.LogInformation("Account is SteamGuard protected"); + if (is2FA) + { + var intervention = new GetAuthCode(GetAuthCode.AuthType.TwoFactorAuth); + _interventionHandler.Raise(intervention); + _twoFactorCode = await intervention.Task; + } + else + { + var intervention = new GetAuthCode(GetAuthCode.AuthType.EmailCode); + _interventionHandler.Raise(intervention); + _authCode = await intervention.Task; + } + + var tcs = Login(_loginTask); + return; + } + + if (callback.Result != EResult.OK) + { + _loginTask.SetException(new SteamException("Unable to log in", callback.Result, callback.ExtendedResult)); + return; + } + + _isLoggedIn = true; + _logger.LogInformation("Logged into Steam"); + if (_haveSigFile) + _loginTask.SetResult(); + }); + } + + private void OnDisconnected(SteamClient.DisconnectedCallback obj) + { + _isConnected = false; + _logger.LogInformation("Logged out"); + } + + private void OnConnected(SteamClient.ConnectedCallback obj) + { + Task.Run(async () => + { + var state = (await _token.Get())!; + _logger.LogInformation("Connected to Steam, logging in as {User}", state.User); + + byte[]? sentryHash = null; + + + if (state.SentryFile != null) + { + _logger.LogInformation("Existing login keys found, reusing"); + sentryHash = CryptoHelper.SHAHash(state.SentryFile); + _haveSigFile = true; + } + else + { + _haveSigFile = false; + } + + + _isConnected = true; + + _steamUser.LogOn(new SteamUser.LogOnDetails + { + Username = state.User, + Password = state.Password, + AuthCode = _authCode, + TwoFactorCode = _twoFactorCode, + SentryFileHash = sentryHash, + RequestSteam2Ticket = true + }); + }); + } + + public Task Connect() + { + _connectTask = new TaskCompletionSource(); + + _client.Connect(); + return _connectTask.Task; + } + + public void Dispose() + { + _httpClient.Dispose(); + _cancellationSource.Cancel(); + _cancellationSource.Dispose(); + } + + public async Task Login(TaskCompletionSource? tcs = null) + { + _loginTask = tcs ?? new TaskCompletionSource(); + _logger.LogInformation("Attempting login"); + _client.Connect(); + + await _loginTask.Task; + await _licenseRequest.Task; + } + + public async Task> GetPackageInfos(IEnumerable packageIds) + { + var packages = packageIds.Where(id => !PackageInfos.ContainsKey(id)).ToList(); + + if (packages.Count > 0) + { + var packageRequests = new List(); + + foreach (var package in packages) + { + var request = new SteamApps.PICSRequest(package); + if (PackageTokens.TryGetValue(package, out var token)) + { + request.AccessToken = token; + } + + packageRequests.Add(request); + } + + _logger.LogInformation("Requesting {Count} package infos", packageRequests.Count); + + var results = await _steamApps.PICSGetProductInfo(new List(), packageRequests); + + if (results.Failed) + throw new SteamException("Exception getting product info", EResult.Invalid, EResult.Invalid); + + foreach (var packageInfo in results.Results!) + { + foreach (var package in packageInfo.Packages.Select(v => v.Value)) + { + PackageInfos[package.ID] = package; + } + + foreach (var package in packageInfo.UnknownPackages) + { + PackageInfos[package] = null; + } + } + } + + return packages.Distinct().ToDictionary(p => p, p => PackageInfos[p]); + + } + + public async Task GetAppInfo(uint appId) + { + if (AppInfo.TryGetValue(appId, out var info)) + return info.AppInfo; + + var result = await _steamApps.PICSGetAccessTokens(new List {appId}, new List()); + + if (result.AppTokensDenied.Contains(appId)) + throw new SteamException($"Cannot get app token for {appId}", EResult.Invalid, EResult.Invalid); + + foreach (var token in result.AppTokens) + { + AppTokens[token.Key] = token.Value; + } + + var request = new SteamApps.PICSRequest(appId); + if (AppTokens.ContainsKey(appId)) + { + request.AccessToken = AppTokens[appId]; + } + + var appResult = await _steamApps.PICSGetProductInfo(new List {request}, + new List()); + + if (appResult.Failed) + throw new SteamException($"Error getting app info for {appId}", EResult.Invalid, EResult.Invalid); + + foreach (var (_, value) in appResult.Results!.SelectMany(v => v.Apps)) + { + var translated = KeyValueTranslator.Translate(value.KeyValues, _dtos); + AppInfo[value.ID] = (value, translated); + } + + return AppInfo[appId].AppInfo; + } + + public async Task> LoadCDNServers() + { + if (_cdnServers.Count > 0) return _cdnServers; + _logger.LogInformation("Loading CDN servers"); + _cdnServers = await ContentServerDirectoryService.LoadAsync(_client.Configuration); + _logger.LogInformation("{Count} servers found", _cdnServers.Count); + + return _cdnServers; + } + + public async Task GetAppManifest(uint appId, uint depotId, ulong manifestId) + { + await LoadCDNServers(); + var client = _cdnServers.First(); + + var uri = new UriBuilder() + { + Host = client.Host, + Port = client.Port, + Scheme = client.Protocol.ToString(), + Path = $"depot/{depotId}/manifest/{manifestId}/5" + }.Uri; + + var rawData = await _httpClient.GetByteArrayAsync(uri); + + using var zip = new ZipArchive(new MemoryStream(rawData)); + var firstEntry = zip.Entries.First(); + var data = new MemoryStream(); + await using var entryStream = firstEntry.Open(); + await entryStream.CopyToAsync(data); + var manifest = DepotManifest.Deserialize(data.ToArray()); + + if (manifest.FilenamesEncrypted) + manifest.DecryptFilenames(await GetDepotKey(depotId, appId)); + + return manifest; + } + + + public async ValueTask GetDepotKey(uint depotId, uint appId) + { + if (DepotKeys.ContainsKey(depotId)) + return DepotKeys[depotId]; + + _logger.LogInformation("Requesting Depot Key for {DepotId}", depotId); + + var result = await _steamApps.GetDepotDecryptionKey(depotId, appId); + if (result.Result != EResult.OK) + throw new SteamException($"Error getting Depot Key for {depotId} {appId}", result.Result, EResult.Invalid); + + DepotKeys[depotId] = result.DepotKey; + return result.DepotKey; + } + + public async Task Download(uint appId, uint depotId, ulong manifest, DepotManifest.FileData fileData, AbsolutePath output) + { + await LoadCDNServers(); + var client = _cdnServers.First(); + var depotKey = await GetDepotKey(depotId, appId); + + await using var os = output.Open(FileMode.Create, FileAccess.Write, FileShare.Read); + + foreach (var chunk in fileData.Chunks.OrderBy(c => c.Offset)) + { + + var chunkId = chunk.ChunkID!.ToHex(); + + + var uri = new UriBuilder() + { + Host = client.Host, + Port = client.Port, + Scheme = client.Protocol.ToString(), + Path = $"depot/{depotId}/chunk/{chunkId}" + }.Uri; + + var data = await _httpClient.GetByteArrayAsync(uri); + var chunkData = new DepotChunk(chunk, data); + chunkData.Process(depotKey); + + await os.WriteAsync(chunkData.Data); + + } + + } +} \ No newline at end of file diff --git a/Wabbajack.Networking.Steam/DTOs/AppInfo.cs b/Wabbajack.Networking.Steam/DTOs/AppInfo.cs new file mode 100644 index 00000000..c4af7bd2 --- /dev/null +++ b/Wabbajack.Networking.Steam/DTOs/AppInfo.cs @@ -0,0 +1,43 @@ +using System.Text.Json.Serialization; +using Wabbajack.DTOs.JsonConverters; + +namespace Wabbajack.Networking.Steam.DTOs; + + +public class AppInfo +{ + + [JsonPropertyName("depots")] + public Dictionary Depots { get; set; } = new(); + +} + +public class Depot +{ + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("config")] + public DepotConfig Config { get; set; } + + [JsonPropertyName("maxsize")] + public ulong MaxSize { get; set; } + + [JsonPropertyName("depotfromapp")] + public uint DepotFromApp { get; set; } + + [JsonPropertyName("sharedinstall")] + public uint SharedInstall { get; set; } + + [JsonPropertyName("manifests")] + public Dictionary Manifests { get; set; } = new(); +} + +public class DepotConfig +{ + [JsonPropertyName("oslist")] + public string OSList { get; set; } + + [JsonPropertyName("language")] + public string Language { get; set; } +} \ No newline at end of file diff --git a/Wabbajack.Networking.Steam/DepotDownloader.cs b/Wabbajack.Networking.Steam/DepotDownloader.cs new file mode 100644 index 00000000..bdd29f3e --- /dev/null +++ b/Wabbajack.Networking.Steam/DepotDownloader.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.Logging; +using SteamKit2; +using Wabbajack.Networking.Steam.DTOs; + +namespace Wabbajack.Networking.Steam; + +public class DepotDownloader +{ + private readonly ILogger _logger; + private readonly Client _steamClient; + + public DepotDownloader(ILogger logger, Client client) + { + _logger = logger; + _steamClient = client; + } + + public async Task AccountHasAccess(uint depotId) + { + var packages = _steamClient.Licenses.Select(l => l.PackageID); + var infos = await _steamClient.GetPackageInfos(packages); + + foreach (var info in infos.Where(i => i.Value != null)) + { + if (info.Value!.KeyValues["appids"].Children.Any(child => child.AsUnsignedInteger() == depotId)) + return true; + + if (info.Value!.KeyValues["depotids"].Children.Any(child => child.AsUnsignedInteger() == depotId)) + return true; + + } + return false; + } + + public async Task GetAppInfo(uint appId) + { + return await _steamClient.GetAppInfo(appId); + } +} \ No newline at end of file diff --git a/Wabbajack.Networking.Steam/KeyValueTranslator.cs b/Wabbajack.Networking.Steam/KeyValueTranslator.cs new file mode 100644 index 00000000..afa9b197 --- /dev/null +++ b/Wabbajack.Networking.Steam/KeyValueTranslator.cs @@ -0,0 +1,47 @@ +using System.Text; +using System.Text.Json; +using SteamKit2; +using Wabbajack.DTOs.JsonConverters; + +namespace Wabbajack.Networking.Steam; + +public class KeyValueTranslator +{ + public static void Translate(Utf8JsonWriter wtr, List kvs) + { + foreach (var kv in kvs) + { + wtr.WritePropertyName(kv.Name!); + if (kv.Value == null) + { + wtr.WriteStartObject(); + Translate(wtr, kv.Children); + wtr.WriteEndObject(); + } + else + { + wtr.WriteStringValue(kv.Value); + } + } + } + + + public static T Translate(KeyValue kv, DTOSerializer dtos) + { + var ms = new MemoryStream(); + var wtr = new Utf8JsonWriter(ms, new JsonWriterOptions() + { + Indented = true, + }); + + wtr.WriteStartObject(); + Translate(wtr, kv.Children); + wtr.WriteEndObject(); + wtr.Flush(); + + var str = Encoding.UTF8.GetString(ms.ToArray()); + + + return JsonSerializer.Deserialize(str, dtos.Options)!; + } +} \ No newline at end of file diff --git a/Wabbajack.Networking.Steam/ServiceExtensions.cs b/Wabbajack.Networking.Steam/ServiceExtensions.cs new file mode 100644 index 00000000..09dd4848 --- /dev/null +++ b/Wabbajack.Networking.Steam/ServiceExtensions.cs @@ -0,0 +1,13 @@ +using Microsoft.Extensions.DependencyInjection; +using Wabbajack.Networking.Steam; + +namespace Wabbajack.Networking.NexusApi; + +public static class ServiceExtensions +{ + public static void AddSteam(this IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + } +} \ No newline at end of file diff --git a/Wabbajack.Networking.Steam/SteamException.cs b/Wabbajack.Networking.Steam/SteamException.cs new file mode 100644 index 00000000..58ee9ae3 --- /dev/null +++ b/Wabbajack.Networking.Steam/SteamException.cs @@ -0,0 +1,16 @@ +using SteamKit2; + +namespace Wabbajack.Networking.Steam; + +public class SteamException : Exception +{ + public EResult Result { get; } + public EResult ExtendedResult { get; } + + public SteamException(string message, EResult result, EResult extendedResult) : base($"{message} {result} / {extendedResult}") + { + Result = result; + ExtendedResult = extendedResult; + } + +} \ No newline at end of file diff --git a/Wabbajack.Networking.Steam/SteamLoginState.cs b/Wabbajack.Networking.Steam/SteamLoginState.cs new file mode 100644 index 00000000..fb2ee469 --- /dev/null +++ b/Wabbajack.Networking.Steam/SteamLoginState.cs @@ -0,0 +1,8 @@ +namespace Wabbajack.Networking.Steam; + +public class SteamLoginState +{ + public byte[]? SentryFile { get; set; } + public string User { get; set; } + public string Password { get; set; } +} \ No newline at end of file diff --git a/Wabbajack.Networking.Steam/UserInterventions/GetAuthCode.cs b/Wabbajack.Networking.Steam/UserInterventions/GetAuthCode.cs new file mode 100644 index 00000000..07254d53 --- /dev/null +++ b/Wabbajack.Networking.Steam/UserInterventions/GetAuthCode.cs @@ -0,0 +1,14 @@ +using Wabbajack.DTOs.Interventions; + +namespace Wabbajack.Networking.Steam.UserInterventions; + +public class GetAuthCode : AUserIntervention +{ + public enum AuthType + { + TwoFactorAuth, + EmailCode + } + public GetAuthCode(AuthType type) => Type = type; + public AuthType Type { get; } +} \ No newline at end of file diff --git a/Wabbajack.Networking.Steam/UserInterventions/GetUsernameAndPassword.cs b/Wabbajack.Networking.Steam/UserInterventions/GetUsernameAndPassword.cs new file mode 100644 index 00000000..3f57de4c --- /dev/null +++ b/Wabbajack.Networking.Steam/UserInterventions/GetUsernameAndPassword.cs @@ -0,0 +1,19 @@ +using Wabbajack.DTOs.Interventions; + +namespace Wabbajack.Networking.Steam.UserInterventions; + +public class GetUsernameAndPassword : IUserIntervention +{ + public void Cancel() + { + throw new NotImplementedException(); + } + + public bool Handled { get; } + public CancellationToken Token { get; } + + public void SetException(Exception exception) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/Wabbajack.Networking.Steam/Wabbajack.Networking.Steam.csproj b/Wabbajack.Networking.Steam/Wabbajack.Networking.Steam.csproj new file mode 100644 index 00000000..26d6cc9a --- /dev/null +++ b/Wabbajack.Networking.Steam/Wabbajack.Networking.Steam.csproj @@ -0,0 +1,20 @@ + + + + net6.0 + enable + enable + + + + + + + + + + + + + + diff --git a/Wabbajack.Networking.WabbajackClientApi/Client.cs b/Wabbajack.Networking.WabbajackClientApi/Client.cs index 6c7439b7..3e562628 100644 --- a/Wabbajack.Networking.WabbajackClientApi/Client.cs +++ b/Wabbajack.Networking.WabbajackClientApi/Client.cs @@ -340,4 +340,10 @@ public class Client { return (await _client.GetFromJsonAsync("https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/configs/forced_removal.json", _dtos.Options, token))!; } + + public async Task GetSteamManifests(Game game, string version) + { + var url = $"https://raw.githubusercontent.com/wabbajack-tools/indexed-game-files/master/{game}/{version}_steam_manifests.json"; + return await _client.GetFromJsonAsync(url, _dtos.Options) ?? Array.Empty(); + } } \ No newline at end of file diff --git a/Wabbajack.Services.OSIntegrated/LoggingRateLimiterReporter.cs b/Wabbajack.Services.OSIntegrated/LoggingRateLimiterReporter.cs index 2767bda1..544385f9 100644 --- a/Wabbajack.Services.OSIntegrated/LoggingRateLimiterReporter.cs +++ b/Wabbajack.Services.OSIntegrated/LoggingRateLimiterReporter.cs @@ -41,13 +41,21 @@ public class LoggingRateLimiterReporter : IDisposable var report = NextReport(); var sb = new StringBuilder(); sb.Append($"[#{_reportNumber}] "); + + var found = false; foreach (var (prev, next, limiter) in _prevReport.Zip(report, _limiters)) { var throughput = next.Transferred - prev.Transferred; - sb.Append($"{limiter.Name}: [{next.Running}/{next.Pending}] {throughput.ToFileSizeString()}/sec "); + if (throughput > 0) + { + found = true; + sb.Append($"{limiter.Name}: [{next.Running}/{next.Pending}] {throughput.ToFileSizeString()}/sec "); + } } - _logger.LogInformation(sb.ToString()); + if (found) + _logger.LogInformation(sb.ToString()); + _prevReport = report; } } \ No newline at end of file diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index c7a50f77..57247400 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -15,13 +15,14 @@ using Wabbajack.Networking.Discord; using Wabbajack.Networking.Http; using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Networking.NexusApi; -using Wabbajack.Networking.WabbajackClientApi; +using Wabbajack.Networking.Steam; using Wabbajack.Paths; using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; using Wabbajack.Services.OSIntegrated.Services; using Wabbajack.Services.OSIntegrated.TokenProviders; using Wabbajack.VFS; +using Client = Wabbajack.Networking.WabbajackClientApi.Client; namespace Wabbajack.Services.OSIntegrated; @@ -106,6 +107,8 @@ public static class ServiceExtensions service.AddSingleton(); service.AddAllSingleton(); + service.AddSteam(); + service.AddSingleton(); service.AddSingleton(); @@ -118,6 +121,10 @@ public static class ServiceExtensions .AddAllSingleton, EncryptedJsonTokenProvider, VectorPlexusTokenProvider>(); + service + .AddAllSingleton, EncryptedJsonTokenProvider, + SteamTokenProvider>(); + service.AddAllSingleton, WabbajackApiTokenProvider>(); service diff --git a/Wabbajack.Services.OSIntegrated/TokenProviders/SteamTokenProvider.cs b/Wabbajack.Services.OSIntegrated/TokenProviders/SteamTokenProvider.cs new file mode 100644 index 00000000..1431fee3 --- /dev/null +++ b/Wabbajack.Services.OSIntegrated/TokenProviders/SteamTokenProvider.cs @@ -0,0 +1,13 @@ +using Microsoft.Extensions.Logging; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Networking.Steam; + +namespace Wabbajack.Services.OSIntegrated.TokenProviders; + +public class SteamTokenProvider : EncryptedJsonTokenProvider +{ + public SteamTokenProvider(ILogger logger, DTOSerializer dtos) : base(logger, dtos, + "steam-login") + { + } +} \ No newline at end of file diff --git a/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj b/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj index 685b6152..71a1455f 100644 --- a/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj +++ b/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj @@ -21,6 +21,7 @@ + diff --git a/Wabbajack.sln b/Wabbajack.sln index 6d9084e1..9393b704 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -116,6 +116,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Launcher", "Wabba EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.App.Wpf", "Wabbajack.App.Wpf\Wabbajack.App.Wpf.csproj", "{372B2DD2-EAA3-4E18-98A7-B9838C7B41F4}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Networking.Steam", "Wabbajack.Networking.Steam\Wabbajack.Networking.Steam.csproj", "{AB9A5C22-10CC-4EE0-A808-FB1DC9E24247}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Networking.Steam.Test", "Wabbajack.Networking.Steam.Test\Wabbajack.Networking.Steam.Test.csproj", "{D6351587-CAF6-4CB6-A2BD-5368E69F297C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -322,6 +326,14 @@ Global {372B2DD2-EAA3-4E18-98A7-B9838C7B41F4}.Debug|Any CPU.Build.0 = Debug|x64 {372B2DD2-EAA3-4E18-98A7-B9838C7B41F4}.Release|Any CPU.ActiveCfg = Release|x64 {372B2DD2-EAA3-4E18-98A7-B9838C7B41F4}.Release|Any CPU.Build.0 = Release|x64 + {AB9A5C22-10CC-4EE0-A808-FB1DC9E24247}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AB9A5C22-10CC-4EE0-A808-FB1DC9E24247}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AB9A5C22-10CC-4EE0-A808-FB1DC9E24247}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AB9A5C22-10CC-4EE0-A808-FB1DC9E24247}.Release|Any CPU.Build.0 = Release|Any CPU + {D6351587-CAF6-4CB6-A2BD-5368E69F297C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D6351587-CAF6-4CB6-A2BD-5368E69F297C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D6351587-CAF6-4CB6-A2BD-5368E69F297C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D6351587-CAF6-4CB6-A2BD-5368E69F297C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {4057B668-8595-44FE-9805-007B284A838F} = {98B731EE-4FC0-4482-A069-BCBA25497871} @@ -358,5 +370,7 @@ Global {29AC8A68-D5EC-43F5-B2CC-72A75545E418} = {98B731EE-4FC0-4482-A069-BCBA25497871} {DEB4B073-4EAA-49FD-9D43-F0F8CB930E7A} = {F01F8595-5FD7-4506-8469-F4A5522DACC1} {4F252332-CA77-41DE-95A8-9DF38A81D675} = {98B731EE-4FC0-4482-A069-BCBA25497871} + {AB9A5C22-10CC-4EE0-A808-FB1DC9E24247} = {F01F8595-5FD7-4506-8469-F4A5522DACC1} + {D6351587-CAF6-4CB6-A2BD-5368E69F297C} = {F01F8595-5FD7-4506-8469-F4A5522DACC1} EndGlobalSection EndGlobal