Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1804 from wabbajack-tools/more-steam-updates

Browse Source 
More steam updates
This commit is contained in:
Timothy Baldridge 2022-01-09 21:23:52 -07:00 committed by GitHub
commit 0fd558fe8c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 61 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/tests.yaml vendored
View File

@ -7,7 +7,7 @@ on:
branches: [ main ]
env:
VERSION: 3.0.0.0-beta2
VERSION: 3.0.0.0-beta3
jobs:
build:

5
Wabbajack.CLI/Verbs/SteamDownloadFile.cs
View File

@ -1,6 +1,7 @@
using System.CommandLine;
using System.CommandLine.Invocation;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FluentFTP.Helpers;
using Microsoft.Extensions.Logging;
@ -81,9 +82,9 @@ public class SteamDownloadFile : IVerb
return 1;
}
_logger.LogInformation("File has is {Size} and {ChunkCount} chunks", fileData.TotalSize.FileSizeToString(), fileData.Chunks.Count);
_logger.LogInformation("File is {Size} and {ChunkCount} chunks", fileData.TotalSize.FileSizeToString(), fileData.Chunks.Count);
await _client.Download(appId, depotManifest!.DepotID, steamManifest!.Manifest, fileData, output);
await _client.Download(appId, depotManifest!.DepotID, steamManifest!.Manifest, fileData, output, CancellationToken.None);
_logger.LogInformation("File downloaded");

4
Wabbajack.Common/CircuitBreaker.cs
View File

@ -11,7 +11,7 @@ public static class CircuitBreaker
public static int DEFAULT_DELAY_MULTIPLIER = 2;
public static int DEFAULT_RETRIES = 5;
public static async ValueTask<TR> WithAutoRetryAsync<TR, TE>(ILogger logger, Func<ValueTask<TR>> f,
public static async ValueTask<TR> WithAutoRetryAsync<TR, TE>(ILogger logger, Func<Task<TR>> f,
TimeSpan? delay = null, int? multipler = null, int? maxRetries = null) where TE : Exception
{
var retries = 0;
@ -37,7 +37,7 @@ public static class CircuitBreaker
}
}
public static async ValueTask WithAutoRetryAsync<TE>(ILogger logger, Func<ValueTask> f, TimeSpan? delay = null,
public static async ValueTask WithAutoRetryAsync<TE>(ILogger logger, Func<Task> f, TimeSpan? delay = null,
int? multipler = null, int? maxRetries = null) where TE : Exception
{
var retries = 0;

78
Wabbajack.Networking.Steam/Client.cs
View File

@ -6,6 +6,7 @@ using Microsoft.VisualBasic.CompilerServices;
using SteamKit2;
using SteamKit2.CDN;
using SteamKit2.Internal;
using Wabbajack.Common;
using Wabbajack.DTOs.Interventions;
using Wabbajack.DTOs.JsonConverters;
using Wabbajack.Hashing.xxHash64;
@ -14,6 +15,7 @@ using Wabbajack.Networking.Steam.DTOs;
using Wabbajack.Networking.Steam.UserInterventions;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
namespace Wabbajack.Networking.Steam;
@ -39,7 +41,8 @@ public class Client : IDisposable
public TaskCompletionSource _licenseRequest = new();
private readonly SteamApps _steamApps;
private readonly DTOSerializer _dtos;
private IReadOnlyCollection<Server> _cdnServers = Array.Empty<Server>();
private Server[] _cdnServers = Array.Empty<Server>();
private readonly IResource<HttpClient> _limiter;
public SteamApps.LicenseListCallback.License[] Licenses { get; private set; }
@ -57,12 +60,13 @@ public class Client : IDisposable
public Client(ILogger<Client> logger, HttpClient client, ITokenProvider<SteamLoginState> token,
IUserInterventionHandler interventionHandler, DTOSerializer dtos)
IUserInterventionHandler interventionHandler, DTOSerializer dtos, IResource<HttpClient> limiter)
{
_logger = logger;
_httpClient = client;
_dtos = dtos;
_interventionHandler = interventionHandler;
_limiter = limiter;
_client = new SteamClient(SteamConfiguration.Create(c =>
{
c.WithProtocolTypes(ProtocolTypes.WebSocket);
@ -354,12 +358,12 @@ public class Client : IDisposable
return AppInfo[appId].AppInfo;
}
public async Task<IReadOnlyCollection<Server>> LoadCDNServers()
public async Task<Server[]> LoadCDNServers()
{
if (_cdnServers.Count > 0) return _cdnServers;
if (_cdnServers.Length > 0) return _cdnServers;
_logger.LogInformation("Loading CDN servers");
_cdnServers = await ContentServerDirectoryService.LoadAsync(_client.Configuration);
_logger.LogInformation("{Count} servers found", _cdnServers.Count);
_cdnServers = (await ContentServerDirectoryService.LoadAsync(_client.Configuration)).ToArray();
_logger.LogInformation("{Count} servers found", _cdnServers.Length);
return _cdnServers;
}
@ -408,35 +412,59 @@ public class Client : IDisposable
return result.DepotKey;
}
public async Task Download(uint appId, uint depotId, ulong manifest, DepotManifest.FileData fileData, AbsolutePath output)
private static readonly Random _random = new();
private Server RandomServer()
{
return _cdnServers[_random.Next(0, _cdnServers.Length)];
}
public async Task Download(uint appId, uint depotId, ulong manifest, DepotManifest.FileData fileData, AbsolutePath output,
CancellationToken token, IJob? parentJob = null)
{
await LoadCDNServers();
var client = _cdnServers.First();
var depotKey = await GetDepotKey(depotId, appId);
await using var os = output.Open(FileMode.Create, FileAccess.Write, FileShare.Read);
foreach (var chunk in fileData.Chunks.OrderBy(c => c.Offset))
{
var chunkId = chunk.ChunkID!.ToHex();
var uri = new UriBuilder()
await fileData.Chunks.OrderBy(c => c.Offset)
.PMapAll(async chunk =>
{
Host = client.Host,
Port = client.Port,
Scheme = client.Protocol.ToString(),
Path = $"depot/{depotId}/chunk/{chunkId}"
}.Uri;
async Task<DepotChunk> AttemptDownload(DepotManifest.ChunkData chunk)
{
var client = RandomServer();
using var job = await _limiter.Begin($"Downloading chunk of {fileData.FileName}",
chunk.CompressedLength, token);
var data = await _httpClient.GetByteArrayAsync(uri);
var chunkData = new DepotChunk(chunk, data);
chunkData.Process(depotKey);
var chunkId = chunk.ChunkID!.ToHex();
await os.WriteAsync(chunkData.Data);
}
var uri = new UriBuilder
{
Host = client.Host,
Port = client.Port,
Scheme = client.Protocol.ToString(),
Path = $"depot/{depotId}/chunk/{chunkId}"
}.Uri;
var data = await _httpClient.GetByteArrayAsync(uri, token);
await job.Report(data.Length, token);
if (parentJob != null)
await parentJob.Report(data.Length, token);
var chunkData = new DepotChunk(chunk, data);
chunkData.Process(depotKey);
return chunkData;
}
return await CircuitBreaker.WithAutoRetryAsync<DepotChunk, HttpRequestException>(_logger, () => AttemptDownload(chunk));
}).Do(async data =>
{
await os.WriteAsync(data.Data, token);
});
}
}

1
Wabbajack.Networking.Steam/Wabbajack.Networking.Steam.csproj
View File

@ -12,6 +12,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Wabbajack.Common\Wabbajack.Common.csproj" />
<ProjectReference Include="..\Wabbajack.DTOs\Wabbajack.DTOs.csproj" />
<ProjectReference Include="..\Wabbajack.Networking.Http.Interfaces\Wabbajack.Networking.Http.Interfaces.csproj" />
<ProjectReference Include="..\Wabbajack.Paths.IO\Wabbajack.Paths.IO.csproj" />