From f178213fee0f8ff1006a0ec3eebcba3a5ebcf517 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 31 Oct 2023 03:04:52 +0000 Subject: [PATCH] Remove Server (#2434) * The server has been replaced with the new CloudFlare Workers backend --- Wabbajack.CLI/VerbRegistration.cs | 4 +- Wabbajack.CLI/Verbs/MegaLogin.cs | 40 +++++++++ Wabbajack.CLI/Verbs/ValidateLists.cs | 57 +++++-------- .../DownloadDispatcher.cs | 8 +- .../GoogleDriveDownloader.cs | 2 +- .../MediaFireDownloader.cs | 2 +- Wabbajack.Downloaders.Mega/MegaDownloader.cs | 41 ++++++--- Wabbajack.Downloaders.Mega/MegaToken.cs | 12 +++ .../StreamExtensions.cs | 84 ++++++++++++++++++- .../ServiceExtensions.cs | 3 + .../TokenProviders/MegaTokenProvider.cs | 12 +++ 11 files changed, 211 insertions(+), 54 deletions(-) create mode 100644 Wabbajack.CLI/Verbs/MegaLogin.cs create mode 100644 Wabbajack.Downloaders.Mega/MegaToken.cs create mode 100644 Wabbajack.Services.OSIntegrated/TokenProviders/MegaTokenProvider.cs diff --git a/Wabbajack.CLI/VerbRegistration.cs b/Wabbajack.CLI/VerbRegistration.cs index b000d201..3854dce4 100644 --- a/Wabbajack.CLI/VerbRegistration.cs +++ b/Wabbajack.CLI/VerbRegistration.cs @@ -1,4 +1,4 @@ - + using Microsoft.Extensions.DependencyInjection; namespace Wabbajack.CLI; using Wabbajack.CLI.Verbs; @@ -37,6 +37,8 @@ CommandLineBuilder.RegisterCommand(ListGames.Definition, c => ((ListG services.AddSingleton(); CommandLineBuilder.RegisterCommand(ListModlists.Definition, c => ((ListModlists)c).Run); services.AddSingleton(); +CommandLineBuilder.RegisterCommand(MegaLogin.Definition, c => ((MegaLogin)c).Run); +services.AddSingleton(); CommandLineBuilder.RegisterCommand(MirrorFile.Definition, c => ((MirrorFile)c).Run); services.AddSingleton(); CommandLineBuilder.RegisterCommand(ModlistReport.Definition, c => ((ModlistReport)c).Run); diff --git a/Wabbajack.CLI/Verbs/MegaLogin.cs b/Wabbajack.CLI/Verbs/MegaLogin.cs new file mode 100644 index 00000000..bb1164f8 --- /dev/null +++ b/Wabbajack.CLI/Verbs/MegaLogin.cs @@ -0,0 +1,40 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.CLI.Builder; +using Wabbajack.Downloaders.ModDB; +using Wabbajack.Networking.Http.Interfaces; + +namespace Wabbajack.CLI.Verbs; + +public class MegaLogin +{ + private readonly ILogger _logger; + + public MegaLogin(ILogger logger, ITokenProvider tokenProvider) + { + _logger = logger; + _tokenProvider = tokenProvider; + } + + public static VerbDefinition Definition = new VerbDefinition("mega-login", + "Hashes a file with Wabbajack's hashing routines", new[] + { + new OptionDefinition(typeof(string), "e", "email", "Email for the user account"), + new OptionDefinition(typeof(string), "p", "password", "Password for the user account"), + }); + + private readonly ITokenProvider _tokenProvider; + + public async Task Run(string email, string password) + { + _logger.LogInformation("Logging into Mega"); + await _tokenProvider.SetToken(new MegaToken + { + Email = email, + Password = password + }); + return 0; + } +} \ No newline at end of file diff --git a/Wabbajack.CLI/Verbs/ValidateLists.cs b/Wabbajack.CLI/Verbs/ValidateLists.cs index 45bf40b2..213ad857 100644 --- a/Wabbajack.CLI/Verbs/ValidateLists.cs +++ b/Wabbajack.CLI/Verbs/ValidateLists.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.CommandLine; using System.CommandLine.NamingConventionBinder; @@ -6,6 +7,7 @@ using System.Diagnostics; using System.IO; using System.Linq; using System.Net.Http; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -55,6 +57,8 @@ public class ValidateLists private readonly IResource _httpLimiter; private readonly AsyncLock _imageProcessLock; + private readonly ConcurrentBag<(Uri, Hash)> _proxyableFiles = new(); + public ValidateLists(ILogger logger, Networking.WabbajackClientApi.Client wjClient, Client gitHubClient, TemporaryFileManager temporaryFileManager, @@ -92,10 +96,7 @@ public class ValidateLists reports.CreateDirectory(); var token = CancellationToken.None; - - _logger.LogInformation("Scanning for existing patches/mirrors"); - var mirroredFiles = (await _wjClient.GetAllMirroredFileDefinitions(token)).Select(m => m.Hash).ToHashSet(); - _logger.LogInformation("Found {Count} mirrored files", mirroredFiles.Count); + var patchFiles = await _wjClient.GetAllPatches(token); _logger.LogInformation("Found {Count} patches", patchFiles.Length); @@ -168,30 +169,7 @@ public class ValidateLists Original = archive }; } - - - if (result.Status == ArchiveStatus.InValid) - { - if (mirroredFiles.Contains(archive.Hash)) - { - return new ValidatedArchive - { - Status = ArchiveStatus.Mirrored, - Original = archive, - PatchedFrom = new Archive - { - State = new WabbajackCDN - { - Url = _wjClient.GetMirrorUrl(archive.Hash)! - }, - Size = archive.Size, - Name = archive.Name, - Hash = archive.Hash - } - }; - } - } - + if (result.Status == ArchiveStatus.InValid) { _logger.LogInformation("Looking for patch for {Hash}", archive.Hash); @@ -210,7 +188,13 @@ public class ValidateLists } } - return new ValidatedArchive() + var downloader = _dispatcher.Downloader(archive); + if (downloader is IProxyable proxyable) + { + _proxyableFiles.Add((proxyable.UnParse(archive.State), archive.Hash)); + } + + return new ValidatedArchive { Status = ArchiveStatus.InValid, Original = archive @@ -255,12 +239,7 @@ public class ValidateLists } await ExportReports(reports, validatedLists, token); - - var usedMirroredFiles = validatedLists.SelectMany(a => a.Archives) - .Where(m => m.Status == ArchiveStatus.Mirrored) - .Select(m => m.Original.Hash) - .ToHashSet(); - await DeleteOldMirrors(mirroredFiles, usedMirroredFiles); + return 0; } @@ -580,7 +559,13 @@ public class ValidateLists .Open(FileMode.Create, FileAccess.Write, FileShare.None); await _dtos.Serialize(upgradedMetas, upgradedMetasFile, true); - + await using var proxyFile = reports.Combine("proxyable.txt") + .Open(FileMode.Create, FileAccess.Write, FileShare.None); + foreach (var file in _proxyableFiles) + { + var str = $"{file.Item1}#name={file.Item2.ToHex()}"; + await proxyFile.WriteAsync(Encoding.UTF8.GetBytes(str), token); + } } private async Task SendDefinitionToLoadOrderLibrary(ValidatedModList validatedModList, CancellationToken token) diff --git a/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs b/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs index cc6f75f3..c01eb31e 100644 --- a/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs +++ b/Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs @@ -40,8 +40,11 @@ public class DownloadDispatcher _limiter = limiter; _useProxyCache = useProxyCache; _verificationCache = verificationCache; + } + public bool UseProxy { get; set; } = false; + public async Task Download(Archive a, AbsolutePath dest, CancellationToken token, bool? proxy = null) { if (token.IsCancellationRequested) @@ -58,6 +61,7 @@ public class DownloadDispatcher public async Task MaybeProxy(Archive a, CancellationToken token) { + if (!UseProxy) return a; var downloader = Downloader(a); if (downloader is not IProxyable p) return a; @@ -134,7 +138,9 @@ public class DownloadDispatcher return true; } - a = await MaybeProxy(a, token); + if (UseProxy) + a = await MaybeProxy(a, token); + var downloader = Downloader(a); using var job = await _limiter.Begin($"Verifying {a.State.PrimaryKeyString}", -1, token); var result = await downloader.Verify(a, job, token); diff --git a/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs b/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs index c4906694..d19bb909 100644 --- a/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs +++ b/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs @@ -80,7 +80,7 @@ public class GoogleDriveDownloader : ADownloader, I var state = archive.State as DTOs.DownloadStates.MediaFire; var url = await Resolve(state!); var msg = new HttpRequestMessage(HttpMethod.Get, url!); - using var result = await _httpClient.SendAsync(msg, token); + using var result = await _httpClient.SendAsync(msg, HttpCompletionOption.ResponseHeadersRead, token); await using var stream = await result.Content.ReadAsStreamAsync(token); return await fn(stream); } diff --git a/Wabbajack.Downloaders.Mega/MegaDownloader.cs b/Wabbajack.Downloaders.Mega/MegaDownloader.cs index 9cdce1fe..d0d34d40 100644 --- a/Wabbajack.Downloaders.Mega/MegaDownloader.cs +++ b/Wabbajack.Downloaders.Mega/MegaDownloader.cs @@ -12,6 +12,7 @@ using Wabbajack.DTOs; using Wabbajack.DTOs.DownloadStates; using Wabbajack.DTOs.Validation; using Wabbajack.Hashing.xxHash64; +using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Paths; using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; @@ -24,17 +25,18 @@ public class MegaDownloader : ADownloader, IUrlDownloader, IProxyable private const string MegaFilePrefix = "https://mega.nz/file/"; private readonly MegaApiClient _apiClient; private readonly ILogger _logger; + private readonly ITokenProvider _tokenProvider; - public MegaDownloader(ILogger logger, MegaApiClient apiClient) + public MegaDownloader(ILogger logger, MegaApiClient apiClient, ITokenProvider tokenProvider) { _logger = logger; _apiClient = apiClient; + _tokenProvider = tokenProvider; } public override async Task Prepare() { - if (!_apiClient.IsLoggedIn) - await _apiClient.LoginAsync(); + await LoginIfNotLoggedIn(); return true; } @@ -64,19 +66,35 @@ public class MegaDownloader : ADownloader, IUrlDownloader, IProxyable public async Task DownloadStream(Archive archive, Func> fn, CancellationToken token) { var state = archive.State as Mega; - if (!_apiClient.IsLoggedIn) - await _apiClient.LoginAsync(); - + await LoginIfNotLoggedIn(); + await using var ins = await _apiClient.DownloadAsync(state!.Url, cancellationToken: token); return await fn(ins); } + private async Task LoginIfNotLoggedIn() + { + if (!_apiClient.IsLoggedIn) + { + if (_tokenProvider.HaveToken()) + { + var authInfo = await _tokenProvider.Get(); + _logger.LogInformation("Logging into Mega with {Email}", authInfo!.Email); + await _apiClient.LoginAsync(authInfo!.Email, authInfo.Password); + } + else + { + _logger.LogInformation("Logging into Mega without credentials"); + await _apiClient.LoginAsync(); + } + } + } + public override async Task Download(Archive archive, Mega state, AbsolutePath destination, IJob job, CancellationToken token) { - if (!_apiClient.IsLoggedIn) - await _apiClient.LoginAsync(); - + await LoginIfNotLoggedIn(); + await using var ous = destination.Open(FileMode.Create, FileAccess.Write, FileShare.None); await using var ins = await _apiClient.DownloadAsync(state.Url, cancellationToken: token); return await ins.HashingCopy(ous, token, job); @@ -93,9 +111,8 @@ public class MegaDownloader : ADownloader, IUrlDownloader, IProxyable public override async Task Verify(Archive archive, Mega archiveState, IJob job, CancellationToken token) { - if (!_apiClient.IsLoggedIn) - await _apiClient.LoginAsync(); - + await LoginIfNotLoggedIn(); + for (var times = 0; times < 5; times++) { try diff --git a/Wabbajack.Downloaders.Mega/MegaToken.cs b/Wabbajack.Downloaders.Mega/MegaToken.cs new file mode 100644 index 00000000..1f14fe42 --- /dev/null +++ b/Wabbajack.Downloaders.Mega/MegaToken.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace Wabbajack.Downloaders.ModDB; + +public class MegaToken +{ + [JsonPropertyName("email")] + public string Email { get; set; } + + [JsonPropertyName("password")] + public string Password { get; set; } +} \ No newline at end of file diff --git a/Wabbajack.Hashing.xxHash64/StreamExtensions.cs b/Wabbajack.Hashing.xxHash64/StreamExtensions.cs index 51d10f92..9bbaa0b0 100644 --- a/Wabbajack.Hashing.xxHash64/StreamExtensions.cs +++ b/Wabbajack.Hashing.xxHash64/StreamExtensions.cs @@ -15,9 +15,9 @@ public static class StreamExtensions } public static async Task HashingCopy(this Stream inputStream, Stream outputStream, - CancellationToken token, IJob? job = null) + CancellationToken token, IJob? job = null, int buffserSize = 1024 * 1024) { - using var rented = MemoryPool.Shared.Rent(1024 * 1024); + using var rented = MemoryPool.Shared.Rent(buffserSize); var buffer = rented.Memory; var hasher = new xxHashAlgorithm(0); @@ -126,4 +126,84 @@ public static class StreamExtensions return new Hash(finalHash); } + + public static (Stream InputStream, Func Fn) HashingPull(this Stream src) + { + var stream = new PullingStream(src); + return (new BufferedStream(stream), () => stream.Hash); + } + + class PullingStream : Stream + { + private readonly Stream _src; + private readonly xxHashAlgorithm _hasher; + private ulong? _hash; + + public PullingStream(Stream src) + { + _src = src; + _hasher = new xxHashAlgorithm(0); + } + + public Hash Hash => new(_hash ?? throw new InvalidOperationException("Hash not yet computed")); + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotImplementedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => _src.Length; + public override long Position + { + get { throw new NotSupportedException(); } + set { throw new NotSupportedException(); } + } + + public override void Flush() + { + throw new NotImplementedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + if (_hash.HasValue) + throw new InvalidDataException("HashingPull can only be read once"); + + var sized = count >> 5 << 5; + if (sized == 0) + throw new ArgumentException("count must be a multiple of 32, got " + count, nameof(count)); + + + var read = _src.ReadAtLeast(buffer, sized); + + if (read == 0) + return 0; + + + if (read == sized) + { + _hasher.TransformByteGroupsInternal(buffer.AsSpan(offset, read)); + } + else + { + _hash = _hasher.FinalizeHashValueInternal(buffer.AsSpan(offset, read)); + return read; + } + + return read; + } + } } \ No newline at end of file diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index 29873e83..14647411 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -13,6 +13,7 @@ using Wabbajack.Compiler; using Wabbajack.Configuration; using Wabbajack.Downloaders; using Wabbajack.Downloaders.GameFile; +using Wabbajack.Downloaders.ModDB; using Wabbajack.Downloaders.VerificationCache; using Wabbajack.DTOs; using Wabbajack.DTOs.Interventions; @@ -166,6 +167,8 @@ public static class ServiceExtensions // Token Providers service.AddAllSingleton, EncryptedJsonTokenProvider, NexusApiTokenProvider>(); + service.AddAllSingleton, EncryptedJsonTokenProvider, MegaTokenProvider>(); + service.AddAllSingleton, EncryptedJsonTokenProvider, BethesdaNetTokenProvider>(); service .AddAllSingleton, EncryptedJsonTokenProvider, diff --git a/Wabbajack.Services.OSIntegrated/TokenProviders/MegaTokenProvider.cs b/Wabbajack.Services.OSIntegrated/TokenProviders/MegaTokenProvider.cs new file mode 100644 index 00000000..2eee5f3e --- /dev/null +++ b/Wabbajack.Services.OSIntegrated/TokenProviders/MegaTokenProvider.cs @@ -0,0 +1,12 @@ +using Microsoft.Extensions.Logging; +using Wabbajack.Downloaders.ModDB; +using Wabbajack.DTOs.JsonConverters; + +namespace Wabbajack.Services.OSIntegrated.TokenProviders; + +public class MegaTokenProvider : EncryptedJsonTokenProvider +{ + public MegaTokenProvider(ILogger logger, DTOSerializer dtos) : base(logger, dtos, "mega-login") + { + } +} \ No newline at end of file