Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove Server (#2434)

Browse Source 
* The server has been replaced with the new CloudFlare Workers backend
This commit is contained in:
Timothy Baldridge 2023-10-31 03:04:52 +00:00 committed by GitHub
parent 0ee3917106
commit f178213fee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 211 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Wabbajack.CLI/VerbRegistration.cs
View File

@ -1,4 +1,4 @@

using Microsoft.Extensions.DependencyInjection;
namespace Wabbajack.CLI;
using Wabbajack.CLI.Verbs;
@ -37,6 +37,8 @@ CommandLineBuilder.RegisterCommand<ListGames>(ListGames.Definition, c => ((ListG
services.AddSingleton<ListGames>();
CommandLineBuilder.RegisterCommand<ListModlists>(ListModlists.Definition, c => ((ListModlists)c).Run);
services.AddSingleton<ListModlists>();
CommandLineBuilder.RegisterCommand<MegaLogin>(MegaLogin.Definition, c => ((MegaLogin)c).Run);
services.AddSingleton<MegaLogin>();
CommandLineBuilder.RegisterCommand<MirrorFile>(MirrorFile.Definition, c => ((MirrorFile)c).Run);
services.AddSingleton<MirrorFile>();
CommandLineBuilder.RegisterCommand<ModlistReport>(ModlistReport.Definition, c => ((ModlistReport)c).Run);

40
Wabbajack.CLI/Verbs/MegaLogin.cs Normal file
View File

@ -0,0 +1,40 @@
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.CLI.Builder;
using Wabbajack.Downloaders.ModDB;
using Wabbajack.Networking.Http.Interfaces;
namespace Wabbajack.CLI.Verbs;
public class MegaLogin
{
private readonly ILogger<MegaLogin> _logger;
public MegaLogin(ILogger<MegaLogin> logger, ITokenProvider<MegaToken> tokenProvider)
{
_logger = logger;
_tokenProvider = tokenProvider;
}
public static VerbDefinition Definition = new VerbDefinition("mega-login",
"Hashes a file with Wabbajack's hashing routines", new[]
{
new OptionDefinition(typeof(string), "e", "email", "Email for the user account"),
new OptionDefinition(typeof(string), "p", "password", "Password for the user account"),
});
private readonly ITokenProvider<MegaToken> _tokenProvider;
public async Task<int> Run(string email, string password)
{
_logger.LogInformation("Logging into Mega");
await _tokenProvider.SetToken(new MegaToken
{
Email = email,
Password = password
});
return 0;
}
}

57
Wabbajack.CLI/Verbs/ValidateLists.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.CommandLine;
using System.CommandLine.NamingConventionBinder;
@ -6,6 +7,7 @@ using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@ -55,6 +57,8 @@ public class ValidateLists
private readonly IResource<HttpClient> _httpLimiter;
private readonly AsyncLock _imageProcessLock;
private readonly ConcurrentBag<(Uri, Hash)> _proxyableFiles = new();
public ValidateLists(ILogger<ValidateLists> logger, Networking.WabbajackClientApi.Client wjClient,
Client gitHubClient, TemporaryFileManager temporaryFileManager,
@ -92,10 +96,7 @@ public class ValidateLists
reports.CreateDirectory();
var token = CancellationToken.None;
_logger.LogInformation("Scanning for existing patches/mirrors");
var mirroredFiles = (await _wjClient.GetAllMirroredFileDefinitions(token)).Select(m => m.Hash).ToHashSet();
_logger.LogInformation("Found {Count} mirrored files", mirroredFiles.Count);
var patchFiles = await _wjClient.GetAllPatches(token);
_logger.LogInformation("Found {Count} patches", patchFiles.Length);
@ -168,30 +169,7 @@ public class ValidateLists
Original = archive
};
}
if (result.Status == ArchiveStatus.InValid)
{
if (mirroredFiles.Contains(archive.Hash))
{
return new ValidatedArchive
{
Status = ArchiveStatus.Mirrored,
Original = archive,
PatchedFrom = new Archive
{
State = new WabbajackCDN
{
Url = _wjClient.GetMirrorUrl(archive.Hash)!
},
Size = archive.Size,
Name = archive.Name,
Hash = archive.Hash
}
};
}
}
if (result.Status == ArchiveStatus.InValid)
{
_logger.LogInformation("Looking for patch for {Hash}", archive.Hash);
@ -210,7 +188,13 @@ public class ValidateLists
}
}
return new ValidatedArchive()
var downloader = _dispatcher.Downloader(archive);
if (downloader is IProxyable proxyable)
{
_proxyableFiles.Add((proxyable.UnParse(archive.State), archive.Hash));
}
return new ValidatedArchive
{
Status = ArchiveStatus.InValid,
Original = archive
@ -255,12 +239,7 @@ public class ValidateLists
}
await ExportReports(reports, validatedLists, token);
var usedMirroredFiles = validatedLists.SelectMany(a => a.Archives)
.Where(m => m.Status == ArchiveStatus.Mirrored)
.Select(m => m.Original.Hash)
.ToHashSet();
await DeleteOldMirrors(mirroredFiles, usedMirroredFiles);
return 0;
}
@ -580,7 +559,13 @@ public class ValidateLists
.Open(FileMode.Create, FileAccess.Write, FileShare.None);
await _dtos.Serialize(upgradedMetas, upgradedMetasFile, true);
await using var proxyFile = reports.Combine("proxyable.txt")
.Open(FileMode.Create, FileAccess.Write, FileShare.None);
foreach (var file in _proxyableFiles)
{
var str = $"{file.Item1}#name={file.Item2.ToHex()}";
await proxyFile.WriteAsync(Encoding.UTF8.GetBytes(str), token);
}
}
private async Task SendDefinitionToLoadOrderLibrary(ValidatedModList validatedModList, CancellationToken token)

8
Wabbajack.Downloaders.Dispatcher/DownloadDispatcher.cs
View File

@ -40,8 +40,11 @@ public class DownloadDispatcher
_limiter = limiter;
_useProxyCache = useProxyCache;
_verificationCache = verificationCache;
}
public bool UseProxy { get; set; } = false;
public async Task<Hash> Download(Archive a, AbsolutePath dest, CancellationToken token, bool? proxy = null)
{
if (token.IsCancellationRequested)
@ -58,6 +61,7 @@ public class DownloadDispatcher
public async Task<Archive> MaybeProxy(Archive a, CancellationToken token)
{
if (!UseProxy) return a;
var downloader = Downloader(a);
if (downloader is not IProxyable p) return a;
@ -134,7 +138,9 @@ public class DownloadDispatcher
return true;
}
a = await MaybeProxy(a, token);
if (UseProxy)
a = await MaybeProxy(a, token);
var downloader = Downloader(a);
using var job = await _limiter.Begin($"Verifying {a.State.PrimaryKeyString}", -1, token);
var result = await downloader.Verify(a, job, token);

2
Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs
View File

@ -80,7 +80,7 @@ public class GoogleDriveDownloader : ADownloader<DTOs.DownloadStates.GoogleDrive
{
var state = archive.State as DTOs.DownloadStates.GoogleDrive;
var msg = await ToMessage(state, true, token);
using var result = await _client.SendAsync(msg, token);
using var result = await _client.SendAsync(msg, HttpCompletionOption.ResponseHeadersRead, token);
HttpException.ThrowOnFailure(result);
await using var stream = await result.Content.ReadAsStreamAsync(token);
return await fn(stream);

2
Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs
View File

@ -79,7 +79,7 @@ public class MediaFireDownloader : ADownloader<DTOs.DownloadStates.MediaFire>, I
var state = archive.State as DTOs.DownloadStates.MediaFire;
var url = await Resolve(state!);
var msg = new HttpRequestMessage(HttpMethod.Get, url!);
using var result = await _httpClient.SendAsync(msg, token);
using var result = await _httpClient.SendAsync(msg, HttpCompletionOption.ResponseHeadersRead, token);
await using var stream = await result.Content.ReadAsStreamAsync(token);
return await fn(stream);
}

41
Wabbajack.Downloaders.Mega/MegaDownloader.cs
View File

@ -12,6 +12,7 @@ using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.DTOs.Validation;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Networking.Http.Interfaces;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
@ -24,17 +25,18 @@ public class MegaDownloader : ADownloader<Mega>, IUrlDownloader, IProxyable
private const string MegaFilePrefix = "https://mega.nz/file/";
private readonly MegaApiClient _apiClient;
private readonly ILogger<MegaDownloader> _logger;
private readonly ITokenProvider<MegaToken> _tokenProvider;
public MegaDownloader(ILogger<MegaDownloader> logger, MegaApiClient apiClient)
public MegaDownloader(ILogger<MegaDownloader> logger, MegaApiClient apiClient, ITokenProvider<MegaToken> tokenProvider)
{
_logger = logger;
_apiClient = apiClient;
_tokenProvider = tokenProvider;
}
public override async Task<bool> Prepare()
{
if (!_apiClient.IsLoggedIn)
await _apiClient.LoginAsync();
await LoginIfNotLoggedIn();
return true;
}
@ -64,19 +66,35 @@ public class MegaDownloader : ADownloader<Mega>, IUrlDownloader, IProxyable
public async Task<T> DownloadStream<T>(Archive archive, Func<Stream, Task<T>> fn, CancellationToken token)
{
var state = archive.State as Mega;
if (!_apiClient.IsLoggedIn)
await _apiClient.LoginAsync();
await LoginIfNotLoggedIn();
await using var ins = await _apiClient.DownloadAsync(state!.Url, cancellationToken: token);
return await fn(ins);
}
private async Task LoginIfNotLoggedIn()
{
if (!_apiClient.IsLoggedIn)
{
if (_tokenProvider.HaveToken())
{
var authInfo = await _tokenProvider.Get();
_logger.LogInformation("Logging into Mega with {Email}", authInfo!.Email);
await _apiClient.LoginAsync(authInfo!.Email, authInfo.Password);
}
else
{
_logger.LogInformation("Logging into Mega without credentials");
await _apiClient.LoginAsync();
}
}
}
public override async Task<Hash> Download(Archive archive, Mega state, AbsolutePath destination, IJob job,
CancellationToken token)
{
if (!_apiClient.IsLoggedIn)
await _apiClient.LoginAsync();
await LoginIfNotLoggedIn();
await using var ous = destination.Open(FileMode.Create, FileAccess.Write, FileShare.None);
await using var ins = await _apiClient.DownloadAsync(state.Url, cancellationToken: token);
return await ins.HashingCopy(ous, token, job);
@ -93,9 +111,8 @@ public class MegaDownloader : ADownloader<Mega>, IUrlDownloader, IProxyable
public override async Task<bool> Verify(Archive archive, Mega archiveState, IJob job, CancellationToken token)
{
if (!_apiClient.IsLoggedIn)
await _apiClient.LoginAsync();
await LoginIfNotLoggedIn();
for (var times = 0; times < 5; times++)
{
try

12
Wabbajack.Downloaders.Mega/MegaToken.cs Normal file
View File

@ -0,0 +1,12 @@
using System.Text.Json.Serialization;
namespace Wabbajack.Downloaders.ModDB;
public class MegaToken
{
[JsonPropertyName("email")]
public string Email { get; set; }
[JsonPropertyName("password")]
public string Password { get; set; }
}

84
Wabbajack.Hashing.xxHash64/StreamExtensions.cs
View File

@ -15,9 +15,9 @@ public static class StreamExtensions
}
public static async Task<Hash> HashingCopy(this Stream inputStream, Stream outputStream,
CancellationToken token, IJob? job = null)
CancellationToken token, IJob? job = null, int buffserSize = 1024 * 1024)
{
using var rented = MemoryPool<byte>.Shared.Rent(1024 * 1024);
using var rented = MemoryPool<byte>.Shared.Rent(buffserSize);
var buffer = rented.Memory;
var hasher = new xxHashAlgorithm(0);
@ -126,4 +126,84 @@ public static class StreamExtensions
return new Hash(finalHash);
}
public static (Stream InputStream, Func<Hash> Fn) HashingPull(this Stream src)
{
var stream = new PullingStream(src);
return (new BufferedStream(stream), () => stream.Hash);
}
class PullingStream : Stream
{
private readonly Stream _src;
private readonly xxHashAlgorithm _hasher;
private ulong? _hash;
public PullingStream(Stream src)
{
_src = src;
_hasher = new xxHashAlgorithm(0);
}
public Hash Hash => new(_hash ?? throw new InvalidOperationException("Hash not yet computed"));
public override long Seek(long offset, SeekOrigin origin)
{
throw new NotImplementedException();
}
public override void SetLength(long value)
{
throw new NotSupportedException();
}
public override void Write(byte[] buffer, int offset, int count)
{
throw new NotSupportedException();
}
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => _src.Length;
public override long Position
{
get { throw new NotSupportedException(); }
set { throw new NotSupportedException(); }
}
public override void Flush()
{
throw new NotImplementedException();
}
public override int Read(byte[] buffer, int offset, int count)
{
if (_hash.HasValue)
throw new InvalidDataException("HashingPull can only be read once");
var sized = count >> 5 << 5;
if (sized == 0)
throw new ArgumentException("count must be a multiple of 32, got " + count, nameof(count));
var read = _src.ReadAtLeast(buffer, sized);
if (read == 0)
return 0;
if (read == sized)
{
_hasher.TransformByteGroupsInternal(buffer.AsSpan(offset, read));
}
else
{
_hash = _hasher.FinalizeHashValueInternal(buffer.AsSpan(offset, read));
return read;
}
return read;
}
}
}

3
Wabbajack.Services.OSIntegrated/ServiceExtensions.cs
View File

@ -13,6 +13,7 @@ using Wabbajack.Compiler;
using Wabbajack.Configuration;
using Wabbajack.Downloaders;
using Wabbajack.Downloaders.GameFile;
using Wabbajack.Downloaders.ModDB;
using Wabbajack.Downloaders.VerificationCache;
using Wabbajack.DTOs;
using Wabbajack.DTOs.Interventions;
@ -166,6 +167,8 @@ public static class ServiceExtensions
// Token Providers
service.AddAllSingleton<ITokenProvider<NexusApiState>, EncryptedJsonTokenProvider<NexusApiState>, NexusApiTokenProvider>();
service.AddAllSingleton<ITokenProvider<MegaToken>, EncryptedJsonTokenProvider<MegaToken>, MegaTokenProvider>();
service.AddAllSingleton<ITokenProvider<BethesdaNetLoginState>, EncryptedJsonTokenProvider<BethesdaNetLoginState>, BethesdaNetTokenProvider>();
service
.AddAllSingleton<ITokenProvider<LoversLabLoginState>, EncryptedJsonTokenProvider<LoversLabLoginState>,

12
Wabbajack.Services.OSIntegrated/TokenProviders/MegaTokenProvider.cs Normal file
View File

@ -0,0 +1,12 @@
using Microsoft.Extensions.Logging;
using Wabbajack.Downloaders.ModDB;
using Wabbajack.DTOs.JsonConverters;
namespace Wabbajack.Services.OSIntegrated.TokenProviders;
public class MegaTokenProvider : EncryptedJsonTokenProvider<MegaToken>
{
public MegaTokenProvider(ILogger<MegaTokenProvider> logger, DTOSerializer dtos) : base(logger, dtos, "mega-login")
{
}
}