diff --git a/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs b/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs index 508eb877..30e6568e 100644 --- a/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs +++ b/Wabbajack.Downloaders.GoogleDrive/GoogleDriveDownloader.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net.Http; using System.Text.Encodings.Web; @@ -63,7 +64,7 @@ public class GoogleDriveDownloader : ADownloader iniData) { if (iniData.ContainsKey("directURL") && Uri.TryCreate(iniData["directURL"], UriKind.Absolute, out var uri)) @@ -72,6 +73,17 @@ public class GoogleDriveDownloader : ADownloader Priority.Normal; + + + public async Task DownloadStream(Archive archive, Func> fn, CancellationToken token) + { + var state = archive.State as DTOs.DownloadStates.GoogleDrive; + var msg = await ToMessage(state, true, token); + using var result = await _client.SendAsync(msg, token); + HttpException.ThrowOnFailure(result); + await using var stream = await result.Content.ReadAsStreamAsync(token); + return await fn(stream); + } public override async Task Download(Archive archive, DTOs.DownloadStates.GoogleDrive state, AbsolutePath destination, IJob job, CancellationToken token) @@ -98,7 +110,10 @@ public class GoogleDriveDownloader : ADownloader c.Key.StartsWith("download_warning_")); @@ -106,6 +121,8 @@ public class GoogleDriveDownloader : ADownloaderGoogle Drive - Quota exceeded")) + throw new Exception("Google Drive - Quota Exceeded"); doc.LoadHtml(txt); @@ -127,13 +144,19 @@ public class GoogleDriveDownloader : ADownloader DownloadStream(Archive archive, Func> fn, CancellationToken token); } \ No newline at end of file diff --git a/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs b/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs index e47e628d..abd16ab2 100644 --- a/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs +++ b/Wabbajack.Downloaders.MediaFire/MediaFireDownloader.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net.Http; using System.Threading; @@ -71,6 +72,16 @@ public class MediaFireDownloader : ADownloader, I return ((DTOs.DownloadStates.MediaFire) state).Url; } + public async Task DownloadStream(Archive archive, Func> fn, CancellationToken token) + { + var state = archive.State as DTOs.DownloadStates.MediaFire; + var url = await Resolve(state!); + var msg = new HttpRequestMessage(HttpMethod.Get, url!); + using var result = await _httpClient.SendAsync(msg, token); + await using var stream = await result.Content.ReadAsStreamAsync(token); + return await fn(stream); + } + public override async Task Download(Archive archive, DTOs.DownloadStates.MediaFire state, AbsolutePath destination, IJob job, CancellationToken token) { @@ -85,7 +96,7 @@ public class MediaFireDownloader : ADownloader, I return await Resolve(archiveState, job, token) != null; } - private async Task Resolve(DTOs.DownloadStates.MediaFire state, IJob job, CancellationToken? token = null) + private async Task Resolve(DTOs.DownloadStates.MediaFire state, IJob? job = null, CancellationToken? token = null) { token ??= CancellationToken.None; using var result = await _httpClient.GetAsync(state.Url, HttpCompletionOption.ResponseHeadersRead, @@ -93,7 +104,8 @@ public class MediaFireDownloader : ADownloader, I if (!result.IsSuccessStatusCode) return null; - job.Size = result.Content.Headers.ContentLength ?? 0; + if (job != null) + job.Size = result.Content.Headers.ContentLength ?? 0; if (result.Content.Headers.ContentType!.MediaType!.StartsWith("text/html", StringComparison.OrdinalIgnoreCase)) diff --git a/Wabbajack.Downloaders.Mega/MegaDownloader.cs b/Wabbajack.Downloaders.Mega/MegaDownloader.cs index 572c88c9..a5ab8eaf 100644 --- a/Wabbajack.Downloaders.Mega/MegaDownloader.cs +++ b/Wabbajack.Downloaders.Mega/MegaDownloader.cs @@ -60,6 +60,16 @@ public class MegaDownloader : ADownloader, IUrlDownloader, IProxyable return ((Mega) state).Url; } + public async Task DownloadStream(Archive archive, Func> fn, CancellationToken token) + { + var state = archive.State as Mega; + if (!_apiClient.IsLoggedIn) + await _apiClient.LoginAsync(); + + await using var ins = await _apiClient.DownloadAsync(state.Url, cancellationToken: token); + return await fn(ins); + } + public override async Task Download(Archive archive, Mega state, AbsolutePath destination, IJob job, CancellationToken token) { diff --git a/Wabbajack.Hashing.xxHash64/StreamExtensions.cs b/Wabbajack.Hashing.xxHash64/StreamExtensions.cs index d1382a83..51d10f92 100644 --- a/Wabbajack.Hashing.xxHash64/StreamExtensions.cs +++ b/Wabbajack.Hashing.xxHash64/StreamExtensions.cs @@ -1,3 +1,4 @@ +using System; using System.Buffers; using System.IO; using System.Threading; @@ -72,4 +73,57 @@ public static class StreamExtensions return new Hash(finalHash); } + + public static async Task HashingCopy(this Stream inputStream, Func, Task> fn, + CancellationToken token) + { + using var rented = MemoryPool.Shared.Rent(1024 * 1024); + var buffer = rented.Memory; + + var hasher = new xxHashAlgorithm(0); + + var running = true; + ulong finalHash = 0; + while (running && !token.IsCancellationRequested) + { + var totalRead = 0; + + while (totalRead != buffer.Length) + { + var read = await inputStream.ReadAsync(buffer.Slice(totalRead, buffer.Length - totalRead), + token); + + if (read == 0) + { + running = false; + break; + } + totalRead += read; + } + + var pendingWrite = fn(buffer[..totalRead]); + if (running) + { + hasher.TransformByteGroupsInternal(buffer.Span); + await pendingWrite; + } + else + { + var preSize = (totalRead >> 5) << 5; + if (preSize > 0) + { + hasher.TransformByteGroupsInternal(buffer[..preSize].Span); + finalHash = hasher.FinalizeHashValueInternal(buffer[preSize..totalRead].Span); + await pendingWrite; + break; + } + + finalHash = hasher.FinalizeHashValueInternal(buffer[..totalRead].Span); + await pendingWrite; + break; + } + } + + return new Hash(finalHash); + } } \ No newline at end of file diff --git a/Wabbajack.Networking.Http/HttpExtension.cs b/Wabbajack.Networking.Http/HttpExtension.cs index b7e4de9e..6346a586 100644 --- a/Wabbajack.Networking.Http/HttpExtension.cs +++ b/Wabbajack.Networking.Http/HttpExtension.cs @@ -20,4 +20,10 @@ public class HttpException : Exception public string Reason { get; set; } public int Code { get; set; } + + public static void ThrowOnFailure(HttpResponseMessage result) + { + if (result.IsSuccessStatusCode) return; + throw new HttpException(result); + } } \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Proxy.cs b/Wabbajack.Server/Controllers/Proxy.cs index edada767..eb3736b8 100644 --- a/Wabbajack.Server/Controllers/Proxy.cs +++ b/Wabbajack.Server/Controllers/Proxy.cs @@ -3,6 +3,7 @@ using FluentFTP.Helpers; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; +using Microsoft.Net.Http.Headers; using Wabbajack.BuildServer; using Wabbajack.Downloaders; using Wabbajack.Downloaders.Interfaces; @@ -107,18 +108,39 @@ public class Proxy : ControllerBase var tempFile = _tempFileManager.CreateFile(deleteOnDispose:false); - var result = await _dispatcher.Download(archive, tempFile.Path, token); - if (hash != default && result != shouldMatch) + var proxyDownloader = _dispatcher.Downloader(archive) as IProxyable; + await using (var of = tempFile.Path.Open(FileMode.Create, FileAccess.Write, FileShare.None)) { - if (tempFile.Path.FileExists()) - tempFile.Path.Delete(); + Response.StatusCode = 200; + if (name != null) + { + Response.Headers.Add(HeaderNames.ContentDisposition, $"attachment; filename=\"{name}\""); + } - return BadRequest(new {Type = "Unmatching Hashes", Expected = shouldMatch.ToHex(), Found = result.ToHex()}); + Response.Headers.Add( HeaderNames.ContentType, "application/octet-stream" ); + + var result = await proxyDownloader.DownloadStream(archive, async s => { + return await s.HashingCopy(async m => + { + var strmA = of.WriteAsync(m, token); + await Response.Body.WriteAsync(m, token); + await Response.Body.FlushAsync(token); + await strmA; + }, token); }, + token); + + + if (hash != default && result != shouldMatch) + { + if (tempFile.Path.FileExists()) + tempFile.Path.Delete(); + } } + await tempFile.Path.MoveToAsync(cacheFile, true, token); _logger.LogInformation("Returning proxy request for {Uri} {Size}", uri, cacheFile.Size().FileSizeToString()); - return new PhysicalFileResult(cacheFile.ToString(), "application/binary"); + return new EmptyResult(); } } \ No newline at end of file