diff --git a/Wabbajack.Server/AppSettings.cs b/Wabbajack.Server/AppSettings.cs index 6b16a846..bbaddce7 100644 --- a/Wabbajack.Server/AppSettings.cs +++ b/Wabbajack.Server/AppSettings.cs @@ -37,18 +37,19 @@ public class AppSettings public CouchDBSetting CesiDB { get; set; } public CouchDBSetting MetricsDB { get; set; } - public S3Settings AuthoredFilesS3 { get; set; } + public S3Settings S3 { get; set; } } public class S3Settings { public string AccessKey { get; set; } public string SecretKey { get; set; } - public string ServiceURL { get; set; } + public string ServiceUrl { get; set; } - public string BucketName { get; set; } + public string AuthoredFilesBucket { get; set; } + public string ProxyFilesBucket { get; set; } - public string BucketCacheFile { get; set; } + public string AuthoredFilesBucketCache { get; set; } } public class CouchDBSetting diff --git a/Wabbajack.Server/Controllers/Proxy.cs b/Wabbajack.Server/Controllers/Proxy.cs index eba4175d..155cfc97 100644 --- a/Wabbajack.Server/Controllers/Proxy.cs +++ b/Wabbajack.Server/Controllers/Proxy.cs @@ -1,4 +1,7 @@ using System.Text; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; using FluentFTP.Helpers; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; @@ -10,7 +13,9 @@ using Wabbajack.Downloaders.Interfaces; using Wabbajack.DTOs; using Wabbajack.DTOs.DownloadStates; using Wabbajack.Hashing.xxHash64; +using Wabbajack.Paths; using Wabbajack.Paths.IO; +using Wabbajack.RateLimiter; using Wabbajack.VFS; namespace Wabbajack.Server.Controllers; @@ -24,14 +29,23 @@ public class Proxy : ControllerBase private readonly TemporaryFileManager _tempFileManager; private readonly AppSettings _appSettings; private readonly FileHashCache _hashCache; + private readonly IAmazonS3 _s3; + private readonly string _bucket; + + private string _redirectUrl = "https://proxy.wabbajack.org/"; + private readonly IResource _resource; - public Proxy(ILogger logger, DownloadDispatcher dispatcher, TemporaryFileManager tempFileManager, FileHashCache hashCache, AppSettings appSettings) + public Proxy(ILogger logger, DownloadDispatcher dispatcher, TemporaryFileManager tempFileManager, + FileHashCache hashCache, AppSettings appSettings, IAmazonS3 s3, IResource resource) { _logger = logger; _dispatcher = dispatcher; _tempFileManager = tempFileManager; _appSettings = appSettings; _hashCache = hashCache; + _s3 = s3; + _bucket = _appSettings.S3.ProxyFilesBucket; + _resource = resource; } [HttpHead] @@ -58,12 +72,14 @@ public class Proxy : ControllerBase [HttpGet] public async Task ProxyGet(CancellationToken token, [FromQuery] Uri uri, [FromQuery] string? name, [FromQuery] string? hash) { + + Hash hashResult = default; var shouldMatch = hash != null ? Hash.FromHex(hash) : default; _logger.LogInformation("Got proxy request for {Uri}", uri); var state = _dispatcher.Parse(uri); var cacheName = (await Encoding.UTF8.GetBytes(uri.ToString()).Hash()).ToHex(); - var cacheFile = _appSettings.ProxyPath.Combine(cacheName); + var cacheFile = await GetCacheEntry(cacheName); if (state == null) { @@ -84,26 +100,27 @@ public class Proxy : ControllerBase return BadRequest(new {Type = "Downloader is not IProxyable", Downloader = downloader.GetType().FullName}); } - if (cacheFile.FileExists() && (DateTime.Now - cacheFile.LastModified()) > TimeSpan.FromHours(4)) + if (cacheFile != null && (DateTime.UtcNow - cacheFile.LastModified) > TimeSpan.FromHours(4)) { try { var verify = await _dispatcher.Verify(archive, token); if (verify) - cacheFile.Touch(); + await TouchCacheEntry(cacheName); } catch (Exception ex) { - _logger.LogInformation(ex, "When trying to verify cached file ({Hash}) {Url}", cacheFile.FileName, uri); - cacheFile.Touch(); + _logger.LogInformation(ex, "When trying to verify cached file ({Hash}) {Url}", + cacheFile.Hash, uri); + await TouchCacheEntry(cacheName); } } - if (cacheFile.FileExists() && (DateTime.Now - cacheFile.LastModified()) > TimeSpan.FromHours(24)) + if (cacheFile != null && (DateTime.Now - cacheFile.LastModified) > TimeSpan.FromHours(24)) { try { - cacheFile.Delete(); + await DeleteCacheEntry(cacheName); } catch (Exception ex) { @@ -112,18 +129,15 @@ public class Proxy : ControllerBase } - if (cacheFile.FileExists()) + var redirectUrl = _redirectUrl + cacheName + "?response-content-disposition=attachment;filename=" + (name ?? "unknown"); + if (cacheFile != null) { if (hash != default) { - var hashResult = await _hashCache.FileHashCachedAsync(cacheFile, token); - if (hashResult != shouldMatch) + if (cacheFile.Hash != shouldMatch) return BadRequest(new {Type = "Unmatching Hashes", Expected = shouldMatch.ToHex(), Found = hashResult.ToHex()}); } - var ret = new PhysicalFileResult(cacheFile.ToString(), "application/octet-stream"); - if (name != null) - ret.FileDownloadName = name; - return ret; + return new RedirectResult(redirectUrl); } _logger.LogInformation("Downloading proxy request for {Uri}", uri); @@ -131,38 +145,87 @@ public class Proxy : ControllerBase var tempFile = _tempFileManager.CreateFile(deleteOnDispose:false); var proxyDownloader = _dispatcher.Downloader(archive) as IProxyable; - await using (var of = tempFile.Path.Open(FileMode.Create, FileAccess.Write, FileShare.None)) + + using var job = await _resource.Begin("Downloading file", 0, token); + hashResult = await proxyDownloader!.Download(archive, tempFile.Path, job, token); + + + if (hash != default && hashResult != shouldMatch) { - Response.StatusCode = 200; - if (name != null) - { - Response.Headers.Add(HeaderNames.ContentDisposition, $"attachment; filename=\"{name}\""); - } - - Response.Headers.Add( HeaderNames.ContentType, "application/octet-stream" ); - - var result = await proxyDownloader!.DownloadStream(archive, async s => { - return await s.HashingCopy(async m => - { - var strmA = of.WriteAsync(m, token); - await Response.Body.WriteAsync(m, token); - await Response.Body.FlushAsync(token); - await strmA; - }, token); }, - token); - - - if (hash != default && result != shouldMatch) - { - if (tempFile.Path.FileExists()) - tempFile.Path.Delete(); - } + if (tempFile.Path.FileExists()) + tempFile.Path.Delete(); } + + await PutCacheEntry(tempFile.Path, cacheName, hashResult); + _logger.LogInformation("Returning proxy request for {Uri}", uri); + return new RedirectResult(redirectUrl); + } - await tempFile.Path.MoveToAsync(cacheFile, true, token); + private async Task GetCacheEntry(string name) + { + var info = await _s3.GetObjectMetadataAsync(new GetObjectMetadataRequest() + { + BucketName = _bucket, + Key = name + }); + if (info.HttpStatusCode == System.Net.HttpStatusCode.NotFound) + return null; + + if (info.Metadata["WJ-Hash"] == null) + return null; + + if (!Hash.TryGetFromHex(info.Metadata["WJ-Hash"], out var hash)) + return null; + + return new CacheStatus + { + LastModified = info.LastModified, + Size = info.ContentLength, + Hash = hash + }; + } + + private async Task TouchCacheEntry(string name) + { + await _s3.CopyObjectAsync(new CopyObjectRequest() + { + SourceBucket = _bucket, + DestinationBucket = _bucket, + SourceKey = name, + DestinationKey = name, + MetadataDirective = S3MetadataDirective.REPLACE, + }); + } + + private async Task PutCacheEntry(AbsolutePath path, string name, Hash hash) + { + var obj = new PutObjectRequest + { + BucketName = _bucket, + Key = name, + FilePath = path.ToString(), + ContentType = "application/octet-stream", + DisablePayloadSigning = true + }; + obj.Metadata.Add("WJ-Hash", hash.ToHex()); + await _s3.PutObjectAsync(obj); + } + + private async Task DeleteCacheEntry(string name) + { + await _s3.DeleteObjectAsync(new DeleteObjectRequest + { + BucketName = _bucket, + Key = name + }); + } - _logger.LogInformation("Returning proxy request for {Uri} {Size}", uri, cacheFile.Size().FileSizeToString()); - return new EmptyResult(); + record CacheStatus + { + public DateTime LastModified { get; init; } + public long Size { get; init; } + + public Hash Hash { get; init; } } } \ No newline at end of file diff --git a/Wabbajack.Server/DataModels/AuthorFiles.cs b/Wabbajack.Server/DataModels/AuthorFiles.cs index c9ff706b..e22b3cd8 100644 --- a/Wabbajack.Server/DataModels/AuthorFiles.cs +++ b/Wabbajack.Server/DataModels/AuthorFiles.cs @@ -31,7 +31,7 @@ public class AuthorFiles private readonly HttpClient _httpClient; private readonly AbsolutePath _cacheFile; - private Uri _baseUri => new($"https://r2.wabbajack.org/"); + private Uri _baseUri => new($"https://authored-files.wabbajack.org/"); public AuthorFiles(ILogger logger, AppSettings settings, DTOSerializer dtos, IAmazonS3 s3, HttpClient client) { @@ -41,10 +41,10 @@ public class AuthorFiles _settings = settings; _dtos = dtos; _fileCache = new ConcurrentDictionary(); - _bucketName = settings.AuthoredFilesS3.BucketName; + _bucketName = settings.S3.AuthoredFilesBucket; _ = PrimeCache(); _streamPool = new RecyclableMemoryStreamManager(); - _cacheFile = _settings.AuthoredFilesS3.BucketCacheFile.ToAbsolutePath(); + _cacheFile = _settings.S3.AuthoredFilesBucket.ToAbsolutePath(); } private async Task PrimeCache() diff --git a/Wabbajack.Server/Resources/Reports/AuthoredFiles.html b/Wabbajack.Server/Resources/Reports/AuthoredFiles.html index fe85202d..a7a216d7 100644 --- a/Wabbajack.Server/Resources/Reports/AuthoredFiles.html +++ b/Wabbajack.Server/Resources/Reports/AuthoredFiles.html @@ -28,7 +28,7 @@ {{$.HumanSize}} {{$.Definition.Author}} {{$.Updated}} - (Slow) HTTP Direct Link + (Slow) HTTP Direct Link {{/each}} diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 14b8c66f..d67d0726 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -101,11 +101,11 @@ public class Startup services.AddSingleton(s => { var appSettings = s.GetRequiredService(); - var settings = new BasicAWSCredentials(appSettings.AuthoredFilesS3.AccessKey, - appSettings.AuthoredFilesS3.SecretKey); + var settings = new BasicAWSCredentials(appSettings.S3.AccessKey, + appSettings.S3.SecretKey); return new AmazonS3Client(settings, new AmazonS3Config { - ServiceURL = appSettings.AuthoredFilesS3.ServiceURL, + ServiceURL = appSettings.S3.ServiceUrl, }); }); services.AddTransient(s => diff --git a/Wabbajack.Server/appsettings.json b/Wabbajack.Server/appsettings.json index 5de64833..1acb7a81 100644 --- a/Wabbajack.Server/appsettings.json +++ b/Wabbajack.Server/appsettings.json @@ -29,12 +29,13 @@ "Username": "wabbajack", "Password": "password" }, - "AuthoredFilesS3": { - "AccessKey": "", - "SecretKey": "", - "ServiceURL": "", - "BucketName": "authored-files", - "BucketCacheFile": "c:\\tmp\\bucket-cache.txt" + "S3": { + "AccessKey": "<>", + "SecretKey": "<>", + "ServiceUrl": "<>", + "ProxyFilesBucket": "proxy-files", + "AuthoredFilesBucket": "authored-files", + "AuthoredFilesBucketCache": "c:\\tmp\\bucket-cache.txt" } }, "AllowedHosts": "*"