Rework the server code a bit to use web workers and r2 in more places

Wabbajack.Server/AppSettings.cs

@@ -37,18 +37,19 @@ public class AppSettings
     public CouchDBSetting CesiDB { get; set; }
     public CouchDBSetting MetricsDB { get; set; }
 
-    public S3Settings AuthoredFilesS3 { get; set; }
+    public S3Settings S3 { get; set; }
 }
 
 public class S3Settings
 {
     public string AccessKey { get; set; }
     public string SecretKey { get; set; }
-    public string ServiceURL { get; set; }
+    public string ServiceUrl { get; set; }
     
-    public string BucketName { get; set; }
+    public string AuthoredFilesBucket { get; set; }
+    public string ProxyFilesBucket { get; set; }
     
-    public string BucketCacheFile { get; set; }
+    public string AuthoredFilesBucketCache { get; set; }
 }
 
 public class CouchDBSetting

Wabbajack.Server/Controllers/Proxy.cs

@@ -1,4 +1,7 @@
 using System.Text;
+using Amazon.Runtime;
+using Amazon.S3;
+using Amazon.S3.Model;
 using FluentFTP.Helpers;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
@@ -10,7 +13,9 @@ using Wabbajack.Downloaders.Interfaces;
 using Wabbajack.DTOs;
 using Wabbajack.DTOs.DownloadStates;
 using Wabbajack.Hashing.xxHash64;
+using Wabbajack.Paths;
 using Wabbajack.Paths.IO;
+using Wabbajack.RateLimiter;
 using Wabbajack.VFS;
 
 namespace Wabbajack.Server.Controllers;
@@ -24,14 +29,23 @@ public class Proxy : ControllerBase
     private readonly TemporaryFileManager _tempFileManager;
     private readonly AppSettings _appSettings;
     private readonly FileHashCache _hashCache;
+    private readonly IAmazonS3 _s3;
+    private readonly string _bucket;
+    
+    private string _redirectUrl = "https://proxy.wabbajack.org/";
+    private readonly IResource<DownloadDispatcher> _resource;
 
-    public Proxy(ILogger<Proxy> logger, DownloadDispatcher dispatcher, TemporaryFileManager tempFileManager, FileHashCache hashCache, AppSettings appSettings)
+    public Proxy(ILogger<Proxy> logger, DownloadDispatcher dispatcher, TemporaryFileManager tempFileManager, 
+        FileHashCache hashCache, AppSettings appSettings, IAmazonS3 s3, IResource<DownloadDispatcher> resource)
     {
         _logger = logger;
         _dispatcher = dispatcher;
         _tempFileManager = tempFileManager;
         _appSettings = appSettings;
         _hashCache = hashCache;
+        _s3 = s3;
+        _bucket = _appSettings.S3.ProxyFilesBucket;
+        _resource = resource;
     }
 
     [HttpHead]
@@ -58,12 +72,14 @@ public class Proxy : ControllerBase
     [HttpGet]
     public async Task<IActionResult> ProxyGet(CancellationToken token, [FromQuery] Uri uri, [FromQuery] string? name, [FromQuery] string? hash)
     {
+
+        Hash hashResult = default;
         var shouldMatch = hash != null ? Hash.FromHex(hash) : default;
         
         _logger.LogInformation("Got proxy request for {Uri}", uri);
         var state = _dispatcher.Parse(uri);
         var cacheName = (await Encoding.UTF8.GetBytes(uri.ToString()).Hash()).ToHex();
-        var cacheFile = _appSettings.ProxyPath.Combine(cacheName);
+        var cacheFile = await GetCacheEntry(cacheName);
 
         if (state == null)
         {
@@ -84,26 +100,27 @@ public class Proxy : ControllerBase
             return BadRequest(new {Type = "Downloader is not IProxyable", Downloader = downloader.GetType().FullName});
         }
         
-        if (cacheFile.FileExists() && (DateTime.Now - cacheFile.LastModified()) > TimeSpan.FromHours(4))
+        if (cacheFile != null && (DateTime.UtcNow - cacheFile.LastModified) > TimeSpan.FromHours(4))
         {
             try
             {
                 var verify = await _dispatcher.Verify(archive, token);
                 if (verify)
-                    cacheFile.Touch();
+                    await TouchCacheEntry(cacheName);
             }
             catch (Exception ex)
             {
-                _logger.LogInformation(ex, "When trying to verify cached file ({Hash}) {Url}", cacheFile.FileName, uri);
+                _logger.LogInformation(ex, "When trying to verify cached file ({Hash}) {Url}", 
-                cacheFile.Touch();
+                    cacheFile.Hash, uri);
+                await TouchCacheEntry(cacheName);
             }
         }
         
-        if (cacheFile.FileExists() && (DateTime.Now - cacheFile.LastModified()) > TimeSpan.FromHours(24))
+        if (cacheFile != null && (DateTime.Now - cacheFile.LastModified) > TimeSpan.FromHours(24))
         {
             try
             {
-                cacheFile.Delete();
+                await DeleteCacheEntry(cacheName);
             }
             catch (Exception ex)
             {
@@ -112,18 +129,15 @@ public class Proxy : ControllerBase
         }
 
 
-        if (cacheFile.FileExists())
+        var redirectUrl = _redirectUrl + cacheName + "?response-content-disposition=attachment;filename=" + (name ?? "unknown");
+        if (cacheFile != null)
         {
             if (hash != default)
             {
-                var hashResult = await _hashCache.FileHashCachedAsync(cacheFile, token);
+                if (cacheFile.Hash != shouldMatch)
-                if (hashResult != shouldMatch)
                     return BadRequest(new {Type = "Unmatching Hashes", Expected = shouldMatch.ToHex(), Found = hashResult.ToHex()});
             }
-            var ret = new PhysicalFileResult(cacheFile.ToString(), "application/octet-stream");
+            return new RedirectResult(redirectUrl);
-            if (name != null)
-                ret.FileDownloadName = name;
-            return ret;
         }
 
         _logger.LogInformation("Downloading proxy request for {Uri}", uri);
@@ -131,38 +145,87 @@ public class Proxy : ControllerBase
         var tempFile = _tempFileManager.CreateFile(deleteOnDispose:false);
 
         var proxyDownloader = _dispatcher.Downloader(archive) as IProxyable;
-        await using (var of = tempFile.Path.Open(FileMode.Create, FileAccess.Write, FileShare.None))
+
+        using var job = await _resource.Begin("Downloading file", 0, token);
+        hashResult = await proxyDownloader!.Download(archive, tempFile.Path, job, token);
+        
+        
+        if (hash != default && hashResult != shouldMatch)
         {
-            Response.StatusCode = 200;
+            if (tempFile.Path.FileExists())
-            if (name != null)
+                tempFile.Path.Delete();
-            {
-                Response.Headers.Add(HeaderNames.ContentDisposition, $"attachment; filename=\"{name}\"");
-            }
-
-            Response.Headers.Add( HeaderNames.ContentType, "application/octet-stream"  );
-            
-            var result = await proxyDownloader!.DownloadStream(archive, async s => { 
-                    return await s.HashingCopy(async m =>
-                {
-                    var strmA = of.WriteAsync(m, token);
-                    await Response.Body.WriteAsync(m, token);
-                    await Response.Body.FlushAsync(token);
-                    await strmA;
-                }, token); },
-                token);
-            
-            
-            if (hash != default && result != shouldMatch)
-            {
-                if (tempFile.Path.FileExists())
-                    tempFile.Path.Delete();
-            }
         }
+        
+        await PutCacheEntry(tempFile.Path, cacheName, hashResult);
 
+        _logger.LogInformation("Returning proxy request for {Uri}", uri);
+        return new RedirectResult(redirectUrl);
+    }
 
-        await tempFile.Path.MoveToAsync(cacheFile, true, token);
+    private async Task<CacheStatus?> GetCacheEntry(string name)
+    {
+        var info = await _s3.GetObjectMetadataAsync(new GetObjectMetadataRequest()
+        {
+            BucketName = _bucket,
+            Key = name
+        });
+        if (info.HttpStatusCode == System.Net.HttpStatusCode.NotFound)
+            return null;
+        
+        if (info.Metadata["WJ-Hash"] == null)
+            return null;
+        
+        if (!Hash.TryGetFromHex(info.Metadata["WJ-Hash"], out var hash))
+            return null;
+        
+        return new CacheStatus
+        {
+            LastModified = info.LastModified,
+            Size = info.ContentLength,
+            Hash = hash
+        };
+    }
+    
+    private async Task TouchCacheEntry(string name)
+    {
+        await _s3.CopyObjectAsync(new CopyObjectRequest()
+        {
+            SourceBucket = _bucket,
+            DestinationBucket = _bucket,
+            SourceKey = name,
+            DestinationKey = name,
+            MetadataDirective = S3MetadataDirective.REPLACE,
+        });
+    }
+    
+    private async Task PutCacheEntry(AbsolutePath path, string name, Hash hash)
+    {
+        var obj = new PutObjectRequest
+        {
+            BucketName = _bucket,
+            Key = name,
+            FilePath = path.ToString(),
+            ContentType = "application/octet-stream",
+            DisablePayloadSigning = true
+        };
+        obj.Metadata.Add("WJ-Hash", hash.ToHex());
+        await _s3.PutObjectAsync(obj);
+    }
+    
+    private async Task DeleteCacheEntry(string name)
+    {
+        await _s3.DeleteObjectAsync(new DeleteObjectRequest
+        {
+            BucketName = _bucket,
+            Key = name
+        });
+    }
 
-        _logger.LogInformation("Returning proxy request for {Uri} {Size}", uri, cacheFile.Size().FileSizeToString());
+    record CacheStatus
-        return new EmptyResult();
+    {
+        public DateTime LastModified { get; init; }
+        public long Size { get; init; }
+        
+        public Hash Hash { get; init; }
     }
 }

Wabbajack.Server/DataModels/AuthorFiles.cs

@@ -31,7 +31,7 @@ public class AuthorFiles
     private readonly HttpClient _httpClient;
     private readonly AbsolutePath _cacheFile;
 
-    private Uri _baseUri => new($"https://r2.wabbajack.org/");
+    private Uri _baseUri => new($"https://authored-files.wabbajack.org/");
     
     public AuthorFiles(ILogger<AuthorFiles> logger, AppSettings settings, DTOSerializer dtos, IAmazonS3 s3, HttpClient client)
     {
@@ -41,10 +41,10 @@ public class AuthorFiles
         _settings = settings;
         _dtos = dtos;
         _fileCache = new ConcurrentDictionary<string, FileDefinitionMetadata>();
-        _bucketName = settings.AuthoredFilesS3.BucketName;
+        _bucketName = settings.S3.AuthoredFilesBucket;
         _ = PrimeCache();
         _streamPool = new RecyclableMemoryStreamManager();
-        _cacheFile = _settings.AuthoredFilesS3.BucketCacheFile.ToAbsolutePath();
+        _cacheFile = _settings.S3.AuthoredFilesBucket.ToAbsolutePath();
     }
 
     private async Task PrimeCache()

Wabbajack.Server/Resources/Reports/AuthoredFiles.html

@@ -28,7 +28,7 @@
         <td>{{$.HumanSize}}</td>
         <td>{{$.Definition.Author}}</td>
         <td>{{$.Updated}}</td>
-        <td><a href='/authored_files/direct_link/{{$.Definition.MungedName}}'>(Slow) HTTP Direct Link</a></td>
+        <td><a href='https://workers.wabbajack.workers.dev/authored_files/stream/{{$.Definition.MungedName}}'>(Slow) HTTP Direct Link</a></td>
     </tr>
     {{/each}}
 </table>

Wabbajack.Server/Startup.cs

@@ -101,11 +101,11 @@ public class Startup
         services.AddSingleton<IAmazonS3>(s =>
         {
             var appSettings = s.GetRequiredService<AppSettings>();
-            var settings = new BasicAWSCredentials(appSettings.AuthoredFilesS3.AccessKey,
+            var settings = new BasicAWSCredentials(appSettings.S3.AccessKey,
-                appSettings.AuthoredFilesS3.SecretKey);
+                appSettings.S3.SecretKey);
             return new AmazonS3Client(settings, new AmazonS3Config
             {
-                ServiceURL = appSettings.AuthoredFilesS3.ServiceURL,
+                ServiceURL = appSettings.S3.ServiceUrl,
             });
         });
         services.AddTransient(s =>

Wabbajack.Server/appsettings.json

@@ -29,12 +29,13 @@
       "Username": "wabbajack",
       "Password": "password"
     },
-    "AuthoredFilesS3": {
+    "S3": {
-      "AccessKey": "<ACCESS_KEY>",
+      "AccessKey": "<>",
-      "SecretKey": "<SECRET_KEY>",
+      "SecretKey": "<>",
-      "ServiceURL": "<SERVICE_URL>",
+      "ServiceUrl": "<>",
-      "BucketName": "authored-files",
+      "ProxyFilesBucket": "proxy-files",
-      "BucketCacheFile": "c:\\tmp\\bucket-cache.txt"
+      "AuthoredFilesBucket": "authored-files",
+      "AuthoredFilesBucketCache": "c:\\tmp\\bucket-cache.txt"
     }
   },
   "AllowedHosts": "*"