Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rework the server code a bit to use web workers and r2 in more places

Browse Source
This commit is contained in:
Timothy Baldridge 2023-10-20 22:40:35 +01:00
parent 90f9df6436
commit cf0dae0049
6 changed files with 125 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Wabbajack.Server/AppSettings.cs
View File

@ -37,18 +37,19 @@ public class AppSettings
public CouchDBSetting CesiDB { get; set; }
public CouchDBSetting MetricsDB { get; set; }
public S3Settings AuthoredFilesS3 { get; set; }
public S3Settings S3 { get; set; }
}
public class S3Settings
{
public string AccessKey { get; set; }
public string SecretKey { get; set; }
public string ServiceURL { get; set; }
public string ServiceUrl { get; set; }
public string BucketName { get; set; }
public string AuthoredFilesBucket { get; set; }
public string ProxyFilesBucket { get; set; }
public string BucketCacheFile { get; set; }
public string AuthoredFilesBucketCache { get; set; }
}
public class CouchDBSetting

149
Wabbajack.Server/Controllers/Proxy.cs
View File

@ -1,4 +1,7 @@
using System.Text;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model;
using FluentFTP.Helpers;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
@ -10,7 +13,9 @@ using Wabbajack.Downloaders.Interfaces;
using Wabbajack.DTOs;
using Wabbajack.DTOs.DownloadStates;
using Wabbajack.Hashing.xxHash64;
using Wabbajack.Paths;
using Wabbajack.Paths.IO;
using Wabbajack.RateLimiter;
using Wabbajack.VFS;
namespace Wabbajack.Server.Controllers;
@ -24,14 +29,23 @@ public class Proxy : ControllerBase
private readonly TemporaryFileManager _tempFileManager;
private readonly AppSettings _appSettings;
private readonly FileHashCache _hashCache;
private readonly IAmazonS3 _s3;
private readonly string _bucket;
private string _redirectUrl = "https://proxy.wabbajack.org/";
private readonly IResource<DownloadDispatcher> _resource;
public Proxy(ILogger<Proxy> logger, DownloadDispatcher dispatcher, TemporaryFileManager tempFileManager, FileHashCache hashCache, AppSettings appSettings)
public Proxy(ILogger<Proxy> logger, DownloadDispatcher dispatcher, TemporaryFileManager tempFileManager,
FileHashCache hashCache, AppSettings appSettings, IAmazonS3 s3, IResource<DownloadDispatcher> resource)
{
_logger = logger;
_dispatcher = dispatcher;
_tempFileManager = tempFileManager;
_appSettings = appSettings;
_hashCache = hashCache;
_s3 = s3;
_bucket = _appSettings.S3.ProxyFilesBucket;
_resource = resource;
}
[HttpHead]
@ -58,12 +72,14 @@ public class Proxy : ControllerBase
[HttpGet]
public async Task<IActionResult> ProxyGet(CancellationToken token, [FromQuery] Uri uri, [FromQuery] string? name, [FromQuery] string? hash)
{
Hash hashResult = default;
var shouldMatch = hash != null ? Hash.FromHex(hash) : default;
_logger.LogInformation("Got proxy request for {Uri}", uri);
var state = _dispatcher.Parse(uri);
var cacheName = (await Encoding.UTF8.GetBytes(uri.ToString()).Hash()).ToHex();
var cacheFile = _appSettings.ProxyPath.Combine(cacheName);
var cacheFile = await GetCacheEntry(cacheName);
if (state == null)
{
@ -84,26 +100,27 @@ public class Proxy : ControllerBase
return BadRequest(new {Type = "Downloader is not IProxyable", Downloader = downloader.GetType().FullName});
}
if (cacheFile.FileExists() && (DateTime.Now - cacheFile.LastModified()) > TimeSpan.FromHours(4))
if (cacheFile != null && (DateTime.UtcNow - cacheFile.LastModified) > TimeSpan.FromHours(4))
{
try
{
var verify = await _dispatcher.Verify(archive, token);
if (verify)
cacheFile.Touch();
await TouchCacheEntry(cacheName);
}
catch (Exception ex)
{
_logger.LogInformation(ex, "When trying to verify cached file ({Hash}) {Url}", cacheFile.FileName, uri);
cacheFile.Touch();
_logger.LogInformation(ex, "When trying to verify cached file ({Hash}) {Url}",
cacheFile.Hash, uri);
await TouchCacheEntry(cacheName);
}
}
if (cacheFile.FileExists() && (DateTime.Now - cacheFile.LastModified()) > TimeSpan.FromHours(24))
if (cacheFile != null && (DateTime.Now - cacheFile.LastModified) > TimeSpan.FromHours(24))
{
try
{
cacheFile.Delete();
await DeleteCacheEntry(cacheName);
}
catch (Exception ex)
{
@ -112,18 +129,15 @@ public class Proxy : ControllerBase
}
if (cacheFile.FileExists())
var redirectUrl = _redirectUrl + cacheName + "?response-content-disposition=attachment;filename=" + (name ?? "unknown");
if (cacheFile != null)
{
if (hash != default)
{
var hashResult = await _hashCache.FileHashCachedAsync(cacheFile, token);
if (hashResult != shouldMatch)
if (cacheFile.Hash != shouldMatch)
return BadRequest(new {Type = "Unmatching Hashes", Expected = shouldMatch.ToHex(), Found = hashResult.ToHex()});
}
var ret = new PhysicalFileResult(cacheFile.ToString(), "application/octet-stream");
if (name != null)
ret.FileDownloadName = name;
return ret;
return new RedirectResult(redirectUrl);
}
_logger.LogInformation("Downloading proxy request for {Uri}", uri);
@ -131,38 +145,87 @@ public class Proxy : ControllerBase
var tempFile = _tempFileManager.CreateFile(deleteOnDispose:false);
var proxyDownloader = _dispatcher.Downloader(archive) as IProxyable;
await using (var of = tempFile.Path.Open(FileMode.Create, FileAccess.Write, FileShare.None))
using var job = await _resource.Begin("Downloading file", 0, token);
hashResult = await proxyDownloader!.Download(archive, tempFile.Path, job, token);
if (hash != default && hashResult != shouldMatch)
{
Response.StatusCode = 200;
if (name != null)
{
Response.Headers.Add(HeaderNames.ContentDisposition, $"attachment; filename=\"{name}\"");
}
Response.Headers.Add( HeaderNames.ContentType, "application/octet-stream" );
var result = await proxyDownloader!.DownloadStream(archive, async s => {
return await s.HashingCopy(async m =>
{
var strmA = of.WriteAsync(m, token);
await Response.Body.WriteAsync(m, token);
await Response.Body.FlushAsync(token);
await strmA;
}, token); },
token);
if (hash != default && result != shouldMatch)
{
if (tempFile.Path.FileExists())
tempFile.Path.Delete();
}
if (tempFile.Path.FileExists())
tempFile.Path.Delete();
}
await PutCacheEntry(tempFile.Path, cacheName, hashResult);
_logger.LogInformation("Returning proxy request for {Uri}", uri);
return new RedirectResult(redirectUrl);
}
await tempFile.Path.MoveToAsync(cacheFile, true, token);
private async Task<CacheStatus?> GetCacheEntry(string name)
{
var info = await _s3.GetObjectMetadataAsync(new GetObjectMetadataRequest()
{
BucketName = _bucket,
Key = name
});
if (info.HttpStatusCode == System.Net.HttpStatusCode.NotFound)
return null;
if (info.Metadata["WJ-Hash"] == null)
return null;
if (!Hash.TryGetFromHex(info.Metadata["WJ-Hash"], out var hash))
return null;
return new CacheStatus
{
LastModified = info.LastModified,
Size = info.ContentLength,
Hash = hash
};
}
private async Task TouchCacheEntry(string name)
{
await _s3.CopyObjectAsync(new CopyObjectRequest()
{
SourceBucket = _bucket,
DestinationBucket = _bucket,
SourceKey = name,
DestinationKey = name,
MetadataDirective = S3MetadataDirective.REPLACE,
});
}
private async Task PutCacheEntry(AbsolutePath path, string name, Hash hash)
{
var obj = new PutObjectRequest
{
BucketName = _bucket,
Key = name,
FilePath = path.ToString(),
ContentType = "application/octet-stream",
DisablePayloadSigning = true
};
obj.Metadata.Add("WJ-Hash", hash.ToHex());
await _s3.PutObjectAsync(obj);
}
private async Task DeleteCacheEntry(string name)
{
await _s3.DeleteObjectAsync(new DeleteObjectRequest
{
BucketName = _bucket,
Key = name
});
}
_logger.LogInformation("Returning proxy request for {Uri} {Size}", uri, cacheFile.Size().FileSizeToString());
return new EmptyResult();
record CacheStatus
{
public DateTime LastModified { get; init; }
public long Size { get; init; }
public Hash Hash { get; init; }
}
}

6
Wabbajack.Server/DataModels/AuthorFiles.cs
View File

@ -31,7 +31,7 @@ public class AuthorFiles
private readonly HttpClient _httpClient;
private readonly AbsolutePath _cacheFile;
private Uri _baseUri => new($"https://r2.wabbajack.org/");
private Uri _baseUri => new($"https://authored-files.wabbajack.org/");
public AuthorFiles(ILogger<AuthorFiles> logger, AppSettings settings, DTOSerializer dtos, IAmazonS3 s3, HttpClient client)
{
@ -41,10 +41,10 @@ public class AuthorFiles
_settings = settings;
_dtos = dtos;
_fileCache = new ConcurrentDictionary<string, FileDefinitionMetadata>();
_bucketName = settings.AuthoredFilesS3.BucketName;
_bucketName = settings.S3.AuthoredFilesBucket;
_ = PrimeCache();
_streamPool = new RecyclableMemoryStreamManager();
_cacheFile = _settings.AuthoredFilesS3.BucketCacheFile.ToAbsolutePath();
_cacheFile = _settings.S3.AuthoredFilesBucket.ToAbsolutePath();
}
private async Task PrimeCache()

2
Wabbajack.Server/Resources/Reports/AuthoredFiles.html
View File

@ -28,7 +28,7 @@
<td>{{$.HumanSize}}</td>
<td>{{$.Definition.Author}}</td>
<td>{{$.Updated}}</td>
<td><a href='/authored_files/direct_link/{{$.Definition.MungedName}}'>(Slow) HTTP Direct Link</a></td>
<td><a href='https://workers.wabbajack.workers.dev/authored_files/stream/{{$.Definition.MungedName}}'>(Slow) HTTP Direct Link</a></td>
</tr>
{{/each}}
</table>

6
Wabbajack.Server/Startup.cs
View File

@ -101,11 +101,11 @@ public class Startup
services.AddSingleton<IAmazonS3>(s =>
{
var appSettings = s.GetRequiredService<AppSettings>();
var settings = new BasicAWSCredentials(appSettings.AuthoredFilesS3.AccessKey,
appSettings.AuthoredFilesS3.SecretKey);
var settings = new BasicAWSCredentials(appSettings.S3.AccessKey,
appSettings.S3.SecretKey);
return new AmazonS3Client(settings, new AmazonS3Config
{
ServiceURL = appSettings.AuthoredFilesS3.ServiceURL,
ServiceURL = appSettings.S3.ServiceUrl,
});
});
services.AddTransient(s =>

13
Wabbajack.Server/appsettings.json
View File

@ -29,12 +29,13 @@
"Username": "wabbajack",
"Password": "password"
},
"AuthoredFilesS3": {
"AccessKey": "<ACCESS_KEY>",
"SecretKey": "<SECRET_KEY>",
"ServiceURL": "<SERVICE_URL>",
"BucketName": "authored-files",
"BucketCacheFile": "c:\\tmp\\bucket-cache.txt"
"S3": {
"AccessKey": "<>",
"SecretKey": "<>",
"ServiceUrl": "<>",
"ProxyFilesBucket": "proxy-files",
"AuthoredFilesBucket": "authored-files",
"AuthoredFilesBucketCache": "c:\\tmp\\bucket-cache.txt"
}
},
"AllowedHosts": "*"