From 7e436818b750ba3c2545c8016f528e6f2aad927c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 21 Jan 2020 05:49:49 -0700 Subject: [PATCH] Server changes for CDN optimization --- CHANGELOG.md | 1 + Wabbajack.BuildServer/AppSettings.cs | 3 ++ .../Controllers/UploadedFiles.cs | 29 +++++++++++++++++-- .../Models/JobQueue/AJobPayload.cs | 3 +- .../Models/Jobs/UploadToCDN.cs | 26 +++++++++++++++++ Wabbajack.BuildServer/Models/UploadedFile.cs | 5 +++- .../Wabbajack.BuildServer.csproj | 1 + Wabbajack.BuildServer/appsettings.json | 4 ++- 8 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 75c6acd5..a447e1ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ### Changelog +#### Version - 1.0 beta 16 - 1/19/2020 * Progress ring displays when downloading modlist images * GUI releases memory of download modlists better when navigating around * Fixed phrasing after failed installations to say "failed". diff --git a/Wabbajack.BuildServer/AppSettings.cs b/Wabbajack.BuildServer/AppSettings.cs index 5470265e..35c24521 100644 --- a/Wabbajack.BuildServer/AppSettings.cs +++ b/Wabbajack.BuildServer/AppSettings.cs @@ -16,5 +16,8 @@ namespace Wabbajack.BuildServer public bool RunFrontEndJobs { get; set; } public bool RunBackEndJobs { get; set; } + + public string BunnyCDNZone { get; set; } + public string BunnyCDNApiKey { get; set; } } } diff --git a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs index 89ae6061..721e9388 100644 --- a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs @@ -15,16 +15,22 @@ using MongoDB.Driver; using MongoDB.Driver.Linq; using Nettle; using Wabbajack.BuildServer.Models; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.BuildServer.Controllers { public class UploadedFiles : AControllerBase { - public UploadedFiles(ILogger logger, DBContext db) : base(logger, db) + private AppSettings _settings; + + public UploadedFiles(ILogger logger, DBContext db, AppSettings settings) : base(logger, db) { - + _settings = settings; } [HttpPut] @@ -68,6 +74,7 @@ namespace Wabbajack.BuildServer.Controllers var final_path = Path.Combine("public", "files", final_name); System.IO.File.Move(Path.Combine("public", "files", Key), final_path); var hash = await final_path.FileHashAsync(); + var record = new UploadedFile { @@ -78,6 +85,23 @@ namespace Wabbajack.BuildServer.Controllers Size = new FileInfo(final_path).Length }; await Db.UploadedFiles.InsertOneAsync(record); + await Db.Jobs.InsertOneAsync(new Job + { + Payload = new IndexJob + { + Archive = new Archive + { + Name = record.MungedName, + Size = record.Size, + Hash = record.Hash, + State = new HTTPDownloader.State + { + Url = record.Uri + } + } + } + }); + return Ok(record.Uri); } @@ -92,6 +116,7 @@ namespace Wabbajack.BuildServer.Controllers "); + [HttpGet] [Route("uploaded_files")] public async Task UploadedFilesGet() diff --git a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs index 76c3181c..7b7169fc 100644 --- a/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs +++ b/Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs @@ -17,7 +17,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue typeof(UpdateModLists), typeof(EnqueueAllArchives), typeof(EnqueueAllGameFiles), - typeof(EnqueueRecentFiles) + typeof(EnqueueRecentFiles), + typeof(UploadToCDN) }; public static Dictionary TypeToName { get; set; } public static Dictionary NameToType { get; set; } diff --git a/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs b/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs new file mode 100644 index 00000000..20891cd2 --- /dev/null +++ b/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs @@ -0,0 +1,26 @@ +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using BunnyCDN.Net.Storage; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Wabbajack.BuildServer.Models.JobQueue; +using Wabbajack.Common; + +namespace Wabbajack.BuildServer.Models.Jobs +{ + public class UploadToCDN : AJobPayload + { + public override string Description => $"Push an uploaded file ({FileId}) to the CDN"; + + public string FileId { get; set; } + + public override async Task Execute(DBContext db, AppSettings settings) + { + var file = await db.UploadedFiles.AsQueryable().Where(f => f.Id == FileId).FirstOrDefaultAsync(); + var cdn = new BunnyCDNStorage(settings.BunnyCDNZone, settings.BunnyCDNApiKey); + Utils.Log($"CDN Push {file.MungedName} to {settings.BunnyCDNZone}"); + await cdn.UploadAsync(Path.Combine("public", "files", file.MungedName), $"{settings.BunnyCDNZone}/{file.MungedName}"); + return JobResult.Success(); + } + } +} diff --git a/Wabbajack.BuildServer/Models/UploadedFile.cs b/Wabbajack.BuildServer/Models/UploadedFile.cs index 42b5c6bd..8c7d9503 100644 --- a/Wabbajack.BuildServer/Models/UploadedFile.cs +++ b/Wabbajack.BuildServer/Models/UploadedFile.cs @@ -16,10 +16,13 @@ namespace Wabbajack.BuildServer.Models public string Hash { get; set; } public string Uploader { get; set; } public DateTime UploadDate { get; set; } = DateTime.UtcNow; + + public string CDNName { get; set; } [BsonIgnore] public string MungedName => $"{Path.GetFileNameWithoutExtension(Name)}-{Id}{Path.GetExtension(Name)}"; - [BsonIgnore] public object Uri => $"https://wabbajack.b-cdn.net/{MungedName}"; + [BsonIgnore] + public string Uri => CDNName == null ? $"https://wabbajack.b-cdn.net/{MungedName}" : $"https://{CDNName}.b-cdn.net/{MungedName}"; } } diff --git a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj index d70fac15..4ebc975f 100644 --- a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj +++ b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj @@ -10,6 +10,7 @@ + diff --git a/Wabbajack.BuildServer/appsettings.json b/Wabbajack.BuildServer/appsettings.json index 96c6a575..166e3f0b 100644 --- a/Wabbajack.BuildServer/appsettings.json +++ b/Wabbajack.BuildServer/appsettings.json @@ -34,7 +34,9 @@ "ArchiveDir": "c:\\archives", "MinimalMode": true, "RunFrontEndJobs": true, - "RunBackEndJobs": true + "RunBackEndJobs": true, + "BunnyCDNZone": "", + "BunnyCDNApiKey": "" }, "AllowedHosts": "*" }