Server changes for CDN optimization

CHANGELOG.md

@@ -1,5 +1,6 @@
 ### Changelog
 
+#### Version - 1.0 beta 16 - 1/19/2020
 * Progress ring displays when downloading modlist images
 * GUI releases memory of download modlists better when navigating around
 * Fixed phrasing after failed installations to say "failed".

Wabbajack.BuildServer/AppSettings.cs

@@ -16,5 +16,8 @@ namespace Wabbajack.BuildServer
         
         public bool RunFrontEndJobs { get; set; }
         public bool RunBackEndJobs { get; set; }
+        
+        public string BunnyCDNZone { get; set; }
+        public string BunnyCDNApiKey { get; set; }
     }
 }

Wabbajack.BuildServer/Controllers/UploadedFiles.cs

@@ -15,16 +15,22 @@ using MongoDB.Driver;
 using MongoDB.Driver.Linq;
 using Nettle;
 using Wabbajack.BuildServer.Models;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.BuildServer.Models.Jobs;
 using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
 using Path = Alphaleonis.Win32.Filesystem.Path;
 
 namespace Wabbajack.BuildServer.Controllers
 {
     public class UploadedFiles : AControllerBase<UploadedFiles>
     {
-        public UploadedFiles(ILogger<UploadedFiles> logger, DBContext db) : base(logger, db)
+        private AppSettings _settings;
+
+        public UploadedFiles(ILogger<UploadedFiles> logger, DBContext db, AppSettings settings) : base(logger, db)
         {
-  
+            _settings = settings;
         }
 
         [HttpPut]
@@ -68,6 +74,7 @@ namespace Wabbajack.BuildServer.Controllers
             var final_path = Path.Combine("public", "files", final_name);
             System.IO.File.Move(Path.Combine("public", "files", Key), final_path);
             var hash = await final_path.FileHashAsync();
+            
 
             var record = new UploadedFile
             {
@@ -78,6 +85,23 @@ namespace Wabbajack.BuildServer.Controllers
                 Size = new FileInfo(final_path).Length
             };
             await Db.UploadedFiles.InsertOneAsync(record);
+            await Db.Jobs.InsertOneAsync(new Job
+            {
+                Payload = new IndexJob
+                {
+                    Archive = new Archive
+                    {
+                        Name = record.MungedName,
+                        Size = record.Size,
+                        Hash = record.Hash,
+                        State = new HTTPDownloader.State
+                        {
+                            Url = record.Uri
+                        }
+                    }
+                }
+            });
+            
             return Ok(record.Uri);
         }
 
@@ -92,6 +116,7 @@ namespace Wabbajack.BuildServer.Controllers
             </body></html>
         ");
 
+
         [HttpGet]
         [Route("uploaded_files")]
         public async Task<ContentResult> UploadedFilesGet()

Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs

@@ -17,7 +17,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue
             typeof(UpdateModLists),
             typeof(EnqueueAllArchives),
             typeof(EnqueueAllGameFiles),
-            typeof(EnqueueRecentFiles)
+            typeof(EnqueueRecentFiles),
+            typeof(UploadToCDN)
         };
         public static Dictionary<Type, string> TypeToName { get; set; }
         public static Dictionary<string, Type> NameToType { get; set; }

Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs

@@ -0,0 +1,26 @@
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using BunnyCDN.Net.Storage;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.Common;
+
+namespace Wabbajack.BuildServer.Models.Jobs
+{
+    public class UploadToCDN : AJobPayload
+    {
+        public override string Description => $"Push an uploaded file ({FileId}) to the CDN";
+        
+        public string FileId { get; set; }
+        
+        public override async Task<JobResult> Execute(DBContext db, AppSettings settings)
+        {
+            var file = await db.UploadedFiles.AsQueryable().Where(f => f.Id == FileId).FirstOrDefaultAsync();
+            var cdn = new BunnyCDNStorage(settings.BunnyCDNZone, settings.BunnyCDNApiKey);
+            Utils.Log($"CDN Push {file.MungedName} to {settings.BunnyCDNZone}");
+            await cdn.UploadAsync(Path.Combine("public", "files", file.MungedName), $"{settings.BunnyCDNZone}/{file.MungedName}");
+            return JobResult.Success();
+        }
+    }
+}

Wabbajack.BuildServer/Models/UploadedFile.cs

@@ -16,10 +16,13 @@ namespace Wabbajack.BuildServer.Models
         public string Hash { get; set; }
         public string Uploader { get; set; }
         public DateTime UploadDate { get; set; } = DateTime.UtcNow;
+        
+        public string CDNName { get; set; }
 
         [BsonIgnore]
         public string MungedName => $"{Path.GetFileNameWithoutExtension(Name)}-{Id}{Path.GetExtension(Name)}";
 
-        [BsonIgnore] public object Uri => $"https://wabbajack.b-cdn.net/{MungedName}";
+        [BsonIgnore]
+        public string Uri => CDNName == null ? $"https://wabbajack.b-cdn.net/{MungedName}" : $"https://{CDNName}.b-cdn.net/{MungedName}";
     }
 }

Wabbajack.BuildServer/Wabbajack.BuildServer.csproj

@@ -10,6 +10,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="BunnyCDN.Net.Storage" Version="1.0.2" />
         <PackageReference Include="graphiql" Version="1.2.0" />
         <PackageReference Include="GraphQL" Version="3.0.0-preview-1352" />
         <PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />

Wabbajack.BuildServer/appsettings.json

@@ -34,7 +34,9 @@
     "ArchiveDir": "c:\\archives",
     "MinimalMode": true,
     "RunFrontEndJobs": true,
-    "RunBackEndJobs": true
+    "RunBackEndJobs": true,
+    "BunnyCDNZone": "",
+    "BunnyCDNApiKey": ""
   },
   "AllowedHosts": "*"
 }