Merge pull request #414 from wabbajack-tools/bunny-cdn-push

Server changes for CDN optimization

CHANGELOG.md

@@ -1,5 +1,6 @@
 ### Changelog
 
+=======
 * Fix for queue size recommendation of 0GB RAM on low-end machines
 * Fix for website readme compilation
 * Fix for compiler downloads folder specification (was always standard path)

Wabbajack.BuildServer/AppSettings.cs

@@ -16,5 +16,8 @@ namespace Wabbajack.BuildServer
         
         public bool RunFrontEndJobs { get; set; }
         public bool RunBackEndJobs { get; set; }
+        
+        public string BunnyCDN_User { get; set; }
+        public string BunnyCDN_Password { get; set; }
     }
 }

Wabbajack.BuildServer/Controllers/UploadedFiles.cs

@@ -15,16 +15,22 @@ using MongoDB.Driver;
 using MongoDB.Driver.Linq;
 using Nettle;
 using Wabbajack.BuildServer.Models;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.BuildServer.Models.Jobs;
 using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
 using Path = Alphaleonis.Win32.Filesystem.Path;
 
 namespace Wabbajack.BuildServer.Controllers
 {
     public class UploadedFiles : AControllerBase<UploadedFiles>
     {
-        public UploadedFiles(ILogger<UploadedFiles> logger, DBContext db) : base(logger, db)
+        private AppSettings _settings;
-        {
 
+        public UploadedFiles(ILogger<UploadedFiles> logger, DBContext db, AppSettings settings) : base(logger, db)
+        {
+            _settings = settings;
         }
 
         [HttpPut]
@@ -46,7 +52,7 @@ namespace Wabbajack.BuildServer.Controllers
             if (!Key.All(a => HexChars.Contains(a)))
                 return BadRequest("NOT A VALID FILENAME");
             Utils.Log($"Writing at position {Offset} in ingest file {Key}");
-            await using (var file = System.IO.File.Open(Path.Combine("public", "files", Key), FileMode.Open, FileAccess.Write))
+            await using (var file = System.IO.File.Open(Path.Combine("public", "files", Key), FileMode.Open, FileAccess.Write, FileShare.ReadWrite))
             {
                 file.Position = Offset;
                 await Request.Body.CopyToAsync(file);
@@ -69,15 +75,23 @@ namespace Wabbajack.BuildServer.Controllers
             System.IO.File.Move(Path.Combine("public", "files", Key), final_path);
             var hash = await final_path.FileHashAsync();
             
+
             var record = new UploadedFile
             {
                 Id = parts[1],
                 Hash = hash, 
                 Name = original_name, 
                 Uploader = user, 
-                Size = new FileInfo(final_path).Length
+                Size = new FileInfo(final_path).Length,
+                CDNName = "wabbajackpush"
             };
             await Db.UploadedFiles.InsertOneAsync(record);
+            await Db.Jobs.InsertOneAsync(new Job
+            {
+                Priority = Job.JobPriority.High, Payload = new UploadToCDN {FileId = record.Id}
+            });
+
+            
             return Ok(record.Uri);
         }
 
@@ -92,6 +106,7 @@ namespace Wabbajack.BuildServer.Controllers
             </body></html>
         ");
 
+
         [HttpGet]
         [Route("uploaded_files")]
         public async Task<ContentResult> UploadedFilesGet()

Wabbajack.BuildServer/Models/JobQueue/AJobPayload.cs

@@ -17,7 +17,8 @@ namespace Wabbajack.BuildServer.Models.JobQueue
             typeof(UpdateModLists),
             typeof(EnqueueAllArchives),
             typeof(EnqueueAllGameFiles),
-            typeof(EnqueueRecentFiles)
+            typeof(EnqueueRecentFiles),
+            typeof(UploadToCDN)
         };
         public static Dictionary<Type, string> TypeToName { get; set; }
         public static Dictionary<string, Type> NameToType { get; set; }

Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Net;
+using System.Threading.Tasks;
+using Alphaleonis.Win32.Filesystem;
+using BunnyCDN.Net.Storage;
+using CG.Web.MegaApiClient;
+using FluentFTP;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Wabbajack.BuildServer.Models.JobQueue;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using File = System.IO.File;
+
+namespace Wabbajack.BuildServer.Models.Jobs
+{
+    public class UploadToCDN : AJobPayload
+    {
+        public override string Description => $"Push an uploaded file ({FileId}) to the CDN";
+        
+        public string FileId { get; set; }
+        
+        public override async Task<JobResult> Execute(DBContext db, AppSettings settings)
+        {
+            var file = await db.UploadedFiles.AsQueryable().Where(f => f.Id == FileId).FirstOrDefaultAsync();
+            using (var client = new FtpClient("storage.bunnycdn.com"))
+            {
+                client.Credentials = new NetworkCredential(settings.BunnyCDN_User, settings.BunnyCDN_Password);
+                await client.ConnectAsync();
+                using (var stream = File.OpenRead(Path.Combine("public", "files", file.MungedName)))
+                {
+                    await client.UploadAsync(stream, file.MungedName, progress: new Progress(file.MungedName));
+                }
+                
+                /*
+                await db.Jobs.InsertOneAsync(new Job
+                {
+                    Payload = new IndexJob
+                    {
+                        Archive = new Archive
+                        {
+                            Name = file.MungedName,
+                            Size = file.Size,
+                            Hash = file.Hash,
+                            State = new HTTPDownloader.State
+                            {
+                                Url = file.Uri
+                            }
+                        }
+                    }
+                });*/
+            }
+            return JobResult.Success();
+        }
+
+        private class Progress : IProgress<FluentFTP.FtpProgress>
+        {
+            private string _name;
+            private DateTime LastUpdate = DateTime.UnixEpoch;
+
+            public Progress(string name)
+            {
+                _name = name;
+            }
+            public void Report(FtpProgress value)
+            {
+                if (DateTime.Now - LastUpdate <= TimeSpan.FromSeconds(5)) return;
+
+                Utils.Log($"Uploading {_name} - {value.Progress}% {(int)((value.TransferSpeed + 1) / 1024 / 1024)} MB/sec ETA: {value.ETA}");
+                LastUpdate = DateTime.Now;
+
+            }
+        }
+    }
+}

Wabbajack.BuildServer/Models/UploadedFile.cs

@@ -17,9 +17,12 @@ namespace Wabbajack.BuildServer.Models
         public string Uploader { get; set; }
         public DateTime UploadDate { get; set; } = DateTime.UtcNow;
         
+        public string CDNName { get; set; }
+
         [BsonIgnore]
         public string MungedName => $"{Path.GetFileNameWithoutExtension(Name)}-{Id}{Path.GetExtension(Name)}";
 
-        [BsonIgnore] public object Uri => $"https://wabbajack.b-cdn.net/{MungedName}";
+        [BsonIgnore]
+        public string Uri => CDNName == null ? $"https://wabbajack.b-cdn.net/{MungedName}" : $"https://{CDNName}.b-cdn.net/{MungedName}";
     }
 }

Wabbajack.BuildServer/Wabbajack.BuildServer.csproj

@@ -10,6 +10,8 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="BunnyCDN.Net.Storage" Version="1.0.2" />
+        <PackageReference Include="FluentFTP" Version="29.0.3" />
         <PackageReference Include="graphiql" Version="1.2.0" />
         <PackageReference Include="GraphQL" Version="3.0.0-preview-1352" />
         <PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />

Wabbajack.BuildServer/appsettings.json

@@ -34,7 +34,9 @@
     "ArchiveDir": "c:\\archives",
     "MinimalMode": true,
     "RunFrontEndJobs": true,
-    "RunBackEndJobs": true
+    "RunBackEndJobs": true,
+    "BunnyCDN_User": "wabbajackcdn",
+    "BunnyCDN_Password": "XXXX"
   },
   "AllowedHosts": "*"
 }

Wabbajack.Lib/FileUploader/AuthorAPI.cs

@@ -1,9 +1,12 @@
 using System;
+using System.IO;
+using System.Linq;
 using System.Net.Http;
 using System.Reactive.Linq;
 using System.Threading.Tasks;
-using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
+using File = Alphaleonis.Win32.Filesystem.File;
+using Path = Alphaleonis.Win32.Filesystem.Path;
 
 namespace Wabbajack.Lib.FileUploader
 {
@@ -29,57 +32,61 @@ namespace Wabbajack.Lib.FileUploader
             var tcs = new TaskCompletionSource<string>();
             queue.QueueTask(async () =>
             {
-                using (var stream =
+
-                    new StatusFileStream(File.OpenRead(filename), $"Uploading {Path.GetFileName(filename)}", queue))
+                var client = new HttpClient();
+                var fsize = new FileInfo(filename).Length;
+                client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey());
+                var response = await client.PutAsync(UploadURL+$"/{Path.GetFileName(filename)}/start", new StringContent(""));
+                if (!response.IsSuccessStatusCode)
                 {
-                    var client = new HttpClient();
+                    tcs.SetResult("FAILED");
-                    client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey());
+                    return;
-                    var response = await client.PutAsync(UploadURL+$"/{Path.GetFileName(filename)}/start", new StringContent(""));
-                    if (!response.IsSuccessStatusCode)
-                    {
-                        tcs.SetResult("FAILED");
-                        return;
-                    }
-
-                    var key = await response.Content.ReadAsStringAsync();
-
-                    var data = new byte[BLOCK_SIZE];
-                    while (stream.Position < stream.Length)
-                    {
-                        var old_offset = stream.Position;
-
-                        var new_size = Math.Min(stream.Length - stream.Position, BLOCK_SIZE);
-                        
-                        if (new_size != data.Length) 
-                            data = new byte[new_size];
-
-                        await stream.ReadAsync(data, 0, data.Length);
-
-                        response = await client.PutAsync(UploadURL + $"/{key}/data/{old_offset}",
-                            new ByteArrayContent(data));
-
-                        if (!response.IsSuccessStatusCode)
-                        {
-                            tcs.SetResult("FAILED");
-                            return;
-                        }
-
-                        var val = long.Parse(await response.Content.ReadAsStringAsync());
-                        if (val != old_offset + data.Length)
-                        {
-                            tcs.SetResult("Sync Error");
-                            return;
-                        }
-
-
-                    }
-
-                    response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent(""));
-                    if (response.IsSuccessStatusCode)
-                        tcs.SetResult(await response.Content.ReadAsStringAsync());
-                    else 
-                        tcs.SetResult("FAILED");
                 }
+
+                var key = await response.Content.ReadAsStringAsync();
+
+                using (var iqueue = new WorkQueue(8))
+                {
+
+                    await Enumerable.Range(0, (int)(fsize / BLOCK_SIZE))
+                        .PMap(iqueue, async block_idx =>
+                        {
+                            var block_offset = block_idx * BLOCK_SIZE;
+                            var block_size = block_offset + BLOCK_SIZE > fsize
+                                ? fsize - block_offset
+                                : BLOCK_SIZE;
+
+                            using (var fs = File.OpenRead(filename))
+                            {
+                                fs.Position = block_offset;
+                                var data = new byte[block_size];
+                                await fs.ReadAsync(data, 0, data.Length);
+
+                                response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}",
+                                    new ByteArrayContent(data));
+
+                                if (!response.IsSuccessStatusCode)
+                                {
+                                    tcs.SetResult("FAILED");
+                                    return;
+                                }
+
+                                var val = long.Parse(await response.Content.ReadAsStringAsync());
+                                if (val != block_offset + data.Length)
+                                {
+                                    tcs.SetResult("Sync Error");
+                                    return;
+                                }
+                            }
+                        });
+                }
+
+                response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent(""));
+                if (response.IsSuccessStatusCode)
+                    tcs.SetResult(await response.Content.ReadAsStringAsync());
+                else 
+                    tcs.SetResult("FAILED");
+
             });
             return tcs.Task;
         }

Wabbajack.Lib/MO2Compiler.cs

@@ -11,6 +11,7 @@ using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Wabbajack.Common;
 using Wabbajack.Lib.CompilationSteps;
+using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.NexusApi;
 using Wabbajack.Lib.Validation;
 using Directory = Alphaleonis.Win32.Filesystem.Directory;
@@ -323,9 +324,17 @@ namespace Wabbajack.Lib
 
         private async Task InferMetas()
         {
-            var to_find = Directory.EnumerateFiles(MO2DownloadsFolder)
+            async Task<bool> HasInvalidMeta(string filename)
+            {
+                string metaname = filename + Consts.MetaFileExtension;
+                if (!File.Exists(metaname)) return true;
+                return (AbstractDownloadState) await DownloadDispatcher.ResolveArchive(metaname.LoadIniFile()) == null;
+            }
+
+            var to_find = (await Directory.EnumerateFiles(MO2DownloadsFolder)
                 .Where(f => !f.EndsWith(Consts.MetaFileExtension) && !f.EndsWith(Consts.HashFileExtension))
-                .Where(f => !File.Exists(f + Consts.MetaFileExtension))
+                .PMap(Queue, async f => await HasInvalidMeta(f) ? f : null))
+                .Where(f => f != null)
                 .ToList();
 
             if (to_find.Count == 0) return;