Fix issues with the server upload dieing, add server-log verb

Wabbajack.BuildServer/Controllers/Heartbeat.cs

@@ -1,11 +1,15 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
+using System.Text;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
+using MongoDB.Bson;
 using Wabbajack.BuildServer.Models;
 using Wabbajack.Common;
+using Wabbajack.Common.StatusFeed;
 
 namespace Wabbajack.BuildServer.Controllers
 {
@@ -23,6 +27,18 @@ namespace Wabbajack.BuildServer.Controllers
         {
         }
 
+        private const int MAX_LOG_SIZE = 128;
+        private static List<string> Log  = new List<string>();
+        public static void AddToLog(IStatusMessage msg)
+        {
+            lock (Log)
+            {
+                Log.Add(msg.ToString());
+                if (Log.Count > MAX_LOG_SIZE)
+                    Log.RemoveAt(0);
+            }
+        }
+
         [HttpGet]
         public async Task<TimeSpan> GetHeartbeat()
         {
@@ -36,5 +52,17 @@ namespace Wabbajack.BuildServer.Controllers
             var message = $"Hello from {nameof(OnlyAuthenticated)}";
             return new ObjectResult(message);
         }
+
+        [HttpGet("logs")]
+        [Authorize]
+        public IActionResult GetLogs()
+        {
+            string[] lst;
+            lock (Log)
+            {
+                lst = Log.ToArray();
+            }
+            return Ok(string.Join("\n", lst));
+        }
     }
 }

Wabbajack.BuildServer/Controllers/NexusCache.cs

@@ -35,7 +35,7 @@ namespace Wabbajack.BuildServer.Controllers
             if (result == null)
             {
                 var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
-                var path = $"/v1/games/{GameName}/mods/{ModId}.json";
+                var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}.json";
                 var body = await api.Get<ModInfo>(path);
                 result = new NexusCacheData<ModInfo> {Data = body, Path = path, Game = GameName, ModId = ModId};
                 try
@@ -63,7 +63,7 @@ namespace Wabbajack.BuildServer.Controllers
             if (result == null)
             {
                 var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
-                var path = $"/v1/games/{GameName}/mods/{ModId}/files.json";
+                var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files.json";
                 var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
                 result = new NexusCacheData<NexusApiClient.GetModFilesResponse>
                 {
@@ -95,7 +95,7 @@ namespace Wabbajack.BuildServer.Controllers
             if (result == null)
             {
                 var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
-                var path = $"/v1/games/{GameName}/mods/{ModId}/files/{FileId}.json";
+                var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files/{FileId}.json";
                 var body = await api.Get<NexusFileInfo>(path);
                 result = new NexusCacheData<NexusFileInfo>
                 {

Wabbajack.BuildServer/Controllers/UploadedFiles.cs

@@ -44,7 +44,7 @@ namespace Wabbajack.BuildServer.Controllers
             
             _writeLocks.GetOrAdd(key, new AsyncLock());
             
-            System.IO.File.Create(Path.Combine("public", "files", key)).Close();
+            System.IO.File.Create(Path.Combine("public", "tmp_files", key)).Close();
             Utils.Log($"Starting Ingest for {key}");
             return Ok(key);
         }
@@ -64,7 +64,7 @@ namespace Wabbajack.BuildServer.Controllers
             
             long position;
             using (var _ = await _writeLocks[Key].Wait())
-            await using (var file = System.IO.File.Open(Path.Combine("public", "files", Key), FileMode.Open, FileAccess.Write, FileShare.ReadWrite))
+            await using (var file = System.IO.File.Open(Path.Combine("public", "tmp_files", Key), FileMode.Open, FileAccess.Write, FileShare.ReadWrite))
             {
                 file.Position = Offset;
                 await ms.CopyToAsync(file);
@@ -86,7 +86,7 @@ namespace Wabbajack.BuildServer.Controllers
             var original_name = $"{parts[0]}{parts[2]}";
 
             var final_path = Path.Combine("public", "files", final_name);
-            System.IO.File.Move(Path.Combine("public", "files", Key), final_path);
+            System.IO.File.Move(Path.Combine("public", "tmp_files", Key), final_path);
             var hash = await final_path.FileHashAsync();
 
             if (expectedHash != hash)

Wabbajack.BuildServer/JobManager.cs

@@ -1,10 +1,12 @@
 using System;
 using System.Linq;
+using System.Reactive.Linq;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using MongoDB.Driver;
 using MongoDB.Driver.Linq;
 using Nettle;
+using Wabbajack.BuildServer.Controllers;
 using Wabbajack.BuildServer.Model.Models;
 using Wabbajack.BuildServer.Models;
 using Wabbajack.BuildServer.Models.JobQueue;
@@ -74,6 +76,8 @@ namespace Wabbajack.BuildServer
         public async Task JobScheduler()
         {
             Utils.LogMessages.Subscribe(msg => Logger.Log(LogLevel.Information, msg.ToString()));
+            Utils.LogMessages.Subscribe(Heartbeat.AddToLog);
+            Utils.LogMessages.OfType<IUserIntervention>().Subscribe(u => u.Cancel());
             if (!Settings.JobScheduler) return;
             while (true)
             {

Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs

@@ -24,14 +24,28 @@ namespace Wabbajack.BuildServer.Models.Jobs
         
         public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
         {
+            int retries = 0;
+            TOP:
             var file = await db.UploadedFiles.AsQueryable().Where(f => f.Id == FileId).FirstOrDefaultAsync();
+            
             using (var client = new FtpClient("storage.bunnycdn.com"))
             {
                 client.Credentials = new NetworkCredential(settings.BunnyCDN_User, settings.BunnyCDN_Password);
                 await client.ConnectAsync();
                 using (var stream = File.OpenRead(Path.Combine("public", "files", file.MungedName)))
                 {
-                    await client.UploadAsync(stream, file.MungedName, progress: new Progress(file.MungedName));
+                    try
+                    {
+                        await client.UploadAsync(stream, file.MungedName, progress: new Progress(file.MungedName));
+                    }
+                    catch (Exception ex)
+                    {
+                        if (retries > 10) throw;
+                        Utils.Log(ex.ToString());
+                        Utils.Log("Retrying FTP Upload");
+                        retries++;
+                        goto TOP;
+                    }
                 }
                 
                 await db.Jobs.InsertOneAsync(new Job

Wabbajack.CLI/OptionsDefinition.cs

@@ -13,7 +13,8 @@ namespace Wabbajack.CLI
             typeof(DownloadUrl), 
             typeof(UpdateModlists), 
             typeof(UpdateNexusCache),
-            typeof(ChangeDownload)
+            typeof(ChangeDownload),
+            typeof(ServerLog)
         };
     }
 }

Wabbajack.CLI/Program.cs

@@ -16,6 +16,7 @@ namespace Wabbajack.CLI
                     (UpdateModlists opts) => opts.Execute(),
                     (UpdateNexusCache opts) => opts.Execute(),
                     (ChangeDownload opts) => opts.Execute(),
+                    (ServerLog opts) => opts.Execute(),
                     errs => 1);
         }
     }

Wabbajack.CLI/Verbs/ServerLog.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Net.Http;
+using System.Threading.Tasks;
+using CommandLine;
+using Wabbajack.Lib.FileUploader;
+
+namespace Wabbajack.CLI.Verbs
+{
+    [Verb("server-log", HelpText = @"Get the latest server log entries", Hidden = false)]
+    public class ServerLog : AVerb
+    {
+        protected override async Task<int> Run()
+        {
+            Console.WriteLine(await AuthorAPI.GetServerLog());
+            return 0;
+        }
+    }
+}

Wabbajack.Lib/FileUploader/AuthorAPI.cs

@@ -171,5 +171,10 @@ namespace Wabbajack.Lib.FileUploader
                 Utils.Log(ex.ToString());
             }
         }
+
+        public static async Task<string> GetServerLog()
+        {
+            return await GetAuthorizedClient().GetStringAsync($"https://{Consts.WabbajackCacheHostname}/heartbeat/logs");
+        }
     }
 }

Wabbajack.Lib/NexusApi/NexusApi.cs

@@ -178,12 +178,12 @@ namespace Wabbajack.Lib.NexusApi
 
                 lock (RemainingLock)
                 {
-                    _dailyRemaining = Math.Min(dailyRemaining, hourlyRemaining);
+                    _dailyRemaining = Math.Min(_dailyRemaining, dailyRemaining);
-                    _hourlyRemaining = Math.Min(dailyRemaining, hourlyRemaining);
+                    _hourlyRemaining = Math.Min(_hourlyRemaining, hourlyRemaining);
                 }
                 
                 if (oldDaily != _dailyRemaining || oldHourly != _hourlyRemaining) 
-                    Utils.Log($"Nexus requests remaining: {dailyRemaining} daily - {hourlyRemaining} hourly");
+                    Utils.Log($"Nexus requests remaining: {_dailyRemaining} daily - {_hourlyRemaining} hourly");
 
                 this.RaisePropertyChanged(nameof(DailyRemaining));
                 this.RaisePropertyChanged(nameof(HourlyRemaining));