Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix issues with the server upload dieing, add server-log verb

Browse Source
This commit is contained in:
Timothy Baldridge 2020-02-25 16:10:41 -07:00
parent 69297d9b8e
commit 42e732559c
10 changed files with 84 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
Wabbajack.BuildServer/Controllers/Heartbeat.cs
View File

@ -1,11 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Bson;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
using Wabbajack.Common.StatusFeed;
namespace Wabbajack.BuildServer.Controllers
{
@ -22,7 +26,19 @@ namespace Wabbajack.BuildServer.Controllers
public Heartbeat(ILogger<Heartbeat> logger, DBContext db) : base(logger, db)
{
}
private const int MAX_LOG_SIZE = 128;
private static List<string> Log = new List<string>();
public static void AddToLog(IStatusMessage msg)
{
lock (Log)
{
Log.Add(msg.ToString());
if (Log.Count > MAX_LOG_SIZE)
Log.RemoveAt(0);
}
}
[HttpGet]
public async Task<TimeSpan> GetHeartbeat()
{
@ -36,5 +52,17 @@ namespace Wabbajack.BuildServer.Controllers
var message = $"Hello from {nameof(OnlyAuthenticated)}";
return new ObjectResult(message);
}
[HttpGet("logs")]
[Authorize]
public IActionResult GetLogs()
{
string[] lst;
lock (Log)
{
lst = Log.ToArray();
}
return Ok(string.Join("\n", lst));
}
}
}

6
Wabbajack.BuildServer/Controllers/NexusCache.cs
View File

@ -35,7 +35,7 @@ namespace Wabbajack.BuildServer.Controllers
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"/v1/games/{GameName}/mods/{ModId}.json";
var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}.json";
var body = await api.Get<ModInfo>(path);
result = new NexusCacheData<ModInfo> {Data = body, Path = path, Game = GameName, ModId = ModId};
try
@ -63,7 +63,7 @@ namespace Wabbajack.BuildServer.Controllers
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"/v1/games/{GameName}/mods/{ModId}/files.json";
var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files.json";
var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
result = new NexusCacheData<NexusApiClient.GetModFilesResponse>
{
@ -95,7 +95,7 @@ namespace Wabbajack.BuildServer.Controllers
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"/v1/games/{GameName}/mods/{ModId}/files/{FileId}.json";
var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files/{FileId}.json";
var body = await api.Get<NexusFileInfo>(path);
result = new NexusCacheData<NexusFileInfo>
{

8
Wabbajack.BuildServer/Controllers/UploadedFiles.cs
View File

@ -29,7 +29,7 @@ namespace Wabbajack.BuildServer.Controllers
{
private static ConcurrentDictionary<string, AsyncLock> _writeLocks = new ConcurrentDictionary<string, AsyncLock>();
private AppSettings _settings;
public UploadedFiles(ILogger<UploadedFiles> logger, DBContext db, AppSettings settings) : base(logger, db)
{
_settings = settings;
@ -44,7 +44,7 @@ namespace Wabbajack.BuildServer.Controllers
_writeLocks.GetOrAdd(key, new AsyncLock());
System.IO.File.Create(Path.Combine("public", "files", key)).Close();
System.IO.File.Create(Path.Combine("public", "tmp_files", key)).Close();
Utils.Log($"Starting Ingest for {key}");
return Ok(key);
}
@ -64,7 +64,7 @@ namespace Wabbajack.BuildServer.Controllers
long position;
using (var _ = await _writeLocks[Key].Wait())
await using (var file = System.IO.File.Open(Path.Combine("public", "files", Key), FileMode.Open, FileAccess.Write, FileShare.ReadWrite))
await using (var file = System.IO.File.Open(Path.Combine("public", "tmp_files", Key), FileMode.Open, FileAccess.Write, FileShare.ReadWrite))
{
file.Position = Offset;
await ms.CopyToAsync(file);
@ -86,7 +86,7 @@ namespace Wabbajack.BuildServer.Controllers
var original_name = $"{parts[0]}{parts[2]}";
var final_path = Path.Combine("public", "files", final_name);
System.IO.File.Move(Path.Combine("public", "files", Key), final_path);
System.IO.File.Move(Path.Combine("public", "tmp_files", Key), final_path);
var hash = await final_path.FileHashAsync();
if (expectedHash != hash)

4
Wabbajack.BuildServer/JobManager.cs
View File

@ -1,10 +1,12 @@
using System;
using System.Linq;
using System.Reactive.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nettle;
using Wabbajack.BuildServer.Controllers;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
@ -74,6 +76,8 @@ namespace Wabbajack.BuildServer
public async Task JobScheduler()
{
Utils.LogMessages.Subscribe(msg => Logger.Log(LogLevel.Information, msg.ToString()));
Utils.LogMessages.Subscribe(Heartbeat.AddToLog);
Utils.LogMessages.OfType<IUserIntervention>().Subscribe(u => u.Cancel());
if (!Settings.JobScheduler) return;
while (true)
{

16
Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs
View File

@ -24,14 +24,28 @@ namespace Wabbajack.BuildServer.Models.Jobs
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
{
int retries = 0;
TOP:
var file = await db.UploadedFiles.AsQueryable().Where(f => f.Id == FileId).FirstOrDefaultAsync();
using (var client = new FtpClient("storage.bunnycdn.com"))
{
client.Credentials = new NetworkCredential(settings.BunnyCDN_User, settings.BunnyCDN_Password);
await client.ConnectAsync();
using (var stream = File.OpenRead(Path.Combine("public", "files", file.MungedName)))
{
await client.UploadAsync(stream, file.MungedName, progress: new Progress(file.MungedName));
try
{
await client.UploadAsync(stream, file.MungedName, progress: new Progress(file.MungedName));
}
catch (Exception ex)
{
if (retries > 10) throw;
Utils.Log(ex.ToString());
Utils.Log("Retrying FTP Upload");
retries++;
goto TOP;
}
}
await db.Jobs.InsertOneAsync(new Job

3
Wabbajack.CLI/OptionsDefinition.cs
View File

@ -13,7 +13,8 @@ namespace Wabbajack.CLI
typeof(DownloadUrl),
typeof(UpdateModlists),
typeof(UpdateNexusCache),
typeof(ChangeDownload)
typeof(ChangeDownload),
typeof(ServerLog)
};
}
}

1
Wabbajack.CLI/Program.cs
View File

@ -16,6 +16,7 @@ namespace Wabbajack.CLI
(UpdateModlists opts) => opts.Execute(),
(UpdateNexusCache opts) => opts.Execute(),
(ChangeDownload opts) => opts.Execute(),
(ServerLog opts) => opts.Execute(),
errs => 1);
}
}

18
Wabbajack.CLI/Verbs/ServerLog.cs Normal file
View File

@ -0,0 +1,18 @@
using System;
using System.Net.Http;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Lib.FileUploader;
namespace Wabbajack.CLI.Verbs
{
[Verb("server-log", HelpText = @"Get the latest server log entries", Hidden = false)]
public class ServerLog : AVerb
{
protected override async Task<int> Run()
{
Console.WriteLine(await AuthorAPI.GetServerLog());
return 0;
}
}
}

5
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -171,5 +171,10 @@ namespace Wabbajack.Lib.FileUploader
Utils.Log(ex.ToString());
}
}
public static async Task<string> GetServerLog()
{
return await GetAuthorizedClient().GetStringAsync($"https://{Consts.WabbajackCacheHostname}/heartbeat/logs");
}
}
}

6
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -178,12 +178,12 @@ namespace Wabbajack.Lib.NexusApi
lock (RemainingLock)
{
_dailyRemaining = Math.Min(dailyRemaining, hourlyRemaining);
_hourlyRemaining = Math.Min(dailyRemaining, hourlyRemaining);
_dailyRemaining = Math.Min(_dailyRemaining, dailyRemaining);
_hourlyRemaining = Math.Min(_hourlyRemaining, hourlyRemaining);
}
if (oldDaily != _dailyRemaining || oldHourly != _hourlyRemaining)
Utils.Log($"Nexus requests remaining: {dailyRemaining} daily - {hourlyRemaining} hourly");
Utils.Log($"Nexus requests remaining: {_dailyRemaining} daily - {_hourlyRemaining} hourly");
this.RaisePropertyChanged(nameof(DailyRemaining));
this.RaisePropertyChanged(nameof(HourlyRemaining));