Rewriting the build server (mostly from scratch)

This commit is contained in:
Timothy Baldridge 2020-05-08 21:56:06 -06:00
parent bba92fd5a2
commit 6abddc68bf
16 changed files with 686 additions and 4 deletions

View File

@ -32,7 +32,5 @@ namespace Wabbajack.BuildServer.Controllers
Timestamp = DateTime.UtcNow
});
}
}
}

View File

@ -118,7 +118,5 @@ namespace Wabbajack.BuildServer.Models.Jobs
LastNexusSync = DateTime.Now;
return updated;
}
}
}

View File

@ -0,0 +1,34 @@
using Microsoft.Extensions.Configuration;
using Wabbajack.Common;
namespace Wabbajack.BuildServer
{
public class AppSettings
{
public AppSettings(IConfiguration config)
{
config.Bind("WabbajackSettings", this);
}
public string DownloadDir { get; set; }
public AbsolutePath DownloadPath => (AbsolutePath)DownloadDir;
public string ArchiveDir { get; set; }
public AbsolutePath ArchivePath => (AbsolutePath)ArchiveDir;
public string TempFolder { get; set; }
public AbsolutePath TempPath => (AbsolutePath)TempFolder;
public bool JobScheduler { get; set; }
public bool JobRunner { get; set; }
public bool RunFrontEndJobs { get; set; }
public bool RunBackEndJobs { get; set; }
public string BunnyCDN_User { get; set; }
public string BunnyCDN_Password { get; set; }
public string SqlConnection { get; set; }
public int MaxJobs { get; set; } = 2;
}
}

View File

@ -0,0 +1,50 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.Common;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using WebSocketSharp;
using LogLevel = Microsoft.Extensions.Logging.LogLevel;
namespace Wabbajack.BuildServer.Controllers
{
[ApiController]
[Route("/metrics")]
public class MetricsController : ControllerBase
{
private SqlService _sql;
private ILogger<MetricsController> _logger;
public MetricsController(ILogger<MetricsController> logger, SqlService sql)
{
_sql = sql;
_logger = logger;
}
[HttpGet]
[Route("{Subject}/{Value}")]
public async Task<Result> LogMetricAsync(string Subject, string Value)
{
var date = DateTime.UtcNow;
await Log(date, Subject, Value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault());
return new Result { Timestamp = date};
}
private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
{
_logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}");
await _sql.IngestMetric(new Metric
{
Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey
});
}
public class Result
{
public DateTime Timestamp { get; set; }
}
}
}

View File

@ -0,0 +1,98 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AngleSharp.Io;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
namespace Wabbajack.BuildServer.Controllers
{
//[Authorize]
[ApiController]
[Route("/v1/games/")]
public class NexusCache : ControllerBase
{
private AppSettings _settings;
private static long CachedCount = 0;
private static long ForwardCount = 0;
private SqlService _sql;
private ILogger<NexusCache> _logger;
public NexusCache(ILogger<NexusCache> logger, SqlService sql, AppSettings settings)
{
_settings = settings;
_sql = sql;
_logger = logger;
}
/// <summary>
/// Looks up the mod details for a given Gamename/ModId pair. If the entry is not found in the cache it will
/// be requested from the server (using the caller's Nexus API key if provided).
/// </summary>
/// <param name="db"></param>
/// <param name="GameName">The Nexus game name</param>
/// <param name="ModId">The Nexus mod id</param>
/// <returns>A Mod Info result</returns>
[HttpGet]
[Route("{GameName}/mods/{ModId}.json")]
public async Task<ModInfo> GetModInfo(string GameName, long ModId)
{
var game = GameRegistry.GetByFuzzyName(GameName).Game;
var result = await _sql.GetNexusModInfoString(game, ModId);
string method = "CACHED";
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
result = await api.GetModInfo(game, ModId, false);
await _sql.AddNexusModInfo(game, ModId, DateTime.UtcNow, result);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);
}
else
{
Interlocked.Increment(ref CachedCount);
}
Response.Headers.Add("x-cache-result", method);
return result;
}
[HttpGet]
[Route("{GameName}/mods/{ModId}/files.json")]
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, long ModId)
{
_logger.Log(LogLevel.Information, $"{GameName} {ModId}");
var game = GameRegistry.GetByFuzzyName(GameName).Game;
var result = await _sql.GetModFiles(game, ModId);
string method = "CACHED";
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
result = await api.GetModFiles(game, ModId, false);
await _sql.AddNexusModFiles(game, ModId, DateTime.UtcNow, result);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);
}
else
{
Interlocked.Increment(ref CachedCount);
}
Response.Headers.Add("x-cache-result", method);
return result;
}
}
}

View File

@ -0,0 +1,12 @@
using System;
namespace Wabbajack.Server.DTOs
{
public class Metric
{
public DateTime Timestamp { get; set; }
public string Action { get; set; }
public string Subject { get; set; }
public string MetricsKey { get; set; }
}
}

View File

@ -0,0 +1,14 @@
using Newtonsoft.Json;
namespace Wabbajack.Server.DTOs
{
public class NexusUpdateEntry
{
[JsonProperty("mod_id")]
public long ModId { get; set; }
[JsonProperty("latest_file_update")]
public long LatestFileUpdate { get; set; }
[JsonProperty("latest_mod_activity")]
public long LastestModActivity { get; set; }
}
}

View File

@ -0,0 +1,15 @@
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task IngestMetric(Metric metric)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", metric);
}
}
}

View File

@ -0,0 +1,93 @@
using System;
using System.Threading.Tasks;
using Dapper;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.Server.DataLayer
{
/// <summary>
/// SQL routines that read/write cached information from the Nexus
/// </summary>
public partial class SqlService
{
public async Task<long> DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date)
{
await using var conn = await Open();
var deleted = await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
return deleted;
}
public async Task<long> DeleteNexusModFilesUpdatedBeforeDate(Game game, long modId, DateTime date)
{
await using var conn = await Open();
var deleted = await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
return deleted;
}
public async Task<ModInfo> GetNexusModInfoString(Game game, long modId)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT Data FROM dbo.NexusModInfos WHERE Game = @Game AND @ModId = ModId",
new {Game = game.MetaData().NexusGameId, ModId = modId});
return result == null ? null : JsonConvert.DeserializeObject<ModInfo>(result);
}
public async Task AddNexusModInfo(Game game, long modId, DateTime lastCheckedUtc, ModInfo data)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"MERGE dbo.NexusModInfos AS Target
USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
ON Target.Game = Source.Game AND Target.ModId = Source.ModId
WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
new
{
Game = game.MetaData().NexusGameId,
ModId = modId,
LastChecked = lastCheckedUtc,
Data = JsonConvert.SerializeObject(data)
});
}
public async Task AddNexusModFiles(Game game, long modId, DateTime lastCheckedUtc, NexusApiClient.GetModFilesResponse data)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"MERGE dbo.NexusModFiles AS Target
USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
ON Target.Game = Source.Game AND Target.ModId = Source.ModId
WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
new
{
Game = game.MetaData().NexusGameId,
ModId = modId,
LastChecked = lastCheckedUtc,
Data = JsonConvert.SerializeObject(data)
});
}
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(Game game, long modId)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT Data FROM dbo.NexusModFiles WHERE Game = @Game AND @ModId = ModId",
new {Game = game.MetaData().NexusGameId, ModId = modId});
return result == null ? null : JsonConvert.DeserializeObject<NexusApiClient.GetModFilesResponse>(result);
}
}
}

View File

@ -0,0 +1,24 @@
using System.Data.SqlClient;
using System.Threading.Tasks;
using Wabbajack.BuildServer;
namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
private AppSettings _settings;
public SqlService(AppSettings settings)
{
_settings = settings;
}
public async Task<SqlConnection> Open()
{
var conn = new SqlConnection(_settings.SqlConnection);
await conn.OpenAsync();
return conn;
}
}
}

View File

@ -0,0 +1,12 @@
using System;
namespace Wabbajack.Server
{
public class GlobalInformation
{
public TimeSpan NexusRSSPollRate = TimeSpan.FromMinutes(1);
public TimeSpan NexusAPIPollRate = TimeSpan.FromHours(2);
public DateTime LastNexusSyncUTC { get; set; }
public TimeSpan TimeSinceLastNexusSync => DateTime.UtcNow - LastNexusSyncUTC;
}
}

View File

@ -0,0 +1,171 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class NexusPoll
{
private SqlService _sql;
private AppSettings _settings;
private GlobalInformation _globalInformation;
private ILogger<NexusPoll> _logger;
public NexusPoll(ILogger<NexusPoll> logger, AppSettings settings, SqlService service, GlobalInformation globalInformation)
{
_sql = service;
_settings = settings;
_globalInformation = globalInformation;
_logger = logger;
}
public async Task UpdateNexusCacheRSS()
{
using var _ = _logger.BeginScope("Nexus Update via RSS");
_logger.Log(LogLevel.Information, "Starting");
var results = await NexusUpdatesFeeds.GetUpdates();
NexusApiClient client = null;
long updated = 0;
foreach (var result in results)
{
try
{
var purgedMods =
await _sql.DeleteNexusModFilesUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
var purgedFiles =
await _sql.DeleteNexusModInfosUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
var totalPurged = purgedFiles + purgedMods;
if (totalPurged > 0)
_logger.Log(LogLevel.Information, $"Purged {totalPurged} cache items");
if (await _sql.GetNexusModInfoString(result.Game, result.ModId) != null) continue;
// Lazily create the client
client ??= await NexusApiClient.Get();
// Cache the info
var files = await client.GetModFiles(result.Game, result.ModId, false);
await _sql.AddNexusModFiles(result.Game, result.ModId, result.TimeStamp, files);
var modInfo = await client.GetModInfo(result.Game, result.ModId, useCache: false);
await _sql.AddNexusModInfo(result.Game, result.ModId, result.TimeStamp, modInfo);
updated++;
}
catch (Exception ex)
{
_logger.LogError(ex, $"Failed Nexus update for {result.Game} - {result.ModId} - {result.TimeStamp}");
}
}
if (updated > 0)
_logger.Log(LogLevel.Information, $"Primed {updated} nexus cache entries");
_globalInformation.LastNexusSyncUTC = DateTime.UtcNow;
}
public async Task UpdateNexusCacheAPI()
{
using var _ = _logger.BeginScope("Nexus Update via API");
_logger.Log(LogLevel.Information, "Starting");
var api = await NexusApiClient.Get();
var gameTasks = GameRegistry.Games.Values
.Where(game => game.NexusName != null)
.Select(async game =>
{
var mods = await api.Get<List<NexusUpdateEntry>>(
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m");
return (game, mods);
})
.Select(async rTask =>
{
var (game, mods) = await rTask;
return mods.Select(mod => new { game = game, mod = mod });
}).ToList();
_logger.Log(LogLevel.Information, $"Getting update list for {gameTasks.Count} games");
var purge = (await Task.WhenAll(gameTasks))
.SelectMany(i => i)
.ToList();
_logger.Log(LogLevel.Information, $"Found {purge.Count} updated mods in the last month");
using var queue = new WorkQueue();
var collected = purge.Select(d =>
{
var a = d.mod.LatestFileUpdate.AsUnixTime();
// Mod activity could hide files
var b = d.mod.LastestModActivity.AsUnixTime();
return new {Game = d.game.Game, Date = (a > b ? a : b), ModId = d.mod.ModId};
});
var purged = await collected.PMap(queue, async t =>
{
var resultA = await _sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date);
var resultB = await _sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date);
return resultA + resultB;
});
_logger.Log(LogLevel.Information, $"Purged {purged.Sum()} cache entries");
}
public void Start()
{
Task.Run(async () =>
{
while (true)
{
try
{
await UpdateNexusCacheRSS();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error polling from Nexus");
}
await Task.Delay(_globalInformation.NexusRSSPollRate);
}
});
Task.Run(async () =>
{
while (true)
{
try
{
await UpdateNexusCacheAPI();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting API feed from Nexus");
}
await Task.Delay(_globalInformation.NexusAPIPollRate);
}
});
}
}
public static class NexusPollExtensions
{
public static void UseNexusPoll(this IApplicationBuilder b)
{
var poll = (NexusPoll)b.ApplicationServices.GetService(typeof(NexusPoll));
poll.Start();
}
}
}

130
Wabbajack.Server/Startup.cs Normal file
View File

@ -0,0 +1,130 @@
using System.Diagnostics;
using System.IO;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http.Features;
using Microsoft.AspNetCore.StaticFiles;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.FileProviders;
using Microsoft.Extensions.Hosting;
using Microsoft.OpenApi.Models;
using Newtonsoft.Json;
using Wabbajack.BuildServer;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
namespace Wabbajack.Server
{
public class TestStartup : Startup
{
public TestStartup(IConfiguration configuration) : base(configuration)
{
}
}
public class Startup
{
public Startup(IConfiguration configuration)
{
Configuration = configuration;
}
public IConfiguration Configuration { get; }
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services.AddSwaggerGen(c =>
{
c.SwaggerDoc("v1", new OpenApiInfo {Title = "Wabbajack Build API", Version = "v1"});
});
services.Configure<FormOptions>(x =>
{
x.ValueLengthLimit = int.MaxValue;
x.MultipartBodyLengthLimit = int.MaxValue;
});
services.AddSingleton<AppSettings>();
services.AddSingleton<SqlService>();
services.AddSingleton<GlobalInformation>();
services.AddSingleton<NexusPoll>();
services.AddMvc();
services.AddControllers()
.AddNewtonsoftJson(o =>
{
o.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore;
});
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
if (!(this is TestStartup))
app.UseHttpsRedirection();
app.UseDeveloperExceptionPage();
var provider = new FileExtensionContentTypeProvider();
provider.Mappings[".rar"] = "application/x-rar-compressed";
provider.Mappings[".7z"] = "application/x-7z-compressed";
provider.Mappings[".zip"] = "application/zip";
provider.Mappings[".wabbajack"] = "application/zip";
app.UseStaticFiles();
app.UseSwagger();
app.UseSwaggerUI(c =>
{
c.SwaggerEndpoint("/swagger/v1/swagger.json", "Wabbajack Build API");
c.RoutePrefix = string.Empty;
});
app.UseRouting();
app.UseAuthentication();
app.UseAuthorization();
app.UseNexusPoll();
app.Use(next =>
{
return async context =>
{
var stopWatch = new Stopwatch();
stopWatch.Start();
context.Response.OnStarting(() =>
{
stopWatch.Stop();
var headers = context.Response.Headers;
headers.Add("Access-Control-Allow-Origin", "*");
headers.Add("Access-Control-Allow-Methods", "POST, GET");
headers.Add("Access-Control-Allow-Headers", "Accept, Origin, Content-type");
headers.Add("X-ResponseTime-Ms", stopWatch.ElapsedMilliseconds.ToString());
return Task.CompletedTask;
});
await next(context);
};
});
app.UseFileServer(new FileServerOptions
{
FileProvider = new PhysicalFileProvider(
Path.Combine(Directory.GetCurrentDirectory(), "public")),
StaticFileOptions = {ServeUnknownFileTypes = true},
});
app.UseEndpoints(endpoints =>
{
endpoints.MapControllers();
});
}
}
}

View File

@ -0,0 +1,22 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
},
"WabbajackSettings": {
"DownloadDir": "c:\\tmp\\downloads",
"ArchiveDir": "w:\\archives",
"TempFolder": "c:\\tmp",
"JobRunner": true,
"JobScheduler": false,
"RunFrontEndJobs": true,
"RunBackEndJobs": true,
"BunnyCDN_User": "wabbajackcdn",
"BunnyCDN_Password": "XXXX",
"SQLConnection": "Data Source=.\\SQLEXPRESS;Integrated Security=True;Initial Catalog=wabbajack_prod;MultipleActiveResultSets=true"
},
"AllowedHosts": "*"
}

View File

@ -0,0 +1 @@
Cheese for Everyone!

View File

@ -44,6 +44,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.App.Test", "Wabba
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.BuildServer.Test", "Wabbajack.BuildServer.Test\Wabbajack.BuildServer.Test.csproj", "{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Server", "Wabbajack.Server\Wabbajack.Server.csproj", "{3E11B700-8405-433D-BF47-6C356087A7C2}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -159,6 +161,14 @@ Global
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|Any CPU.Build.0 = Release|Any CPU
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|x64.ActiveCfg = Release|Any CPU
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|x64.Build.0 = Release|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|x64.ActiveCfg = Debug|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|x64.Build.0 = Debug|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|Any CPU.Build.0 = Release|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|x64.ActiveCfg = Release|Any CPU
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE