mirror of
https://github.com/wabbajack-tools/wabbajack.git
synced 2024-08-30 18:42:17 +00:00
Rewriting the build server (mostly from scratch)
This commit is contained in:
parent
bba92fd5a2
commit
6abddc68bf
@ -32,7 +32,5 @@ namespace Wabbajack.BuildServer.Controllers
|
|||||||
Timestamp = DateTime.UtcNow
|
Timestamp = DateTime.UtcNow
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -118,7 +118,5 @@ namespace Wabbajack.BuildServer.Models.Jobs
|
|||||||
LastNexusSync = DateTime.Now;
|
LastNexusSync = DateTime.Now;
|
||||||
return updated;
|
return updated;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
34
Wabbajack.Server/AppSettings.cs
Normal file
34
Wabbajack.Server/AppSettings.cs
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
|
using Wabbajack.Common;
|
||||||
|
|
||||||
|
namespace Wabbajack.BuildServer
|
||||||
|
{
|
||||||
|
public class AppSettings
|
||||||
|
{
|
||||||
|
public AppSettings(IConfiguration config)
|
||||||
|
{
|
||||||
|
config.Bind("WabbajackSettings", this);
|
||||||
|
}
|
||||||
|
|
||||||
|
public string DownloadDir { get; set; }
|
||||||
|
public AbsolutePath DownloadPath => (AbsolutePath)DownloadDir;
|
||||||
|
public string ArchiveDir { get; set; }
|
||||||
|
public AbsolutePath ArchivePath => (AbsolutePath)ArchiveDir;
|
||||||
|
|
||||||
|
public string TempFolder { get; set; }
|
||||||
|
|
||||||
|
public AbsolutePath TempPath => (AbsolutePath)TempFolder;
|
||||||
|
|
||||||
|
public bool JobScheduler { get; set; }
|
||||||
|
public bool JobRunner { get; set; }
|
||||||
|
|
||||||
|
public bool RunFrontEndJobs { get; set; }
|
||||||
|
public bool RunBackEndJobs { get; set; }
|
||||||
|
|
||||||
|
public string BunnyCDN_User { get; set; }
|
||||||
|
public string BunnyCDN_Password { get; set; }
|
||||||
|
public string SqlConnection { get; set; }
|
||||||
|
|
||||||
|
public int MaxJobs { get; set; } = 2;
|
||||||
|
}
|
||||||
|
}
|
50
Wabbajack.Server/Controllers/Metrics.cs
Normal file
50
Wabbajack.Server/Controllers/Metrics.cs
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
using System;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Microsoft.AspNetCore.Mvc;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Wabbajack.Common;
|
||||||
|
using Wabbajack.Server.DataLayer;
|
||||||
|
using Wabbajack.Server.DTOs;
|
||||||
|
using WebSocketSharp;
|
||||||
|
using LogLevel = Microsoft.Extensions.Logging.LogLevel;
|
||||||
|
|
||||||
|
namespace Wabbajack.BuildServer.Controllers
|
||||||
|
{
|
||||||
|
[ApiController]
|
||||||
|
[Route("/metrics")]
|
||||||
|
public class MetricsController : ControllerBase
|
||||||
|
{
|
||||||
|
private SqlService _sql;
|
||||||
|
private ILogger<MetricsController> _logger;
|
||||||
|
|
||||||
|
public MetricsController(ILogger<MetricsController> logger, SqlService sql)
|
||||||
|
{
|
||||||
|
_sql = sql;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
[HttpGet]
|
||||||
|
[Route("{Subject}/{Value}")]
|
||||||
|
public async Task<Result> LogMetricAsync(string Subject, string Value)
|
||||||
|
{
|
||||||
|
var date = DateTime.UtcNow;
|
||||||
|
await Log(date, Subject, Value, Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault());
|
||||||
|
return new Result { Timestamp = date};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null)
|
||||||
|
{
|
||||||
|
_logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}");
|
||||||
|
await _sql.IngestMetric(new Metric
|
||||||
|
{
|
||||||
|
Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public class Result
|
||||||
|
{
|
||||||
|
public DateTime Timestamp { get; set; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
98
Wabbajack.Server/Controllers/NexusCache.cs
Normal file
98
Wabbajack.Server/Controllers/NexusCache.cs
Normal file
@ -0,0 +1,98 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Threading;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using AngleSharp.Io;
|
||||||
|
using Microsoft.AspNetCore.Authorization;
|
||||||
|
using Microsoft.AspNetCore.Mvc;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Newtonsoft.Json;
|
||||||
|
|
||||||
|
using Wabbajack.Common;
|
||||||
|
using Wabbajack.Lib;
|
||||||
|
using Wabbajack.Lib.NexusApi;
|
||||||
|
using Wabbajack.Server.DataLayer;
|
||||||
|
|
||||||
|
namespace Wabbajack.BuildServer.Controllers
|
||||||
|
{
|
||||||
|
//[Authorize]
|
||||||
|
[ApiController]
|
||||||
|
[Route("/v1/games/")]
|
||||||
|
public class NexusCache : ControllerBase
|
||||||
|
{
|
||||||
|
private AppSettings _settings;
|
||||||
|
private static long CachedCount = 0;
|
||||||
|
private static long ForwardCount = 0;
|
||||||
|
private SqlService _sql;
|
||||||
|
private ILogger<NexusCache> _logger;
|
||||||
|
|
||||||
|
public NexusCache(ILogger<NexusCache> logger, SqlService sql, AppSettings settings)
|
||||||
|
{
|
||||||
|
_settings = settings;
|
||||||
|
_sql = sql;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Looks up the mod details for a given Gamename/ModId pair. If the entry is not found in the cache it will
|
||||||
|
/// be requested from the server (using the caller's Nexus API key if provided).
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="db"></param>
|
||||||
|
/// <param name="GameName">The Nexus game name</param>
|
||||||
|
/// <param name="ModId">The Nexus mod id</param>
|
||||||
|
/// <returns>A Mod Info result</returns>
|
||||||
|
[HttpGet]
|
||||||
|
[Route("{GameName}/mods/{ModId}.json")]
|
||||||
|
public async Task<ModInfo> GetModInfo(string GameName, long ModId)
|
||||||
|
{
|
||||||
|
var game = GameRegistry.GetByFuzzyName(GameName).Game;
|
||||||
|
var result = await _sql.GetNexusModInfoString(game, ModId);
|
||||||
|
|
||||||
|
string method = "CACHED";
|
||||||
|
if (result == null)
|
||||||
|
{
|
||||||
|
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
|
||||||
|
result = await api.GetModInfo(game, ModId, false);
|
||||||
|
await _sql.AddNexusModInfo(game, ModId, DateTime.UtcNow, result);
|
||||||
|
|
||||||
|
|
||||||
|
method = "NOT_CACHED";
|
||||||
|
Interlocked.Increment(ref ForwardCount);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Interlocked.Increment(ref CachedCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
Response.Headers.Add("x-cache-result", method);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
[HttpGet]
|
||||||
|
[Route("{GameName}/mods/{ModId}/files.json")]
|
||||||
|
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, long ModId)
|
||||||
|
{
|
||||||
|
_logger.Log(LogLevel.Information, $"{GameName} {ModId}");
|
||||||
|
var game = GameRegistry.GetByFuzzyName(GameName).Game;
|
||||||
|
var result = await _sql.GetModFiles(game, ModId);
|
||||||
|
|
||||||
|
string method = "CACHED";
|
||||||
|
if (result == null)
|
||||||
|
{
|
||||||
|
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
|
||||||
|
result = await api.GetModFiles(game, ModId, false);
|
||||||
|
await _sql.AddNexusModFiles(game, ModId, DateTime.UtcNow, result);
|
||||||
|
|
||||||
|
method = "NOT_CACHED";
|
||||||
|
Interlocked.Increment(ref ForwardCount);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Interlocked.Increment(ref CachedCount);
|
||||||
|
}
|
||||||
|
Response.Headers.Add("x-cache-result", method);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
12
Wabbajack.Server/DTOs/Metric.cs
Normal file
12
Wabbajack.Server/DTOs/Metric.cs
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
using System;
|
||||||
|
|
||||||
|
namespace Wabbajack.Server.DTOs
|
||||||
|
{
|
||||||
|
public class Metric
|
||||||
|
{
|
||||||
|
public DateTime Timestamp { get; set; }
|
||||||
|
public string Action { get; set; }
|
||||||
|
public string Subject { get; set; }
|
||||||
|
public string MetricsKey { get; set; }
|
||||||
|
}
|
||||||
|
}
|
14
Wabbajack.Server/DTOs/NexusUpdateEntry.cs
Normal file
14
Wabbajack.Server/DTOs/NexusUpdateEntry.cs
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
using Newtonsoft.Json;
|
||||||
|
|
||||||
|
namespace Wabbajack.Server.DTOs
|
||||||
|
{
|
||||||
|
public class NexusUpdateEntry
|
||||||
|
{
|
||||||
|
[JsonProperty("mod_id")]
|
||||||
|
public long ModId { get; set; }
|
||||||
|
[JsonProperty("latest_file_update")]
|
||||||
|
public long LatestFileUpdate { get; set; }
|
||||||
|
[JsonProperty("latest_mod_activity")]
|
||||||
|
public long LastestModActivity { get; set; }
|
||||||
|
}
|
||||||
|
}
|
15
Wabbajack.Server/DataLayer/Metrics.cs
Normal file
15
Wabbajack.Server/DataLayer/Metrics.cs
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
using System.Threading.Tasks;
|
||||||
|
using Dapper;
|
||||||
|
using Wabbajack.Server.DTOs;
|
||||||
|
|
||||||
|
namespace Wabbajack.Server.DataLayer
|
||||||
|
{
|
||||||
|
public partial class SqlService
|
||||||
|
{
|
||||||
|
public async Task IngestMetric(Metric metric)
|
||||||
|
{
|
||||||
|
await using var conn = await Open();
|
||||||
|
await conn.ExecuteAsync(@"INSERT INTO dbo.Metrics (Timestamp, Action, Subject, MetricsKey) VALUES (@Timestamp, @Action, @Subject, @MetricsKey)", metric);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
93
Wabbajack.Server/DataLayer/Nexus.cs
Normal file
93
Wabbajack.Server/DataLayer/Nexus.cs
Normal file
@ -0,0 +1,93 @@
|
|||||||
|
using System;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Dapper;
|
||||||
|
using Newtonsoft.Json;
|
||||||
|
using Wabbajack.Common;
|
||||||
|
using Wabbajack.Lib.NexusApi;
|
||||||
|
|
||||||
|
namespace Wabbajack.Server.DataLayer
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// SQL routines that read/write cached information from the Nexus
|
||||||
|
/// </summary>
|
||||||
|
public partial class SqlService
|
||||||
|
{
|
||||||
|
public async Task<long> DeleteNexusModInfosUpdatedBeforeDate(Game game, long modId, DateTime date)
|
||||||
|
{
|
||||||
|
await using var conn = await Open();
|
||||||
|
var deleted = await conn.ExecuteScalarAsync<long>(
|
||||||
|
@"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
|
||||||
|
SELECT @@ROWCOUNT AS Deleted",
|
||||||
|
new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
|
||||||
|
return deleted;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<long> DeleteNexusModFilesUpdatedBeforeDate(Game game, long modId, DateTime date)
|
||||||
|
{
|
||||||
|
await using var conn = await Open();
|
||||||
|
var deleted = await conn.ExecuteScalarAsync<long>(
|
||||||
|
@"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
|
||||||
|
SELECT @@ROWCOUNT AS Deleted",
|
||||||
|
new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
|
||||||
|
return deleted;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<ModInfo> GetNexusModInfoString(Game game, long modId)
|
||||||
|
{
|
||||||
|
await using var conn = await Open();
|
||||||
|
var result = await conn.QueryFirstOrDefaultAsync<string>(
|
||||||
|
"SELECT Data FROM dbo.NexusModInfos WHERE Game = @Game AND @ModId = ModId",
|
||||||
|
new {Game = game.MetaData().NexusGameId, ModId = modId});
|
||||||
|
return result == null ? null : JsonConvert.DeserializeObject<ModInfo>(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task AddNexusModInfo(Game game, long modId, DateTime lastCheckedUtc, ModInfo data)
|
||||||
|
{
|
||||||
|
await using var conn = await Open();
|
||||||
|
|
||||||
|
await conn.ExecuteAsync(
|
||||||
|
@"MERGE dbo.NexusModInfos AS Target
|
||||||
|
USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
|
||||||
|
ON Target.Game = Source.Game AND Target.ModId = Source.ModId
|
||||||
|
WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
|
||||||
|
WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
|
||||||
|
new
|
||||||
|
{
|
||||||
|
Game = game.MetaData().NexusGameId,
|
||||||
|
ModId = modId,
|
||||||
|
LastChecked = lastCheckedUtc,
|
||||||
|
Data = JsonConvert.SerializeObject(data)
|
||||||
|
});
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task AddNexusModFiles(Game game, long modId, DateTime lastCheckedUtc, NexusApiClient.GetModFilesResponse data)
|
||||||
|
{
|
||||||
|
await using var conn = await Open();
|
||||||
|
|
||||||
|
await conn.ExecuteAsync(
|
||||||
|
@"MERGE dbo.NexusModFiles AS Target
|
||||||
|
USING (SELECT @Game Game, @ModId ModId, @LastChecked LastChecked, @Data Data) AS Source
|
||||||
|
ON Target.Game = Source.Game AND Target.ModId = Source.ModId
|
||||||
|
WHEN MATCHED THEN UPDATE SET Target.Data = @Data, Target.LastChecked = @LastChecked
|
||||||
|
WHEN NOT MATCHED THEN INSERT (Game, ModId, LastChecked, Data) VALUES (@Game, @ModId, @LastChecked, @Data);",
|
||||||
|
new
|
||||||
|
{
|
||||||
|
Game = game.MetaData().NexusGameId,
|
||||||
|
ModId = modId,
|
||||||
|
LastChecked = lastCheckedUtc,
|
||||||
|
Data = JsonConvert.SerializeObject(data)
|
||||||
|
});
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(Game game, long modId)
|
||||||
|
{
|
||||||
|
await using var conn = await Open();
|
||||||
|
var result = await conn.QueryFirstOrDefaultAsync<string>(
|
||||||
|
"SELECT Data FROM dbo.NexusModFiles WHERE Game = @Game AND @ModId = ModId",
|
||||||
|
new {Game = game.MetaData().NexusGameId, ModId = modId});
|
||||||
|
return result == null ? null : JsonConvert.DeserializeObject<NexusApiClient.GetModFilesResponse>(result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
24
Wabbajack.Server/DataLayer/SqlService.cs
Normal file
24
Wabbajack.Server/DataLayer/SqlService.cs
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
using System.Data.SqlClient;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Wabbajack.BuildServer;
|
||||||
|
|
||||||
|
namespace Wabbajack.Server.DataLayer
|
||||||
|
{
|
||||||
|
public partial class SqlService
|
||||||
|
{
|
||||||
|
private AppSettings _settings;
|
||||||
|
|
||||||
|
public SqlService(AppSettings settings)
|
||||||
|
{
|
||||||
|
_settings = settings;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<SqlConnection> Open()
|
||||||
|
{
|
||||||
|
var conn = new SqlConnection(_settings.SqlConnection);
|
||||||
|
await conn.OpenAsync();
|
||||||
|
return conn;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
12
Wabbajack.Server/GlobalInformation.cs
Normal file
12
Wabbajack.Server/GlobalInformation.cs
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
using System;
|
||||||
|
|
||||||
|
namespace Wabbajack.Server
|
||||||
|
{
|
||||||
|
public class GlobalInformation
|
||||||
|
{
|
||||||
|
public TimeSpan NexusRSSPollRate = TimeSpan.FromMinutes(1);
|
||||||
|
public TimeSpan NexusAPIPollRate = TimeSpan.FromHours(2);
|
||||||
|
public DateTime LastNexusSyncUTC { get; set; }
|
||||||
|
public TimeSpan TimeSinceLastNexusSync => DateTime.UtcNow - LastNexusSyncUTC;
|
||||||
|
}
|
||||||
|
}
|
171
Wabbajack.Server/Services/NexusPoll.cs
Normal file
171
Wabbajack.Server/Services/NexusPoll.cs
Normal file
@ -0,0 +1,171 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Microsoft.AspNetCore.Builder;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Wabbajack.BuildServer;
|
||||||
|
using Wabbajack.Common;
|
||||||
|
using Wabbajack.Lib.NexusApi;
|
||||||
|
using Wabbajack.Server.DataLayer;
|
||||||
|
using Wabbajack.Server.DTOs;
|
||||||
|
|
||||||
|
namespace Wabbajack.Server.Services
|
||||||
|
{
|
||||||
|
public class NexusPoll
|
||||||
|
{
|
||||||
|
private SqlService _sql;
|
||||||
|
private AppSettings _settings;
|
||||||
|
private GlobalInformation _globalInformation;
|
||||||
|
private ILogger<NexusPoll> _logger;
|
||||||
|
|
||||||
|
public NexusPoll(ILogger<NexusPoll> logger, AppSettings settings, SqlService service, GlobalInformation globalInformation)
|
||||||
|
{
|
||||||
|
_sql = service;
|
||||||
|
_settings = settings;
|
||||||
|
_globalInformation = globalInformation;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task UpdateNexusCacheRSS()
|
||||||
|
{
|
||||||
|
using var _ = _logger.BeginScope("Nexus Update via RSS");
|
||||||
|
_logger.Log(LogLevel.Information, "Starting");
|
||||||
|
|
||||||
|
var results = await NexusUpdatesFeeds.GetUpdates();
|
||||||
|
NexusApiClient client = null;
|
||||||
|
long updated = 0;
|
||||||
|
foreach (var result in results)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var purgedMods =
|
||||||
|
await _sql.DeleteNexusModFilesUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
|
||||||
|
var purgedFiles =
|
||||||
|
await _sql.DeleteNexusModInfosUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
|
||||||
|
|
||||||
|
var totalPurged = purgedFiles + purgedMods;
|
||||||
|
if (totalPurged > 0)
|
||||||
|
_logger.Log(LogLevel.Information, $"Purged {totalPurged} cache items");
|
||||||
|
|
||||||
|
if (await _sql.GetNexusModInfoString(result.Game, result.ModId) != null) continue;
|
||||||
|
|
||||||
|
// Lazily create the client
|
||||||
|
client ??= await NexusApiClient.Get();
|
||||||
|
|
||||||
|
// Cache the info
|
||||||
|
var files = await client.GetModFiles(result.Game, result.ModId, false);
|
||||||
|
await _sql.AddNexusModFiles(result.Game, result.ModId, result.TimeStamp, files);
|
||||||
|
|
||||||
|
var modInfo = await client.GetModInfo(result.Game, result.ModId, useCache: false);
|
||||||
|
await _sql.AddNexusModInfo(result.Game, result.ModId, result.TimeStamp, modInfo);
|
||||||
|
updated++;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, $"Failed Nexus update for {result.Game} - {result.ModId} - {result.TimeStamp}");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
if (updated > 0)
|
||||||
|
_logger.Log(LogLevel.Information, $"Primed {updated} nexus cache entries");
|
||||||
|
|
||||||
|
_globalInformation.LastNexusSyncUTC = DateTime.UtcNow;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task UpdateNexusCacheAPI()
|
||||||
|
{
|
||||||
|
using var _ = _logger.BeginScope("Nexus Update via API");
|
||||||
|
_logger.Log(LogLevel.Information, "Starting");
|
||||||
|
var api = await NexusApiClient.Get();
|
||||||
|
|
||||||
|
var gameTasks = GameRegistry.Games.Values
|
||||||
|
.Where(game => game.NexusName != null)
|
||||||
|
.Select(async game =>
|
||||||
|
{
|
||||||
|
var mods = await api.Get<List<NexusUpdateEntry>>(
|
||||||
|
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m");
|
||||||
|
|
||||||
|
return (game, mods);
|
||||||
|
})
|
||||||
|
.Select(async rTask =>
|
||||||
|
{
|
||||||
|
var (game, mods) = await rTask;
|
||||||
|
return mods.Select(mod => new { game = game, mod = mod });
|
||||||
|
}).ToList();
|
||||||
|
|
||||||
|
_logger.Log(LogLevel.Information, $"Getting update list for {gameTasks.Count} games");
|
||||||
|
|
||||||
|
var purge = (await Task.WhenAll(gameTasks))
|
||||||
|
.SelectMany(i => i)
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
_logger.Log(LogLevel.Information, $"Found {purge.Count} updated mods in the last month");
|
||||||
|
using var queue = new WorkQueue();
|
||||||
|
var collected = purge.Select(d =>
|
||||||
|
{
|
||||||
|
var a = d.mod.LatestFileUpdate.AsUnixTime();
|
||||||
|
// Mod activity could hide files
|
||||||
|
var b = d.mod.LastestModActivity.AsUnixTime();
|
||||||
|
|
||||||
|
return new {Game = d.game.Game, Date = (a > b ? a : b), ModId = d.mod.ModId};
|
||||||
|
});
|
||||||
|
|
||||||
|
var purged = await collected.PMap(queue, async t =>
|
||||||
|
{
|
||||||
|
var resultA = await _sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date);
|
||||||
|
var resultB = await _sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date);
|
||||||
|
return resultA + resultB;
|
||||||
|
});
|
||||||
|
|
||||||
|
_logger.Log(LogLevel.Information, $"Purged {purged.Sum()} cache entries");
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Start()
|
||||||
|
{
|
||||||
|
Task.Run(async () =>
|
||||||
|
{
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await UpdateNexusCacheRSS();
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Error polling from Nexus");
|
||||||
|
}
|
||||||
|
await Task.Delay(_globalInformation.NexusRSSPollRate);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Task.Run(async () =>
|
||||||
|
{
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await UpdateNexusCacheAPI();
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "Error getting API feed from Nexus");
|
||||||
|
}
|
||||||
|
|
||||||
|
await Task.Delay(_globalInformation.NexusAPIPollRate);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class NexusPollExtensions
|
||||||
|
{
|
||||||
|
public static void UseNexusPoll(this IApplicationBuilder b)
|
||||||
|
{
|
||||||
|
var poll = (NexusPoll)b.ApplicationServices.GetService(typeof(NexusPoll));
|
||||||
|
poll.Start();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
130
Wabbajack.Server/Startup.cs
Normal file
130
Wabbajack.Server/Startup.cs
Normal file
@ -0,0 +1,130 @@
|
|||||||
|
using System.Diagnostics;
|
||||||
|
using System.IO;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Microsoft.AspNetCore.Builder;
|
||||||
|
using Microsoft.AspNetCore.Hosting;
|
||||||
|
using Microsoft.AspNetCore.Http.Features;
|
||||||
|
using Microsoft.AspNetCore.StaticFiles;
|
||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Extensions.FileProviders;
|
||||||
|
using Microsoft.Extensions.Hosting;
|
||||||
|
using Microsoft.OpenApi.Models;
|
||||||
|
using Newtonsoft.Json;
|
||||||
|
using Wabbajack.BuildServer;
|
||||||
|
using Wabbajack.Server.DataLayer;
|
||||||
|
using Wabbajack.Server.Services;
|
||||||
|
|
||||||
|
namespace Wabbajack.Server
|
||||||
|
{
|
||||||
|
public class TestStartup : Startup
|
||||||
|
{
|
||||||
|
public TestStartup(IConfiguration configuration) : base(configuration)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
|
public class Startup
|
||||||
|
{
|
||||||
|
public Startup(IConfiguration configuration)
|
||||||
|
{
|
||||||
|
Configuration = configuration;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IConfiguration Configuration { get; }
|
||||||
|
|
||||||
|
// This method gets called by the runtime. Use this method to add services to the container.
|
||||||
|
public void ConfigureServices(IServiceCollection services)
|
||||||
|
{
|
||||||
|
services.AddSwaggerGen(c =>
|
||||||
|
{
|
||||||
|
c.SwaggerDoc("v1", new OpenApiInfo {Title = "Wabbajack Build API", Version = "v1"});
|
||||||
|
});
|
||||||
|
|
||||||
|
services.Configure<FormOptions>(x =>
|
||||||
|
{
|
||||||
|
x.ValueLengthLimit = int.MaxValue;
|
||||||
|
x.MultipartBodyLengthLimit = int.MaxValue;
|
||||||
|
});
|
||||||
|
|
||||||
|
services.AddSingleton<AppSettings>();
|
||||||
|
services.AddSingleton<SqlService>();
|
||||||
|
services.AddSingleton<GlobalInformation>();
|
||||||
|
services.AddSingleton<NexusPoll>();
|
||||||
|
services.AddMvc();
|
||||||
|
services.AddControllers()
|
||||||
|
.AddNewtonsoftJson(o =>
|
||||||
|
{
|
||||||
|
|
||||||
|
o.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore;
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
|
||||||
|
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
|
||||||
|
{
|
||||||
|
if (env.IsDevelopment())
|
||||||
|
{
|
||||||
|
app.UseDeveloperExceptionPage();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!(this is TestStartup))
|
||||||
|
app.UseHttpsRedirection();
|
||||||
|
|
||||||
|
app.UseDeveloperExceptionPage();
|
||||||
|
|
||||||
|
var provider = new FileExtensionContentTypeProvider();
|
||||||
|
provider.Mappings[".rar"] = "application/x-rar-compressed";
|
||||||
|
provider.Mappings[".7z"] = "application/x-7z-compressed";
|
||||||
|
provider.Mappings[".zip"] = "application/zip";
|
||||||
|
provider.Mappings[".wabbajack"] = "application/zip";
|
||||||
|
app.UseStaticFiles();
|
||||||
|
|
||||||
|
app.UseSwagger();
|
||||||
|
app.UseSwaggerUI(c =>
|
||||||
|
{
|
||||||
|
c.SwaggerEndpoint("/swagger/v1/swagger.json", "Wabbajack Build API");
|
||||||
|
c.RoutePrefix = string.Empty;
|
||||||
|
});
|
||||||
|
app.UseRouting();
|
||||||
|
|
||||||
|
app.UseAuthentication();
|
||||||
|
app.UseAuthorization();
|
||||||
|
app.UseNexusPoll();
|
||||||
|
|
||||||
|
app.Use(next =>
|
||||||
|
{
|
||||||
|
return async context =>
|
||||||
|
{
|
||||||
|
var stopWatch = new Stopwatch();
|
||||||
|
stopWatch.Start();
|
||||||
|
context.Response.OnStarting(() =>
|
||||||
|
{
|
||||||
|
stopWatch.Stop();
|
||||||
|
var headers = context.Response.Headers;
|
||||||
|
headers.Add("Access-Control-Allow-Origin", "*");
|
||||||
|
headers.Add("Access-Control-Allow-Methods", "POST, GET");
|
||||||
|
headers.Add("Access-Control-Allow-Headers", "Accept, Origin, Content-type");
|
||||||
|
headers.Add("X-ResponseTime-Ms", stopWatch.ElapsedMilliseconds.ToString());
|
||||||
|
return Task.CompletedTask;
|
||||||
|
});
|
||||||
|
await next(context);
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
app.UseFileServer(new FileServerOptions
|
||||||
|
{
|
||||||
|
FileProvider = new PhysicalFileProvider(
|
||||||
|
Path.Combine(Directory.GetCurrentDirectory(), "public")),
|
||||||
|
StaticFileOptions = {ServeUnknownFileTypes = true},
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
app.UseEndpoints(endpoints =>
|
||||||
|
{
|
||||||
|
endpoints.MapControllers();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
22
Wabbajack.Server/appsettings.json
Normal file
22
Wabbajack.Server/appsettings.json
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
{
|
||||||
|
"Logging": {
|
||||||
|
"LogLevel": {
|
||||||
|
"Default": "Information",
|
||||||
|
"Microsoft": "Warning",
|
||||||
|
"Microsoft.Hosting.Lifetime": "Information"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"WabbajackSettings": {
|
||||||
|
"DownloadDir": "c:\\tmp\\downloads",
|
||||||
|
"ArchiveDir": "w:\\archives",
|
||||||
|
"TempFolder": "c:\\tmp",
|
||||||
|
"JobRunner": true,
|
||||||
|
"JobScheduler": false,
|
||||||
|
"RunFrontEndJobs": true,
|
||||||
|
"RunBackEndJobs": true,
|
||||||
|
"BunnyCDN_User": "wabbajackcdn",
|
||||||
|
"BunnyCDN_Password": "XXXX",
|
||||||
|
"SQLConnection": "Data Source=.\\SQLEXPRESS;Integrated Security=True;Initial Catalog=wabbajack_prod;MultipleActiveResultSets=true"
|
||||||
|
},
|
||||||
|
"AllowedHosts": "*"
|
||||||
|
}
|
1
Wabbajack.Server/public/WABBAJACK_TEST_FILE.txt
Normal file
1
Wabbajack.Server/public/WABBAJACK_TEST_FILE.txt
Normal file
@ -0,0 +1 @@
|
|||||||
|
Cheese for Everyone!
|
@ -44,6 +44,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.App.Test", "Wabba
|
|||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.BuildServer.Test", "Wabbajack.BuildServer.Test\Wabbajack.BuildServer.Test.csproj", "{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.BuildServer.Test", "Wabbajack.BuildServer.Test\Wabbajack.BuildServer.Test.csproj", "{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}"
|
||||||
EndProject
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Server", "Wabbajack.Server\Wabbajack.Server.csproj", "{3E11B700-8405-433D-BF47-6C356087A7C2}"
|
||||||
|
EndProject
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|Any CPU = Debug|Any CPU
|
Debug|Any CPU = Debug|Any CPU
|
||||||
@ -159,6 +161,14 @@ Global
|
|||||||
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|Any CPU.Build.0 = Release|Any CPU
|
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|x64.ActiveCfg = Release|Any CPU
|
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|x64.Build.0 = Release|Any CPU
|
{160D3A0F-68E1-4AFF-8625-E5E0FFBB2058}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{3E11B700-8405-433D-BF47-6C356087A7C2}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{3E11B700-8405-433D-BF47-6C356087A7C2}.Release|x64.Build.0 = Release|Any CPU
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
|
Loading…
Reference in New Issue
Block a user