Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes for nexus cache priming

Browse Source
This commit is contained in:
Timothy Baldridge 2020-04-10 21:16:10 -06:00
parent 26a42d3ceb
commit 508eb32230
8 changed files with 96 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Wabbajack.BuildServer.Test/JobTests.cs
View File

@ -47,7 +47,15 @@ namespace Wabbajack.BuildServer.Test
Assert.Null(await sql.GetModFiles(record.Game, record.ModId));
Assert.Null(await sql.GetNexusModInfoString(record.Game, record.ModId));
}
}
[Fact]
public async Task CanPrimeTheNexusCache()
{
var sql = Fixture.GetService<SqlService>();
Assert.True(await GetNexusUpdatesJob.UpdateNexusCacheFast(sql) > 0);
Assert.True(await GetNexusUpdatesJob.UpdateNexusCacheFast(sql) == 0);
}
}
}

9
Wabbajack.BuildServer/Controllers/Heartbeat.cs
View File

@ -5,6 +5,7 @@ using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common.StatusFeed;
namespace Wabbajack.BuildServer.Controllers
@ -36,9 +37,13 @@ namespace Wabbajack.BuildServer.Controllers
}
[HttpGet]
public async Task<TimeSpan> GetHeartbeat()
public async Task<IActionResult> GetHeartbeat()
{
return DateTime.Now - _startTime;
return Ok(new
{
Uptime = DateTime.Now - _startTime,
LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync
});
}
[HttpGet("only-authenticated")]

12
Wabbajack.BuildServer/Controllers/NexusCache.cs
View File

@ -49,10 +49,10 @@ namespace Wabbajack.BuildServer.Controllers
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{ModId}.json";
var body = await api.Get<ModInfo>(path);
await SQL.AddNexusModInfo(game, ModId, DateTime.Now, body);
result = await api.GetModInfo(game, ModId, false);
await SQL.AddNexusModInfo(game, ModId, DateTime.UtcNow, result);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);
}
@ -78,10 +78,8 @@ namespace Wabbajack.BuildServer.Controllers
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files.json";
var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
await SQL.AddNexusModFiles(game, ModId, DateTime.Now, body);
result = await api.GetModFiles(game, ModId, false);
await SQL.AddNexusModFiles(game, ModId, DateTime.UtcNow, result);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);

13
Wabbajack.BuildServer/JobManager.cs
View File

@ -10,6 +10,7 @@ using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.BuildServer
{
@ -74,6 +75,9 @@ namespace Wabbajack.BuildServer
Utils.LogMessages.Subscribe(Heartbeat.AddToLog);
Utils.LogMessages.OfType<IUserIntervention>().Subscribe(u => u.Cancel());
if (!Settings.JobScheduler) return;
var task = RunNexusCacheLoop();
while (true)
{
await KillOrphanedJobs();
@ -86,6 +90,15 @@ namespace Wabbajack.BuildServer
}
}
private async Task RunNexusCacheLoop()
{
while (true)
{
await GetNexusUpdatesJob.UpdateNexusCacheFast(Sql);
await Task.Delay(TimeSpan.FromMinutes(1));
}
}
private async Task KillOrphanedJobs()
{
try

40
Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Models.JobQueue;
@ -69,6 +70,43 @@ namespace Wabbajack.BuildServer.Models.Jobs
return JobResult.Success();
}
public static DateTime LastNexusSync { get; set; } = DateTime.Now;
public static async Task<long> UpdateNexusCacheFast(SqlService sql)
{
var results = await NexusUpdatesFeeds.GetUpdates();
NexusApiClient client = null;
long updated = 0;
foreach (var result in results)
{
var purgedMods = await sql.DeleteNexusModFilesUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
var purgedFiles = await sql.DeleteNexusModInfosUpdatedBeforeDate(result.Game, result.ModId, result.TimeStamp);
var totalPurged = purgedFiles + purgedMods;
if (totalPurged > 0)
Utils.Log($"Purged {totalPurged} cache items");
if (await sql.GetNexusModInfoString(result.Game, result.ModId) != null) continue;
// Lazily create the client
client ??= await NexusApiClient.Get();
// Cache the info
var files = await client.GetModFiles(result.Game, result.ModId, false);
await sql.AddNexusModFiles(result.Game, result.ModId, result.TimeStamp, files);
var modInfo = await client.GetModInfo(result.Game, result.ModId);
await sql.AddNexusModInfo(result.Game, result.ModId, result.TimeStamp, modInfo);
updated++;
}
if (updated > 0)
Utils.Log($"Primed {updated} nexus cache entries");
LastNexusSync = DateTime.Now;
return updated;
}
}

6
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -575,7 +575,7 @@ namespace Wabbajack.BuildServer.Model.Models
{
await using var conn = await Open();
var deleted = await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date
@"DELETE FROM dbo.NexusModInfos WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
return deleted;
@ -585,9 +585,9 @@ namespace Wabbajack.BuildServer.Model.Models
{
await using var conn = await Open();
var deleted = await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked <= @Date
@"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, @Date = date});
new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
return deleted;
}

17
Wabbajack.Common/Json.cs
View File

@ -27,11 +27,14 @@ namespace Wabbajack.Common
};
public static JsonSerializerSettings JsonSettings =>
new JsonSerializerSettings {
TypeNameHandling = TypeNameHandling.Objects,
SerializationBinder = new JsonNameSerializationBinder(),
Converters = Converters};
new JsonSerializerSettings {
TypeNameHandling = TypeNameHandling.Objects,
SerializationBinder = new JsonNameSerializationBinder(),
Converters = Converters};
public static JsonSerializerSettings GenericJsonSettings =>
new JsonSerializerSettings { };
public static void ToJson<T>(this T obj, string filename)
{
@ -73,11 +76,11 @@ namespace Wabbajack.Common
return JsonConvert.DeserializeObject<T>(data, JsonSettings)!;
}
public static T FromJson<T>(this Stream stream)
public static T FromJson<T>(this Stream stream, bool genericReader = false)
{
using var tr = new StreamReader(stream, Encoding.UTF8, leaveOpen: true);
using var reader = new JsonTextReader(tr);
var ser = JsonSerializer.Create(JsonSettings);
var ser = JsonSerializer.Create(genericReader ? GenericJsonSettings : JsonSettings);
return ser.Deserialize<T>(reader);
}

16
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -252,7 +252,7 @@ namespace Wabbajack.Lib.NexusApi
await using var stream = await response.Content.ReadAsStreamAsync();
return stream.FromJson<T>();
return stream.FromJson<T>(genericReader:true);
}
catch (TimeoutException)
{
@ -321,10 +321,11 @@ namespace Wabbajack.Lib.NexusApi
public List<NexusFileInfo> files { get; set; }
}
public async Task<GetModFilesResponse> GetModFiles(Game game, long modid)
public async Task<GetModFilesResponse> GetModFiles(Game game, long modid, bool useCache = true)
{
var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modid}/files.json";
var result = await GetCached<GetModFilesResponse>(url);
var result = useCache ? await GetCached<GetModFilesResponse>(url) : await Get<GetModFilesResponse>(url);
if (result.files == null)
throw new InvalidOperationException("Got Null data from the Nexus while finding mod files");
return result;
@ -336,10 +337,15 @@ namespace Wabbajack.Lib.NexusApi
return await Get<List<MD5Response>>(url);
}
public async Task<ModInfo> GetModInfo(Game game, long modId)
public async Task<ModInfo> GetModInfo(Game game, long modId, bool useCache = true)
{
var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modId}.json";
return await GetCached<ModInfo>(url);
if (useCache)
{
return await GetCached<ModInfo>(url);
}
return await Get<ModInfo>(url);
}
private class DownloadLink