diff --git a/Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs b/Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs index b07f9ab2..f86e5165 100644 --- a/Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs +++ b/Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs @@ -34,7 +34,8 @@ namespace Wabbajack.BuildServer.Test $"WabbajackSettings:BunnyCDN_User=TEST", $"WabbajackSettings:BunnyCDN_Password=TEST", "WabbajackSettings:JobScheduler=false", - "WabbajackSettings:JobRunner=false" + "WabbajackSettings:JobRunner=false", + "WabbajackSettinss:DisableNexusForwarding=true" }, true); _host = builder.Build(); _token = new CancellationTokenSource(); diff --git a/Wabbajack.BuildServer/AppSettings.cs b/Wabbajack.BuildServer/AppSettings.cs index 9037b977..196ecc06 100644 --- a/Wabbajack.BuildServer/AppSettings.cs +++ b/Wabbajack.BuildServer/AppSettings.cs @@ -25,7 +25,6 @@ namespace Wabbajack.BuildServer public string BunnyCDN_User { get; set; } public string BunnyCDN_Password { get; set; } - public string SqlConnection { get; set; } public int MaxJobs { get; set; } = 2; diff --git a/Wabbajack.BuildServer/Controllers/NexusCache.cs b/Wabbajack.BuildServer/Controllers/NexusCache.cs index b8f1ad4e..68fe9160 100644 --- a/Wabbajack.BuildServer/Controllers/NexusCache.cs +++ b/Wabbajack.BuildServer/Controllers/NexusCache.cs @@ -1,13 +1,12 @@ using System.Linq; +using System.Threading; using System.Threading.Tasks; -using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using MongoDB.Driver; using Newtonsoft.Json; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; -using Wabbajack.Common; using Wabbajack.Lib.NexusApi; namespace Wabbajack.BuildServer.Controllers @@ -31,37 +30,46 @@ namespace Wabbajack.BuildServer.Controllers /// A Mod Info result [HttpGet] [Route("{GameName}/mods/{ModId}.json")] - public async Task GetModInfo(string GameName, string ModId) + public async Task GetModInfo(string GameName, long ModId) { - var result = await Db.NexusModInfos.FindOneAsync(info => info.Game == GameName && info.ModId == ModId); - + Utils.Log($"Nexus Mod Info {GameName} {ModId}"); + var game = GameRegistry.GetByFuzzyName(GameName).Game; + var result = await SQL.GetNexusModInfoString(game, ModId); + string method = "CACHED"; if (result == null) { var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}.json"; + var path = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{ModId}.json"; var body = await api.Get(path); - result = new NexusCacheData {Data = body, Path = path, Game = GameName, ModId = ModId}; try { - await Db.NexusModInfos.InsertOneAsync(result); + await SQL.AddNexusModInfo(game, ModId, DateTime.Now, body); } catch (MongoWriteException) { } method = "NOT_CACHED"; + Interlocked.Increment(ref ForwardCount); + } + else + { + Interlocked.Increment(ref CachedCount); } Response.Headers.Add("x-cache-result", method); - return result.Data; + return result; } [HttpGet] [Route("{GameName}/mods/{ModId}/files.json")] - public async Task GetModFiles(string GameName, string ModId) + public async Task GetModFiles(string GameName, long ModId) { - var result = await Db.NexusModFiles.FindOneAsync(info => info.Game == GameName && info.ModId == ModId); + Utils.Log($"Nexus Mod Files {GameName} {ModId}"); + + var game = GameRegistry.GetByFuzzyName(GameName).Game; + var result = await SQL.GetModFiles(game, ModId); string method = "CACHED"; if (result == null) @@ -69,75 +77,79 @@ namespace Wabbajack.BuildServer.Controllers var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files.json"; var body = await api.Get(path); - result = new NexusCacheData - { - Data = body, Path = path, Game = GameName, ModId = ModId - }; try { - await Db.NexusModFiles.InsertOneAsync(result); + await SQL.AddNexusModFiles(game, ModId, DateTime.Now, body); } catch (MongoWriteException) { } method = "NOT_CACHED"; + Interlocked.Increment(ref ForwardCount); + } + else + { + Interlocked.Increment(ref CachedCount); + } + Response.Headers.Add("x-cache-result", method); + return result; + } + + private class NexusIngestHeader + { + public List> ModInfos { get; set; } + public List> FileInfos { get; set; } + public List> ModFiles { get; set; } + } + + [HttpGet] + [Route("/nexus_cache/ingest")] + [Authorize] + public async Task IngestNexusFile() + { + long totalRows = 0; + + var dataPath = @"nexus_export.json".RelativeTo(_settings.TempPath); + + var data = JsonConvert.DeserializeObject(await dataPath.ReadAllTextAsync()); + + foreach (var record in data.ModInfos) + { + await SQL.AddNexusModInfo(GameRegistry.GetByFuzzyName(record.Game).Game, long.Parse(record.ModId), + record.LastCheckedUTC, record.Data); + totalRows += 1; + } + + foreach (var record in data.FileInfos) + { + await SQL.AddNexusFileInfo(GameRegistry.GetByFuzzyName(record.Game).Game, long.Parse(record.ModId), + long.Parse(record.FileId), + record.LastCheckedUTC, record.Data); + totalRows += 1; + } + + foreach (var record in data.ModFiles) + { + await SQL.AddNexusModFiles(GameRegistry.GetByFuzzyName(record.Game).Game, long.Parse(record.ModId), + record.LastCheckedUTC, record.Data); + totalRows += 1; } - Response.Headers.Add("x-cache-result", method); - return result.Data; + return Ok(totalRows); } [HttpGet] - [Route("{GameName}/mods/{ModId}/files/{FileId}.json")] - public async Task GetFileInfo(string GameName, string ModId, string FileId) + [Route("/nexus_cache/stats")] + public async Task NexusCacheStats() { - var result = await Db.NexusFileInfos.FindOneAsync(info => - info.Game == GameName && info.ModId == ModId && info.FileId == FileId); - - string method = "CACHED"; - if (result == null) + return Ok(new ClientAPI.NexusCacheStats { - var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files/{FileId}.json"; - var body = await api.Get(path); - result = new NexusCacheData - { - Data = body, - Path = path, - Game = GameName, - ModId = ModId, - FileId = FileId - }; - try - { - await Db.NexusFileInfos.InsertOneAsync(result); - } - catch (MongoWriteException) - { - } - - method = "NOT_CACHED"; - } - - Response.Headers.Add("x-cache-method", method); - return result.Data; - } - - [HttpPost] - [Authorize] - [Route("/nexus_api_cache/export")] - public async Task ExportNexusCache() - { - Utils.Log("Exporting Nexus Info"); - var file_infos = Db.NexusFileInfos.AsQueryable().ToListAsync(); - var mod_infos = Db.NexusModInfos.AsQueryable().ToListAsync(); - var mod_files = Db.NexusModFiles.AsQueryable().ToListAsync(); - - var data = new {FileInfos = await file_infos, ModInfos = await mod_infos, Modfiles = await mod_files,}; - Utils.Log("Writing Data"); - return Ok(JsonConvert.SerializeObject(data)); - + CachedCount = CachedCount, + ForwardCount = ForwardCount, + CacheRatio = (double)CachedCount / (ForwardCount == 0 ? 1 : ForwardCount) + }); } + } } diff --git a/Wabbajack.BuildServer/Models/Sql/SqlService.cs b/Wabbajack.BuildServer/Models/Sql/SqlService.cs index 8d813dad..4702a463 100644 --- a/Wabbajack.BuildServer/Models/Sql/SqlService.cs +++ b/Wabbajack.BuildServer/Models/Sql/SqlService.cs @@ -10,6 +10,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.VisualBasic; using Newtonsoft.Json; using ReactiveUI; +using Wabbajack.BuildServer.GraphQL; using Wabbajack.BuildServer.Model.Models.Results; using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; diff --git a/Wabbajack.Common/Http/Client.cs b/Wabbajack.Common/Http/Client.cs index ae79c2ac..fed539f2 100644 --- a/Wabbajack.Common/Http/Client.cs +++ b/Wabbajack.Common/Http/Client.cs @@ -98,5 +98,11 @@ namespace Wabbajack.Common.Http return new_message; } + + public async Task GetJsonAsync(string s) + { + var result = await GetStringAsync(s); + return result.FromJSONString(); + } } } diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index 3f08fc13..67fd70fb 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -39,5 +39,18 @@ namespace Wabbajack.Lib return null; } } + + public class NexusCacheStats + { + public long CachedCount { get; set; } + public long ForwardCount { get; set; } + public double CacheRatio { get; set; } + } + + public static async Task GetNexusCacheStats() + { + return await GetClient() + .GetJsonAsync($"{Consts.WabbajackBuildServerUri}nexus_cache/stats"); + } } } diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index e80ae7d4..50dba8ce 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -254,7 +254,12 @@ namespace Wabbajack.Lib.NexusApi { try { - var builder = new UriBuilder(url) { Host = Consts.WabbajackCacheHostname, Scheme = "https" }; + var builder = new UriBuilder(url) + { + Host = Consts.WabbajackBuildServerUri.Host, + Scheme = Consts.WabbajackBuildServerUri.Scheme, + Port = Consts.WabbajackBuildServerUri.Port + }; return await Get(builder.ToString()); } catch (Exception) @@ -297,7 +302,7 @@ namespace Wabbajack.Lib.NexusApi public List files { get; set; } } - public async Task GetModFiles(Game game, int modid) + public async Task GetModFiles(Game game, long modid) { var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modid}/files.json"; var result = await GetCached(url); @@ -312,7 +317,7 @@ namespace Wabbajack.Lib.NexusApi return await Get>(url); } - public async Task GetModInfo(Game game, string modId) + public async Task GetModInfo(Game game, long modId) { var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modId}.json"; return await GetCached(url);