Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Nexus Cache is now SQL Based

Browse Source
This commit is contained in:
Timothy Baldridge 2020-04-02 15:16:46 -06:00
parent be2d24a578
commit 55fb7e1467
7 changed files with 107 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs
View File

@ -34,7 +34,8 @@ namespace Wabbajack.BuildServer.Test
$"WabbajackSettings:BunnyCDN_User=TEST",
$"WabbajackSettings:BunnyCDN_Password=TEST",
"WabbajackSettings:JobScheduler=false",
"WabbajackSettings:JobRunner=false"
"WabbajackSettings:JobRunner=false",
"WabbajackSettinss:DisableNexusForwarding=true"
}, true);
_host = builder.Build();
_token = new CancellationTokenSource();

1
Wabbajack.BuildServer/AppSettings.cs
View File

@ -25,7 +25,6 @@ namespace Wabbajack.BuildServer
public string BunnyCDN_User { get; set; }
public string BunnyCDN_Password { get; set; }
public string SqlConnection { get; set; }
public int MaxJobs { get; set; } = 2;

142
Wabbajack.BuildServer/Controllers/NexusCache.cs
View File

@ -1,13 +1,12 @@
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using Newtonsoft.Json;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.BuildServer.Controllers
@ -31,37 +30,46 @@ namespace Wabbajack.BuildServer.Controllers
/// <returns>A Mod Info result</returns>
[HttpGet]
[Route("{GameName}/mods/{ModId}.json")]
public async Task<ModInfo> GetModInfo(string GameName, string ModId)
public async Task<ModInfo> GetModInfo(string GameName, long ModId)
{
var result = await Db.NexusModInfos.FindOneAsync(info => info.Game == GameName && info.ModId == ModId);
Utils.Log($"Nexus Mod Info {GameName} {ModId}");
var game = GameRegistry.GetByFuzzyName(GameName).Game;
var result = await SQL.GetNexusModInfoString(game, ModId);
string method = "CACHED";
if (result == null)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}.json";
var path = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{ModId}.json";
var body = await api.Get<ModInfo>(path);
result = new NexusCacheData<ModInfo> {Data = body, Path = path, Game = GameName, ModId = ModId};
try
{
await Db.NexusModInfos.InsertOneAsync(result);
await SQL.AddNexusModInfo(game, ModId, DateTime.Now, body);
}
catch (MongoWriteException)
{
}
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);
}
else
{
Interlocked.Increment(ref CachedCount);
}
Response.Headers.Add("x-cache-result", method);
return result.Data;
return result;
}
[HttpGet]
[Route("{GameName}/mods/{ModId}/files.json")]
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, string ModId)
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(string GameName, long ModId)
{
var result = await Db.NexusModFiles.FindOneAsync(info => info.Game == GameName && info.ModId == ModId);
Utils.Log($"Nexus Mod Files {GameName} {ModId}");
var game = GameRegistry.GetByFuzzyName(GameName).Game;
var result = await SQL.GetModFiles(game, ModId);
string method = "CACHED";
if (result == null)
@ -69,75 +77,79 @@ namespace Wabbajack.BuildServer.Controllers
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files.json";
var body = await api.Get<NexusApiClient.GetModFilesResponse>(path);
result = new NexusCacheData<NexusApiClient.GetModFilesResponse>
{
Data = body, Path = path, Game = GameName, ModId = ModId
};
try
{
await Db.NexusModFiles.InsertOneAsync(result);
await SQL.AddNexusModFiles(game, ModId, DateTime.Now, body);
}
catch (MongoWriteException)
{
}
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);
}
else
{
Interlocked.Increment(ref CachedCount);
}
Response.Headers.Add("x-cache-result", method);
return result;
}
private class NexusIngestHeader
{
public List<NexusCacheData<ModInfo>> ModInfos { get; set; }
public List<NexusCacheData<NexusFileInfo>> FileInfos { get; set; }
public List<NexusCacheData<NexusApiClient.GetModFilesResponse>> ModFiles { get; set; }
}
[HttpGet]
[Route("/nexus_cache/ingest")]
[Authorize]
public async Task<IActionResult> IngestNexusFile()
{
long totalRows = 0;
var dataPath = @"nexus_export.json".RelativeTo(_settings.TempPath);
var data = JsonConvert.DeserializeObject<NexusIngestHeader>(await dataPath.ReadAllTextAsync());
foreach (var record in data.ModInfos)
{
await SQL.AddNexusModInfo(GameRegistry.GetByFuzzyName(record.Game).Game, long.Parse(record.ModId),
record.LastCheckedUTC, record.Data);
totalRows += 1;
}
foreach (var record in data.FileInfos)
{
await SQL.AddNexusFileInfo(GameRegistry.GetByFuzzyName(record.Game).Game, long.Parse(record.ModId),
long.Parse(record.FileId),
record.LastCheckedUTC, record.Data);
totalRows += 1;
}
foreach (var record in data.ModFiles)
{
await SQL.AddNexusModFiles(GameRegistry.GetByFuzzyName(record.Game).Game, long.Parse(record.ModId),
record.LastCheckedUTC, record.Data);
totalRows += 1;
}
Response.Headers.Add("x-cache-result", method);
return result.Data;
return Ok(totalRows);
}
[HttpGet]
[Route("{GameName}/mods/{ModId}/files/{FileId}.json")]
public async Task<object> GetFileInfo(string GameName, string ModId, string FileId)
[Route("/nexus_cache/stats")]
public async Task<IActionResult> NexusCacheStats()
{
var result = await Db.NexusFileInfos.FindOneAsync(info =>
info.Game == GameName && info.ModId == ModId && info.FileId == FileId);
string method = "CACHED";
if (result == null)
return Ok(new ClientAPI.NexusCacheStats
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
var path = $"https://api.nexusmods.com/v1/games/{GameName}/mods/{ModId}/files/{FileId}.json";
var body = await api.Get<NexusFileInfo>(path);
result = new NexusCacheData<NexusFileInfo>
{
Data = body,
Path = path,
Game = GameName,
ModId = ModId,
FileId = FileId
};
try
{
await Db.NexusFileInfos.InsertOneAsync(result);
}
catch (MongoWriteException)
{
}
method = "NOT_CACHED";
}
Response.Headers.Add("x-cache-method", method);
return result.Data;
}
[HttpPost]
[Authorize]
[Route("/nexus_api_cache/export")]
public async Task<IActionResult> ExportNexusCache()
{
Utils.Log("Exporting Nexus Info");
var file_infos = Db.NexusFileInfos.AsQueryable().ToListAsync();
var mod_infos = Db.NexusModInfos.AsQueryable().ToListAsync();
var mod_files = Db.NexusModFiles.AsQueryable().ToListAsync();
var data = new {FileInfos = await file_infos, ModInfos = await mod_infos, Modfiles = await mod_files,};
Utils.Log("Writing Data");
return Ok(JsonConvert.SerializeObject(data));
CachedCount = CachedCount,
ForwardCount = ForwardCount,
CacheRatio = (double)CachedCount / (ForwardCount == 0 ? 1 : ForwardCount)
});
}
}
}

1
Wabbajack.BuildServer/Models/Sql/SqlService.cs
View File

@ -10,6 +10,7 @@ using Microsoft.Extensions.Configuration;
using Microsoft.VisualBasic;
using Newtonsoft.Json;
using ReactiveUI;
using Wabbajack.BuildServer.GraphQL;
using Wabbajack.BuildServer.Model.Models.Results;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;

6
Wabbajack.Common/Http/Client.cs
View File

@ -98,5 +98,11 @@ namespace Wabbajack.Common.Http
return new_message;
}
public async Task<T> GetJsonAsync<T>(string s)
{
var result = await GetStringAsync(s);
return result.FromJSONString<T>();
}
}
}

13
Wabbajack.Lib/ClientAPI.cs
View File

@ -39,5 +39,18 @@ namespace Wabbajack.Lib
return null;
}
}
public class NexusCacheStats
{
public long CachedCount { get; set; }
public long ForwardCount { get; set; }
public double CacheRatio { get; set; }
}
public static async Task<NexusCacheStats> GetNexusCacheStats()
{
return await GetClient()
.GetJsonAsync<NexusCacheStats>($"{Consts.WabbajackBuildServerUri}nexus_cache/stats");
}
}
}

11
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -254,7 +254,12 @@ namespace Wabbajack.Lib.NexusApi
{
try
{
var builder = new UriBuilder(url) { Host = Consts.WabbajackCacheHostname, Scheme = "https" };
var builder = new UriBuilder(url)
{
Host = Consts.WabbajackBuildServerUri.Host,
Scheme = Consts.WabbajackBuildServerUri.Scheme,
Port = Consts.WabbajackBuildServerUri.Port
};
return await Get<T>(builder.ToString());
}
catch (Exception)
@ -297,7 +302,7 @@ namespace Wabbajack.Lib.NexusApi
public List<NexusFileInfo> files { get; set; }
}
public async Task<GetModFilesResponse> GetModFiles(Game game, int modid)
public async Task<GetModFilesResponse> GetModFiles(Game game, long modid)
{
var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modid}/files.json";
var result = await GetCached<GetModFilesResponse>(url);
@ -312,7 +317,7 @@ namespace Wabbajack.Lib.NexusApi
return await Get<List<MD5Response>>(url);
}
public async Task<ModInfo> GetModInfo(Game game, string modId)
public async Task<ModInfo> GetModInfo(Game game, long modId)
{
var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modId}.json";
return await GetCached<ModInfo>(url);