using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; using System.Reflection; using System.Text; using System.Threading.Tasks; using MongoDB.Driver; using Nancy; using Newtonsoft.Json; using Wabbajack.CacheServer.DTOs; using Wabbajack.Common; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.NexusApi; namespace Wabbajack.CacheServer { public class NexusCacheModule : NancyModule { public NexusCacheModule() : base("/") { Get("/v1/games/{GameName}/mods/{ModID}/files/{FileID}.json", HandleFileID); Get("/v1/games/{GameName}/mods/{ModID}/files.json", HandleGetFiles); Get("/v1/games/{GameName}/mods/{ModID}.json", HandleModInfo); Get("/nexus_api_cache/{request}.json", HandleCacheCall); Get("/nexus_api_cache", ListCache); Get("/nexus_api_cache/update", UpdateCache); Get("/nexus_api_cache/ingest/{Folder}", HandleIngestCache); } class UpdatedMod { public long mod_id; public long latest_file_update; public long latest_mod_activity; } public async Task UpdateCache(object arg) { var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); var gameTasks = GameRegistry.Games.Values .Where(game => game.NexusName != null) .Select(async game => { return (game, mods: await api.Get>( $"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m")); }) .Select(async rTask => { var (game, mods) = await rTask; return mods.Select(mod => new { game = game, mod = mod }); }).ToList(); Utils.Log($"Getting update list for {gameTasks.Count} games"); var purge = (await Task.WhenAll(gameTasks)) .SelectMany(i => i) .ToList(); Utils.Log($"Found {purge.Count} updated mods in the last month"); using (var queue = new WorkQueue()) { var collected = await purge.Select(d => { var a = d.mod.latest_file_update.AsUnixTime(); // Mod activity could hide files var b = d.mod.latest_mod_activity.AsUnixTime(); return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.mod_id.ToString()}; }).PMap(queue, async t => { var resultA = await Server.Config.NexusModInfos.Connect().DeleteManyAsync(f => f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); var resultB = await Server.Config.NexusModFiles.Connect().DeleteManyAsync(f => f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); var resultC = await Server.Config.NexusFileInfos.Connect().DeleteManyAsync(f => f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date); return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount; }); Utils.Log($"Purged {collected.Sum()} cache entries"); } return "Done"; } private string ListCache(object arg) { Utils.Log($"{DateTime.Now} - List Cache"); return String.Join("", Directory.EnumerateFiles(NexusApiClient.LocalCacheDir) .Select(f => new FileInfo(f)) .OrderByDescending(fi => fi.LastWriteTime) .Select(fi => { var decoded = Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(fi.Name).FromHex()); return $"{fi.LastWriteTime} \t {fi.Length.ToFileSizeString()} \t {decoded} \n"; })); } private async Task HandleModInfo(dynamic arg) { Utils.Log($"{DateTime.Now} - Mod Info - {arg.GameName}/{arg.ModID}/"); string gameName = arg.GameName; string modId = arg.ModId; var result = await Server.Config.NexusModInfos.Connect() .FindOneAsync(info => info.Game == gameName && info.ModId == modId); string method = "CACHED"; if (result == null) { var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); var path = $"/v1/games/{gameName}/mods/{modId}.json"; var body = await api.Get(path); result = new NexusCacheData { Data = body, Path = path, Game = gameName, ModId = modId }; try { await Server.Config.NexusModInfos.Connect().InsertOneAsync(result); } catch (MongoWriteException) { } method = "NOT_CACHED"; } Response response = result.Data.ToJSON(); response.Headers.Add("WABBAJACK_CACHE_FROM", method); response.ContentType = "application/json"; return response; } private async Task HandleFileID(dynamic arg) { Utils.Log($"{DateTime.Now} - File Info - {arg.GameName}/{arg.ModID}/{arg.FileID}"); string gameName = arg.GameName; string modId = arg.ModId; string fileId = arg.FileId; var result = await Server.Config.NexusFileInfos.Connect() .FindOneAsync(info => info.Game == gameName && info.ModId == modId && info.FileId == fileId); string method = "CACHED"; if (result == null) { var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); var path = $"/v1/games/{gameName}/mods/{modId}/files/{fileId}.json"; var body = await api.Get(path); result = new NexusCacheData { Data = body, Path = path, Game = gameName, ModId = modId, FileId = fileId }; try { await Server.Config.NexusFileInfos.Connect().InsertOneAsync(result); } catch (MongoWriteException) { } method = "NOT_CACHED"; } Response response = result.Data.ToJSON(); response.Headers.Add("WABBAJACK_CACHE_FROM", method); response.ContentType = "application/json"; return response; } private async Task HandleGetFiles(dynamic arg) { Utils.Log($"{DateTime.Now} - Mod Files - {arg.GameName} {arg.ModID}"); string gameName = arg.GameName; string modId = arg.ModId; var result = await Server.Config.NexusModFiles.Connect() .FindOneAsync(info => info.Game == gameName && info.ModId == modId); string method = "CACHED"; if (result == null) { var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); var path = $"/v1/games/{gameName}/mods/{modId}/files.json"; var body = await api.Get(path); result = new NexusCacheData { Data = body, Path = path, Game = gameName, ModId = modId }; try { await Server.Config.NexusModFiles.Connect().InsertOneAsync(result); } catch (MongoWriteException) { } method = "NOT_CACHED"; } Response response = result.Data.ToJSON(); response.Headers.Add("WABBAJACK_CACHE_FROM", method); response.ContentType = "application/json"; return response; } private async Task HandleCacheCall(dynamic arg) { try { string param = (string)arg.request; var url = new Uri(Encoding.UTF8.GetString(param.FromHex())); var client = new HttpClient(); var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 8080, Scheme = "http"}; client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]); return await client.GetStringAsync(builder.Uri.ToString()); } catch (Exception ex) { Utils.Log(ex.ToString()); return "ERROR"; } } private async Task HandleIngestCache(dynamic arg) { int count = 0; int failed = 0; using (var queue = new WorkQueue()) { await Directory.EnumerateFiles(Path.Combine(Server.Config.Settings.TempDir, (string)arg.Folder)).PMap(queue, async file => { Utils.Log($"Ingesting {file}"); if (!file.EndsWith(".json")) return; var fileInfo = new FileInfo(file); count++; var url = new Url( Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(file).FromHex())); var split = url.Path.Split(new[] {'/'}, StringSplitOptions.RemoveEmptyEntries); try { switch (split.Length) { case 5 when split[3] == "mods": { var body = file.FromJSON(); var payload = new NexusCacheData(); payload.Data = body; payload.Game = split[2]; payload.Path = url.Path; payload.ModId = body.mod_id; payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc; try { await Server.Config.NexusModInfos.Connect().InsertOneAsync(payload); } catch (MongoWriteException ex) { } break; } case 6 when split[5] == "files.json": { var body = file.FromJSON(); var payload = new NexusCacheData(); payload.Path = url.Path; payload.Data = body; payload.Game = split[2]; payload.ModId = split[4]; payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc; try { await Server.Config.NexusModFiles.Connect().InsertOneAsync(payload); } catch (MongoWriteException ex) { } break; } case 7 when split[5] == "files": { var body = file.FromJSON(); var payload = new NexusCacheData(); payload.Data = body; payload.Path = url.Path; payload.Game = split[2]; payload.FileId = Path.GetFileNameWithoutExtension(split[6]); payload.ModId = split[4]; payload.LastCheckedUTC = fileInfo.LastWriteTimeUtc; try { await Server.Config.NexusFileInfos.Connect().InsertOneAsync(payload); } catch (MongoWriteException ex) { } break; } } } catch (Exception ex) { failed++; } }); } return $"Inserted {count} caches, {failed} failed"; } } }