Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement MongoDB based mod cleaning

Browse Source
This commit is contained in:
Timothy Baldridge 2020-01-01 10:11:48 -07:00
parent 717ad8c70a
commit 55f90bd485
2 changed files with 55 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
Wabbajack.CacheServer/NexusCacheModule.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
@ -7,6 +8,7 @@ using System.Text;
using System.Threading.Tasks;
using MongoDB.Driver;
using Nancy;
using Newtonsoft.Json;
using Wabbajack.CacheServer.DTOs;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
@ -28,10 +30,62 @@ namespace Wabbajack.CacheServer
Get("/nexus_api_cache/ingest/{Folder}", HandleIngestCache);
}
class UpdatedMod
{
public long mod_id;
public long latest_file_update;
public long latest_mod_activity;
}
public async Task<object> UpdateCache(object arg)
{
var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault());
await api.ClearUpdatedModsInCache();
var gameTasks = GameRegistry.Games.Values
.Where(game => game.NexusName != null)
.Select(async game =>
{
return (game,
mods: await api.Get<List<UpdatedMod>>(
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m"));
})
.Select(async rTask =>
{
var (game, mods) = await rTask;
return mods.Select(mod => new { game = game, mod = mod });
}).ToList();
Utils.Log($"Getting update lits for {gameTasks.Count} games");
var purge = (await Task.WhenAll(gameTasks))
.SelectMany(i => i)
.ToList();
Utils.Log($"Found {purge.Count} updated mods in the last month");
using (var queue = new WorkQueue())
{
var collected = await purge.Select(d =>
{
var a = d.mod.latest_file_update.AsUnixTime();
// Mod activity could hide files
var b = d.mod.latest_mod_activity.AsUnixTime();
return new {Game = d.game.NexusName, Date = (a > b ? a : b), ModId = d.mod.mod_id.ToString()};
}).PMap(queue, async t =>
{
var resultA = await Server.Config.NexusModInfos.Connect().DeleteManyAsync(f =>
f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
var resultB = await Server.Config.NexusModFiles.Connect().DeleteManyAsync(f =>
f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
var resultC = await Server.Config.NexusFileInfos.Connect().DeleteManyAsync(f =>
f.Game == t.Game && f.ModId == t.ModId && f.LastCheckedUTC <= t.Date);
return resultA.DeletedCount + resultB.DeletedCount + resultC.DeletedCount;
});
Utils.Log($"Purged {collected.Sum()} cache entries");
}
return "Done";
}
@ -175,7 +229,6 @@ namespace Wabbajack.CacheServer
string param = (string)arg.request;
var url = new Uri(Encoding.UTF8.GetString(param.FromHex()));
Utils.Log($"{DateTime.Now} - Not Cached - {url}");
var client = new HttpClient();
var builder = new UriBuilder(url) {Host = "localhost", Port = Request.Url.Port ?? 8080, Scheme = "http"};
client.DefaultRequestHeaders.Add("apikey", Request.Headers["apikey"]);

90
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -353,13 +353,6 @@ namespace Wabbajack.Lib.NexusApi
public string URI { get; set; }
}
private class UpdatedMod
{
public long mod_id;
public long latest_file_update;
public long latest_mod_activity;
}
private static bool? _useLocalCache;
public static MethodInfo CacheMethod { get; set; }
@ -374,88 +367,5 @@ namespace Wabbajack.Lib.NexusApi
}
set => _localCacheDir = value;
}
public async Task ClearUpdatedModsInCache()
{
if (NexusApiClient.CacheMethod == null) return;
using (var queue = new WorkQueue())
{
var invalid_json = (await Directory.EnumerateFiles(LocalCacheDir, "*.json")
.PMap(queue, f =>
{
var s = JsonSerializer.Create();
try
{
using (var tr = File.OpenText(f))
s.Deserialize(new JsonTextReader(tr));
return null;
}
catch (JsonReaderException)
{
return f;
}
})).Where(f => f != null).ToList();
Utils.Log($"Found {invalid_json.Count} bad json files");
foreach (var file in invalid_json)
File.Delete(file);
}
var gameTasks = GameRegistry.Games.Values
.Where(game => game.NexusName != null)
.Select(async game =>
{
return (game,
mods: await Get<List<UpdatedMod>>(
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m"));
})
.Select(async rTask =>
{
var (game, mods) = await rTask;
return mods.Select(mod => new { game = game, mod = mod });
});
var purge = (await Task.WhenAll(gameTasks))
.SelectMany(i => i)
.ToList();
Utils.Log($"Found {purge.Count} updated mods in the last month");
using (var queue = new WorkQueue())
{
var to_purge = (await Directory.EnumerateFiles(LocalCacheDir, "*.json")
.PMap(queue, f =>
{
Utils.Status("Cleaning Nexus cache for");
var uri = new Uri(Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(f).FromHex()));
var parts = uri.PathAndQuery.Split('/', '.').ToHashSet();
var found = purge.FirstOrDefault(p =>
parts.Contains(p.game.NexusName) && parts.Contains(p.mod.mod_id.ToString()));
if (found != null)
{
var a = found.mod.latest_file_update.AsUnixTime();
// Mod activity could hide files
var b = found.mod.latest_mod_activity.AsUnixTime();
var should_remove = File.GetLastWriteTimeUtc(f) <= (a > b ? a : b);
return (should_remove, f);
}
// ToDo
// Can improve to not read the entire file to see if it starts with null
if (File.ReadAllText(f).StartsWith("null"))
return (true, f);
return (false, f);
}))
.Where(p => p.Item1)
.ToList();
Utils.Log($"Purging {to_purge.Count} cache entries");
await to_purge.PMap(queue, f =>
{
var uri = new Uri(Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(f.f).FromHex()));
Utils.Log($"Purging {uri}");
File.Delete(f.f);
});
}
}
}
}