Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rework Nexus API caching logic to use build server cache

Browse Source
This commit is contained in:
Timothy Baldridge 2019-11-04 22:08:47 -07:00
parent c43bcc7d89
commit 3d9cf4cc65
5 changed files with 121 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.Common/Consts.cs
View File

@ -81,6 +81,8 @@ namespace Wabbajack.Common
}
}
public static string WabbajackCacheLocation = "http://build.wabbajack.org/nexus_api_cache/";
public static TimeSpan NexusCacheExpiry = new TimeSpan(1, 0, 0, 0);
}
}

3
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -56,7 +56,8 @@ namespace Wabbajack.Lib.Downloaders
return;
}
var updated = client.GetModsUpdatedSince(Game.Skyrim,DateTime.Now - TimeSpan.FromDays(30));
client.ClearUpdatedModsInCache();
//var updated = client.GetModsUpdatedSince(Game.Skyrim,DateTime.Now - TimeSpan.FromDays(30));
}
public class State : AbstractDownloadState

102
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -197,16 +197,33 @@ namespace Wabbajack.Lib.NexusApi
private T GetCached<T>(string url)
{
var code = Encoding.UTF8.GetBytes(url).ToHex();
var cache_file = Path.Combine(Consts.NexusCacheDirectory, code + ".json");
if (File.Exists(cache_file) && DateTime.Now - File.GetLastWriteTime(cache_file) < Consts.NexusCacheExpiry)
var code = Encoding.UTF8.GetBytes(url).ToHex() + ".json";
if (UseLocalCache)
{
return cache_file.FromJSON<T>();
if (!Directory.Exists(LocalCacheDir))
Directory.CreateDirectory(LocalCacheDir);
var cache_file = Path.Combine(LocalCacheDir, code);
if (File.Exists(cache_file))
{
return cache_file.FromJSON<T>();
}
var result = Get<T>(url);
result.ToJSON(cache_file);
return result;
}
try
{
return Get<T>(Consts.WabbajackCacheLocation + code);
}
catch (Exception)
{
return Get<T>(url);
}
var result = Get<T>(url);
result.ToJSON(cache_file);
return result;
}
@ -318,31 +335,68 @@ namespace Wabbajack.Lib.NexusApi
public long latest_mod_activity;
}
public IEnumerable<long> GetModsUpdatedSince(Game game, DateTime since)
private static bool? _useLocalCache;
public static bool UseLocalCache
{
var result =
Get<List<UpdatedMod>>(
$"https://api.nexusmods.com/v1/games/{GameRegistry.Games[game].NexusName}/mods/updated.json?period=1m");
return result.Where(r => r.latest_file_update.AsUnixTime() >= since)
.Select(m => m.mod_id)
.ToList();
get
{
if (_useLocalCache == null) return LocalCacheDir != null;
return _useLocalCache ?? false;
}
set => _useLocalCache = value;
}
public static void ClearCacheFor(HashSet<(Game, long)> mods)
private static string _localCacheDir;
public static string LocalCacheDir
{
Directory.EnumerateFiles(Consts.NexusCacheDirectory, "*.json")
.PMap(f =>
get
{
if (_localCacheDir == null)
_localCacheDir = Environment.GetEnvironmentVariable("NEXUSCACHEDIR");
return _localCacheDir;
}
set => _localCacheDir = value;
}
public void ClearUpdatedModsInCache()
{
if (!UseLocalCache) return;
var purge = GameRegistry.Games.Values
.Where(game => game.NexusName != null)
.Select(game => new
{
game = game,
mods = Get<List<UpdatedMod>>(
$"https://api.nexusmods.com/v1/games/{game.NexusName}/mods/updated.json?period=1m")
})
.SelectMany(r => r.mods.Select(mod => new {game = r.game,
mod = mod}))
.ToList();
Utils.Log($"Found {purge.Count} updated mods in the last month");
var to_purge = Directory.EnumerateFiles(LocalCacheDir, "*.json")
.Select(f =>
{
Utils.Status("Cleaning Nexus cache for");
var filename = Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(f).FromHex());
foreach (var (game, modid) in mods)
var uri = new Uri(Encoding.UTF8.GetString(Path.GetFileNameWithoutExtension(f).FromHex()));
var parts = uri.PathAndQuery.Split('/', '.').ToHashSet();
var found = purge.FirstOrDefault(p => parts.Contains(p.game.NexusName) && parts.Contains(p.mod.mod_id.ToString()));
if (found != null)
{
if (filename.Contains(GameRegistry.Games[game].NexusName) &&
(filename.Contains("\\" + modid + "\\") ||
filename.Contains("\\" + modid + ".")))
File.Delete(f);
var should_remove = File.GetLastWriteTimeUtc(f) <= found.mod.latest_file_update.AsUnixTime();
return (should_remove, f);
}
});
return (false, f);
})
.Where(p => p.Item1)
.ToList();
Utils.Log($"Purging {to_purge.Count} cache entries");
to_purge.PMap(f => File.Delete(f.f));
}
}

5
Wabbajack.Test.ListValidation/ListValidation.cs
View File

@ -7,6 +7,7 @@ using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.Test.ListValidation
{
@ -19,8 +20,8 @@ namespace Wabbajack.Test.ListValidation
Directory.CreateDirectory(Consts.ModListDownloadFolder);
Utils.SetLoggerFn(s => TestContext.WriteLine(s));
WorkQueue.Init();
Utils.ToJSON("ff");
var api = new NexusApiClient();
api.ClearUpdatedModsInCache();
}
private TestContext testContextInstance;

53
Wabbajack.Test/DownloaderTests.cs
View File

@ -7,6 +7,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
using File = Alphaleonis.Win32.Filesystem.File;
@ -15,6 +16,12 @@ namespace Wabbajack.Test
[TestClass]
public class DownloaderTests
{
[TestInitialize]
public void Setup()
{
WorkQueue.Init();
}
[TestMethod]
public void TestAllPrepares()
{
@ -145,9 +152,9 @@ namespace Wabbajack.Test
public void MediaFireDownload()
{
var ini = @"[General]
directURL=http://www.mediafire.com/file/agiqzm1xwebczpx/WABBAJACK_TEST_FILE.txt";
directURL=http://www.mediafire.com/file/agiqzm1xwebczpx/WABBAJACK_TEST_FILE.txt";
var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString());
var state = (AbstractDownloadState) DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.IsNotNull(state);
@ -155,44 +162,56 @@ namespace Wabbajack.Test
"http://www.mediafire.com/file/agiqzm1xwebczpx/WABBAJACK_TEST_FILE.txt");
Assert.AreEqual("http://www.mediafire.com/file/agiqzm1xwebczpx/WABBAJACK_TEST_FILE.txt",
((MediaFireDownloader.State)url_state).Url);
((MediaFireDownloader.State) url_state).Url);
var converted = state.ViaJSON();
Assert.IsTrue(converted.Verify());
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> { "http://www.mediafire.com/file/agiqzm1xwebczpx/" } }));
Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string>() }));
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist
{AllowedPrefixes = new List<string> {"http://www.mediafire.com/file/agiqzm1xwebczpx/"}}));
Assert.IsFalse(converted.IsWhitelisted(new ServerWhitelist {AllowedPrefixes = new List<string>()}));
converted.Download(new Archive { Name = "Media Fire Test.txt" }, filename);
converted.Download(new Archive {Name = "Media Fire Test.txt"}, filename);
Assert.AreEqual(File.ReadAllText(filename), "Cheese for Everyone!");
}
[TestMethod]
public void NexusDownload()
{
var ini = @"[General]
var old_val = NexusApiClient.UseLocalCache;
try
{
NexusApiClient.UseLocalCache = false;
var ini = @"[General]
gameName=SkyrimSE
modID = 12604
fileID=35407";
var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString());
var state = (AbstractDownloadState)DownloadDispatcher.ResolveArchive(ini.LoadIniString());
Assert.IsNotNull(state);
Assert.IsNotNull(state);
var converted = state.ViaJSON();
Assert.IsTrue(converted.Verify());
// Exercise the cache code
Assert.IsTrue(converted.Verify());
var filename = Guid.NewGuid().ToString();
var converted = state.ViaJSON();
Assert.IsTrue(converted.Verify());
// Exercise the cache code
Assert.IsTrue(converted.Verify());
var filename = Guid.NewGuid().ToString();
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> () }));
Assert.IsTrue(converted.IsWhitelisted(new ServerWhitelist { AllowedPrefixes = new List<string> () }));
converted.Download(new Archive { Name = "SkyUI.7z" }, filename);
converted.Download(new Archive { Name = "SkyUI.7z" }, filename);
Assert.AreEqual(filename.FileHash(), "dF2yafV2Oks=");
Assert.AreEqual(filename.FileHash(), "dF2yafV2Oks=");
}
finally
{
NexusApiClient.UseLocalCache = old_val;
}
}
[TestMethod]