Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Slightly improved nexus caching code

Browse Source
This commit is contained in:
Timothy Baldridge 2020-05-14 16:21:56 -06:00
parent 23fecce38b
commit 9f9becf19f
12 changed files with 183 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
Wabbajack.Common/ConcurrentHashSet.cs Normal file
View File

@ -0,0 +1,30 @@
using System.Collections.Generic;
namespace Wabbajack.Common
{
public class ConcurrentHashSet<T> where T : notnull
{
private Dictionary<T, bool> _inner;
public ConcurrentHashSet()
{
_inner = new Dictionary<T, bool>();
}
public ConcurrentHashSet(IEnumerable<T> input)
{
_inner = new Dictionary<T, bool>();
foreach (var itm in input)
Add(itm);
}
public bool Contains(T key)
{
return _inner.ContainsKey(key);
}
public void Add(T key)
{
_inner[key] = true;
}
}
}

4
Wabbajack.Common/Paths.cs
View File

@ -77,7 +77,7 @@ namespace Wabbajack.Common
return;
}
throw new InvalidDataException("Absolute path must be absolute");
throw new InvalidDataException($"Absolute path must be absolute, got {_path}");
}
public string Normalize()
@ -487,7 +487,7 @@ namespace Wabbajack.Common
{
if (Path.IsPathRooted(_path))
{
throw new InvalidDataException("Cannot create relative path from absolute path string");
throw new InvalidDataException($"Cannot create relative path from absolute path string, got {_path}");
}
}

42
Wabbajack.Server.Test/NexusCacheTests.cs
View File

@ -3,8 +3,11 @@ using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
@ -54,5 +57,44 @@ namespace Wabbajack.BuildServer.Test
Assert.Single(modInfoResponse.files);
Assert.Equal("blerg", modInfoResponse.files.First().file_name);
}
[Fact]
public async Task CanQueryAndFindNexusModfilesSlow()
{
var startTime = DateTime.UtcNow;
var sql = Fixture.GetService<SqlService>();
var validator = Fixture.GetService<ListValidator>();
await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow);
await sql.DeleteNexusModInfosUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow);
var result = await validator.SlowNexusModStats(new ValidationData(),
new NexusDownloader.State {Game = Game.SkyrimSpecialEdition, ModID = 1137, FileID = 121449});
Assert.Equal(ArchiveStatus.Valid, result);
var gameId = Game.SkyrimSpecialEdition.MetaData().NexusGameId;
var hs = await sql.AllNexusFiles();
var found = hs.FirstOrDefault(h =>
h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449);
Assert.True(found != default);
Assert.True(found.LastChecked > startTime && found.LastChecked < DateTime.UtcNow);
// Delete with exactly the same date, shouldn't clear out the record
await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, found.LastChecked);
var hs2 = await sql.AllNexusFiles();
var found2 = hs2.FirstOrDefault(h =>
h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449);
Assert.True(found != default);
Assert.True(found2.LastChecked == found.LastChecked);
// Delete all the records, it should now be gone
await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow);
var hs3 = await sql.AllNexusFiles();
Assert.DoesNotContain(hs3, f => f.NexusGameId == gameId && f.ModId == 1137);
}
}
}

16
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

@ -535,6 +535,22 @@ CREATE UNIQUE NONCLUSTERED INDEX [IX_IndexedFile_By_SHA256] ON [dbo].[IndexedFil
[Sha256] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
GO
/****** Object: Table [dbo].[NexusModFilesSlow] Script Date: 5/14/2020 2:23:15 PM ******/
CREATE TABLE [dbo].[NexusModFilesSlow](
[GameId] [bigint] NOT NULL,
[FileId] [bigint] NOT NULL,
[ModId] [bigint] NOT NULL,
[LastChecked] [datetime] NOT NULL,
CONSTRAINT [PK_NexusModFilesSlow] PRIMARY KEY CLUSTERED
(
[GameId] ASC,
[FileId] ASC,
[ModId] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY]
GO
/****** Object: StoredProcedure [dbo].[MergeAllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/
SET ANSI_NULLS ON
GO

2
Wabbajack.Server/DTOs/ArchiveStatus.cs
View File

@ -1,6 +1,6 @@
namespace Wabbajack.Server.DTOs
{
enum ArchiveStatus
public enum ArchiveStatus
{
Valid,
InValid,

4
Wabbajack.Server/DTOs/ValidationData.cs
View File

@ -7,8 +7,10 @@ namespace Wabbajack.Server.DTOs
{
public class ValidationData
{
public HashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; }
public ConcurrentHashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; } = new ConcurrentHashSet<(long Game, long ModId, long FileId)>();
public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; }
public List<(ModlistMetadata Metadata, ModList ModList)> ModLists { get; set; }
public ConcurrentHashSet<(Game Game, long ModId)> SlowQueriedFor { get; set; } = new ConcurrentHashSet<(Game Game, long ModId)>();
}
}

26
Wabbajack.Server/DataLayer/Nexus.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Dapper;
using Newtonsoft.Json;
using Wabbajack.Common;
@ -29,6 +30,11 @@ namespace Wabbajack.Server.DataLayer
@"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
deleted += await conn.ExecuteScalarAsync<long>(
@"DELETE FROM dbo.NexusModFilesSlow WHERE GameId = @Game AND ModID = @ModId AND LastChecked < @Date
SELECT @@ROWCOUNT AS Deleted",
new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date});
return deleted;
}
@ -78,7 +84,25 @@ namespace Wabbajack.Server.DataLayer
LastChecked = lastCheckedUtc,
Data = JsonConvert.SerializeObject(data)
});
}
public async Task AddNexusModFileSlow(Game game, long modId, long fileId, DateTime lastCheckedUtc)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"MERGE dbo.NexusModFilesSlow AS Target
USING (SELECT @GameId GameId, @ModId ModId, @LastChecked LastChecked, @FileId FileId) AS Source
ON Target.GameId = Source.GameId AND Target.ModId = Source.ModId AND Target.FileId = Source.FileId
WHEN MATCHED THEN UPDATE SET Target.LastChecked = @LastChecked
WHEN NOT MATCHED THEN INSERT (GameId, ModId, LastChecked, FileId) VALUES (@GameId, @ModId, @LastChecked, FileId);",
new
{
GameId = game.MetaData().NexusGameId,
ModId = modId,
FileId = fileId,
LastChecked = lastCheckedUtc,
});
}
public async Task<NexusApiClient.GetModFilesResponse> GetModFiles(Game game, long modId)

20
Wabbajack.Server/DataLayer/ValidationData.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
@ -18,7 +19,7 @@ namespace Wabbajack.Server.DataLayer
var modLists = AllModLists();
return new ValidationData
{
NexusFiles = await nexusFiles,
NexusFiles = new ConcurrentHashSet<(long Game, long ModId, long FileId)>((await nexusFiles).Select(f => (f.NexusGameId, f.ModId, f.FileId))),
ArchiveStatus = await archiveStatus,
ModLists = await modLists,
};
@ -33,14 +34,17 @@ namespace Wabbajack.Server.DataLayer
return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3);
}
public async Task<HashSet<(long NexusGameId, long ModId, long FileId)>> AllNexusFiles()
public async Task<HashSet<(long NexusGameId, long ModId, long FileId, DateTime LastChecked)>> AllNexusFiles()
{
await using var conn = await Open();
var results = await conn.QueryAsync<(long, long, long)>(@"SELECT Game, ModId, p.file_id
FROM [NexusModFiles] files
CROSS APPLY
OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p
WHERE p.category is not null");
var results = await conn.QueryAsync<(long, long, long, DateTime)>(@"SELECT Game, ModId, p.file_id, LastChecked
FROM [NexusModFiles] files
CROSS APPLY
OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p
WHERE p.category is not null
UNION
SELECT GameId, ModId, FileId, LastChecked FROM dbo.NexusModFilesSlow
");
return results.ToHashSet();
}

2
Wabbajack.Server/GlobalInformation.cs
View File

@ -5,7 +5,7 @@ namespace Wabbajack.Server
public class GlobalInformation
{
public TimeSpan NexusRSSPollRate = TimeSpan.FromMinutes(1);
public TimeSpan NexusAPIPollRate = TimeSpan.FromHours(2);
public TimeSpan NexusAPIPollRate = TimeSpan.FromHours(24);
public DateTime LastNexusSyncUTC { get; set; }
public TimeSpan TimeSinceLastNexusSync => DateTime.UtcNow - LastNexusSyncUTC;
}

5
Wabbajack.Server/Services/ArchiveDownloader.cs
View File

@ -30,8 +30,9 @@ namespace Wabbajack.Server.Services
while (true)
{
var (daily, hourly) = await _nexusClient.GetRemainingApiCalls();
bool ignoreNexus = hourly < 25;
//var (daily, hourly) = await _nexusClient.GetRemainingApiCalls();
//bool ignoreNexus = hourly < 25;
var ignoreNexus = true;
if (ignoreNexus)
_logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})");
else

49
Wabbajack.Server/Services/ListValidator.cs
View File

@ -1,9 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc.Filters;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Org.BouncyCastle.Crypto.Digests;
using RocksDbSharp;
using Wabbajack.BuildServer;
using Wabbajack.Common;
@ -91,7 +94,7 @@ namespace Wabbajack.Server.Services
nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)):
return (archive, ArchiveStatus.Valid);
case NexusDownloader.State ns:
return (archive, await FastNexusModStats(ns));
return (archive, await SlowNexusModStats(data, ns));
case ManualDownloader.State _:
return (archive, ArchiveStatus.Valid);
default:
@ -106,6 +109,50 @@ namespace Wabbajack.Server.Services
}
}
}
private readonly AsyncLock _slowQueryLock = new AsyncLock();
public async Task<ArchiveStatus> SlowNexusModStats(ValidationData data, NexusDownloader.State ns)
{
var gameId = ns.Game.MetaData().NexusGameId;
using var _ = await _slowQueryLock.WaitAsync();
_logger.Log(LogLevel.Warning, $"Slow querying for {ns.Game} {ns.ModID} {ns.FileID}");
if (data.NexusFiles.Contains((gameId, ns.ModID, ns.FileID)))
return ArchiveStatus.Valid;
if (data.SlowQueriedFor.Contains((ns.Game, ns.ModID)))
return ArchiveStatus.InValid;
var queryTime = DateTime.UtcNow;
var regex = new Regex("(?<=[?;&]file_id\\=)\\d+");
var client = new Common.Http.Client();
var result =
await client.GetHtmlAsync(
$"https://www.nexusmods.com/{ns.Game.MetaData().NexusName}/mods/{ns.ModID}/?tab=files");
var fileIds = result.DocumentNode.Descendants()
.Select(f => f.GetAttributeValue("href", ""))
.Select(f =>
{
var match = regex.Match(f);
return !match.Success ? null : match.Value;
})
.Where(m => m != null)
.Select(m => long.Parse(m))
.Distinct()
.ToList();
_logger.Log(LogLevel.Warning, $"Slow queried {fileIds.Count} files");
foreach (var id in fileIds)
{
await _sql.AddNexusModFileSlow(ns.Game, ns.ModID, id, queryTime);
data.NexusFiles.Add((gameId, ns.ModID, id));
}
return fileIds.Contains(ns.FileID) ? ArchiveStatus.Valid : ArchiveStatus.InValid;
}
private async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns)
{

11
Wabbajack.Server/Services/NexusPoll.cs
View File

@ -48,17 +48,6 @@ namespace Wabbajack.Server.Services
if (totalPurged > 0)
_logger.Log(LogLevel.Information, $"Purged {totalPurged} cache items {result.Game} {result.ModId} {result.TimeStamp}");
if (await _sql.GetNexusModInfoString(result.Game, result.ModId) != null) continue;
// Lazily create the client
client ??= await NexusApiClient.Get();
// Cache the info
var files = await client.GetModFiles(result.Game, result.ModId, false);
await _sql.AddNexusModFiles(result.Game, result.ModId, result.TimeStamp, files);
var modInfo = await client.GetModInfo(result.Game, result.ModId, useCache: false);
await _sql.AddNexusModInfo(result.Game, result.ModId, result.TimeStamp, modInfo);
updated++;
}
catch (Exception ex)