Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Server side fixes for the new Nexus API

Browse Source
This commit is contained in:
Timothy Baldridge 2021-06-24 17:01:03 -06:00
parent 14b15adf2a
commit ab49b08f19
13 changed files with 150 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Wabbajack.Lib/Downloaders/AbstractDownloadState.cs
View File

@ -27,6 +27,7 @@ namespace Wabbajack.Lib.Downloaders
public static List<Type> KnownSubTypes = new List<Type>
{
typeof(DeprecatedLoversLabDownloader.State),
typeof(DeprecatedVectorPlexusDownloader),
typeof(HTTPDownloader.State),
typeof(GameFileSourceDownloader.State),
typeof(GoogleDriveDownloader.State),

21
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -74,8 +74,12 @@ namespace Wabbajack.Lib.Downloaders
}
catch (Exception)
{
Utils.Error($"Error getting mod info for Nexus mod with {general.modID}");
throw;
return new State
{
Game = GameRegistry.GetByFuzzyName((string)general.gameName).Game,
ModID = long.Parse(general.modID),
FileID = long.Parse(general.fileID),
};
}
try
@ -220,15 +224,8 @@ namespace Wabbajack.Lib.Downloaders
var nclient = DownloadDispatcher.GetInstance<NexusDownloader>();
await nclient.Prepare();
var client = nclient.Client!;
var modInfo = await client.GetModInfo(Game, ModID);
if (!modInfo.available) return false;
var modFiles = await client.GetModFiles(Game, ModID);
var found = modFiles.files
.FirstOrDefault(file => file.file_id == FileID && file.category_name != null);
return found != null;
var file = await client.GetModFile(Game, ModID, FileID);
return file?.category_name != null;
}
catch (Exception ex)
{
@ -245,7 +242,7 @@ namespace Wabbajack.Lib.Downloaders
public override string GetManifestURL(Archive a)
{
return $"http://nexusmods.com/{Game.MetaData().NexusName}/mods/{ModID}";
return $"https://www.nexusmods.com/{Game.MetaData().NexusName}/mods/{ModID}/?tab=files&file_id={FileID}";
}
public override string[] GetMetaIni()

1
Wabbajack.Lib/NexusApi/INexusApi.cs
View File

@ -8,6 +8,7 @@ namespace Wabbajack.Lib.NexusApi
{
public Task<string> GetNexusDownloadLink(NexusDownloader.State archive);
public Task<NexusApiClient.GetModFilesResponse> GetModFiles(Game game, long modid, bool useCache = true);
public Task<NexusFileInfo> GetModFile(Game game, long modid, long fileId, bool useCache = true);
public Task<ModInfo> GetModInfo(Game game, long modId, bool useCache = true);
public Task<UserStatus> GetUserStatus();

7
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -357,6 +357,13 @@ namespace Wabbajack.Lib.NexusApi
return result;
}
public async Task<NexusFileInfo> GetModFile(Game game, long modId, long fileId, bool useCache = true)
{
var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/{modId}/files/{fileId}.json";
var result = useCache ? await GetCached<NexusFileInfo>(url) : await Get<NexusFileInfo>(url);
return result;
}
public async Task<List<MD5Response>> GetModInfoFromMD5(Game game, string md5Hash)
{
var url = $"https://api.nexusmods.com/v1/games/{game.MetaData().NexusName}/mods/md5_search/{md5Hash}.json";

10
Wabbajack.Server.Test/NexusCacheTests.cs
View File

@ -103,23 +103,13 @@ namespace Wabbajack.BuildServer.Test
h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449);
Assert.True(found != default);
Assert.True(found.LastChecked > startTime && found.LastChecked < DateTime.UtcNow);
// Delete with exactly the same date, shouldn't clear out the record
await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, found.LastChecked);
var hs2 = await sql.AllNexusFiles();
var found2 = hs2.FirstOrDefault(h =>
h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449);
Assert.True(found != default);
Assert.True(found2.LastChecked == found.LastChecked);
// Delete all the records, it should now be gone
await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow);
var hs3 = await sql.AllNexusFiles();
Assert.DoesNotContain(hs3, f => f.NexusGameId == gameId && f.ModId == 1137);
}

23
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

@ -806,6 +806,29 @@ CREATE TABLE [dbo].[NexusModFiles](
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
/****** Object: Table [dbo].[NexusModFile] Script Date: 6/24/2021 2:39:17 PM ******/
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[NexusModFile](
[Game] [int] NOT NULL,
[ModId] [bigint] NOT NULL,
[FileId] [bigint] NOT NULL,
[Data] [nvarchar](max) NOT NULL,
[LastChecked] [datetime] NOT NULL,
CONSTRAINT [PK_NexusModFile] PRIMARY KEY CLUSTERED
(
[Game] ASC,
[ModId] ASC,
[FileId] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
/****** Object: Table [dbo].[NexusModFilesSlow] Script Date: 3/9/2021 11:12:53 PM ******/
SET ANSI_NULLS ON
GO

2
Wabbajack.Server/Controllers/Heartbeat.cs
View File

@ -41,7 +41,7 @@ namespace Wabbajack.BuildServer.Controllers
}
private const int MAX_LOG_SIZE = 128;
private static List<string> Log = new List<string>();
private static List<string> Log = new();
private GlobalInformation _globalInformation;
private SqlService _sql;
private ILogger<Heartbeat> _logger;

37
Wabbajack.Server/Controllers/NexusCache.cs
View File

@ -126,6 +126,43 @@ namespace Wabbajack.BuildServer.Controllers
Response.Headers.Add("x-cache-result", method);
return result;
}
[HttpGet]
[Route("{GameName}/mods/{ModId}/files/{FileId}.json")]
public async Task<ActionResult<NexusFileInfo>> GetModFile(string GameName, long ModId, long FileId)
{
try
{
var game = GameRegistry.GetByFuzzyName(GameName).Game;
var result = await _sql.GetModFile(game, ModId, FileId);
string method = "CACHED";
if (result == null)
{
var api = await GetClient();
result = await api.GetModFile(game, ModId, FileId, false);
var date = result.uploaded_time;
date = date == default ? DateTime.UtcNow : date;
await _sql.AddNexusModFile(game, ModId, FileId, date, result);
method = "NOT_CACHED";
Interlocked.Increment(ref ForwardCount);
}
else
{
Interlocked.Increment(ref CachedCount);
}
Response.Headers.Add("x-cache-result", method);
return result;
}
catch (Exception ex)
{
_logger.LogInformation("Unable to find mod file {GameName} {ModId}, {FileId}", GameName, ModId, FileId);
return NotFound();
}
}
[HttpGet]
[Authorize(Roles ="Author")]

3
Wabbajack.Server/DTOs/ValidationData.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading.Tasks;
using Wabbajack.Common;
@ -9,7 +10,7 @@ namespace Wabbajack.Server.DTOs
{
public class ValidationData
{
public ConcurrentHashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; } = new ConcurrentHashSet<(long Game, long ModId, long FileId)>();
public Dictionary<(long Game, long ModId, long FileId), string> NexusFiles { get; set; } = new ();
public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; }
public List<ModlistMetadata> ModLists { get; set; }

27
Wabbajack.Server/DataLayer/Nexus.cs
View File

@ -195,5 +195,32 @@ namespace Wabbajack.Server.DataLayer
await tx.CommitAsync();
}
public async Task<NexusFileInfo> GetModFile(Game game, long modId, long fileId)
{
await using var conn = await Open();
var result = await conn.QueryFirstOrDefaultAsync<string>(
"SELECT Data FROM dbo.NexusModFile WHERE Game = @Game AND @ModId = ModId AND @FileId = FileId",
new {Game = game.MetaData().NexusGameId, ModId = modId, FileId = fileId});
return result == null ? null : JsonConvert.DeserializeObject<NexusFileInfo>(result);
}
public async Task AddNexusModFile(Game game, long modId, long fileId, DateTime lastCheckedUtc, NexusFileInfo data)
{
await using var conn = await Open();
await conn.ExecuteAsync(
@"INSERT INTO dbo.NexusModFile (Game, ModId, FileId, LastChecked, Data)
VALUES (@Game, @ModId, @FileId, @LastChecked, @Data)",
new
{
Game = game.MetaData().NexusGameId,
ModId = modId,
FileId = fileId,
LastChecked = lastCheckedUtc,
Data = JsonConvert.SerializeObject(data)
});
}
}
}

17
Wabbajack.Server/DataLayer/ValidationData.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
@ -14,14 +15,15 @@ namespace Wabbajack.Server.DataLayer
{
public async Task<ValidationData> GetValidationData()
{
var nexusFiles = AllNexusFiles();
var archiveStatus = AllModListArchivesStatus();
var modLists = AllModLists();
var mirrors = GetAllMirroredHashes();
var authoredFiles = AllAuthoredFiles();
var nexusFiles = await AllNexusFiles();
return new ValidationData
{
NexusFiles = new ConcurrentHashSet<(long Game, long ModId, long FileId)>((await nexusFiles).Select(f => (f.NexusGameId, f.ModId, f.FileId))),
NexusFiles = nexusFiles.ToDictionary(nf => (nf.NexusGameId, nf.ModId, nf.FileId), nf => nf.category),
ArchiveStatus = await archiveStatus,
ModLists = await modLists,
Mirrors = await mirrors,
@ -39,17 +41,10 @@ namespace Wabbajack.Server.DataLayer
return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3);
}
public async Task<HashSet<(long NexusGameId, long ModId, long FileId, DateTime LastChecked)>> AllNexusFiles()
public async Task<HashSet<(long NexusGameId, long ModId, long FileId, string category)>> AllNexusFiles()
{
await using var conn = await Open();
var results = await conn.QueryAsync<(long, long, long, DateTime)>(@"SELECT Game, ModId, p.file_id, LastChecked
FROM [NexusModFiles] files
CROSS APPLY
OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p
WHERE p.category is not null
UNION
SELECT GameId, ModId, FileId, LastChecked FROM dbo.NexusModFilesSlow
");
var results = await conn.QueryAsync<(long, long, long, string)>(@"SELECT Game, ModId, FileId, JSON_VALUE(Data, '$.category') FROM dbo.NexusModFile");
return results.ToHashSet();
}

104
Wabbajack.Server/Services/ListValidator.cs
View File

@ -102,6 +102,9 @@ namespace Wabbajack.Server.Services
await _sql.StartMirror((archive.Hash, reason));
return (archive, ArchiveStatus.Updating);
}
if (archive.State is NexusDownloader.State)
return (archive, result);
return await TryToHeal(data, archive, metadata);
}
@ -349,9 +352,9 @@ namespace Wabbajack.Server.Services
case GoogleDriveDownloader.State _:
// Disabled for now due to GDrive rate-limiting the build server
return (archive, ArchiveStatus.Valid);
case NexusDownloader.State nexusState when data.NexusFiles.Contains((
nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)):
return (archive, ArchiveStatus.Valid);
case NexusDownloader.State nexusState when data.NexusFiles.TryGetValue(
(nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID), out var category):
return (archive, category != null ? ArchiveStatus.Valid : ArchiveStatus.InValid);
case NexusDownloader.State ns:
return (archive, await FastNexusModStats(ns));
case ManualDownloader.State _:
@ -362,6 +365,10 @@ namespace Wabbajack.Server.Services
return (archive, ArchiveStatus.Valid);
case MediaFireDownloader.State _:
return (archive, ArchiveStatus.Valid);
case DeprecatedLoversLabDownloader.State _:
return (archive, ArchiveStatus.Valid);
case DeprecatedVectorPlexusDownloader.State _:
return (archive, ArchiveStatus.Valid);
default:
{
if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash),
@ -374,94 +381,45 @@ namespace Wabbajack.Server.Services
}
}
}
private AsyncLock _lock = new();
public async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns)
{
// Check if some other thread has added them
var mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID);
var files = await _sql.GetModFiles(ns.Game, ns.ModID);
var file = await _sql.GetModFile(ns.Game, ns.ModID, ns.FileID);
if (mod == null || files == null)
if (file == null)
{
// Acquire the lock
using var lck = await _lock.WaitAsync();
// Check again
mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID);
files = await _sql.GetModFiles(ns.Game, ns.ModID);
if (mod == null || files == null)
try
{
NexusApiClient nexusClient = await _nexus.GetClient();
var queryTime = DateTime.UtcNow;
_logger.Log(LogLevel.Information, "Found missing Nexus file info {Game} {ModID} {FileID}", ns.Game, ns.ModID, ns.FileID);
try
{
file = await nexusClient.GetModFile(ns.Game, ns.ModID, ns.FileID, false);
}
catch
{
file = new NexusFileInfo() {category_name = null};
}
try
{
NexusApiClient nexusClient = await _nexus.GetClient();
var queryTime = DateTime.UtcNow;
if (mod == null)
{
_logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
try
{
mod = await nexusClient.GetModInfo(ns.Game, ns.ModID, false);
}
catch (Exception ex)
{
Utils.Log("Exception in Nexus Validation " + ex);
mod = new ModInfo
{
mod_id = ns.ModID.ToString(),
game_id = ns.Game.MetaData().NexusGameId,
available = false
};
}
try
{
await _sql.AddNexusModInfo(ns.Game, ns.ModID, queryTime, mod);
}
catch (Exception)
{
// Could be a PK constraint failure
}
}
if (files == null)
{
_logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
try
{
files = await nexusClient.GetModFiles(ns.Game, ns.ModID, false);
}
catch
{
files = new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>()};
}
try
{
await _sql.AddNexusModFiles(ns.Game, ns.ModID, queryTime, files);
}
catch (Exception)
{
// Could be a PK constraint failure
}
}
await _sql.AddNexusModFile(ns.Game, ns.ModID, ns.FileID, queryTime, file);
}
catch (Exception)
{
return ArchiveStatus.InValid;
// Could be a PK constraint failure
}
}
catch (Exception)
{
return ArchiveStatus.InValid;
}
}
if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID))
return ArchiveStatus.Valid;
return ArchiveStatus.InValid;
return file?.category_name != null ? ArchiveStatus.Valid : ArchiveStatus.InValid;
}
}

5
Wabbajack.Test/DownloaderTests.cs
View File

@ -256,6 +256,11 @@ namespace Wabbajack.Test
await converted.Download(new Archive(state: null!) { Name = "SkyUI.7z" }, filename.Path);
Assert.Equal(Hash.FromBase64("dF2yafV2Oks="), await filename.Path.FileHashAsync());
// Verify that we can see a older file
var data = await (await NexusApiClient.Get()).GetModFile(Game.SkyrimSpecialEdition, 45221, 185392, useCache:false);
Assert.Equal("Smooth Combat - Non Combat Animation System 2.3", data.name);
}
[Fact]