Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Re-enable nexus archived file support again

Browse Source
This commit is contained in:
Timothy Baldridge 2021-07-04 15:37:55 -06:00
parent 89c0c651b5
commit 24d10305a4
7 changed files with 43 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -221,18 +221,12 @@ namespace Wabbajack.Lib.Downloaders
{ {
try try
{ {
var nclient = DownloadDispatcher.GetInstance<NexusDownloader>(); var nexusClient = DownloadDispatcher.GetInstance<NexusDownloader>();
await nclient.Prepare(); await nexusClient.Prepare();
var client = nclient.Client!; var client = nexusClient.Client!;
var modInfo = await client.GetModInfo(Game, ModID); var file = await client.GetModFile(Game, ModID, FileID);
if (!modInfo.available) return false; return file?.category_name != null;
var modFiles = await client.GetModFiles(Game, ModID);
var found = modFiles.files
.FirstOrDefault(file => file.file_id == FileID && file.category_name != null);
return found != null;
} }
catch (Exception ex) catch (Exception ex)
{ {

16
Wabbajack.Server.Test/NexusCacheTests.cs
View File

@ -102,23 +102,7 @@ namespace Wabbajack.BuildServer.Test
var found = hs.FirstOrDefault(h => var found = hs.FirstOrDefault(h =>
h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449); h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449);
Assert.True(found != default); Assert.True(found != default);
Assert.True(found.LastChecked > startTime && found.LastChecked < DateTime.UtcNow);
// Delete with exactly the same date, shouldn't clear out the record
await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, found.LastChecked);
var hs2 = await sql.AllNexusFiles();
var found2 = hs2.FirstOrDefault(h =>
h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449);
Assert.True(found != default);
Assert.True(found2.LastChecked == found.LastChecked);
// Delete all the records, it should now be gone
await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow);
var hs3 = await sql.AllNexusFiles();
Assert.DoesNotContain(hs3, f => f.NexusGameId == gameId && f.ModId == 1137);
} }

2
Wabbajack.Server/DTOs/ValidationData.cs
View File

@ -9,7 +9,7 @@ namespace Wabbajack.Server.DTOs
{ {
public class ValidationData public class ValidationData
{ {
public ConcurrentHashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; } = new ConcurrentHashSet<(long Game, long ModId, long FileId)>(); public Dictionary<(long Game, long ModId, long FileId), string> NexusFiles { get; set; } = new ();
public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; } public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; }
public List<ModlistMetadata> ModLists { get; set; } public List<ModlistMetadata> ModLists { get; set; }

1
Wabbajack.Server/DataLayer/Nexus.cs
View File

@ -118,6 +118,7 @@ namespace Wabbajack.Server.DataLayer
await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId}); await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId}); await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModPermissions WHERE ModId = @ModId", new {ModId = modId}); await conn.ExecuteAsync("DELETE FROM dbo.NexusModPermissions WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModFile WHERE ModId = @ModID", new {ModId = modId});
} }
public async Task<Dictionary<(Game, long), HTMLInterface.PermissionValue>> GetNexusPermissions() public async Task<Dictionary<(Game, long), HTMLInterface.PermissionValue>> GetNexusPermissions()

13
Wabbajack.Server/DataLayer/ValidationData.cs
View File

@ -23,7 +23,7 @@ namespace Wabbajack.Server.DataLayer
var nexusFiles = await AllNexusFiles(); var nexusFiles = await AllNexusFiles();
return new ValidationData return new ValidationData
{ {
NexusFiles = new ConcurrentHashSet<(long Game, long ModId, long FileId)>(nexusFiles.Select(f => (f.NexusGameId, f.ModId, f.FileId))), NexusFiles = nexusFiles.ToDictionary(nf => (nf.NexusGameId, nf.ModId, nf.FileId), nf => nf.category),
ArchiveStatus = await archiveStatus, ArchiveStatus = await archiveStatus,
ModLists = await modLists, ModLists = await modLists,
Mirrors = await mirrors, Mirrors = await mirrors,
@ -41,17 +41,10 @@ namespace Wabbajack.Server.DataLayer
return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3); return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3);
} }
public async Task<HashSet<(long NexusGameId, long ModId, long FileId, DateTime LastChecked)>> AllNexusFiles() public async Task<HashSet<(long NexusGameId, long ModId, long FileId, string category)>> AllNexusFiles()
{ {
await using var conn = await Open(); await using var conn = await Open();
var results = await conn.QueryAsync<(long, long, long, DateTime)>(@"SELECT Game, ModId, p.file_id, LastChecked var results = await conn.QueryAsync<(long, long, long, string)>(@"SELECT Game, ModId, FileId, JSON_VALUE(Data, '$.category_name') FROM dbo.NexusModFile");
FROM [NexusModFiles] files
CROSS APPLY
OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p
WHERE p.category is not null
UNION
SELECT GameId, ModId, FileId, LastChecked FROM dbo.NexusModFilesSlow
");
return results.ToHashSet(); return results.ToHashSet();
} }

104
Wabbajack.Server/Services/ListValidator.cs
View File

@ -41,6 +41,7 @@ namespace Wabbajack.Server.Services
public override async Task<int> Execute() public override async Task<int> Execute()
{ {
var data = await _sql.GetValidationData(); var data = await _sql.GetValidationData();
_logger.LogInformation("Found {count} nexus files", data.NexusFiles.Count);
using var queue = new WorkQueue(); using var queue = new WorkQueue();
var oldSummaries = Summaries; var oldSummaries = Summaries;
@ -102,6 +103,8 @@ namespace Wabbajack.Server.Services
await _sql.StartMirror((archive.Hash, reason)); await _sql.StartMirror((archive.Hash, reason));
return (archive, ArchiveStatus.Updating); return (archive, ArchiveStatus.Updating);
} }
if (archive.State is NexusDownloader.State)
return (archive, result);
return await TryToHeal(data, archive, metadata); return await TryToHeal(data, archive, metadata);
} }
@ -349,9 +352,9 @@ namespace Wabbajack.Server.Services
case GoogleDriveDownloader.State _: case GoogleDriveDownloader.State _:
// Disabled for now due to GDrive rate-limiting the build server // Disabled for now due to GDrive rate-limiting the build server
return (archive, ArchiveStatus.Valid); return (archive, ArchiveStatus.Valid);
case NexusDownloader.State nexusState when data.NexusFiles.Contains(( case NexusDownloader.State nexusState when data.NexusFiles.TryGetValue(
nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)): (nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID), out var category):
return (archive, ArchiveStatus.Valid); return (archive, category != null ? ArchiveStatus.Valid : ArchiveStatus.InValid);
case NexusDownloader.State ns: case NexusDownloader.State ns:
return (archive, await FastNexusModStats(ns)); return (archive, await FastNexusModStats(ns));
case ManualDownloader.State _: case ManualDownloader.State _:
@ -362,6 +365,10 @@ namespace Wabbajack.Server.Services
return (archive, ArchiveStatus.Valid); return (archive, ArchiveStatus.Valid);
case MediaFireDownloader.State _: case MediaFireDownloader.State _:
return (archive, ArchiveStatus.Valid); return (archive, ArchiveStatus.Valid);
case DeprecatedLoversLabDownloader.State _:
return (archive, ArchiveStatus.InValid);
case DeprecatedVectorPlexusDownloader.State _:
return (archive, ArchiveStatus.InValid);
default: default:
{ {
if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash), if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash),
@ -375,93 +382,44 @@ namespace Wabbajack.Server.Services
} }
} }
private AsyncLock _lock = new();
public async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns) public async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns)
{ {
// Check if some other thread has added them // Check if some other thread has added them
var mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID); var file = await _sql.GetModFile(ns.Game, ns.ModID, ns.FileID);
var files = await _sql.GetModFiles(ns.Game, ns.ModID);
if (mod == null || files == null) if (file == null)
{ {
// Acquire the lock try
//using var lck = await _lock.WaitAsync();
// Check again
mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID);
files = await _sql.GetModFiles(ns.Game, ns.ModID);
if (mod == null || files == null)
{ {
NexusApiClient nexusClient = await _nexus.GetClient();
var queryTime = DateTime.UtcNow;
_logger.Log(LogLevel.Information, "Found missing Nexus file info {Game} {ModID} {FileID}", ns.Game, ns.ModID, ns.FileID);
try
{
file = await nexusClient.GetModFile(ns.Game, ns.ModID, ns.FileID, false);
}
catch
{
file = new NexusFileInfo() {category_name = null};
}
try try
{ {
NexusApiClient nexusClient = await _nexus.GetClient(); await _sql.AddNexusModFile(ns.Game, ns.ModID, ns.FileID, queryTime, file);
var queryTime = DateTime.UtcNow;
if (mod == null)
{
_logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
try
{
mod = await nexusClient.GetModInfo(ns.Game, ns.ModID, false);
}
catch (Exception ex)
{
Utils.Log("Exception in Nexus Validation " + ex);
mod = new ModInfo
{
mod_id = ns.ModID.ToString(),
game_id = ns.Game.MetaData().NexusGameId,
available = false
};
}
try
{
await _sql.AddNexusModInfo(ns.Game, ns.ModID, queryTime, mod);
}
catch (Exception)
{
// Could be a PK constraint failure
}
}
if (files == null)
{
_logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
try
{
files = await nexusClient.GetModFiles(ns.Game, ns.ModID, false);
}
catch
{
files = new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>()};
}
try
{
await _sql.AddNexusModFiles(ns.Game, ns.ModID, queryTime, files);
}
catch (Exception)
{
// Could be a PK constraint failure
}
}
} }
catch (Exception) catch (Exception)
{ {
return ArchiveStatus.InValid; // Could be a PK constraint failure
} }
} }
catch (Exception)
{
return ArchiveStatus.InValid;
}
} }
if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID)) return file?.category_name != null ? ArchiveStatus.Valid : ArchiveStatus.InValid;
return ArchiveStatus.Valid;
return ArchiveStatus.InValid;
} }
} }

2
Wabbajack.Server/Startup.cs
View File

@ -13,6 +13,7 @@ using Microsoft.Extensions.FileProviders;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
using Newtonsoft.Json; using Newtonsoft.Json;
using Wabbajack.BuildServer; using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib.LibCefHelpers; using Wabbajack.Lib.LibCefHelpers;
using Wabbajack.Server.DataLayer; using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services; using Wabbajack.Server.Services;
@ -37,6 +38,7 @@ namespace Wabbajack.Server
// This method gets called by the runtime. Use this method to add services to the container. // This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services) public void ConfigureServices(IServiceCollection services)
{ {
Consts.UseNetworkWorkaroundMode = true;
Helpers.Init(); Helpers.Init();
/*services.AddSwaggerGen(c => /*services.AddSwaggerGen(c =>
{ {