Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Re-enable nexus archived file support again

Browse Source
This commit is contained in:
Timothy Baldridge 2021-07-04 15:37:55 -06:00
parent 89c0c651b5
commit 24d10305a4
7 changed files with 43 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -221,18 +221,12 @@ namespace Wabbajack.Lib.Downloaders
{
try
{
var nclient = DownloadDispatcher.GetInstance<NexusDownloader>();
await nclient.Prepare();
var client = nclient.Client!;
var nexusClient = DownloadDispatcher.GetInstance<NexusDownloader>();
await nexusClient.Prepare();
var client = nexusClient.Client!;
var modInfo = await client.GetModInfo(Game, ModID);
if (!modInfo.available) return false;
var modFiles = await client.GetModFiles(Game, ModID);
var found = modFiles.files
.FirstOrDefault(file => file.file_id == FileID && file.category_name != null);
return found != null;
var file = await client.GetModFile(Game, ModID, FileID);
return file?.category_name != null;
}
catch (Exception ex)
{

16
Wabbajack.Server.Test/NexusCacheTests.cs
View File

@ -103,22 +103,6 @@ namespace Wabbajack.BuildServer.Test
h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449);
Assert.True(found != default);
Assert.True(found.LastChecked > startTime && found.LastChecked < DateTime.UtcNow);
// Delete with exactly the same date, shouldn't clear out the record
await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, found.LastChecked);
var hs2 = await sql.AllNexusFiles();
var found2 = hs2.FirstOrDefault(h =>
h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449);
Assert.True(found != default);
Assert.True(found2.LastChecked == found.LastChecked);
// Delete all the records, it should now be gone
await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow);
var hs3 = await sql.AllNexusFiles();
Assert.DoesNotContain(hs3, f => f.NexusGameId == gameId && f.ModId == 1137);
}

2
Wabbajack.Server/DTOs/ValidationData.cs
View File

@ -9,7 +9,7 @@ namespace Wabbajack.Server.DTOs
{
public class ValidationData
{
public ConcurrentHashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; } = new ConcurrentHashSet<(long Game, long ModId, long FileId)>();
public Dictionary<(long Game, long ModId, long FileId), string> NexusFiles { get; set; } = new ();
public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; }
public List<ModlistMetadata> ModLists { get; set; }

1
Wabbajack.Server/DataLayer/Nexus.cs
View File

@ -118,6 +118,7 @@ namespace Wabbajack.Server.DataLayer
await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModPermissions WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModFile WHERE ModId = @ModID", new {ModId = modId});
}
public async Task<Dictionary<(Game, long), HTMLInterface.PermissionValue>> GetNexusPermissions()

13
Wabbajack.Server/DataLayer/ValidationData.cs
View File

@ -23,7 +23,7 @@ namespace Wabbajack.Server.DataLayer
var nexusFiles = await AllNexusFiles();
return new ValidationData
{
NexusFiles = new ConcurrentHashSet<(long Game, long ModId, long FileId)>(nexusFiles.Select(f => (f.NexusGameId, f.ModId, f.FileId))),
NexusFiles = nexusFiles.ToDictionary(nf => (nf.NexusGameId, nf.ModId, nf.FileId), nf => nf.category),
ArchiveStatus = await archiveStatus,
ModLists = await modLists,
Mirrors = await mirrors,
@ -41,17 +41,10 @@ namespace Wabbajack.Server.DataLayer
return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3);
}
public async Task<HashSet<(long NexusGameId, long ModId, long FileId, DateTime LastChecked)>> AllNexusFiles()
public async Task<HashSet<(long NexusGameId, long ModId, long FileId, string category)>> AllNexusFiles()
{
await using var conn = await Open();
var results = await conn.QueryAsync<(long, long, long, DateTime)>(@"SELECT Game, ModId, p.file_id, LastChecked
FROM [NexusModFiles] files
CROSS APPLY
OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p
WHERE p.category is not null
UNION
SELECT GameId, ModId, FileId, LastChecked FROM dbo.NexusModFilesSlow
");
var results = await conn.QueryAsync<(long, long, long, string)>(@"SELECT Game, ModId, FileId, JSON_VALUE(Data, '$.category_name') FROM dbo.NexusModFile");
return results.ToHashSet();
}

76
Wabbajack.Server/Services/ListValidator.cs
View File

@ -41,6 +41,7 @@ namespace Wabbajack.Server.Services
public override async Task<int> Execute()
{
var data = await _sql.GetValidationData();
_logger.LogInformation("Found {count} nexus files", data.NexusFiles.Count);
using var queue = new WorkQueue();
var oldSummaries = Summaries;
@ -102,6 +103,8 @@ namespace Wabbajack.Server.Services
await _sql.StartMirror((archive.Hash, reason));
return (archive, ArchiveStatus.Updating);
}
if (archive.State is NexusDownloader.State)
return (archive, result);
return await TryToHeal(data, archive, metadata);
}
@ -349,9 +352,9 @@ namespace Wabbajack.Server.Services
case GoogleDriveDownloader.State _:
// Disabled for now due to GDrive rate-limiting the build server
return (archive, ArchiveStatus.Valid);
case NexusDownloader.State nexusState when data.NexusFiles.Contains((
nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)):
return (archive, ArchiveStatus.Valid);
case NexusDownloader.State nexusState when data.NexusFiles.TryGetValue(
(nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID), out var category):
return (archive, category != null ? ArchiveStatus.Valid : ArchiveStatus.InValid);
case NexusDownloader.State ns:
return (archive, await FastNexusModStats(ns));
case ManualDownloader.State _:
@ -362,6 +365,10 @@ namespace Wabbajack.Server.Services
return (archive, ArchiveStatus.Valid);
case MediaFireDownloader.State _:
return (archive, ArchiveStatus.Valid);
case DeprecatedLoversLabDownloader.State _:
return (archive, ArchiveStatus.InValid);
case DeprecatedVectorPlexusDownloader.State _:
return (archive, ArchiveStatus.InValid);
default:
{
if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash),
@ -375,93 +382,44 @@ namespace Wabbajack.Server.Services
}
}
private AsyncLock _lock = new();
public async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns)
{
// Check if some other thread has added them
var mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID);
var files = await _sql.GetModFiles(ns.Game, ns.ModID);
var file = await _sql.GetModFile(ns.Game, ns.ModID, ns.FileID);
if (mod == null || files == null)
if (file == null)
{
// Acquire the lock
//using var lck = await _lock.WaitAsync();
// Check again
mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID);
files = await _sql.GetModFiles(ns.Game, ns.ModID);
if (mod == null || files == null)
{
try
{
NexusApiClient nexusClient = await _nexus.GetClient();
var queryTime = DateTime.UtcNow;
if (mod == null)
{
_logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
_logger.Log(LogLevel.Information, "Found missing Nexus file info {Game} {ModID} {FileID}", ns.Game, ns.ModID, ns.FileID);
try
{
mod = await nexusClient.GetModInfo(ns.Game, ns.ModID, false);
}
catch (Exception ex)
{
Utils.Log("Exception in Nexus Validation " + ex);
mod = new ModInfo
{
mod_id = ns.ModID.ToString(),
game_id = ns.Game.MetaData().NexusGameId,
available = false
};
}
try
{
await _sql.AddNexusModInfo(ns.Game, ns.ModID, queryTime, mod);
}
catch (Exception)
{
// Could be a PK constraint failure
}
}
if (files == null)
{
_logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
try
{
files = await nexusClient.GetModFiles(ns.Game, ns.ModID, false);
file = await nexusClient.GetModFile(ns.Game, ns.ModID, ns.FileID, false);
}
catch
{
files = new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>()};
file = new NexusFileInfo() {category_name = null};
}
try
{
await _sql.AddNexusModFiles(ns.Game, ns.ModID, queryTime, files);
await _sql.AddNexusModFile(ns.Game, ns.ModID, ns.FileID, queryTime, file);
}
catch (Exception)
{
// Could be a PK constraint failure
}
}
}
catch (Exception)
{
return ArchiveStatus.InValid;
}
}
}
if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID))
return ArchiveStatus.Valid;
return ArchiveStatus.InValid;
return file?.category_name != null ? ArchiveStatus.Valid : ArchiveStatus.InValid;
}
}

2
Wabbajack.Server/Startup.cs
View File

@ -13,6 +13,7 @@ using Microsoft.Extensions.FileProviders;
using Microsoft.Extensions.Hosting;
using Newtonsoft.Json;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib.LibCefHelpers;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
@ -37,6 +38,7 @@ namespace Wabbajack.Server
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
Consts.UseNetworkWorkaroundMode = true;
Helpers.Init();
/*services.AddSwaggerGen(c =>
{