Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Latest work on LL attachments and Nexus API fixes

Browse Source
This commit is contained in:
Timothy Baldridge 2021-07-02 21:15:27 -06:00
parent 943b460d9d
commit c1169fcbc4
9 changed files with 133 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Wabbajack.Lib/ACompiler.cs
View File

@ -165,13 +165,7 @@ namespace Wabbajack.Lib
return;
var b = await metaState.LoadMetaData();
Utils.Log(b
? $"Getting meta data for {a.Name} was successful!"
: $"Getting meta data for {a.Name} failed!");
}
else
{
Utils.Log($"Archive {a.Name} is not an AbstractMetaState!");
if (b) Utils.Log($"Getting meta data for {a.Name} was successful!");
}
});

32
Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs
View File

@ -140,7 +140,7 @@ namespace Wabbajack.Lib.Downloaders
}
var data = await Utils.FromEncryptedJson<OAuthResultState>(EncryptedKeyName);
await data.Refresh();
await data.Refresh(SiteName);
var client = new Http.Client();
client.Headers.Add(("Authorization", $"Bearer {data.AccessToken}"));
return client;
@ -240,6 +240,7 @@ namespace Wabbajack.Lib.Downloaders
public string? Description { get; set; }
public async Task<bool> LoadMetaData()
{
if (IsAttachment) return false;
var data = await TypedDownloader.GetDownloads(IPS4Mod);
Name = data.Title;
Author = data.Author?.Name;
@ -321,7 +322,7 @@ namespace Wabbajack.Lib.Downloaders
}
public async Task<bool> Refresh()
public async Task<bool> Refresh(string siteName = "")
{
if (ExpiresAt > DateTime.UtcNow + TimeSpan.FromHours(6))
return true;
@ -333,13 +334,28 @@ namespace Wabbajack.Lib.Downloaders
new ("refresh_token", RefreshToken),
new ("client_id", ClientID)
};
using var response = await client.PostAsync(TokenEndpoint!.ToString(), new FormUrlEncodedContent(formData.ToList()));
var responseData = (await response.Content.ReadAsStringAsync()).FromJsonString<OAuthResultState>();
try
{
using var response = await client.PostAsync(TokenEndpoint!.ToString(),
new FormUrlEncodedContent(formData.ToList()));
var responseData = (await response.Content.ReadAsStringAsync()).FromJsonString<OAuthResultState>();
AccessToken = responseData.AccessToken;
ExpiresIn = responseData.ExpiresIn;
ExpiresAt = DateTime.UtcNow + TimeSpan.FromSeconds(ExpiresIn);
return true;
}
catch (HttpException ex)
{
if (ex.Code == 400)
{
throw new CriticalFailureIntervention(
$"You have been logged out of {siteName} for reasons out of our control, please log back in via the settings panel",
$"Logged out of {siteName}");
}
}
AccessToken = responseData.AccessToken;
ExpiresIn = responseData.ExpiresIn;
ExpiresAt = DateTime.UtcNow + TimeSpan.FromSeconds(ExpiresIn);
return true;
}

12
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -224,15 +224,21 @@ namespace Wabbajack.Lib.Downloaders
var nclient = DownloadDispatcher.GetInstance<NexusDownloader>();
await nclient.Prepare();
var client = nclient.Client!;
var file = await client.GetModFile(Game, ModID, FileID);
return file?.category_name != null;
var modInfo = await client.GetModInfo(Game, ModID);
if (!modInfo.available) return false;
var modFiles = await client.GetModFiles(Game, ModID);
var found = modFiles.files
.FirstOrDefault(file => file.file_id == FileID && file.category_name != null);
return found != null;
}
catch (Exception ex)
{
Utils.Log($"{Name} - {Game} - {ModID} - {FileID} - Error getting Nexus download URL - {ex}");
return false;
}
}
public override IDownloader GetDownloader()

1
Wabbajack.Lib/Http/ClientFactory.cs
View File

@ -25,7 +25,6 @@ namespace Wabbajack.Lib.Http
AutomaticDecompression = DecompressionMethods.All,
};
Utils.Log($"Configuring with SSL {_socketsHandler.SslOptions.EnabledSslProtocols}");
ServicePointManager.ServerCertificateValidationCallback +=
(sender, certificate, chain, errors) =>

12
Wabbajack.Server.Test/NexusCacheTests.cs
View File

@ -103,12 +103,22 @@ namespace Wabbajack.BuildServer.Test
h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449);
Assert.True(found != default);
Assert.True(found.LastChecked > startTime && found.LastChecked < DateTime.UtcNow);
// Delete with exactly the same date, shouldn't clear out the record
await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, found.LastChecked);
var hs2 = await sql.AllNexusFiles();
var found2 = hs2.FirstOrDefault(h =>
h.NexusGameId == gameId && h.ModId == 1137 && h.FileId == 121449);
Assert.True(found != default);
Assert.True(found2.LastChecked == found.LastChecked);
// Delete all the records, it should now be gone
await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow);
var hs3 = await sql.AllNexusFiles();
Assert.DoesNotContain(hs3, f => f.NexusGameId == gameId && f.ModId == 1137);
}

3
Wabbajack.Server/DTOs/ValidationData.cs
View File

@ -1,5 +1,4 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading.Tasks;
using Wabbajack.Common;
@ -10,7 +9,7 @@ namespace Wabbajack.Server.DTOs
{
public class ValidationData
{
public Dictionary<(long Game, long ModId, long FileId), string> NexusFiles { get; set; } = new ();
public ConcurrentHashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; } = new ConcurrentHashSet<(long Game, long ModId, long FileId)>();
public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; }
public List<ModlistMetadata> ModLists { get; set; }

13
Wabbajack.Server/DataLayer/ValidationData.cs
View File

@ -23,7 +23,7 @@ namespace Wabbajack.Server.DataLayer
var nexusFiles = await AllNexusFiles();
return new ValidationData
{
NexusFiles = nexusFiles.ToDictionary(nf => (nf.NexusGameId, nf.ModId, nf.FileId), nf => nf.category),
NexusFiles = new ConcurrentHashSet<(long Game, long ModId, long FileId)>(nexusFiles.Select(f => (f.NexusGameId, f.ModId, f.FileId))),
ArchiveStatus = await archiveStatus,
ModLists = await modLists,
Mirrors = await mirrors,
@ -41,10 +41,17 @@ namespace Wabbajack.Server.DataLayer
return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3);
}
public async Task<HashSet<(long NexusGameId, long ModId, long FileId, string category)>> AllNexusFiles()
public async Task<HashSet<(long NexusGameId, long ModId, long FileId, DateTime LastChecked)>> AllNexusFiles()
{
await using var conn = await Open();
var results = await conn.QueryAsync<(long, long, long, string)>(@"SELECT Game, ModId, FileId, JSON_VALUE(Data, '$.category_name') FROM dbo.NexusModFile");
var results = await conn.QueryAsync<(long, long, long, DateTime)>(@"SELECT Game, ModId, p.file_id, LastChecked
FROM [NexusModFiles] files
CROSS APPLY
OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p
WHERE p.category is not null
UNION
SELECT GameId, ModId, FileId, LastChecked FROM dbo.NexusModFilesSlow
");
return results.ToHashSet();
}

104
Wabbajack.Server/Services/ListValidator.cs
View File

@ -102,9 +102,6 @@ namespace Wabbajack.Server.Services
await _sql.StartMirror((archive.Hash, reason));
return (archive, ArchiveStatus.Updating);
}
if (archive.State is NexusDownloader.State)
return (archive, result);
return await TryToHeal(data, archive, metadata);
}
@ -352,9 +349,9 @@ namespace Wabbajack.Server.Services
case GoogleDriveDownloader.State _:
// Disabled for now due to GDrive rate-limiting the build server
return (archive, ArchiveStatus.Valid);
case NexusDownloader.State nexusState when data.NexusFiles.TryGetValue(
(nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID), out var category):
return (archive, category != null ? ArchiveStatus.Valid : ArchiveStatus.InValid);
case NexusDownloader.State nexusState when data.NexusFiles.Contains((
nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)):
return (archive, ArchiveStatus.Valid);
case NexusDownloader.State ns:
return (archive, await FastNexusModStats(ns));
case ManualDownloader.State _:
@ -365,10 +362,6 @@ namespace Wabbajack.Server.Services
return (archive, ArchiveStatus.Valid);
case MediaFireDownloader.State _:
return (archive, ArchiveStatus.Valid);
case DeprecatedLoversLabDownloader.State _:
return (archive, ArchiveStatus.Valid);
case DeprecatedVectorPlexusDownloader.State _:
return (archive, ArchiveStatus.Valid);
default:
{
if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash),
@ -381,45 +374,94 @@ namespace Wabbajack.Server.Services
}
}
}
private AsyncLock _lock = new();
public async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns)
{
// Check if some other thread has added them
var file = await _sql.GetModFile(ns.Game, ns.ModID, ns.FileID);
var mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID);
var files = await _sql.GetModFiles(ns.Game, ns.ModID);
if (file == null)
if (mod == null || files == null)
{
try
// Acquire the lock
//using var lck = await _lock.WaitAsync();
// Check again
mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID);
files = await _sql.GetModFiles(ns.Game, ns.ModID);
if (mod == null || files == null)
{
NexusApiClient nexusClient = await _nexus.GetClient();
var queryTime = DateTime.UtcNow;
_logger.Log(LogLevel.Information, "Found missing Nexus file info {Game} {ModID} {FileID}", ns.Game, ns.ModID, ns.FileID);
try
{
file = await nexusClient.GetModFile(ns.Game, ns.ModID, ns.FileID, false);
}
catch
{
file = new NexusFileInfo() {category_name = null};
}
try
{
await _sql.AddNexusModFile(ns.Game, ns.ModID, ns.FileID, queryTime, file);
NexusApiClient nexusClient = await _nexus.GetClient();
var queryTime = DateTime.UtcNow;
if (mod == null)
{
_logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
try
{
mod = await nexusClient.GetModInfo(ns.Game, ns.ModID, false);
}
catch (Exception ex)
{
Utils.Log("Exception in Nexus Validation " + ex);
mod = new ModInfo
{
mod_id = ns.ModID.ToString(),
game_id = ns.Game.MetaData().NexusGameId,
available = false
};
}
try
{
await _sql.AddNexusModInfo(ns.Game, ns.ModID, queryTime, mod);
}
catch (Exception)
{
// Could be a PK constraint failure
}
}
if (files == null)
{
_logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}");
try
{
files = await nexusClient.GetModFiles(ns.Game, ns.ModID, false);
}
catch
{
files = new NexusApiClient.GetModFilesResponse {files = new List<NexusFileInfo>()};
}
try
{
await _sql.AddNexusModFiles(ns.Game, ns.ModID, queryTime, files);
}
catch (Exception)
{
// Could be a PK constraint failure
}
}
}
catch (Exception)
{
// Could be a PK constraint failure
return ArchiveStatus.InValid;
}
}
catch (Exception)
{
return ArchiveStatus.InValid;
}
}
return file?.category_name != null ? ArchiveStatus.Valid : ArchiveStatus.InValid;
if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID))
return ArchiveStatus.Valid;
return ArchiveStatus.InValid;
}
}

4
Wabbajack.Test/DownloaderTests.cs
View File

@ -384,6 +384,10 @@ namespace Wabbajack.Test
await converted.Download(new Archive(state: null!) { Name = "LoversLab Test.txt" }, filename.Path);
Assert.Equal(Hash.FromBase64("gLJDxGDaeQ0="), await filename.Path.FileHashAsync());
Assert.False(await ((LoversLabOAuthDownloader.State) state).LoadMetaData());
}