Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cache nexus permissions / mod hidden status. Use hidden status to purge the nexus cache

Browse Source
This commit is contained in:
Timothy Baldridge 2020-07-25 12:09:02 -06:00
parent c28ec4a796
commit 2e0c13f854
11 changed files with 215 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Wabbajack.Common/GameMetaData.cs
View File

@ -452,5 +452,11 @@ namespace Wabbajack.Common
}
};
public static Dictionary<long, Game> ByNexusID =
Games.Values.Where(g => g.NexusGameId != 0)
.GroupBy(g => g.NexusGameId)
.Select(g => g.First())
.ToDictionary(d => d.NexusGameId, d => d.Game);
}
}

6
Wabbajack.Lib/Http/Client.cs
View File

@ -8,6 +8,7 @@ using System.Threading.Tasks;
using HtmlAgilityPack;
using Wabbajack.Common;
using Wabbajack.Common.Exceptions;
using Wabbajack.Lib.LibCefHelpers;
namespace Wabbajack.Lib.Http
{
@ -151,5 +152,10 @@ namespace Wabbajack.Lib.Http
var client = new Client {Headers = newHeaders, Cookies = Cookies,};
return client;
}
public void AddCookies(Helpers.Cookie[] cookies)
{
Cookies.AddRange(cookies.Select(c => new Cookie {Domain = c.Domain, Name = c.Name, Value = c.Value, Path = c.Path}));
}
}
}

51
Wabbajack.Lib/NexusApi/HtmlInterface.cs Normal file
View File

@ -0,0 +1,51 @@
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.LibCefHelpers;
namespace Wabbajack.Lib.NexusApi
{
public class HTMLInterface
{
public static async Task<PermissionValue> GetUploadPermissions(Game game, long modId)
{
var client = new Lib.Http.Client();
if (Utils.HaveEncryptedJson("nexus-cookies"))
{
var cookies = await Utils.FromEncryptedJson<Helpers.Cookie[]>("nexus-cookies");
client.AddCookies(cookies);
}
var response = await client.GetHtmlAsync($"https://nexusmods.com/{game.MetaData().NexusName}/mods/{modId}");
var hidden = response.DocumentNode
.Descendants()
.Any(n => n.Id == $"{modId}-title" && n.InnerText == "Hidden mod");
if (hidden) return PermissionValue.Hidden;
var perm = response.DocumentNode
.Descendants()
.Where(d => d.HasClass("permissions-title") && d.InnerHtml == "Upload permission")
.SelectMany(d => d.ParentNode.ParentNode.GetClasses())
.FirstOrDefault(perm => perm.StartsWith("permission-"));
return perm switch
{
"permission-no" => PermissionValue.No,
"permission-maybe" => PermissionValue.Maybe,
"permission-yes" => PermissionValue.Yes,
_ => PermissionValue.No
};
}
public enum PermissionValue : int
{
No = 0,
Yes = 1,
Maybe = 2,
Hidden = 3,
}
}
}

10
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -11,6 +11,7 @@ using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using System.Threading;
using Wabbajack.Common.Exceptions;
using Wabbajack.Lib.LibCefHelpers;
using Wabbajack.Lib.WebAutomation;
namespace Wabbajack.Lib.NexusApi
@ -85,9 +86,11 @@ namespace Wabbajack.Lib.NexusApi
{
updateStatus("Please log into the Nexus");
await browser.NavigateTo(new Uri("https://users.nexusmods.com/auth/continue?client_id=nexus&redirect_uri=https://www.nexusmods.com/oauth/callback&response_type=code&referrer=//www.nexusmods.com"));
Helpers.Cookie[] cookies = {};
while (true)
{
var cookies = await browser.GetCookies("nexusmods.com");
cookies = await browser.GetCookies("nexusmods.com");
if (cookies.Any(c => c.Name == "member_id"))
break;
cancel.ThrowIfCancellationRequested();
@ -97,8 +100,13 @@ namespace Wabbajack.Lib.NexusApi
await browser.NavigateTo(new Uri("https://www.nexusmods.com/users/myaccount?tab=api"));
updateStatus("Saving login info");
await cookies.ToEcryptedJson("nexus-cookies");
updateStatus("Looking for API Key");
var apiKey = new TaskCompletionSource<string>();

3
Wabbajack.Server.Test/ModListValidationTests.cs
View File

@ -150,9 +150,6 @@ namespace Wabbajack.BuildServer.Test
Assert.Equal(0, data.ValidationSummary.Failed);
Assert.Equal(1, data.ValidationSummary.Passed);
Assert.Equal(0, data.ValidationSummary.Updating);
}
private async Task RevalidateLists(bool runNonNexus)

18
Wabbajack.Server.Test/sql/wabbajack_db.sql
View File

@ -589,6 +589,8 @@ CONSTRAINT [PK_NexusModFilesSlow] PRIMARY KEY CLUSTERED
) ON [PRIMARY]
GO
/****** Object: Table [dbo].[Patches] Script Date: 5/18/2020 6:26:07 AM ******/
CREATE TABLE [dbo].[Patches](
@ -653,6 +655,22 @@ CREATE TABLE [dbo].[VirusScanResults](
) ON [PRIMARY]
GO
/****** Object: Table [dbo].[NexusModPermissions] Script Date: 7/25/2020 11:42:04 AM ******/
CREATE TABLE [dbo].[NexusModPermissions](
[NexusGameID] [int] NOT NULL,
[ModID] [bigint] NOT NULL,
[Permissions] [int] NOT NULL,
CONSTRAINT [PK_NexusModPermissions] PRIMARY KEY CLUSTERED
(
[NexusGameID] ASC,
[ModID] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY]
GO
/****** Object: StoredProcedure [dbo].[MergeAllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/
SET ANSI_NULLS ON
GO

14
Wabbajack.Server/DataLayer/ModLists.cs
View File

@ -80,7 +80,19 @@ namespace Wabbajack.Server.DataLayer
{
await using var conn = await Open();
var archives = await conn.QueryAsync<(string, Hash, long, AbstractDownloadState)>("SELECT Name, Hash, Size, State FROM dbo.ModListArchives WHERE MachineUrl = @MachineUrl",
new {MachineUrl = machineURL});
new {MachineUrl = machineURL});
return archives.Select(t => new Archive(t.Item4)
{
Name = string.IsNullOrWhiteSpace(t.Item1) ? t.Item4.PrimaryKeyString : t.Item1,
Size = t.Item3,
Hash = t.Item2
}).ToList();
}
public async Task<List<Archive>> ModListArchives()
{
await using var conn = await Open();
var archives = await conn.QueryAsync<(string, Hash, long, AbstractDownloadState)>("SELECT Name, Hash, Size, State FROM dbo.ModListArchives");
return archives.Select(t => new Archive(t.Item4)
{
Name = string.IsNullOrWhiteSpace(t.Item1) ? t.Item4.PrimaryKeyString : t.Item1,

31
Wabbajack.Server/DataLayer/Nexus.cs
View File

@ -1,10 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Dapper;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Lib.Validation;
namespace Wabbajack.Server.DataLayer
{
@ -115,5 +118,33 @@ namespace Wabbajack.Server.DataLayer
await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId});
await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId});
}
public async Task<Dictionary<(Game, long), HTMLInterface.PermissionValue>> GetNexusPermissions()
{
await using var conn = await Open();
var results =
await conn.QueryAsync<(int, long, int)>("SELECT NexusGameID, ModID, Permissions FROM NexusModPermissions");
return results.ToDictionary(f => (GameRegistry.ByNexusID[f.Item1], f.Item2),
f => (HTMLInterface.PermissionValue)f.Item3);
}
public async Task SetNexusPermissions(IEnumerable<(Game, long, HTMLInterface.PermissionValue)> permissions)
{
await using var conn = await Open();
var tx = await conn.BeginTransactionAsync();
await conn.ExecuteAsync("DELETE FROM NexusModPermissions", transaction:tx);
foreach (var (game, modId, perm) in permissions)
{
await conn.ExecuteAsync(
"INSERT INTO NexusModPermissions (NexusGameID, ModID, Permissions) VALUES (@NexusGameID, @ModID, @Permissions)",
new {NexusGameID = game.MetaData().NexusGameId, ModID = modId, Permissions = (int)perm}, tx);
}
await tx.CommitAsync();
}
}
}

70
Wabbajack.Server/Services/NexusPermissionsUpdater.cs Normal file
View File

@ -0,0 +1,70 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class NexusPermissionsUpdater : AbstractService<NexusKeyMaintainance, int>
{
private DiscordWebHook _discord;
private SqlService _sql;
public NexusPermissionsUpdater(ILogger<NexusKeyMaintainance> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromHours(4))
{
_discord = discord;
_sql = sql;
}
public override async Task<int> Execute()
{
var permissions = await _sql.GetNexusPermissions();
var data = await _sql.ModListArchives();
var nexusArchives = data.Select(a => a.State).OfType<NexusDownloader.State>().Select(d => (d.Game, d.ModID))
.Distinct()
.ToList();
_logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods");
using var queue = new WorkQueue();
var results = await nexusArchives.PMap(queue, async archive =>
{
var permissions = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID);
return (archive.Game, archive.ModID, permissions);
});
var updated = 0;
foreach (var result in results)
{
if (permissions.TryGetValue((result.Game, result.ModID), out var oldPermission))
{
if (oldPermission != result.permissions)
{
await _discord.Send(Channel.Spam,
new DiscordMessage {
Content = $"Permissions status of {result.Game} {result.ModID} was {oldPermission} is now {result.permissions} "
});
await _sql.PurgeNexusCache(result.ModID);
updated += 1;
}
}
}
await _sql.SetNexusPermissions(results);
if (updated > 0)
await _quickSync.Notify<ListValidator>();
return updated;
}
}
}

2
Wabbajack.Server/Startup.cs
View File

@ -68,6 +68,7 @@ namespace Wabbajack.Server
services.AddSingleton<NexusKeyMaintainance>();
services.AddSingleton<PatchBuilder>();
services.AddSingleton<CDNMirrorList>();
services.AddSingleton<NexusPermissionsUpdater>();
services.AddMvc();
services.AddControllers()
@ -123,6 +124,7 @@ namespace Wabbajack.Server
app.UseService<NexusKeyMaintainance>();
app.UseService<PatchBuilder>();
app.UseService<CDNMirrorList>();
app.UseService<NexusPermissionsUpdater>();
app.Use(next =>
{

9
Wabbajack.Test/ContentRightsManagementTests.cs
View File

@ -7,6 +7,7 @@ using Wabbajack.Lib.Validation;
using Game = Wabbajack.Common.Game;
using Wabbajack.Common;
using System.Threading.Tasks;
using Wabbajack.Lib.NexusApi;
using Xunit;
namespace Wabbajack.Test
@ -116,5 +117,13 @@ namespace Wabbajack.Test
await new ValidateModlist().LoadListsFromGithub();
}
}
[Fact]
public async Task CanGetReuploadRights()
{
Assert.Equal(HTMLInterface.PermissionValue.No, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 266));
Assert.Equal(HTMLInterface.PermissionValue.Yes, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 1137));
Assert.Equal(HTMLInterface.PermissionValue.Hidden, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 34604));
}
}
}