From 2e0c13f854a72d9ec0546155ed3fb57b36e91786 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 25 Jul 2020 12:09:02 -0600 Subject: [PATCH] Cache nexus permissions / mod hidden status. Use hidden status to purge the nexus cache --- Wabbajack.Common/GameMetaData.cs | 6 ++ Wabbajack.Lib/Http/Client.cs | 6 ++ Wabbajack.Lib/NexusApi/HtmlInterface.cs | 51 ++++++++++++++ Wabbajack.Lib/NexusApi/NexusApi.cs | 10 ++- .../ModListValidationTests.cs | 3 - Wabbajack.Server.Test/sql/wabbajack_db.sql | 18 +++++ Wabbajack.Server/DataLayer/ModLists.cs | 14 +++- Wabbajack.Server/DataLayer/Nexus.cs | 31 ++++++++ .../Services/NexusPermissionsUpdater.cs | 70 +++++++++++++++++++ Wabbajack.Server/Startup.cs | 2 + .../ContentRightsManagementTests.cs | 9 +++ 11 files changed, 215 insertions(+), 5 deletions(-) create mode 100644 Wabbajack.Lib/NexusApi/HtmlInterface.cs create mode 100644 Wabbajack.Server/Services/NexusPermissionsUpdater.cs diff --git a/Wabbajack.Common/GameMetaData.cs b/Wabbajack.Common/GameMetaData.cs index eb04f528..509b558b 100644 --- a/Wabbajack.Common/GameMetaData.cs +++ b/Wabbajack.Common/GameMetaData.cs @@ -452,5 +452,11 @@ namespace Wabbajack.Common } }; + public static Dictionary ByNexusID = + Games.Values.Where(g => g.NexusGameId != 0) + .GroupBy(g => g.NexusGameId) + .Select(g => g.First()) + .ToDictionary(d => d.NexusGameId, d => d.Game); + } } diff --git a/Wabbajack.Lib/Http/Client.cs b/Wabbajack.Lib/Http/Client.cs index abf4a87c..ee37589b 100644 --- a/Wabbajack.Lib/Http/Client.cs +++ b/Wabbajack.Lib/Http/Client.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using HtmlAgilityPack; using Wabbajack.Common; using Wabbajack.Common.Exceptions; +using Wabbajack.Lib.LibCefHelpers; namespace Wabbajack.Lib.Http { @@ -151,5 +152,10 @@ namespace Wabbajack.Lib.Http var client = new Client {Headers = newHeaders, Cookies = Cookies,}; return client; } + + public void AddCookies(Helpers.Cookie[] cookies) + { + Cookies.AddRange(cookies.Select(c => new Cookie {Domain = c.Domain, Name = c.Name, Value = c.Value, Path = c.Path})); + } } } diff --git a/Wabbajack.Lib/NexusApi/HtmlInterface.cs b/Wabbajack.Lib/NexusApi/HtmlInterface.cs new file mode 100644 index 00000000..9cb14a65 --- /dev/null +++ b/Wabbajack.Lib/NexusApi/HtmlInterface.cs @@ -0,0 +1,51 @@ +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Lib.LibCefHelpers; + +namespace Wabbajack.Lib.NexusApi +{ + public class HTMLInterface + { + public static async Task GetUploadPermissions(Game game, long modId) + { + var client = new Lib.Http.Client(); + if (Utils.HaveEncryptedJson("nexus-cookies")) + { + var cookies = await Utils.FromEncryptedJson("nexus-cookies"); + client.AddCookies(cookies); + } + + var response = await client.GetHtmlAsync($"https://nexusmods.com/{game.MetaData().NexusName}/mods/{modId}"); + + var hidden = response.DocumentNode + .Descendants() + .Any(n => n.Id == $"{modId}-title" && n.InnerText == "Hidden mod"); + + if (hidden) return PermissionValue.Hidden; + + var perm = response.DocumentNode + .Descendants() + .Where(d => d.HasClass("permissions-title") && d.InnerHtml == "Upload permission") + .SelectMany(d => d.ParentNode.ParentNode.GetClasses()) + .FirstOrDefault(perm => perm.StartsWith("permission-")); + + return perm switch + { + "permission-no" => PermissionValue.No, + "permission-maybe" => PermissionValue.Maybe, + "permission-yes" => PermissionValue.Yes, + _ => PermissionValue.No + }; + } + + public enum PermissionValue : int + { + No = 0, + Yes = 1, + Maybe = 2, + Hidden = 3, + } + } +} diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index d833703a..40e25835 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -11,6 +11,7 @@ using Wabbajack.Common; using Wabbajack.Lib.Downloaders; using System.Threading; using Wabbajack.Common.Exceptions; +using Wabbajack.Lib.LibCefHelpers; using Wabbajack.Lib.WebAutomation; namespace Wabbajack.Lib.NexusApi @@ -85,9 +86,11 @@ namespace Wabbajack.Lib.NexusApi { updateStatus("Please log into the Nexus"); await browser.NavigateTo(new Uri("https://users.nexusmods.com/auth/continue?client_id=nexus&redirect_uri=https://www.nexusmods.com/oauth/callback&response_type=code&referrer=//www.nexusmods.com")); + + Helpers.Cookie[] cookies = {}; while (true) { - var cookies = await browser.GetCookies("nexusmods.com"); + cookies = await browser.GetCookies("nexusmods.com"); if (cookies.Any(c => c.Name == "member_id")) break; cancel.ThrowIfCancellationRequested(); @@ -97,8 +100,13 @@ namespace Wabbajack.Lib.NexusApi await browser.NavigateTo(new Uri("https://www.nexusmods.com/users/myaccount?tab=api")); + updateStatus("Saving login info"); + + await cookies.ToEcryptedJson("nexus-cookies"); + updateStatus("Looking for API Key"); + var apiKey = new TaskCompletionSource(); diff --git a/Wabbajack.Server.Test/ModListValidationTests.cs b/Wabbajack.Server.Test/ModListValidationTests.cs index d61cb82f..727aaec7 100644 --- a/Wabbajack.Server.Test/ModListValidationTests.cs +++ b/Wabbajack.Server.Test/ModListValidationTests.cs @@ -150,9 +150,6 @@ namespace Wabbajack.BuildServer.Test Assert.Equal(0, data.ValidationSummary.Failed); Assert.Equal(1, data.ValidationSummary.Passed); Assert.Equal(0, data.ValidationSummary.Updating); - - - } private async Task RevalidateLists(bool runNonNexus) diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index b76c82e1..1bda2ece 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -589,6 +589,8 @@ CONSTRAINT [PK_NexusModFilesSlow] PRIMARY KEY CLUSTERED ) ON [PRIMARY] GO + + /****** Object: Table [dbo].[Patches] Script Date: 5/18/2020 6:26:07 AM ******/ CREATE TABLE [dbo].[Patches]( @@ -653,6 +655,22 @@ CREATE TABLE [dbo].[VirusScanResults]( ) ON [PRIMARY] GO + +/****** Object: Table [dbo].[NexusModPermissions] Script Date: 7/25/2020 11:42:04 AM ******/ +CREATE TABLE [dbo].[NexusModPermissions]( +[NexusGameID] [int] NOT NULL, +[ModID] [bigint] NOT NULL, +[Permissions] [int] NOT NULL, +CONSTRAINT [PK_NexusModPermissions] PRIMARY KEY CLUSTERED + ( + [NexusGameID] ASC, + [ModID] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO + + + /****** Object: StoredProcedure [dbo].[MergeAllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/ SET ANSI_NULLS ON GO diff --git a/Wabbajack.Server/DataLayer/ModLists.cs b/Wabbajack.Server/DataLayer/ModLists.cs index 4f527f7d..43f23acd 100644 --- a/Wabbajack.Server/DataLayer/ModLists.cs +++ b/Wabbajack.Server/DataLayer/ModLists.cs @@ -80,7 +80,19 @@ namespace Wabbajack.Server.DataLayer { await using var conn = await Open(); var archives = await conn.QueryAsync<(string, Hash, long, AbstractDownloadState)>("SELECT Name, Hash, Size, State FROM dbo.ModListArchives WHERE MachineUrl = @MachineUrl", - new {MachineUrl = machineURL}); + new {MachineUrl = machineURL}); + return archives.Select(t => new Archive(t.Item4) + { + Name = string.IsNullOrWhiteSpace(t.Item1) ? t.Item4.PrimaryKeyString : t.Item1, + Size = t.Item3, + Hash = t.Item2 + }).ToList(); + } + + public async Task> ModListArchives() + { + await using var conn = await Open(); + var archives = await conn.QueryAsync<(string, Hash, long, AbstractDownloadState)>("SELECT Name, Hash, Size, State FROM dbo.ModListArchives"); return archives.Select(t => new Archive(t.Item4) { Name = string.IsNullOrWhiteSpace(t.Item1) ? t.Item4.PrimaryKeyString : t.Item1, diff --git a/Wabbajack.Server/DataLayer/Nexus.cs b/Wabbajack.Server/DataLayer/Nexus.cs index c4ae531d..55f1d11d 100644 --- a/Wabbajack.Server/DataLayer/Nexus.cs +++ b/Wabbajack.Server/DataLayer/Nexus.cs @@ -1,10 +1,13 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; using Dapper; using Newtonsoft.Json; using Wabbajack.Common; using Wabbajack.Lib.NexusApi; +using Wabbajack.Lib.Validation; namespace Wabbajack.Server.DataLayer { @@ -115,5 +118,33 @@ namespace Wabbajack.Server.DataLayer await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId}); await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId}); } + + public async Task> GetNexusPermissions() + { + await using var conn = await Open(); + + var results = + await conn.QueryAsync<(int, long, int)>("SELECT NexusGameID, ModID, Permissions FROM NexusModPermissions"); + return results.ToDictionary(f => (GameRegistry.ByNexusID[f.Item1], f.Item2), + f => (HTMLInterface.PermissionValue)f.Item3); + } + + public async Task SetNexusPermissions(IEnumerable<(Game, long, HTMLInterface.PermissionValue)> permissions) + { + await using var conn = await Open(); + var tx = await conn.BeginTransactionAsync(); + + await conn.ExecuteAsync("DELETE FROM NexusModPermissions", transaction:tx); + + foreach (var (game, modId, perm) in permissions) + { + await conn.ExecuteAsync( + "INSERT INTO NexusModPermissions (NexusGameID, ModID, Permissions) VALUES (@NexusGameID, @ModID, @Permissions)", + new {NexusGameID = game.MetaData().NexusGameId, ModID = modId, Permissions = (int)perm}, tx); + } + + await tx.CommitAsync(); + + } } } diff --git a/Wabbajack.Server/Services/NexusPermissionsUpdater.cs b/Wabbajack.Server/Services/NexusPermissionsUpdater.cs new file mode 100644 index 00000000..59268655 --- /dev/null +++ b/Wabbajack.Server/Services/NexusPermissionsUpdater.cs @@ -0,0 +1,70 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.BuildServer; +using Wabbajack.Common; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.NexusApi; +using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DTOs; + +namespace Wabbajack.Server.Services +{ + public class NexusPermissionsUpdater : AbstractService + { + private DiscordWebHook _discord; + private SqlService _sql; + + public NexusPermissionsUpdater(ILogger logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromHours(4)) + { + _discord = discord; + _sql = sql; + } + + public override async Task Execute() + { + var permissions = await _sql.GetNexusPermissions(); + + var data = await _sql.ModListArchives(); + var nexusArchives = data.Select(a => a.State).OfType().Select(d => (d.Game, d.ModID)) + .Distinct() + .ToList(); + + _logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods"); + + using var queue = new WorkQueue(); + + var results = await nexusArchives.PMap(queue, async archive => + { + var permissions = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID); + return (archive.Game, archive.ModID, permissions); + }); + + var updated = 0; + foreach (var result in results) + { + if (permissions.TryGetValue((result.Game, result.ModID), out var oldPermission)) + { + if (oldPermission != result.permissions) + { + await _discord.Send(Channel.Spam, + new DiscordMessage { + Content = $"Permissions status of {result.Game} {result.ModID} was {oldPermission} is now {result.permissions} " + }); + await _sql.PurgeNexusCache(result.ModID); + updated += 1; + } + } + } + + await _sql.SetNexusPermissions(results); + + if (updated > 0) + await _quickSync.Notify(); + + + return updated; + } + } +} diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 9fae9f72..27b74615 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -68,6 +68,7 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddMvc(); services.AddControllers() @@ -123,6 +124,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); + app.UseService(); app.Use(next => { diff --git a/Wabbajack.Test/ContentRightsManagementTests.cs b/Wabbajack.Test/ContentRightsManagementTests.cs index 7da956cc..3eebcabb 100644 --- a/Wabbajack.Test/ContentRightsManagementTests.cs +++ b/Wabbajack.Test/ContentRightsManagementTests.cs @@ -7,6 +7,7 @@ using Wabbajack.Lib.Validation; using Game = Wabbajack.Common.Game; using Wabbajack.Common; using System.Threading.Tasks; +using Wabbajack.Lib.NexusApi; using Xunit; namespace Wabbajack.Test @@ -116,5 +117,13 @@ namespace Wabbajack.Test await new ValidateModlist().LoadListsFromGithub(); } } + + [Fact] + public async Task CanGetReuploadRights() + { + Assert.Equal(HTMLInterface.PermissionValue.No, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 266)); + Assert.Equal(HTMLInterface.PermissionValue.Yes, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 1137)); + Assert.Equal(HTMLInterface.PermissionValue.Hidden, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 34604)); + } } }