Cache nexus permissions / mod hidden status. Use hidden status to purge the nexus cache

Wabbajack.Common/GameMetaData.cs

@@ -452,5 +452,11 @@ namespace Wabbajack.Common
             }	
         };
 
+        public static Dictionary<long, Game> ByNexusID =
+            Games.Values.Where(g => g.NexusGameId != 0)
+                .GroupBy(g => g.NexusGameId)
+                .Select(g => g.First())
+                .ToDictionary(d => d.NexusGameId, d => d.Game);
+
     }
 }

Wabbajack.Lib/Http/Client.cs

@@ -8,6 +8,7 @@ using System.Threading.Tasks;
 using HtmlAgilityPack;
 using Wabbajack.Common;
 using Wabbajack.Common.Exceptions;
+using Wabbajack.Lib.LibCefHelpers;
 
 namespace Wabbajack.Lib.Http
 {
@@ -151,5 +152,10 @@ namespace Wabbajack.Lib.Http
             var client = new Client {Headers = newHeaders, Cookies = Cookies,};
             return client;
         }
+
+        public void AddCookies(Helpers.Cookie[] cookies)
+        {
+            Cookies.AddRange(cookies.Select(c => new Cookie {Domain = c.Domain, Name = c.Name, Value = c.Value, Path = c.Path}));
+        }
     }
 }

Wabbajack.Lib/NexusApi/HtmlInterface.cs

@@ -0,0 +1,51 @@
+using System.Linq;
+using System.Net;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.Lib.LibCefHelpers;
+
+namespace Wabbajack.Lib.NexusApi
+{
+    public class HTMLInterface
+    {
+        public static async Task<PermissionValue> GetUploadPermissions(Game game, long modId)
+        {
+            var client = new Lib.Http.Client();
+            if (Utils.HaveEncryptedJson("nexus-cookies"))
+            {
+                var cookies = await Utils.FromEncryptedJson<Helpers.Cookie[]>("nexus-cookies");
+                client.AddCookies(cookies);
+            }
+
+            var response = await client.GetHtmlAsync($"https://nexusmods.com/{game.MetaData().NexusName}/mods/{modId}");
+
+            var hidden = response.DocumentNode
+                .Descendants()
+                .Any(n => n.Id == $"{modId}-title" && n.InnerText == "Hidden mod");
+
+            if (hidden) return PermissionValue.Hidden;
+
+            var perm = response.DocumentNode
+                .Descendants()
+                .Where(d => d.HasClass("permissions-title") && d.InnerHtml == "Upload permission")
+                .SelectMany(d => d.ParentNode.ParentNode.GetClasses())
+                .FirstOrDefault(perm => perm.StartsWith("permission-"));
+
+            return perm switch
+            {
+                "permission-no" => PermissionValue.No,
+                "permission-maybe" => PermissionValue.Maybe,
+                "permission-yes" => PermissionValue.Yes,
+                _ => PermissionValue.No
+            };
+        }
+
+        public enum PermissionValue : int
+        {
+            No = 0,
+            Yes = 1,
+            Maybe = 2,
+            Hidden = 3,
+        }
+    }
+}

Wabbajack.Lib/NexusApi/NexusApi.cs

@@ -11,6 +11,7 @@ using Wabbajack.Common;
 using Wabbajack.Lib.Downloaders;
 using System.Threading;
 using Wabbajack.Common.Exceptions;
+using Wabbajack.Lib.LibCefHelpers;
 using Wabbajack.Lib.WebAutomation;
 
 namespace Wabbajack.Lib.NexusApi
@@ -85,9 +86,11 @@ namespace Wabbajack.Lib.NexusApi
         {
             updateStatus("Please log into the Nexus");
             await browser.NavigateTo(new Uri("https://users.nexusmods.com/auth/continue?client_id=nexus&redirect_uri=https://www.nexusmods.com/oauth/callback&response_type=code&referrer=//www.nexusmods.com"));
+
+            Helpers.Cookie[] cookies = {};
             while (true)
             {
-                var cookies = await browser.GetCookies("nexusmods.com");
+                cookies = await browser.GetCookies("nexusmods.com");
                 if (cookies.Any(c => c.Name == "member_id"))
                     break;
                 cancel.ThrowIfCancellationRequested();
@@ -97,9 +100,14 @@ namespace Wabbajack.Lib.NexusApi
 
             await browser.NavigateTo(new Uri("https://www.nexusmods.com/users/myaccount?tab=api"));
 
+            updateStatus("Saving login info");
+
+            await cookies.ToEcryptedJson("nexus-cookies");
+            
             updateStatus("Looking for API Key");
 
 
+            
             var apiKey = new TaskCompletionSource<string>();
 
             while (true)

Wabbajack.Server.Test/ModListValidationTests.cs

@@ -150,9 +150,6 @@ namespace Wabbajack.BuildServer.Test
             Assert.Equal(0, data.ValidationSummary.Failed);
             Assert.Equal(1, data.ValidationSummary.Passed);
             Assert.Equal(0, data.ValidationSummary.Updating);
-            
-
-
         }
         
         private async Task RevalidateLists(bool runNonNexus)

Wabbajack.Server.Test/sql/wabbajack_db.sql

@@ -589,6 +589,8 @@ CONSTRAINT [PK_NexusModFilesSlow] PRIMARY KEY CLUSTERED
 ) ON [PRIMARY]
 GO
 
+
+
 /****** Object:  Table [dbo].[Patches]    Script Date: 5/18/2020 6:26:07 AM ******/
 
 CREATE TABLE [dbo].[Patches](
@@ -653,6 +655,22 @@ CREATE TABLE [dbo].[VirusScanResults](
 ) ON [PRIMARY]
 GO
 
+
+/****** Object:  Table [dbo].[NexusModPermissions]    Script Date: 7/25/2020 11:42:04 AM ******/
+CREATE TABLE [dbo].[NexusModPermissions](
+[NexusGameID] [int] NOT NULL,
+[ModID] [bigint] NOT NULL,
+[Permissions] [int] NOT NULL,
+CONSTRAINT [PK_NexusModPermissions] PRIMARY KEY CLUSTERED
+    (
+     [NexusGameID] ASC,
+     [ModID] ASC
+        )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+) ON [PRIMARY]
+GO
+
+
+
 /****** Object:  StoredProcedure [dbo].[MergeAllFilesInArchive]    Script Date: 3/28/2020 4:58:59 PM ******/
 SET ANSI_NULLS ON
 GO

Wabbajack.Server/DataLayer/ModLists.cs

@@ -88,5 +88,17 @@ namespace Wabbajack.Server.DataLayer
                 Hash = t.Item2
             }).ToList();
         }
+        
+        public async Task<List<Archive>> ModListArchives()
+        {
+            await using var conn = await Open();
+            var archives = await conn.QueryAsync<(string, Hash, long, AbstractDownloadState)>("SELECT Name, Hash, Size, State FROM dbo.ModListArchives");
+            return archives.Select(t => new Archive(t.Item4)
+            {
+                Name = string.IsNullOrWhiteSpace(t.Item1) ? t.Item4.PrimaryKeyString : t.Item1, 
+                Size = t.Item3, 
+                Hash = t.Item2
+            }).ToList();
+        }
     }
 }

Wabbajack.Server/DataLayer/Nexus.cs

@@ -1,10 +1,13 @@
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using Alphaleonis.Win32.Filesystem;
 using Dapper;
 using Newtonsoft.Json;
 using Wabbajack.Common;
 using Wabbajack.Lib.NexusApi;
+using Wabbajack.Lib.Validation;
 
 namespace Wabbajack.Server.DataLayer
 {
@@ -115,5 +118,33 @@ namespace Wabbajack.Server.DataLayer
             await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId});
             await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId});
         }
+
+        public async Task<Dictionary<(Game, long), HTMLInterface.PermissionValue>> GetNexusPermissions()
+        {
+            await using var conn = await Open();
+
+            var results =
+                await conn.QueryAsync<(int, long, int)>("SELECT NexusGameID, ModID, Permissions FROM NexusModPermissions");
+            return results.ToDictionary(f => (GameRegistry.ByNexusID[f.Item1], f.Item2),
+                f => (HTMLInterface.PermissionValue)f.Item3);
+        }
+
+        public async Task SetNexusPermissions(IEnumerable<(Game, long, HTMLInterface.PermissionValue)> permissions)
+        {
+            await using var conn = await Open();
+            var tx = await conn.BeginTransactionAsync();
+
+            await conn.ExecuteAsync("DELETE FROM NexusModPermissions", transaction:tx);
+
+            foreach (var (game, modId, perm) in permissions)
+            {
+                await conn.ExecuteAsync(
+                    "INSERT INTO NexusModPermissions (NexusGameID, ModID, Permissions) VALUES (@NexusGameID, @ModID, @Permissions)",
+                    new {NexusGameID = game.MetaData().NexusGameId, ModID = modId, Permissions = (int)perm}, tx);
+            }
+
+            await tx.CommitAsync();
+
+        }
     }
 }

Wabbajack.Server/Services/NexusPermissionsUpdater.cs

@@ -0,0 +1,70 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer;
+using Wabbajack.Common;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.NexusApi;
+using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DTOs;
+
+namespace Wabbajack.Server.Services
+{
+    public class NexusPermissionsUpdater : AbstractService<NexusKeyMaintainance, int>
+    {
+        private DiscordWebHook _discord;
+        private SqlService _sql;
+
+        public NexusPermissionsUpdater(ILogger<NexusKeyMaintainance> logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromHours(4))
+        {
+            _discord = discord;
+            _sql = sql;
+        }
+
+        public override async Task<int> Execute()
+        {
+            var permissions = await _sql.GetNexusPermissions();
+
+            var data = await _sql.ModListArchives();
+            var nexusArchives = data.Select(a => a.State).OfType<NexusDownloader.State>().Select(d => (d.Game, d.ModID))
+                .Distinct()
+                .ToList();
+            
+            _logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods");
+            
+            using var queue = new WorkQueue();
+
+            var results = await nexusArchives.PMap(queue, async archive =>
+            {
+                var permissions = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID);
+                return (archive.Game, archive.ModID, permissions);
+            });
+
+            var updated = 0;
+            foreach (var result in results)
+            {
+                if (permissions.TryGetValue((result.Game, result.ModID), out var oldPermission))
+                {
+                    if (oldPermission != result.permissions)
+                    {
+                        await _discord.Send(Channel.Spam,
+                            new DiscordMessage {
+                                Content = $"Permissions status of {result.Game} {result.ModID} was {oldPermission} is now {result.permissions} "
+                            });
+                        await _sql.PurgeNexusCache(result.ModID);
+                        updated += 1;
+                    }
+                }
+            }
+
+            await _sql.SetNexusPermissions(results);
+
+            if (updated > 0)
+                await _quickSync.Notify<ListValidator>();
+
+
+            return updated;
+        }
+    }
+}

Wabbajack.Server/Startup.cs

@@ -68,6 +68,7 @@ namespace Wabbajack.Server
             services.AddSingleton<NexusKeyMaintainance>();
             services.AddSingleton<PatchBuilder>();
             services.AddSingleton<CDNMirrorList>();
+            services.AddSingleton<NexusPermissionsUpdater>();
             
             services.AddMvc();
             services.AddControllers()
@@ -123,6 +124,7 @@ namespace Wabbajack.Server
             app.UseService<NexusKeyMaintainance>();
             app.UseService<PatchBuilder>();
             app.UseService<CDNMirrorList>();
+            app.UseService<NexusPermissionsUpdater>();
 
             app.Use(next =>
             {

Wabbajack.Test/ContentRightsManagementTests.cs

@@ -7,6 +7,7 @@ using Wabbajack.Lib.Validation;
 using Game = Wabbajack.Common.Game;
 using Wabbajack.Common;
 using System.Threading.Tasks;
+using Wabbajack.Lib.NexusApi;
 using Xunit;
 
 namespace Wabbajack.Test
@@ -116,5 +117,13 @@ namespace Wabbajack.Test
                 await new ValidateModlist().LoadListsFromGithub();
             }
         }
+        
+        [Fact]
+        public async Task CanGetReuploadRights()
+        {
+            Assert.Equal(HTMLInterface.PermissionValue.No, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 266));
+            Assert.Equal(HTMLInterface.PermissionValue.Yes, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 1137));
+            Assert.Equal(HTMLInterface.PermissionValue.Hidden, await HTMLInterface.GetUploadPermissions(Game.SkyrimSpecialEdition, 34604));
+        }
     }
 }