Discord integration and more cache stuff

Wabbajack.Common/Json.cs

@@ -63,9 +63,9 @@ namespace Wabbajack.Common
             obj.ToJson(fs);
         }
 
-        public static string ToJson<T>(this T obj)
+        public static string ToJson<T>(this T obj, bool useGenericSettings = false)
         {
-            return JsonConvert.SerializeObject(obj, JsonSettings);
+            return JsonConvert.SerializeObject(obj, useGenericSettings ? GenericJsonSettings : JsonSettings);
         }
 
         public static T FromJson<T>(this AbsolutePath filename)

Wabbajack.Server/AppSettings.cs

@@ -29,5 +29,8 @@ namespace Wabbajack.BuildServer
         public string SqlConnection { get; set; }
 
         public int MaxJobs { get; set; } = 2;
+
+        public string SpamWebHook { get; set; } = null;
+        public string HamWebHook { get; set; } = null;
     }
 }

Wabbajack.Server/DTOs/DiscordMessage.cs

@@ -0,0 +1,103 @@
+using System;
+using Newtonsoft.Json;
+using Wabbajack.Common.Serialization.Json;
+
+namespace Wabbajack.Server.DTOs
+{
+    [JsonName("DiscordMessage")]
+    public class DiscordMessage
+    {
+        [JsonProperty("username")]
+        public string UserName { get; set; }
+        
+        [JsonProperty("avatar_url")]
+        public Uri AvatarUrl { get; set; }
+        
+        [JsonProperty("content")]
+        public string Content { get; set; }
+        
+        [JsonProperty("embeds")]
+        public DiscordEmbed[] Embeds { get; set; }
+    }
+
+    [JsonName("DiscordEmbed")]
+    public class DiscordEmbed
+    {
+        [JsonProperty("color")]
+        public int Color { get; set; }
+
+        [JsonProperty("author")]
+        public DiscordAuthor Author { get; set; }
+        
+        [JsonProperty("url")]
+        public Uri Url { get; set; }
+
+        [JsonProperty("description")]
+        public string Description { get; set; }
+        
+        [JsonProperty("fields")]
+        public DiscordField Field { get; set; }
+
+        [JsonProperty("thumbnail")]
+        public DiscordNumbnail Thumbnail { get; set; }
+        
+        [JsonProperty("image")]
+        public DiscordImage Image { get; set; }
+        
+        [JsonProperty("footer")]
+        public DiscordFooter Footer { get; set; }
+        
+        [JsonProperty("timestamp")]
+        public DateTime Timestamp { get; set; } = DateTime.UtcNow;
+    }
+
+    [JsonName("DiscordAuthor")]
+    public class DiscordAuthor
+    {
+        [JsonProperty("name")]
+        public string Name { get; set; }
+
+        [JsonProperty("url")]
+        public Uri Url { get; set; }
+
+        [JsonProperty("icon_url")]
+        public Uri IconUrl { get; set; }
+    }
+
+    [JsonName("DiscordField")]
+    public class DiscordField
+    {
+        [JsonProperty("name")]
+        public string Name { get; set; }
+
+        [JsonProperty("value")]
+        public string Value { get; set; }
+
+        [JsonProperty("inline")]
+        public bool Inline { get; set; }
+    }
+
+    [JsonName("DiscordThumbnail")]
+    public class DiscordNumbnail
+    {
+        [JsonProperty("Url")]
+        public Uri Url { get; set; }
+    }
+    
+    [JsonName("DiscordImage")]
+    public class DiscordImage
+    {
+        [JsonProperty("Url")]
+        public Uri Url { get; set; }
+    }
+    
+    [JsonName("DiscordFooter")]
+    public class DiscordFooter
+    {
+        [JsonProperty("text")]
+        public string Text { get; set; }
+        
+        [JsonProperty("icon_url")]
+        public Uri icon_url { get; set; }
+    }
+}

Wabbajack.Server/DataLayer/Nexus.cs

@@ -95,7 +95,7 @@ namespace Wabbajack.Server.DataLayer
                       USING (SELECT @GameId GameId, @ModId ModId, @LastChecked LastChecked, @FileId FileId) AS Source
                       ON Target.GameId = Source.GameId AND Target.ModId = Source.ModId AND Target.FileId = Source.FileId
                       WHEN MATCHED THEN UPDATE SET Target.LastChecked = @LastChecked
-                      WHEN NOT MATCHED THEN INSERT (GameId, ModId, LastChecked, FileId) VALUES (@GameId, @ModId, @LastChecked, FileId);",
+                      WHEN NOT MATCHED THEN INSERT (GameId, ModId, LastChecked, FileId) VALUES (@GameId, @ModId, @LastChecked, @FileId);",
                 new
                 {
                     GameId = game.MetaData().NexusGameId,

Wabbajack.Server/Services/ArchiveDownloader.cs

@@ -30,11 +30,11 @@ namespace Wabbajack.Server.Services
 
             while (true)
             {
-                //var (daily, hourly) = await _nexusClient.GetRemainingApiCalls();
+                var (daily, hourly) = await _nexusClient.GetRemainingApiCalls();
                 //bool ignoreNexus = hourly < 25;
                 var ignoreNexus = true;
                 if (ignoreNexus)
-                    _logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})");
+                    _logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {daily}, Hourly:{hourly})");
                 else
                     _logger.LogInformation($"Looking for any download (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})");
 

Wabbajack.Server/Services/DiscordWebHook.cs

@@ -0,0 +1,73 @@
+using System;
+using System.Net.Http;
+using System.Reflection;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer;
+using Wabbajack.Common;
+using Wabbajack.Server.DTOs;
+
+namespace Wabbajack.Server.Services
+{
+    public enum Channel
+    {
+        // High volume messaging, really only useful for internal devs
+        Spam,
+        // Low volume messages designed for admins
+        Ham
+    }
+    public class DiscordWebHook : AbstractService<DiscordWebHook, int>
+    {
+        private AppSettings _settings;
+        private ILogger<DiscordWebHook> _logger;
+        private Random _random = new Random();
+
+        public DiscordWebHook(ILogger<DiscordWebHook> logger, AppSettings settings) : base(logger, settings, TimeSpan.FromHours(1))
+        {
+            _settings = settings;
+            _logger = logger;
+
+            var message = new DiscordMessage
+            {
+                Content = $"\"{GetQuote()}\" - Sheogorath (as he brings the server online)",
+            }; 
+            var a = Send(Channel.Ham, message);
+            var b = Send(Channel.Spam, message);
+
+        }
+
+        public async Task Send(Channel channel, DiscordMessage message)
+        {
+            try
+            {
+                string url = channel switch
+                {
+                    Channel.Spam => _settings.SpamWebHook,
+                    Channel.Ham => _settings.HamWebHook,
+                    _ => null
+                };
+                if (url == null) return;
+
+                var client = new Common.Http.Client();
+                await client.PostAsync(url, new StringContent(message.ToJson(true), Encoding.UTF8, "application/json"));
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, ex.ToJson());
+            }
+        }
+
+        private string GetQuote()
+        {
+            var data = Assembly.GetExecutingAssembly()!.GetManifestResourceStream("Wabbajack.Server.sheo_quotes.txt");
+            var lines = Encoding.UTF8.GetString(data.ReadAll()).Split('\n');
+            return lines[_random.Next(lines.Length)].Trim();
+        }
+
+        public override async Task<int> Execute()
+        {
+            return 0;
+        }
+    }
+}

Wabbajack.Server/Services/ListValidator.cs

@@ -115,18 +115,22 @@ namespace Wabbajack.Server.Services
         public async Task<ArchiveStatus> SlowNexusModStats(ValidationData data, NexusDownloader.State ns)
         {
             var gameId = ns.Game.MetaData().NexusGameId;
-            using var _ = await _slowQueryLock.WaitAsync();
+            //using var _ = await _slowQueryLock.WaitAsync();
             _logger.Log(LogLevel.Warning, $"Slow querying for {ns.Game} {ns.ModID} {ns.FileID}");
 
 
             if (data.NexusFiles.Contains((gameId, ns.ModID, ns.FileID)))
                 return ArchiveStatus.Valid;
 
+            if (data.NexusFiles.Contains((gameId, ns.ModID, -1)))
+                return ArchiveStatus.InValid;
+
             if (data.SlowQueriedFor.Contains((ns.Game, ns.ModID)))
                 return ArchiveStatus.InValid;
             
             var queryTime = DateTime.UtcNow;
-            var regex = new Regex("(?<=[?;&]file_id\\=)\\d+");
+            var regex_id = new Regex("(?<=[?;&]id\\=)\\d+");
+            var regex_file_id = new Regex("(?<=[?;&]file_id\\=)\\d+");
             var client = new Common.Http.Client();
             var result =
                 await client.GetHtmlAsync(
@@ -136,14 +140,19 @@ namespace Wabbajack.Server.Services
                 .Select(f => f.GetAttributeValue("href", ""))
                 .Select(f =>
                 {
-                    var match = regex.Match(f);
+                    var match = regex_id.Match(f);
-                    return !match.Success ? null : match.Value;
+                    if (match.Success)
+                        return match.Value;
+                    match = regex_file_id.Match(f);
+                    if (match.Success)
+                        return match.Value;
+                    return null;
                 })
                 .Where(m => m != null)
                 .Select(m => long.Parse(m))
                 .Distinct()
                 .ToList();
-
+            
             _logger.Log(LogLevel.Warning, $"Slow queried {fileIds.Count} files");
             foreach (var id in fileIds)
             {
@@ -151,6 +160,10 @@ namespace Wabbajack.Server.Services
                 data.NexusFiles.Add((gameId, ns.ModID, id));
             }
 
+            // Add in the default marker
+            await _sql.AddNexusModFileSlow(ns.Game, ns.ModID, -1, queryTime);
+            data.NexusFiles.Add((gameId, ns.ModID, -1));
+
             return fileIds.Contains(ns.FileID) ? ArchiveStatus.Valid : ArchiveStatus.InValid;
         }
         

Wabbajack.Server/Services/NexusPoll.cs

@@ -48,7 +48,7 @@ namespace Wabbajack.Server.Services
                     if (totalPurged > 0)
                         _logger.Log(LogLevel.Information, $"Purged {totalPurged} cache items {result.Game} {result.ModId} {result.TimeStamp}");
 
-                    updated++;
+                    updated += totalPurged;
                 }
                 catch (Exception ex)
                 {
@@ -58,7 +58,7 @@ namespace Wabbajack.Server.Services
             }
 
             if (updated > 0) 
-                _logger.Log(LogLevel.Information, $"Primed {updated} nexus cache entries");
+                _logger.Log(LogLevel.Information, $"RSS Purged {updated} nexus cache entries");
 
             _globalInformation.LastNexusSyncUTC = DateTime.UtcNow;
         }

Wabbajack.Server/Startup.cs

@@ -63,6 +63,7 @@ namespace Wabbajack.Server
             services.AddSingleton<NonNexusDownloadValidator>();
             services.AddSingleton<ListValidator>();
             services.AddSingleton<ArchiveDownloader>();
+            services.AddSingleton<DiscordWebHook>();
             
             services.AddMvc();
             services.AddControllers()
@@ -112,6 +113,7 @@ namespace Wabbajack.Server
             app.UseService<NonNexusDownloadValidator>();
             app.UseService<ListValidator>();
             app.UseService<ArchiveDownloader>();
+            app.UseService<DiscordWebHook>();
 
             app.Use(next =>
             {

Wabbajack.Server/Wabbajack.Server.csproj

@@ -42,5 +42,10 @@
       </None>
     </ItemGroup>
 
+    <ItemGroup>
+      <None Remove="sheo_quotes.txt" />
+      <EmbeddedResource Include="sheo_quotes.txt" />
+    </ItemGroup>
+
 
 </Project>

Wabbajack.Server/sheo_quotes.txt

@@ -0,0 +1,55 @@
+I see you have completed my little errand. Well done. Perhaps you’ve gotten a taste of madness aswell? Do not believe madness to be a curse, mortal. For some it is the greatest of blessings. A bitter mercy perhaps, but mercy non the less. Give me the Fork of Horripilation, I believe I have something more suitable for your needs. Go now. Remember what you have seen.
+Use the fork wisely, mortal. Few have wield to have not come away changed. Use the fork to strike a deathblow to the giant Bullnetch that resides near the hermit. Do this, return the Fork of Horripilation to me, and Sheogorath will reward you well.
+What is it, mortal? Have you come to be of the service to Sheogorath? That in and of itself speaks toward your madness. This pleases me. Fetch the Fork of Horripliation from the mad hermit near Ald Redaynia. Take care with him. He’s not the most... stable man.
+Unworthy, unworthy, unworthy! Useless mortal meat. Walking bag of dung!
+Bring me a champion! Rend the flesh of my foes! A mortal champion to wade through the entrails of my enemies!
+Really, do come in. It’s lovely in the Isles right now. Perfect time for a visit.
+Greetings! Salutations! Welcome! Now go away. Leave. Run. Or die.
+Isn't that a hoot? I love it, myself. Best part of being a Daedric Prince, really. Go ahead, try it again. He loves it!
+Marvellous, marvellous! Self-immolation is a wonderful thing, isn't it? But now that we've had our fun, off to the Sacellum with you.
+I suppose an introduction is in order. I'm Sheogorath, Prince of Madness. And other things. I'm not talking about them.
+You should be off like the wind, solving problems and doing good deeds!
+Time. Time is an artificial construct. An arbitrary system based on the idea that events occur in a linear direction at all times.
+Always forward, never back. Is the concept of time correct? Is time relevant? It matters not. One way or another, I fear that our time has run out.
+A new Gatekeeper! Excellent. We might be onto something with you, after all. That should keep out the stragglers.
+A little busy here! I'm trying to decide what to have for dinner. Oh, how I love eating. One of my favorite things to do.
+It's Jyggalag's time, and not a good time at all. You're going to help me stop it. First, though, you need to get your feet wet.
+Another Daedric Prince. Not a nice one. I don't think ANY of the other Princes like him, actually. I mean, Malacath is more popular at parties.
+The Daedric Prince of Order. Or biscuits... No. Order. And not in a good way. Bleak. Colorless. Dead. Boring, boring, boring.
+The Greymarch comes, and Jyggalag walks. Or runs. Never skips, sidles, or struts. Mostly, he just destroys everything around him.
+Once you understand what My Realm is, you might understand why it's important to keep it intact.
+Two halves, two rulers, two places. Meet and greet. Do what they will, so you know what they're about.
+Ask? ASK? I don't ask. I tell. This is My Realm, remember? My creation, My place, My rules.
+Wonderful! Time for a celebration... Cheese for everyone!
+Makes all of my subjects uneasy. Tense. Homicidal. Some of them, at least. We need to get that Torch relit, before the place falls apart.
+You're going to stop the Greymarch by becoming Me. Or a version of Me. You'll be powerful. Powerful enough to stop Jyggalag.
+You know what would be a good sign? "Free Sweetrolls!" Who wouldn't like that?
+You'll be my champion. You'll grow powerful. You'll grow to be me. Prince of Madness, a new Sheogorath. Or you'll die trying. I love that about you.
+Oh, don't forget to make use of dear Haskill. Between you and me, if he's not summoned three or four times a day, I don't think he feels appreciated.
+I hate indecision! Or maybe I don't. Make up your mind, or I'll have your skin made into a hat -- one of those arrowcatchers. I love those hats!
+So, which is it? What will it be? Mania? Dementia? The suspense is killing me. Or you, if I have to keep waiting.
+Except where the backbone is an actual backbone. Ever been to Malacath's realm...? Nasty stuff. But, back to the business at hand.
+Happens every time. The Greymarch starts, Order appears, and I become Jyggalag and wipe out My whole Realm.
+Flee while you can, mortal. When we next meet I will not know you, and I will slay you like the others.
+Ah... New Sheoth. My home away from places that aren't my home. The current location is much better than some of the prior ones. Don't you think?
+The Isles, the Isles. A wonderful place! Except when it's horrible. Then it's horribly wonderful. Good for a visit. Or for an eternity.
+Time to save the Realm! Rescue the damsel! Slay the beast! Or die trying. Your help is required.
+Daedra are the embodiment of change. Change and permanency. I'm no different, except in the ways that I am.
+Was it Molag? No, no... Little Tim, the toymaker's son? The ghost of King Lysandus? Or was it... Yes! Stanley, that talking grapefruit from Passwall.
+Reeaaaallllyyyy?
+Well? Spit it out, mortal. I haven't got an eternity! Oh, wait! I do.
+I am a part of you, little mortal. I am a shadow in your subconscious, a blemish on your fragile little psyche. You know me. You just don't know it.
+Sheogorath, Daedric Prince of Madness. At your service.
+Yaaawwwwnn....
+Oh, pardon me. Were you saying something? I do apologize, it's just that I find myself suddenly and irrevocably...
+Bored!
+I mean, really. Here you stand, before Sheogorath himself, Daedric Prince of Madness, and all you deem fit to do is... deliver a message? How sad.
+Now you. You can call me Ann Marie.
+Oh... lovely. Now all my dear Pelagius has to worry about are the several hundred legitimate threats...
+Ah, wonderful, wonderful! Why waste all that hatred on yourself when it can so easily be directed at others!
+Mortal? Insufferable.
+Yes, yes, you're entirely brilliant. Conquering madness and all that. Blah blah blah.
+Ah, so now my dear Pelagius can hate himself for being legitimately afraid of things that actually threaten his existence...
+Conquering paranoia should be a snap after that ordeal, hmm?
+Welcome to the deceptively verdant mind of the Emperor Pelagius III. That's right! You're in the head of a dead, homicidally insane monarch.
+The Wabbajack! Huh? Huh? Didn't see that coming, did you?