From 29d327d747776c635c70b898c6c81c1499f93c83 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 14 May 2020 23:25:02 -0600 Subject: [PATCH 1/2] Discord integration and more cache stuff --- Wabbajack.Common/Json.cs | 4 +- Wabbajack.Server/AppSettings.cs | 3 + Wabbajack.Server/DTOs/DiscordMessage.cs | 103 ++++++++++++++++++ Wabbajack.Server/DataLayer/Nexus.cs | 2 +- .../Services/ArchiveDownloader.cs | 4 +- Wabbajack.Server/Services/DiscordWebHook.cs | 73 +++++++++++++ Wabbajack.Server/Services/ListValidator.cs | 23 +++- Wabbajack.Server/Services/NexusPoll.cs | 4 +- Wabbajack.Server/Startup.cs | 2 + Wabbajack.Server/Wabbajack.Server.csproj | 5 + Wabbajack.Server/sheo_quotes.txt | 55 ++++++++++ 11 files changed, 266 insertions(+), 12 deletions(-) create mode 100644 Wabbajack.Server/DTOs/DiscordMessage.cs create mode 100644 Wabbajack.Server/Services/DiscordWebHook.cs create mode 100644 Wabbajack.Server/sheo_quotes.txt diff --git a/Wabbajack.Common/Json.cs b/Wabbajack.Common/Json.cs index 7bc93385..090e110e 100644 --- a/Wabbajack.Common/Json.cs +++ b/Wabbajack.Common/Json.cs @@ -63,9 +63,9 @@ namespace Wabbajack.Common obj.ToJson(fs); } - public static string ToJson(this T obj) + public static string ToJson(this T obj, bool useGenericSettings = false) { - return JsonConvert.SerializeObject(obj, JsonSettings); + return JsonConvert.SerializeObject(obj, useGenericSettings ? GenericJsonSettings : JsonSettings); } public static T FromJson(this AbsolutePath filename) diff --git a/Wabbajack.Server/AppSettings.cs b/Wabbajack.Server/AppSettings.cs index f651dd96..415b42bc 100644 --- a/Wabbajack.Server/AppSettings.cs +++ b/Wabbajack.Server/AppSettings.cs @@ -29,5 +29,8 @@ namespace Wabbajack.BuildServer public string SqlConnection { get; set; } public int MaxJobs { get; set; } = 2; + + public string SpamWebHook { get; set; } = null; + public string HamWebHook { get; set; } = null; } } diff --git a/Wabbajack.Server/DTOs/DiscordMessage.cs b/Wabbajack.Server/DTOs/DiscordMessage.cs new file mode 100644 index 00000000..f9903fa6 --- /dev/null +++ b/Wabbajack.Server/DTOs/DiscordMessage.cs @@ -0,0 +1,103 @@ +using System; +using Newtonsoft.Json; +using Wabbajack.Common.Serialization.Json; + +namespace Wabbajack.Server.DTOs +{ + [JsonName("DiscordMessage")] + public class DiscordMessage + { + [JsonProperty("username")] + public string UserName { get; set; } + + [JsonProperty("avatar_url")] + public Uri AvatarUrl { get; set; } + + [JsonProperty("content")] + public string Content { get; set; } + + [JsonProperty("embeds")] + public DiscordEmbed[] Embeds { get; set; } + } + + [JsonName("DiscordEmbed")] + public class DiscordEmbed + { + [JsonProperty("color")] + public int Color { get; set; } + + [JsonProperty("author")] + public DiscordAuthor Author { get; set; } + + [JsonProperty("url")] + public Uri Url { get; set; } + + [JsonProperty("description")] + public string Description { get; set; } + + [JsonProperty("fields")] + public DiscordField Field { get; set; } + + [JsonProperty("thumbnail")] + public DiscordNumbnail Thumbnail { get; set; } + + [JsonProperty("image")] + public DiscordImage Image { get; set; } + + [JsonProperty("footer")] + public DiscordFooter Footer { get; set; } + + [JsonProperty("timestamp")] + public DateTime Timestamp { get; set; } = DateTime.UtcNow; + } + + [JsonName("DiscordAuthor")] + public class DiscordAuthor + { + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("url")] + public Uri Url { get; set; } + + [JsonProperty("icon_url")] + public Uri IconUrl { get; set; } + } + + [JsonName("DiscordField")] + public class DiscordField + { + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("value")] + public string Value { get; set; } + + [JsonProperty("inline")] + public bool Inline { get; set; } + } + + [JsonName("DiscordThumbnail")] + public class DiscordNumbnail + { + [JsonProperty("Url")] + public Uri Url { get; set; } + } + + [JsonName("DiscordImage")] + public class DiscordImage + { + [JsonProperty("Url")] + public Uri Url { get; set; } + } + + [JsonName("DiscordFooter")] + public class DiscordFooter + { + [JsonProperty("text")] + public string Text { get; set; } + + [JsonProperty("icon_url")] + public Uri icon_url { get; set; } + } +} diff --git a/Wabbajack.Server/DataLayer/Nexus.cs b/Wabbajack.Server/DataLayer/Nexus.cs index a5ecb0a3..43a3c57e 100644 --- a/Wabbajack.Server/DataLayer/Nexus.cs +++ b/Wabbajack.Server/DataLayer/Nexus.cs @@ -95,7 +95,7 @@ namespace Wabbajack.Server.DataLayer USING (SELECT @GameId GameId, @ModId ModId, @LastChecked LastChecked, @FileId FileId) AS Source ON Target.GameId = Source.GameId AND Target.ModId = Source.ModId AND Target.FileId = Source.FileId WHEN MATCHED THEN UPDATE SET Target.LastChecked = @LastChecked - WHEN NOT MATCHED THEN INSERT (GameId, ModId, LastChecked, FileId) VALUES (@GameId, @ModId, @LastChecked, FileId);", + WHEN NOT MATCHED THEN INSERT (GameId, ModId, LastChecked, FileId) VALUES (@GameId, @ModId, @LastChecked, @FileId);", new { GameId = game.MetaData().NexusGameId, diff --git a/Wabbajack.Server/Services/ArchiveDownloader.cs b/Wabbajack.Server/Services/ArchiveDownloader.cs index 7335bfeb..e9e28d31 100644 --- a/Wabbajack.Server/Services/ArchiveDownloader.cs +++ b/Wabbajack.Server/Services/ArchiveDownloader.cs @@ -30,11 +30,11 @@ namespace Wabbajack.Server.Services while (true) { - //var (daily, hourly) = await _nexusClient.GetRemainingApiCalls(); + var (daily, hourly) = await _nexusClient.GetRemainingApiCalls(); //bool ignoreNexus = hourly < 25; var ignoreNexus = true; if (ignoreNexus) - _logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})"); + _logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {daily}, Hourly:{hourly})"); else _logger.LogInformation($"Looking for any download (Daily: {_nexusClient.DailyRemaining}, Hourly:{_nexusClient.HourlyRemaining})"); diff --git a/Wabbajack.Server/Services/DiscordWebHook.cs b/Wabbajack.Server/Services/DiscordWebHook.cs new file mode 100644 index 00000000..750e01db --- /dev/null +++ b/Wabbajack.Server/Services/DiscordWebHook.cs @@ -0,0 +1,73 @@ +using System; +using System.Net.Http; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.BuildServer; +using Wabbajack.Common; +using Wabbajack.Server.DTOs; + +namespace Wabbajack.Server.Services +{ + public enum Channel + { + // High volume messaging, really only useful for internal devs + Spam, + // Low volume messages designed for admins + Ham + } + public class DiscordWebHook : AbstractService + { + private AppSettings _settings; + private ILogger _logger; + private Random _random = new Random(); + + public DiscordWebHook(ILogger logger, AppSettings settings) : base(logger, settings, TimeSpan.FromHours(1)) + { + _settings = settings; + _logger = logger; + + var message = new DiscordMessage + { + Content = $"\"{GetQuote()}\" - Sheogorath (as he brings the server online)", + }; + var a = Send(Channel.Ham, message); + var b = Send(Channel.Spam, message); + + } + + public async Task Send(Channel channel, DiscordMessage message) + { + try + { + string url = channel switch + { + Channel.Spam => _settings.SpamWebHook, + Channel.Ham => _settings.HamWebHook, + _ => null + }; + if (url == null) return; + + var client = new Common.Http.Client(); + await client.PostAsync(url, new StringContent(message.ToJson(true), Encoding.UTF8, "application/json")); + } + catch (Exception ex) + { + _logger.LogError(ex, ex.ToJson()); + } + } + + private string GetQuote() + { + var data = Assembly.GetExecutingAssembly()!.GetManifestResourceStream("Wabbajack.Server.sheo_quotes.txt"); + var lines = Encoding.UTF8.GetString(data.ReadAll()).Split('\n'); + return lines[_random.Next(lines.Length)].Trim(); + } + + public override async Task Execute() + { + return 0; + } + } +} diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 6063931a..821bcc71 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -115,18 +115,22 @@ namespace Wabbajack.Server.Services public async Task SlowNexusModStats(ValidationData data, NexusDownloader.State ns) { var gameId = ns.Game.MetaData().NexusGameId; - using var _ = await _slowQueryLock.WaitAsync(); + //using var _ = await _slowQueryLock.WaitAsync(); _logger.Log(LogLevel.Warning, $"Slow querying for {ns.Game} {ns.ModID} {ns.FileID}"); if (data.NexusFiles.Contains((gameId, ns.ModID, ns.FileID))) return ArchiveStatus.Valid; + if (data.NexusFiles.Contains((gameId, ns.ModID, -1))) + return ArchiveStatus.InValid; + if (data.SlowQueriedFor.Contains((ns.Game, ns.ModID))) return ArchiveStatus.InValid; var queryTime = DateTime.UtcNow; - var regex = new Regex("(?<=[?;&]file_id\\=)\\d+"); + var regex_id = new Regex("(?<=[?;&]id\\=)\\d+"); + var regex_file_id = new Regex("(?<=[?;&]file_id\\=)\\d+"); var client = new Common.Http.Client(); var result = await client.GetHtmlAsync( @@ -136,14 +140,19 @@ namespace Wabbajack.Server.Services .Select(f => f.GetAttributeValue("href", "")) .Select(f => { - var match = regex.Match(f); - return !match.Success ? null : match.Value; + var match = regex_id.Match(f); + if (match.Success) + return match.Value; + match = regex_file_id.Match(f); + if (match.Success) + return match.Value; + return null; }) .Where(m => m != null) .Select(m => long.Parse(m)) .Distinct() .ToList(); - + _logger.Log(LogLevel.Warning, $"Slow queried {fileIds.Count} files"); foreach (var id in fileIds) { @@ -151,6 +160,10 @@ namespace Wabbajack.Server.Services data.NexusFiles.Add((gameId, ns.ModID, id)); } + // Add in the default marker + await _sql.AddNexusModFileSlow(ns.Game, ns.ModID, -1, queryTime); + data.NexusFiles.Add((gameId, ns.ModID, -1)); + return fileIds.Contains(ns.FileID) ? ArchiveStatus.Valid : ArchiveStatus.InValid; } diff --git a/Wabbajack.Server/Services/NexusPoll.cs b/Wabbajack.Server/Services/NexusPoll.cs index bd2902cc..987bed19 100644 --- a/Wabbajack.Server/Services/NexusPoll.cs +++ b/Wabbajack.Server/Services/NexusPoll.cs @@ -48,7 +48,7 @@ namespace Wabbajack.Server.Services if (totalPurged > 0) _logger.Log(LogLevel.Information, $"Purged {totalPurged} cache items {result.Game} {result.ModId} {result.TimeStamp}"); - updated++; + updated += totalPurged; } catch (Exception ex) { @@ -58,7 +58,7 @@ namespace Wabbajack.Server.Services } if (updated > 0) - _logger.Log(LogLevel.Information, $"Primed {updated} nexus cache entries"); + _logger.Log(LogLevel.Information, $"RSS Purged {updated} nexus cache entries"); _globalInformation.LastNexusSyncUTC = DateTime.UtcNow; } diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index a2de2e82..f33f2e9d 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -63,6 +63,7 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddMvc(); services.AddControllers() @@ -112,6 +113,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); + app.UseService(); app.Use(next => { diff --git a/Wabbajack.Server/Wabbajack.Server.csproj b/Wabbajack.Server/Wabbajack.Server.csproj index 13adc236..e98fb232 100644 --- a/Wabbajack.Server/Wabbajack.Server.csproj +++ b/Wabbajack.Server/Wabbajack.Server.csproj @@ -42,5 +42,10 @@ + + + + + diff --git a/Wabbajack.Server/sheo_quotes.txt b/Wabbajack.Server/sheo_quotes.txt new file mode 100644 index 00000000..917d1bc3 --- /dev/null +++ b/Wabbajack.Server/sheo_quotes.txt @@ -0,0 +1,55 @@ +I see you have completed my little errand. Well done. Perhaps you’ve gotten a taste of madness aswell? Do not believe madness to be a curse, mortal. For some it is the greatest of blessings. A bitter mercy perhaps, but mercy non the less. Give me the Fork of Horripilation, I believe I have something more suitable for your needs. Go now. Remember what you have seen. +Use the fork wisely, mortal. Few have wield to have not come away changed. Use the fork to strike a deathblow to the giant Bullnetch that resides near the hermit. Do this, return the Fork of Horripilation to me, and Sheogorath will reward you well. +What is it, mortal? Have you come to be of the service to Sheogorath? That in and of itself speaks toward your madness. This pleases me. Fetch the Fork of Horripliation from the mad hermit near Ald Redaynia. Take care with him. He’s not the most... stable man. +Unworthy, unworthy, unworthy! Useless mortal meat. Walking bag of dung! +Bring me a champion! Rend the flesh of my foes! A mortal champion to wade through the entrails of my enemies! +Really, do come in. It’s lovely in the Isles right now. Perfect time for a visit. +Greetings! Salutations! Welcome! Now go away. Leave. Run. Or die. +Isn't that a hoot? I love it, myself. Best part of being a Daedric Prince, really. Go ahead, try it again. He loves it! +Marvellous, marvellous! Self-immolation is a wonderful thing, isn't it? But now that we've had our fun, off to the Sacellum with you. +I suppose an introduction is in order. I'm Sheogorath, Prince of Madness. And other things. I'm not talking about them. +You should be off like the wind, solving problems and doing good deeds! +Time. Time is an artificial construct. An arbitrary system based on the idea that events occur in a linear direction at all times. +Always forward, never back. Is the concept of time correct? Is time relevant? It matters not. One way or another, I fear that our time has run out. +A new Gatekeeper! Excellent. We might be onto something with you, after all. That should keep out the stragglers. +A little busy here! I'm trying to decide what to have for dinner. Oh, how I love eating. One of my favorite things to do. +It's Jyggalag's time, and not a good time at all. You're going to help me stop it. First, though, you need to get your feet wet. +Another Daedric Prince. Not a nice one. I don't think ANY of the other Princes like him, actually. I mean, Malacath is more popular at parties. +The Daedric Prince of Order. Or biscuits... No. Order. And not in a good way. Bleak. Colorless. Dead. Boring, boring, boring. +The Greymarch comes, and Jyggalag walks. Or runs. Never skips, sidles, or struts. Mostly, he just destroys everything around him. +Once you understand what My Realm is, you might understand why it's important to keep it intact. +Two halves, two rulers, two places. Meet and greet. Do what they will, so you know what they're about. +Ask? ASK? I don't ask. I tell. This is My Realm, remember? My creation, My place, My rules. +Wonderful! Time for a celebration... Cheese for everyone! +Makes all of my subjects uneasy. Tense. Homicidal. Some of them, at least. We need to get that Torch relit, before the place falls apart. +You're going to stop the Greymarch by becoming Me. Or a version of Me. You'll be powerful. Powerful enough to stop Jyggalag. +You know what would be a good sign? "Free Sweetrolls!" Who wouldn't like that? +You'll be my champion. You'll grow powerful. You'll grow to be me. Prince of Madness, a new Sheogorath. Or you'll die trying. I love that about you. +Oh, don't forget to make use of dear Haskill. Between you and me, if he's not summoned three or four times a day, I don't think he feels appreciated. +I hate indecision! Or maybe I don't. Make up your mind, or I'll have your skin made into a hat -- one of those arrowcatchers. I love those hats! +So, which is it? What will it be? Mania? Dementia? The suspense is killing me. Or you, if I have to keep waiting. +Except where the backbone is an actual backbone. Ever been to Malacath's realm...? Nasty stuff. But, back to the business at hand. +Happens every time. The Greymarch starts, Order appears, and I become Jyggalag and wipe out My whole Realm. +Flee while you can, mortal. When we next meet I will not know you, and I will slay you like the others. +Ah... New Sheoth. My home away from places that aren't my home. The current location is much better than some of the prior ones. Don't you think? +The Isles, the Isles. A wonderful place! Except when it's horrible. Then it's horribly wonderful. Good for a visit. Or for an eternity. +Time to save the Realm! Rescue the damsel! Slay the beast! Or die trying. Your help is required. +Daedra are the embodiment of change. Change and permanency. I'm no different, except in the ways that I am. +Was it Molag? No, no... Little Tim, the toymaker's son? The ghost of King Lysandus? Or was it... Yes! Stanley, that talking grapefruit from Passwall. +Reeaaaallllyyyy? +Well? Spit it out, mortal. I haven't got an eternity! Oh, wait! I do. +I am a part of you, little mortal. I am a shadow in your subconscious, a blemish on your fragile little psyche. You know me. You just don't know it. +Sheogorath, Daedric Prince of Madness. At your service. +Yaaawwwwnn.... +Oh, pardon me. Were you saying something? I do apologize, it's just that I find myself suddenly and irrevocably... +Bored! +I mean, really. Here you stand, before Sheogorath himself, Daedric Prince of Madness, and all you deem fit to do is... deliver a message? How sad. +Now you. You can call me Ann Marie. +Oh... lovely. Now all my dear Pelagius has to worry about are the several hundred legitimate threats... +Ah, wonderful, wonderful! Why waste all that hatred on yourself when it can so easily be directed at others! +Mortal? Insufferable. +Yes, yes, you're entirely brilliant. Conquering madness and all that. Blah blah blah. +Ah, so now my dear Pelagius can hate himself for being legitimately afraid of things that actually threaten his existence... +Conquering paranoia should be a snap after that ordeal, hmm? +Welcome to the deceptively verdant mind of the Emperor Pelagius III. That's right! You're in the head of a dead, homicidally insane monarch. +The Wabbajack! Huh? Huh? Didn't see that coming, did you? \ No newline at end of file From 88e3db0b2a5833e68bd579397423d8eaaeca5d7f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 16 May 2020 09:08:40 -0600 Subject: [PATCH 2/2] Several server-side changes to make the system more durable to API call limits. --- CHANGELOG.md | 3 + Wabbajack.Lib/AInstaller.cs | 13 +- Wabbajack.Lib/Downloaders/NexusDownloader.cs | 3 +- Wabbajack.Lib/NexusApi/NexusApi.cs | 26 ++- Wabbajack.Server.Test/NexusCacheTests.cs | 12 +- Wabbajack.Server.Test/sql/wabbajack_db.sql | 18 ++ Wabbajack.Server/AppSettings.cs | 3 + Wabbajack.Server/Controllers/NexusCache.cs | 17 +- Wabbajack.Server/DataLayer/Nexus.cs | 5 - Wabbajack.Server/DataLayer/NexusKeys.cs | 51 +++++ .../Services/ArchiveDownloader.cs | 5 +- Wabbajack.Server/Services/ListValidator.cs | 174 +++++++----------- .../Services/ModListDownloader.cs | 23 ++- .../Services/NexusKeyMaintainance.cs | 87 +++++++++ Wabbajack.Server/Services/NexusPoll.cs | 9 +- Wabbajack.Server/Startup.cs | 2 + Wabbajack/View Models/MainWindowVM.cs | 1 + 17 files changed, 322 insertions(+), 130 deletions(-) create mode 100644 Wabbajack.Server/DataLayer/NexusKeys.cs create mode 100644 Wabbajack.Server/Services/NexusKeyMaintainance.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index f8def467..7182b963 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ### Changelog +#### Version - 2.0.5.1 - 5/16/2020 +* Close automatically Wabbajack when opening the CLI, resolves the "RocksDB is in use" errors + #### Version - 2.0.5.0 - 5/14/2020 * Make the CDN downloads multi-threaded * Optimize installation of included files diff --git a/Wabbajack.Lib/AInstaller.cs b/Wabbajack.Lib/AInstaller.cs index ff2cb1b0..50bf4976 100644 --- a/Wabbajack.Lib/AInstaller.cs +++ b/Wabbajack.Lib/AInstaller.cs @@ -292,11 +292,18 @@ namespace Wabbajack.Lib public async Task HashArchives() { - var hashResults = await - DownloadFolder.EnumerateFiles() + Utils.Log("Looking for files to hash"); + var toHash = DownloadFolder.EnumerateFiles() .Concat(Game.GameLocation().EnumerateFiles()) .Where(e => e.Extension != Consts.HashFileExtension) - .PMap(Queue, async e => (await e.FileHashCachedAsync(), e)); + .ToList(); + + Utils.Log($"Found {toHash} files to hash"); + + var hashResults = await + toHash + .PMap(Queue, async e => (await e.FileHashCachedAsync(), e)); + HashedArchives.SetTo(hashResults .OrderByDescending(e => e.Item2.LastModified) .GroupBy(e => e.Item1) diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 04283602..d4749087 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -121,7 +121,8 @@ namespace Wabbajack.Lib.Downloaders $"Authenticating for the Nexus failed. A nexus account is required to automatically download mods.")); return; } - + + await Metrics.Send("nexus_login", _client.ApiKey!); if (!await _client.IsPremium()) { diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index 3dc808a2..1fce03b5 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -157,8 +157,11 @@ namespace Wabbajack.Lib.NexusApi { var url = "https://api.nexusmods.com/v1/users/validate.json"; using var response = await HttpClient.GetAsync(url); - return (int.Parse(response.Headers.GetValues("X-RL-Daily-Remaining").First()), + var result = (int.Parse(response.Headers.GetValues("X-RL-Daily-Remaining").First()), int.Parse(response.Headers.GetValues("X-RL-Hourly-Remaining").First())); + _dailyRemaining = result.Item1; + _hourlyRemaining = result.Item2; + return result; } #endregion @@ -177,6 +180,13 @@ namespace Wabbajack.Lib.NexusApi return _dailyRemaining; } } + protected set + { + lock (RemainingLock) + { + _dailyRemaining = value; + } + } } private int _hourlyRemaining; @@ -189,10 +199,18 @@ namespace Wabbajack.Lib.NexusApi return _hourlyRemaining; } } + protected set + { + lock (RemainingLock) + { + _hourlyRemaining = value; + } + } + } - private void UpdateRemaining(HttpResponseMessage response) + protected virtual async Task UpdateRemaining(HttpResponseMessage response) { try { @@ -221,7 +239,7 @@ namespace Wabbajack.Lib.NexusApi #endregion - private NexusApiClient(string? apiKey = null) + protected NexusApiClient(string? apiKey = null) { ApiKey = apiKey; @@ -250,7 +268,7 @@ namespace Wabbajack.Lib.NexusApi try { using var response = await HttpClient.GetAsync(url); - UpdateRemaining(response); + await UpdateRemaining(response); if (!response.IsSuccessStatusCode) { Utils.Log($"Nexus call failed: {response.RequestMessage.RequestUri}"); diff --git a/Wabbajack.Server.Test/NexusCacheTests.cs b/Wabbajack.Server.Test/NexusCacheTests.cs index 97ffcf73..e8b81665 100644 --- a/Wabbajack.Server.Test/NexusCacheTests.cs +++ b/Wabbajack.Server.Test/NexusCacheTests.cs @@ -60,7 +60,7 @@ namespace Wabbajack.BuildServer.Test } [Fact] - public async Task CanQueryAndFindNexusModfilesSlow() + public async Task CanQueryAndFindNexusModfilesFast() { var startTime = DateTime.UtcNow; var sql = Fixture.GetService(); @@ -68,8 +68,7 @@ namespace Wabbajack.BuildServer.Test await sql.DeleteNexusModFilesUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow); await sql.DeleteNexusModInfosUpdatedBeforeDate(Game.SkyrimSpecialEdition, 1137, DateTime.UtcNow); - var result = await validator.SlowNexusModStats(new ValidationData(), - new NexusDownloader.State {Game = Game.SkyrimSpecialEdition, ModID = 1137, FileID = 121449}); + var result = await validator.FastNexusModStats(new NexusDownloader.State {Game = Game.SkyrimSpecialEdition, ModID = 1137, FileID = 121449}); Assert.Equal(ArchiveStatus.Valid, result); var gameId = Game.SkyrimSpecialEdition.MetaData().NexusGameId; @@ -119,6 +118,13 @@ namespace Wabbajack.BuildServer.Test Assert.Equal(b, new Box{Value = b}.ToJson().FromJsonString().Value); Assert.NotEqual(a.Hour, b.Hour); Assert.Equal(b.Hour, new Box{Value = a}.ToJson().FromJsonString().Value.Hour); + + + var ts = (long)1589528640; + var ds = DateTime.Parse("2020-05-15 07:44:00.000"); + Assert.Equal(ds, ts.AsUnixTime()); + Assert.Equal(ts, (long)ds.AsUnixTime()); + Assert.Equal(ts, (long)ts.AsUnixTime().AsUnixTime()); } } diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index 56c61b95..09a9754a 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -551,6 +551,24 @@ CONSTRAINT [PK_NexusModFilesSlow] PRIMARY KEY CLUSTERED ) ON [PRIMARY] GO +/****** Object: Table [dbo].[NexusKeys] Script Date: 5/15/2020 5:20:02 PM ******/ +SET ANSI_NULLS ON +GO + +SET QUOTED_IDENTIFIER ON +GO + +CREATE TABLE [dbo].[NexusKeys]( + [ApiKey] [nvarchar](162) NOT NULL, + [DailyRemain] [int] NOT NULL, + [HourlyRemain] [int] NOT NULL, + CONSTRAINT [PK_NexusKeys] PRIMARY KEY CLUSTERED + ( + [ApiKey] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] +GO + /****** Object: StoredProcedure [dbo].[MergeAllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/ SET ANSI_NULLS ON GO diff --git a/Wabbajack.Server/AppSettings.cs b/Wabbajack.Server/AppSettings.cs index 415b42bc..9da5f051 100644 --- a/Wabbajack.Server/AppSettings.cs +++ b/Wabbajack.Server/AppSettings.cs @@ -25,6 +25,9 @@ namespace Wabbajack.BuildServer public bool RunFrontEndJobs { get; set; } public bool RunBackEndJobs { get; set; } + public bool RunNexusPolling { get; set; } + public bool RunDownloader { get; set; } + public string BunnyCDN_StorageZone { get; set; } public string SqlConnection { get; set; } diff --git a/Wabbajack.Server/Controllers/NexusCache.cs b/Wabbajack.Server/Controllers/NexusCache.cs index b5ed0414..c2c8830a 100644 --- a/Wabbajack.Server/Controllers/NexusCache.cs +++ b/Wabbajack.Server/Controllers/NexusCache.cs @@ -52,7 +52,7 @@ namespace Wabbajack.BuildServer.Controllers string method = "CACHED"; if (result == null) { - var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); + var api = await GetClient(); result = await api.GetModInfo(game, ModId, false); await _sql.AddNexusModInfo(game, ModId, result.updated_time, result); @@ -69,6 +69,21 @@ namespace Wabbajack.BuildServer.Controllers return result; } + private async Task GetClient() + { + var key = Request.Headers["apikey"].FirstOrDefault(); + if (key == null) + return await NexusApiClient.Get(null); + + if (await _sql.HaveKey(key)) + return await NexusApiClient.Get(key); + + var client = await NexusApiClient.Get(key); + var (daily, hourly) = await client.GetRemainingApiCalls(); + await _sql.SetNexusAPIKey(key, daily, hourly); + return client; + } + [HttpGet] [Route("{GameName}/mods/{ModId}/files.json")] public async Task GetModFiles(string GameName, long ModId) diff --git a/Wabbajack.Server/DataLayer/Nexus.cs b/Wabbajack.Server/DataLayer/Nexus.cs index 43a3c57e..d6cc0938 100644 --- a/Wabbajack.Server/DataLayer/Nexus.cs +++ b/Wabbajack.Server/DataLayer/Nexus.cs @@ -30,11 +30,6 @@ namespace Wabbajack.Server.DataLayer @"DELETE FROM dbo.NexusModFiles WHERE Game = @Game AND ModID = @ModId AND LastChecked < @Date SELECT @@ROWCOUNT AS Deleted", new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date}); - - deleted += await conn.ExecuteScalarAsync( - @"DELETE FROM dbo.NexusModFilesSlow WHERE GameId = @Game AND ModID = @ModId AND LastChecked < @Date - SELECT @@ROWCOUNT AS Deleted", - new {Game = game.MetaData().NexusGameId, ModId = modId, Date = date}); return deleted; } diff --git a/Wabbajack.Server/DataLayer/NexusKeys.cs b/Wabbajack.Server/DataLayer/NexusKeys.cs new file mode 100644 index 00000000..0ace9f5e --- /dev/null +++ b/Wabbajack.Server/DataLayer/NexusKeys.cs @@ -0,0 +1,51 @@ +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Dapper; + +namespace Wabbajack.Server.DataLayer +{ + public partial class SqlService + { + public async Task SetNexusAPIKey(string key, long daily, long hourly) + { + await using var conn = await Open(); + await using var trans = await conn.BeginTransactionAsync(); + await conn.ExecuteAsync(@"DELETE FROM NexusKeys WHERE ApiKey = @ApiKey", new {ApiKey = key}, trans); + await conn.ExecuteAsync(@"INSERT INTO NexusKeys (ApiKey, DailyRemain, HourlyRemain) VALUES (@ApiKey, @DailyRemain, @HourlyRemain)", + new {ApiKey = key, DailyRemain = daily, HourlyRemain = hourly}, trans); + await trans.CommitAsync(); + } + + + public async Task DeleteNexusAPIKey(string key) + { + await using var conn = await Open(); + await conn.ExecuteAsync(@"DELETE FROM NexusKeys WHERE ApiKey = @ApiKey", new {ApiKey = key}); + } + + public async Task> GetNexusApiKeys(int threshold = 1500) + { + await using var conn = await Open(); + return (await conn.QueryAsync(@"SELECT ApiKey FROM NexusKeys WHERE DailyRemain >= @Threshold ORDER BY DailyRemain DESC", + new {Threshold = threshold})).ToList(); + } + + public async Task> GetNexusApiKeysWithCounts(int threshold = 1500) + { + await using var conn = await Open(); + return (await conn.QueryAsync<(string, int, int)>(@"SELECT ApiKey, DailyRemain, HourlyRemain FROM NexusKeys WHERE DailyRemain >= @Threshold ORDER BY DailyRemain DESC", + new {Threshold = threshold})).ToList(); + } + + + public async Task HaveKey(string key) + { + await using var conn = await Open(); + return (await conn.QueryAsync(@"SELECT ApiKey FROM NexusKeys WHERE ApiKey = @ApiKey", + new {ApiKey = key})).Any(); + } + + } +} diff --git a/Wabbajack.Server/Services/ArchiveDownloader.cs b/Wabbajack.Server/Services/ArchiveDownloader.cs index e9e28d31..fbf733ad 100644 --- a/Wabbajack.Server/Services/ArchiveDownloader.cs +++ b/Wabbajack.Server/Services/ArchiveDownloader.cs @@ -31,8 +31,8 @@ namespace Wabbajack.Server.Services while (true) { var (daily, hourly) = await _nexusClient.GetRemainingApiCalls(); - //bool ignoreNexus = hourly < 25; - var ignoreNexus = true; + bool ignoreNexus = (daily < 100 && hourly < 10); + //var ignoreNexus = true; if (ignoreNexus) _logger.LogWarning($"Ignoring Nexus Downloads due to low hourly api limit (Daily: {daily}, Hourly:{hourly})"); else @@ -89,6 +89,7 @@ namespace Wabbajack.Server.Services } catch (Exception ex) { + _logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}"); await nextDownload.Fail(_sql, ex.ToString()); } diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 821bcc71..ce2955c5 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -22,22 +22,25 @@ namespace Wabbajack.Server.Services public class ListValidator : AbstractService { private SqlService _sql; - private NexusApiClient _nexusClient; + private DiscordWebHook _discord; + private NexusKeyMaintainance _nexus; public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries { get; private set; } = new (ModListSummary Summary, DetailedStatus Detailed)[0]; - public ListValidator(ILogger logger, AppSettings settings, SqlService sql) + public ListValidator(ILogger logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus) : base(logger, settings, TimeSpan.FromMinutes(10)) { _sql = sql; + _discord = discord; + _nexus = nexus; } public override async Task Execute() { var data = await _sql.GetValidationData(); - + using var queue = new WorkQueue(); var results = await data.ModLists.PMap(queue, async list => @@ -94,7 +97,7 @@ namespace Wabbajack.Server.Services nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)): return (archive, ArchiveStatus.Valid); case NexusDownloader.State ns: - return (archive, await SlowNexusModStats(data, ns)); + return (archive, await FastNexusModStats(ns)); case ManualDownloader.State _: return (archive, ArchiveStatus.Valid); default: @@ -109,125 +112,88 @@ namespace Wabbajack.Server.Services } } } - - private readonly AsyncLock _slowQueryLock = new AsyncLock(); - public async Task SlowNexusModStats(ValidationData data, NexusDownloader.State ns) + private AsyncLock _lock = new AsyncLock(); + + public async Task FastNexusModStats(NexusDownloader.State ns) { - var gameId = ns.Game.MetaData().NexusGameId; - //using var _ = await _slowQueryLock.WaitAsync(); - _logger.Log(LogLevel.Warning, $"Slow querying for {ns.Game} {ns.ModID} {ns.FileID}"); - - - if (data.NexusFiles.Contains((gameId, ns.ModID, ns.FileID))) - return ArchiveStatus.Valid; - - if (data.NexusFiles.Contains((gameId, ns.ModID, -1))) - return ArchiveStatus.InValid; - - if (data.SlowQueriedFor.Contains((ns.Game, ns.ModID))) - return ArchiveStatus.InValid; - - var queryTime = DateTime.UtcNow; - var regex_id = new Regex("(?<=[?;&]id\\=)\\d+"); - var regex_file_id = new Regex("(?<=[?;&]file_id\\=)\\d+"); - var client = new Common.Http.Client(); - var result = - await client.GetHtmlAsync( - $"https://www.nexusmods.com/{ns.Game.MetaData().NexusName}/mods/{ns.ModID}/?tab=files"); - - var fileIds = result.DocumentNode.Descendants() - .Select(f => f.GetAttributeValue("href", "")) - .Select(f => - { - var match = regex_id.Match(f); - if (match.Success) - return match.Value; - match = regex_file_id.Match(f); - if (match.Success) - return match.Value; - return null; - }) - .Where(m => m != null) - .Select(m => long.Parse(m)) - .Distinct() - .ToList(); - - _logger.Log(LogLevel.Warning, $"Slow queried {fileIds.Count} files"); - foreach (var id in fileIds) - { - await _sql.AddNexusModFileSlow(ns.Game, ns.ModID, id, queryTime); - data.NexusFiles.Add((gameId, ns.ModID, id)); - } - - // Add in the default marker - await _sql.AddNexusModFileSlow(ns.Game, ns.ModID, -1, queryTime); - data.NexusFiles.Add((gameId, ns.ModID, -1)); - - return fileIds.Contains(ns.FileID) ? ArchiveStatus.Valid : ArchiveStatus.InValid; - } - - private async Task FastNexusModStats(NexusDownloader.State ns) - { - + // Check if some other thread has added them var mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID); var files = await _sql.GetModFiles(ns.Game, ns.ModID); - try + if (mod == null || files == null) { - if (mod == null) + // Aquire the lock + using var lck = await _lock.WaitAsync(); + + // Check again + mod = await _sql.GetNexusModInfoString(ns.Game, ns.ModID); + files = await _sql.GetModFiles(ns.Game, ns.ModID); + + if (mod == null || files == null) { - _nexusClient ??= await NexusApiClient.Get(); - _logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}"); + + NexusApiClient nexusClient = await _nexus.GetClient(); + var queryTime = DateTime.UtcNow; + try { - mod = await _nexusClient.GetModInfo(ns.Game, ns.ModID, false); - } - catch - { - mod = new ModInfo + if (mod == null) { - mod_id = ns.ModID.ToString(), game_id = ns.Game.MetaData().NexusGameId, available = false - }; - } + _logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}"); + try + { + mod = await nexusClient.GetModInfo(ns.Game, ns.ModID, false); + } + catch + { + mod = new ModInfo + { + mod_id = ns.ModID.ToString(), + game_id = ns.Game.MetaData().NexusGameId, + available = false + }; + } - try - { - await _sql.AddNexusModInfo(ns.Game, ns.ModID, mod.updated_time, mod); - } - catch (Exception _) - { - // Could be a PK constraint failure - } + try + { + await _sql.AddNexusModInfo(ns.Game, ns.ModID, queryTime, mod); + } + catch (Exception _) + { + // Could be a PK constraint failure + } - } + } - if (files == null) - { - _logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}"); - try - { - files = await _nexusClient.GetModFiles(ns.Game, ns.ModID, false); - } - catch - { - files = new NexusApiClient.GetModFilesResponse {files = new List()}; - } + if (files == null) + { + _logger.Log(LogLevel.Information, $"Found missing Nexus mod info {ns.Game} {ns.ModID}"); + try + { + files = await nexusClient.GetModFiles(ns.Game, ns.ModID, false); + } + catch + { + files = new NexusApiClient.GetModFilesResponse {files = new List()}; + } - try - { - await _sql.AddNexusModFiles(ns.Game, ns.ModID, mod.updated_time, files); + try + { + await _sql.AddNexusModFiles(ns.Game, ns.ModID, queryTime, files); + } + catch (Exception _) + { + // Could be a PK constraint failure + } + } } - catch (Exception _) + catch (Exception ex) { - // Could be a PK constraint failure + return ArchiveStatus.InValid; } } } - catch (Exception ex) - { - return ArchiveStatus.InValid; - } if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID)) return ArchiveStatus.Valid; diff --git a/Wabbajack.Server/Services/ModListDownloader.cs b/Wabbajack.Server/Services/ModListDownloader.cs index d8d4b17c..2c611e3e 100644 --- a/Wabbajack.Server/Services/ModListDownloader.cs +++ b/Wabbajack.Server/Services/ModListDownloader.cs @@ -11,6 +11,7 @@ using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.ModListRegistry; using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DTOs; namespace Wabbajack.Server.Services { @@ -20,13 +21,15 @@ namespace Wabbajack.Server.Services private AppSettings _settings; private ArchiveMaintainer _maintainer; private SqlService _sql; + private DiscordWebHook _discord; - public ModListDownloader(ILogger logger, AppSettings settings, ArchiveMaintainer maintainer, SqlService sql) + public ModListDownloader(ILogger logger, AppSettings settings, ArchiveMaintainer maintainer, SqlService sql, DiscordWebHook discord) { _logger = logger; _settings = settings; _maintainer = maintainer; _sql = sql; + _discord = discord; } public void Start() @@ -69,6 +72,8 @@ namespace Wabbajack.Server.Services if (!_maintainer.HaveArchive(list.DownloadMetadata!.Hash)) { _logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}"); + await _discord.Send(Channel.Ham, + new DiscordMessage {Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"}); var tf = new TempFile(); var state = DownloadDispatcher.ResolveArchive(list.Links.Download); if (state == null) @@ -100,7 +105,9 @@ namespace Wabbajack.Server.Services { if (entry == null) { - Utils.Log($"Bad Modlist {list.Links.MachineURL}"); + _logger.LogWarning($"Bad Modlist {list.Links.MachineURL}"); + await _discord.Send(Channel.Ham, + new DiscordMessage {Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"}); continue; } @@ -110,7 +117,9 @@ namespace Wabbajack.Server.Services } catch (JsonReaderException ex) { - Utils.Log($"Bad JSON format for {list.Links.MachineURL}"); + _logger.LogWarning($"Bad Modlist {list.Links.MachineURL}"); + await _discord.Send(Channel.Ham, + new DiscordMessage {Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"}); continue; } } @@ -120,12 +129,20 @@ namespace Wabbajack.Server.Services catch (Exception ex) { _logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}"); + await _discord.Send(Channel.Ham, + new DiscordMessage {Content = $"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"}); } } _logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists"); + if (downloaded > 0) + await _discord.Send(Channel.Ham, + new DiscordMessage {Content = $"Downloaded {downloaded} new lists"}); var fc = await _sql.EnqueueModListFilesForIndexing(); _logger.Log(LogLevel.Information, $"Enqueing {fc} files for downloading"); + if (fc > 0) + await _discord.Send(Channel.Ham, + new DiscordMessage {Content = $"Enqueing {fc} files for downloading"}); return downloaded; } diff --git a/Wabbajack.Server/Services/NexusKeyMaintainance.cs b/Wabbajack.Server/Services/NexusKeyMaintainance.cs new file mode 100644 index 00000000..bf21b232 --- /dev/null +++ b/Wabbajack.Server/Services/NexusKeyMaintainance.cs @@ -0,0 +1,87 @@ +using System; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.BuildServer; +using Wabbajack.Lib.NexusApi; +using Wabbajack.Server.DataLayer; + +namespace Wabbajack.Server.Services +{ + public class NexusKeyMaintainance : AbstractService + { + private SqlService _sql; + + public NexusKeyMaintainance(ILogger logger, AppSettings settings, SqlService sql) : base(logger, settings, TimeSpan.FromHours(1)) + { + _sql = sql; + } + + public async Task GetClient() + { + var keys = await _sql.GetNexusApiKeysWithCounts(1500); + foreach (var key in keys) + { + return new TrackingClient(_sql, key); + } + + return await NexusApiClient.Get(); + } + + public override async Task Execute() + { + var keys = await _sql.GetNexusApiKeysWithCounts(0); + _logger.Log(LogLevel.Information, $"Verifying {keys.Count} API Keys"); + foreach (var key in keys) + { + try + { + var client = new TrackingClient(_sql, key); + + var status = await client.GetUserStatus(); + if (!status.is_premium) + { + await _sql.DeleteNexusAPIKey(key.Key); + continue; + } + + var (daily, hourly) = await client.GetRemainingApiCalls(); + await _sql.SetNexusAPIKey(key.Key, daily, hourly); + } + catch (Exception ex) + { + _logger.Log(LogLevel.Warning, "Update error, purging API key"); + await _sql.DeleteNexusAPIKey(key.Key); + } + } + + return keys.Count; + } + } + + public class TrackingClient : NexusApiClient + { + private SqlService _sql; + public TrackingClient(SqlService sql, (string Key, int Daily, int Hourly) key) : base(key.Key) + { + _sql = sql; + DailyRemaining = key.Daily; + HourlyRemaining = key.Hourly; + } + + protected virtual async Task UpdateRemaining(HttpResponseMessage response) + { + await base.UpdateRemaining(response); + try + { + var dailyRemaining = int.Parse(response.Headers.GetValues("x-rl-daily-remaining").First()); + var hourlyRemaining = int.Parse(response.Headers.GetValues("x-rl-hourly-remaining").First()); + await _sql.SetNexusAPIKey(ApiKey, dailyRemaining, hourlyRemaining); + } + catch (Exception ex) + { + } + } + } +} diff --git a/Wabbajack.Server/Services/NexusPoll.cs b/Wabbajack.Server/Services/NexusPoll.cs index 987bed19..ef68b47e 100644 --- a/Wabbajack.Server/Services/NexusPoll.cs +++ b/Wabbajack.Server/Services/NexusPoll.cs @@ -98,14 +98,15 @@ namespace Wabbajack.Server.Services // Mod activity could hide files var b = d.mod.LastestModActivity.AsUnixTime(); - return new {Game = d.game.Game, Date = (a > b ? a : b), ModId = d.mod.ModId}; + return new {Game = d.game.Game, Date = (a > b) ? a : b, ModId = d.mod.ModId}; }); var purged = await collected.PMap(queue, async t => { - var resultA = await _sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date); - var resultB = await _sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date); - return resultA + resultB; + long purgeCount = 0; + purgeCount += await _sql.DeleteNexusModInfosUpdatedBeforeDate(t.Game, t.ModId, t.Date); + purgeCount += await _sql.DeleteNexusModFilesUpdatedBeforeDate(t.Game, t.ModId, t.Date); + return purgeCount; }); _logger.Log(LogLevel.Information, $"Purged {purged.Sum()} cache entries"); diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index f33f2e9d..e97dcd6b 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -64,6 +64,7 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddMvc(); services.AddControllers() @@ -114,6 +115,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); + app.UseService(); app.Use(next => { diff --git a/Wabbajack/View Models/MainWindowVM.cs b/Wabbajack/View Models/MainWindowVM.cs index a0886b98..ddfa5292 100644 --- a/Wabbajack/View Models/MainWindowVM.cs +++ b/Wabbajack/View Models/MainWindowVM.cs @@ -155,6 +155,7 @@ namespace Wabbajack WorkingDirectory = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location) }; Process.Start(process); + ShutdownApplication(); } private static bool IsStartingFromModlist(out AbsolutePath modlistPath)