From bc94ec43215e11e0547f77e9a3cce78dd79bb311 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 11 Aug 2020 22:25:12 -0600 Subject: [PATCH] Tons of server-side fixes and tweaks to deal with strange nexus states --- Wabbajack.CLI/OptionsDefinition.cs | 3 +- Wabbajack.CLI/Verbs/NexusPermissions.cs | 27 +++++ Wabbajack.Common/OctoDiff.cs | 25 +++- Wabbajack.Lib/Downloaders/NexusDownloader.cs | 35 +++--- .../ModListRegistry/ModListMetadata.cs | 2 +- Wabbajack.Lib/NexusApi/INexusApi.cs | 17 +++ Wabbajack.Lib/NexusApi/NexusApi.cs | 26 ++-- Wabbajack.Server.Test/NexusCacheTests.cs | 14 +++ Wabbajack.Server.Test/sql/wabbajack_db.sql | 1 + Wabbajack.Server/Controllers/Metrics.cs | 2 +- Wabbajack.Server/Controllers/ModUpgrade.cs | 14 +-- Wabbajack.Server/Controllers/NexusCache.cs | 27 ++++- .../DataLayer/ArchiveDownloads.cs | 19 +++ Wabbajack.Server/DataLayer/Nexus.cs | 14 ++- Wabbajack.Server/Services/ListValidator.cs | 111 ++++++++++++++---- .../Services/NexusKeyMaintainance.cs | 7 +- .../Services/NexusPermissionsUpdater.cs | 41 +++---- Wabbajack.Server/Services/NexusPoll.cs | 6 +- Wabbajack.Server/Services/PatchBuilder.cs | 4 +- Wabbajack.Server/Services/QuickSync.cs | 14 ++- 20 files changed, 295 insertions(+), 114 deletions(-) create mode 100644 Wabbajack.CLI/Verbs/NexusPermissions.cs create mode 100644 Wabbajack.Lib/NexusApi/INexusApi.cs diff --git a/Wabbajack.CLI/OptionsDefinition.cs b/Wabbajack.CLI/OptionsDefinition.cs index 8c73a6da..50398100 100644 --- a/Wabbajack.CLI/OptionsDefinition.cs +++ b/Wabbajack.CLI/OptionsDefinition.cs @@ -29,7 +29,8 @@ namespace Wabbajack.CLI typeof(ForceHealing), typeof(HashVariants), typeof(ParseMeta), - typeof(NoPatch) + typeof(NoPatch), + typeof(NexusPermissions) }; } } diff --git a/Wabbajack.CLI/Verbs/NexusPermissions.cs b/Wabbajack.CLI/Verbs/NexusPermissions.cs new file mode 100644 index 00000000..99fdf2bd --- /dev/null +++ b/Wabbajack.CLI/Verbs/NexusPermissions.cs @@ -0,0 +1,27 @@ +using System; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Lib.NexusApi; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("nexus-permissions", HelpText = "Get the nexus permissions for a mod")] + public class NexusPermissions : AVerb + { + [Option('m', "mod-id", Required = true, HelpText = "Mod Id")] + public long ModId { get; set; } = 0; + + [Option('g', "game", Required = true, HelpText = "Game Name")] + public string GameName { get; set; } = ""; + protected override async Task Run() + { + var game = GameRegistry.GetByFuzzyName(GameName).Game; + var p = await HTMLInterface.GetUploadPermissions(game, ModId); + Console.WriteLine($"Game: {game}"); + Console.WriteLine($"ModId: {ModId}"); + Console.WriteLine($"Permissions: {p}"); + return ExitCode.Ok; + } + } +} diff --git a/Wabbajack.Common/OctoDiff.cs b/Wabbajack.Common/OctoDiff.cs index 271274bb..7958ffc2 100644 --- a/Wabbajack.Common/OctoDiff.cs +++ b/Wabbajack.Common/OctoDiff.cs @@ -36,24 +36,39 @@ namespace Wabbajack.Common sigStream.Position = 0; } - public static void Create(Stream oldData, Stream newData, Stream signature, Stream output) + public static void Create(Stream oldData, Stream newData, Stream signature, Stream output, ProgressReporter? reporter = null) { CreateSignature(oldData, signature); - var db = new DeltaBuilder {ProgressReporter = reporter}; + var db = new DeltaBuilder {ProgressReporter = reporter ?? new ProgressReporter()}; db.BuildDelta(newData, new SignatureReader(signature, reporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(output))); } - private class ProgressReporter : IProgressReporter + public class ProgressReporter : IProgressReporter { private DateTime _lastUpdate = DateTime.UnixEpoch; - private readonly TimeSpan _updateInterval = TimeSpan.FromMilliseconds(100); + private TimeSpan _updateInterval; + private Action _report; + + public ProgressReporter() + { + _updateInterval = TimeSpan.FromMilliseconds(100); + _report = (s, percent) => Utils.Status(s, percent); + } + + public ProgressReporter(TimeSpan updateInterval, Action report) + { + _updateInterval = updateInterval; + _report = report; + } + + public void ReportProgress(string operation, long currentPosition, long total) { if (DateTime.Now - _lastUpdate < _updateInterval) return; _lastUpdate = DateTime.Now; if (currentPosition >= total || total < 1 || currentPosition < 0) return; - Utils.Status(operation, new Percent(total, currentPosition)); + _report(operation, new Percent(total, currentPosition)); } } diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index 11a6a73f..f1a15ead 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders private bool _prepared; private AsyncLock _lock = new AsyncLock(); private UserStatus? _status; - private NexusApiClient? _client; + public INexusApi? Client; public IObservable IsLoggedIn => Utils.HaveEncryptedJsonObservable("nexusapikey"); @@ -115,9 +115,9 @@ namespace Wabbajack.Lib.Downloaders await CLIArguments.ApiKey.ToEcryptedJson("nexusapikey"); } - _client = await NexusApiClient.Get(); - _status = await _client.GetUserStatus(); - if (!_client.IsAuthenticated) + Client = await NexusApiClient.Get(); + _status = await Client.GetUserStatus(); + if (!Client.IsAuthenticated) { Utils.ErrorThrow(new UnconvertedError( $"Authenticating for the Nexus failed. A nexus account is required to automatically download mods.")); @@ -205,20 +205,13 @@ namespace Wabbajack.Lib.Downloaders try { var client = await NexusApiClient.Get(); + var modInfo = await client.GetModInfo(Game, ModID); + if (!modInfo.available) return false; var modFiles = await client.GetModFiles(Game, ModID); var found = modFiles.files .FirstOrDefault(file => file.file_id == FileID && file.category_name != null); - if (found != null) - return true; - - Utils.Log($"Could not validate {URL} with cache, validating manually"); - modFiles = await client.GetModFiles(Game, ModID, false); - - found = modFiles.files - .FirstOrDefault(file => file.file_id == FileID && file.category_name != null); - return found != null; } catch (Exception ex) @@ -244,11 +237,15 @@ namespace Wabbajack.Lib.Downloaders return new[] {"[General]", $"gameName={Game.MetaData().MO2ArchiveName}", $"modID={ModID}", $"fileID={FileID}"}; } + public static Func> DownloadShortcut = async a => default; public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a) { var client = await NexusApiClient.Get(); var mod = await client.GetModInfo(Game, ModID); + if (!mod.available) + return default; + var files = await client.GetModFiles(Game, ModID); var oldFile = files.files.FirstOrDefault(f => f.file_id == FileID); var nl = new Levenshtein(); @@ -265,13 +262,21 @@ namespace Wabbajack.Lib.Downloaders return default; } - var tempFile = new TempFile(); - var newArchive = new Archive(new State {Game = Game, ModID = ModID, FileID = newFile.file_id}) { Name = newFile.file_name, }; + var fastPath = await DownloadShortcut(newArchive); + if (fastPath != default) + { + newArchive.Size = fastPath.Size; + newArchive.Hash = await fastPath.FileHashAsync(); + return (newArchive, new TempFile()); + } + + var tempFile = new TempFile(); + await newArchive.State.Download(newArchive, tempFile.Path); newArchive.Size = tempFile.Path.Size; diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index 8107a7ee..07619a82 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -89,7 +89,7 @@ namespace Wabbajack.Lib.ModListRegistry try { var client = new Http.Client(); - return (await client.GetStringAsync(Consts.ModlistMetadataURL)).FromJsonString>(); + return (await client.GetStringAsync(Consts.UnlistedModlistMetadataURL)).FromJsonString>(); } catch (Exception ex) { diff --git a/Wabbajack.Lib/NexusApi/INexusApi.cs b/Wabbajack.Lib/NexusApi/INexusApi.cs new file mode 100644 index 00000000..e6bb9efd --- /dev/null +++ b/Wabbajack.Lib/NexusApi/INexusApi.cs @@ -0,0 +1,17 @@ +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Lib.Downloaders; + +namespace Wabbajack.Lib.NexusApi +{ + public interface INexusApi + { + public Task GetNexusDownloadLink(NexusDownloader.State archive); + public Task GetModFiles(Game game, long modid, bool useCache = true); + public Task GetModInfo(Game game, long modId, bool useCache = true); + + public Task GetUserStatus(); + public Task IsPremium(); + public bool IsAuthenticated { get; } + } +} diff --git a/Wabbajack.Lib/NexusApi/NexusApi.cs b/Wabbajack.Lib/NexusApi/NexusApi.cs index 40e25835..c2be42d9 100644 --- a/Wabbajack.Lib/NexusApi/NexusApi.cs +++ b/Wabbajack.Lib/NexusApi/NexusApi.cs @@ -16,7 +16,7 @@ using Wabbajack.Lib.WebAutomation; namespace Wabbajack.Lib.NexusApi { - public class NexusApiClient : ViewModel + public class NexusApiClient : ViewModel, INexusApi { private static readonly string API_KEY_CACHE_FILE = "nexus.key_cache"; @@ -24,7 +24,7 @@ namespace Wabbajack.Lib.NexusApi #region Authentication - public string? ApiKey { get; } + public static string? ApiKey { get; set; } public bool IsAuthenticated => ApiKey != null; @@ -313,7 +313,11 @@ namespace Wabbajack.Lib.NexusApi public async Task GetNexusDownloadLink(NexusDownloader.State archive) { ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; - + + var info = await GetModInfo(archive.Game, archive.ModID); + if (!info.available) + throw new Exception("Mod unavailable"); + var url = $"https://api.nexusmods.com/v1/games/{archive.Game.MetaData().NexusName}/mods/{archive.ModID}/files/{archive.FileID}/download_link.json"; try { @@ -321,6 +325,8 @@ namespace Wabbajack.Lib.NexusApi } catch (HttpException ex) { + + if (ex.Code != 403 || await IsPremium()) { throw; @@ -383,20 +389,6 @@ namespace Wabbajack.Lib.NexusApi public string URI { get; set; } = string.Empty; } - private static string? _localCacheDir; - public static string LocalCacheDir - { - get - { - if (_localCacheDir == null) - _localCacheDir = Environment.GetEnvironmentVariable("NEXUSCACHEDIR"); - if (_localCacheDir == null) - throw new ArgumentNullException($"Enviornment variable could not be located: NEXUSCACHEDIR"); - return _localCacheDir; - } - set => _localCacheDir = value; - } - public static Uri ManualDownloadUrl(NexusDownloader.State state) { return new Uri($"https://www.nexusmods.com/{state.Game.MetaData().NexusName}/mods/{state.ModID}?tab=files"); diff --git a/Wabbajack.Server.Test/NexusCacheTests.cs b/Wabbajack.Server.Test/NexusCacheTests.cs index 2433a969..0c3ae82c 100644 --- a/Wabbajack.Server.Test/NexusCacheTests.cs +++ b/Wabbajack.Server.Test/NexusCacheTests.cs @@ -152,5 +152,19 @@ namespace Wabbajack.BuildServer.Test Assert.Equal(ts, (long)ts.AsUnixTime().AsUnixTime()); } + + [Fact] + public async Task CanGetAndSetPermissions() + { + var game = Game.Oblivion; + var modId = 4424; + var sql = Fixture.GetService(); + + foreach (HTMLInterface.PermissionValue result in Enum.GetValues(typeof(HTMLInterface.PermissionValue))) + { + await sql.SetNexusPermission(game, modId, result); + Assert.Equal(result, (await sql.GetNexusPermissions())[(game, modId)]); + } + } } } diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index 9ba90c06..d350b2dd 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -637,6 +637,7 @@ CREATE TABLE [dbo].[NexusKeys]( [ApiKey] [nvarchar](162) NOT NULL, [DailyRemain] [int] NOT NULL, [HourlyRemain] [int] NOT NULL, + [IsPremium] [tinyint] NOT NULL, CONSTRAINT [PK_NexusKeys] PRIMARY KEY CLUSTERED ( [ApiKey] ASC diff --git a/Wabbajack.Server/Controllers/Metrics.cs b/Wabbajack.Server/Controllers/Metrics.cs index e371e078..7889573c 100644 --- a/Wabbajack.Server/Controllers/Metrics.cs +++ b/Wabbajack.Server/Controllers/Metrics.cs @@ -127,7 +127,7 @@ namespace Wabbajack.BuildServer.Controllers private async Task Log(DateTime timestamp, string action, string subject, string metricsKey = null) { - _logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}"); + //_logger.Log(LogLevel.Information, $"Log - {timestamp} {action} {subject} {metricsKey}"); await _sql.IngestMetric(new Metric { Timestamp = timestamp, Action = action, Subject = subject, MetricsKey = metricsKey diff --git a/Wabbajack.Server/Controllers/ModUpgrade.cs b/Wabbajack.Server/Controllers/ModUpgrade.cs index 00c2f873..e56be45b 100644 --- a/Wabbajack.Server/Controllers/ModUpgrade.cs +++ b/Wabbajack.Server/Controllers/ModUpgrade.cs @@ -73,15 +73,15 @@ namespace Wabbajack.BuildServer.Controllers { if (await request.OldArchive.State.Verify(request.OldArchive)) { - _logger.LogInformation( - $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), old archive is valid"); + //_logger.LogInformation( + // $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), old archive is valid"); return NotFound("File is Valid"); } } catch (Exception ex) { - _logger.LogInformation( - $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), due to upgrade failure"); + //_logger.LogInformation( + // $"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), due to upgrade failure"); return NotFound("File is Valid"); } @@ -99,13 +99,13 @@ namespace Wabbajack.BuildServer.Controllers { if (patch.PatchSize != 0) { - _logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch Found"); + //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch Found"); await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id); return Ok( $"https://{(await _creds).Username}.b-cdn.net/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); } - _logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed"); + //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed"); return NotFound("Patch creation failed"); } @@ -119,7 +119,7 @@ namespace Wabbajack.BuildServer.Controllers await _quickSync.Notify(); } - _logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found is processing"); + //_logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found is processing"); // Still processing return Accepted(); } diff --git a/Wabbajack.Server/Controllers/NexusCache.cs b/Wabbajack.Server/Controllers/NexusCache.cs index 03f4e5ff..aa3db466 100644 --- a/Wabbajack.Server/Controllers/NexusCache.cs +++ b/Wabbajack.Server/Controllers/NexusCache.cs @@ -10,9 +10,11 @@ using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Wabbajack.Common; +using Wabbajack.Common.Exceptions; using Wabbajack.Lib; using Wabbajack.Lib.NexusApi; using Wabbajack.Server.DataLayer; +using Wabbajack.Server.Services; namespace Wabbajack.BuildServer.Controllers { @@ -27,12 +29,14 @@ namespace Wabbajack.BuildServer.Controllers private static long ForwardCount = 0; private SqlService _sql; private ILogger _logger; + private NexusKeyMaintainance _keys; - public NexusCache(ILogger logger, SqlService sql, AppSettings settings) + public NexusCache(ILogger logger, SqlService sql, AppSettings settings, NexusKeyMaintainance keys) { _settings = settings; _sql = sql; _logger = logger; + _keys = keys; } /// @@ -74,7 +78,7 @@ namespace Wabbajack.BuildServer.Controllers { var key = Request.Headers["apikey"].FirstOrDefault(); if (key == null) - return await NexusApiClient.Get(null); + return await _keys.GetClient(); if (await _sql.HaveKey(key)) return await NexusApiClient.Get(key); @@ -89,18 +93,31 @@ namespace Wabbajack.BuildServer.Controllers [Route("{GameName}/mods/{ModId}/files.json")] public async Task GetModFiles(string GameName, long ModId) { - _logger.Log(LogLevel.Information, $"{GameName} {ModId}"); + //_logger.Log(LogLevel.Information, $"{GameName} {ModId}"); var game = GameRegistry.GetByFuzzyName(GameName).Game; var result = await _sql.GetModFiles(game, ModId); string method = "CACHED"; if (result == null) { - var api = await NexusApiClient.Get(Request.Headers["apikey"].FirstOrDefault()); - result = await api.GetModFiles(game, ModId, false); + var api = await GetClient(); + var permission = HTMLInterface.GetUploadPermissions(game, ModId); + try + { + result = await api.GetModFiles(game, ModId, false); + } + catch (HttpException ex) + { + if (ex.Code == 403) + result = new NexusApiClient.GetModFilesResponse {files = new List()}; + else + throw; + } + var date = result.files.Select(f => f.uploaded_time).OrderByDescending(o => o).FirstOrDefault(); date = date == default ? DateTime.UtcNow : date; await _sql.AddNexusModFiles(game, ModId, date, result); + await _sql.SetNexusPermission(game, ModId, await permission); method = "NOT_CACHED"; Interlocked.Increment(ref ForwardCount); diff --git a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs index 508a508f..0af2f679 100644 --- a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs +++ b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs @@ -83,6 +83,25 @@ namespace Wabbajack.Server.DataLayer } + public async Task GetArchiveDownload(string primaryKeyString) + { + await using var conn = await Open(); + var result = await conn.QueryFirstOrDefaultAsync<(Guid, long?, Hash?, bool?, AbstractDownloadState, DateTime?)>( + "SELECT Id, Size, Hash, IsFailed, DownloadState, DownloadFinished FROM dbo.ArchiveDownloads WHERE PrimaryKeyString = @PrimaryKeyString AND IsFailed = 0", + new {PrimaryKeyString = primaryKeyString}); + if (result == default) + return null; + + return new ArchiveDownload + { + Id = result.Item1, + IsFailed = result.Item4, + DownloadFinished = result.Item6, + Archive = new Archive(result.Item5) {Size = result.Item2 ?? 0, Hash = result.Item3 ?? default} + }; + + } + public async Task GetArchiveDownload(string primaryKeyString, Hash hash, long size) { await using var conn = await Open(); diff --git a/Wabbajack.Server/DataLayer/Nexus.cs b/Wabbajack.Server/DataLayer/Nexus.cs index 487ca3a5..317045da 100644 --- a/Wabbajack.Server/DataLayer/Nexus.cs +++ b/Wabbajack.Server/DataLayer/Nexus.cs @@ -117,6 +117,7 @@ namespace Wabbajack.Server.DataLayer await using var conn = await Open(); await conn.ExecuteAsync("DELETE FROM dbo.NexusModFiles WHERE ModId = @ModId", new {ModId = modId}); await conn.ExecuteAsync("DELETE FROM dbo.NexusModInfos WHERE ModId = @ModId", new {ModId = modId}); + await conn.ExecuteAsync("DELETE FROM dbo.NexusModPermissions WHERE ModId = @ModId", new {ModId = modId}); } public async Task> GetNexusPermissions() @@ -128,6 +129,17 @@ namespace Wabbajack.Server.DataLayer return results.ToDictionary(f => (GameRegistry.ByNexusID[f.Item1], f.Item2), f => (HTMLInterface.PermissionValue)f.Item3); } + + public async Task> GetHiddenNexusMods() + { + await using var conn = await Open(); + + var results = + await conn.QueryAsync<(int, long, int)>("SELECT NexusGameID, ModID, Permissions FROM NexusModPermissions WHERE Permissions = @Permissions", + new {Permissions = (int)HTMLInterface.PermissionValue.Hidden}); + return results.ToDictionary(f => (GameRegistry.ByNexusID[f.Item1], f.Item2), + f => (HTMLInterface.PermissionValue)f.Item3); + } public async Task SetNexusPermissions(IEnumerable<(Game, long, HTMLInterface.PermissionValue)> permissions) { @@ -166,7 +178,7 @@ namespace Wabbajack.Server.DataLayer await using var conn = await Open(); var tx = await conn.BeginTransactionAsync(); - await conn.ExecuteAsync("DELETE FROM NexusModPermissions WHERE GameID = @GameID AND ModID = @ModID", new + await conn.ExecuteAsync("DELETE FROM NexusModPermissions WHERE NexusGameID = @GameID AND ModID = @ModID", new { GameID = game.MetaData().NexusGameId, ModID = modId diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 500c9006..9ad41b2f 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -46,6 +47,9 @@ namespace Wabbajack.Server.Services using var queue = new WorkQueue(); var oldSummaries = Summaries; + var stopwatch = new Stopwatch(); + stopwatch.Start(); + var results = await data.ModLists.PMap(queue, async metadata => { var oldSummary = @@ -54,15 +58,24 @@ namespace Wabbajack.Server.Services var listArchives = await _sql.ModListArchives(metadata.Links.MachineURL); var archives = await listArchives.PMap(queue, async archive => { - var (_, result) = await ValidateArchive(data, archive); - if (result == ArchiveStatus.InValid) + try { - if (data.Mirrors.Contains(archive.Hash)) - return (archive, ArchiveStatus.Mirrored); - return await TryToHeal(data, archive, metadata); - } + var (_, result) = await ValidateArchive(data, archive); + if (result == ArchiveStatus.InValid) + { + if (data.Mirrors.Contains(archive.Hash)) + return (archive, ArchiveStatus.Mirrored); + return await TryToHeal(data, archive, metadata); + } - return (archive, result); + + return (archive, result); + } + catch (Exception ex) + { + _logger.LogError(ex, $"During Validation of {archive.Hash} {archive.State.PrimaryKeyString}"); + return (archive, ArchiveStatus.InValid); + } }); var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid); @@ -141,20 +154,16 @@ namespace Wabbajack.Server.Services return (summary, detailed); }); Summaries = results; + + stopwatch.Stop(); + _logger.LogInformation($"Finished Validation in {stopwatch.Elapsed}"); + return Summaries.Count(s => s.Summary.HasFailures); } private AsyncLock _healLock = new AsyncLock(); private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive, ModlistMetadata modList) { - using var _ = await _healLock.WaitAsync(); - - if (!(archive.State is IUpgradingState)) - { - _logger.Log(LogLevel.Information, $"Cannot heal {archive.State.PrimaryKeyString} because it's not a healable state"); - return (archive, ArchiveStatus.InValid); - } - var srcDownload = await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size); if (srcDownload == null || srcDownload.IsFailed == true) { @@ -163,7 +172,7 @@ namespace Wabbajack.Server.Services } - var patches = await _sql.PatchesForSource(srcDownload.Id); + var patches = await _sql.PatchesForSource(archive.Hash); foreach (var patch in patches) { if (patch.Finished is null) @@ -176,28 +185,80 @@ namespace Wabbajack.Server.Services if (status == ArchiveStatus.Valid) return (archive, ArchiveStatus.Updated); } - - + + using var _ = await _healLock.WaitAsync(); var upgradeTime = DateTime.UtcNow; + _logger.LogInformation($"Validator Finding Upgrade for {archive.Hash} {archive.State.PrimaryKeyString}"); + + NexusDownloader.State.DownloadShortcut = async findIt => + { + _logger.LogInformation($"Quick find for {findIt.State.PrimaryKeyString}"); + var foundArchive = await _sql.GetArchiveDownload(findIt.State.PrimaryKeyString); + if (foundArchive == null) + { + _logger.LogInformation($"No Quick find for {findIt.State.PrimaryKeyString}"); + return default; + } + + return _archives.TryGetPath(foundArchive.Archive.Hash, out var path) ? path : default; + }; + var upgrade = await (archive.State as IUpgradingState)?.FindUpgrade(archive); + + if (upgrade == default) { _logger.Log(LogLevel.Information, $"Cannot heal {archive.State.PrimaryKeyString} because an alternative wasn't found"); return (archive, ArchiveStatus.InValid); } + + _logger.LogInformation($"Upgrade {upgrade.Archive.State.PrimaryKeyString} found for {archive.State.PrimaryKeyString}"); - await _archives.Ingest(upgrade.NewFile.Path); - var id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime); + { + } + + var found = await _sql.GetArchiveDownload(upgrade.Archive.State.PrimaryKeyString, upgrade.Archive.Hash, + upgrade.Archive.Size); + Guid id; + if (found == null) + { + if (upgrade.NewFile.Path.Exists) + await _archives.Ingest(upgrade.NewFile.Path); + id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime); + } + else + { + id = found.Id; + } + var destDownload = await _sql.GetArchiveDownload(id); - - await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload}); - - _logger.Log(LogLevel.Information, $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}"); - await _discord.Send(Channel.Ham, new DiscordMessage { Content = $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`" }); + + if (destDownload.Archive.Hash == srcDownload.Archive.Hash && destDownload.Archive.State.PrimaryKeyString == srcDownload.Archive.State.PrimaryKeyString) + { + _logger.Log(LogLevel.Information, $"Can't heal because src and dest match"); + return (archive, ArchiveStatus.InValid); + } + + + var existing = await _sql.FindPatch(srcDownload.Id, destDownload.Id); + if (existing == null) + { + await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload}); + + _logger.Log(LogLevel.Information, + $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}"); + await _discord.Send(Channel.Ham, + new DiscordMessage + { + Content = + $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`" + }); + } await upgrade.NewFile.DisposeAsync(); + _logger.LogInformation($"Patch in progress {archive.Hash} {archive.State.PrimaryKeyString}"); return (archive, ArchiveStatus.Updating); } diff --git a/Wabbajack.Server/Services/NexusKeyMaintainance.cs b/Wabbajack.Server/Services/NexusKeyMaintainance.cs index ea033203..969248b9 100644 --- a/Wabbajack.Server/Services/NexusKeyMaintainance.cs +++ b/Wabbajack.Server/Services/NexusKeyMaintainance.cs @@ -4,6 +4,7 @@ using System.Net.Http; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Wabbajack.BuildServer; +using Wabbajack.Common; using Wabbajack.Lib.NexusApi; using Wabbajack.Server.DataLayer; @@ -12,6 +13,7 @@ namespace Wabbajack.Server.Services public class NexusKeyMaintainance : AbstractService { private SqlService _sql; + private string _selfKey; public NexusKeyMaintainance(ILogger logger, AppSettings settings, SqlService sql, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromHours(4)) { @@ -21,7 +23,7 @@ namespace Wabbajack.Server.Services public async Task GetClient() { var keys = await _sql.GetNexusApiKeysWithCounts(1500); - foreach (var key in keys) + foreach (var key in keys.Where(k => k.Key != _selfKey)) { return new TrackingClient(_sql, key); } @@ -31,6 +33,7 @@ namespace Wabbajack.Server.Services public override async Task Execute() { + _selfKey ??= await Utils.FromEncryptedJson("nexusapikey"); var keys = await _sql.GetNexusApiKeysWithCounts(0); _logger.Log(LogLevel.Information, $"Verifying {keys.Count} API Keys"); foreach (var key in keys) @@ -70,7 +73,7 @@ namespace Wabbajack.Server.Services HourlyRemaining = key.Hourly; } - protected virtual async Task UpdateRemaining(HttpResponseMessage response) + protected override async Task UpdateRemaining(HttpResponseMessage response) { await base.UpdateRemaining(response); try diff --git a/Wabbajack.Server/Services/NexusPermissionsUpdater.cs b/Wabbajack.Server/Services/NexusPermissionsUpdater.cs index 6190c1ae..5879940a 100644 --- a/Wabbajack.Server/Services/NexusPermissionsUpdater.cs +++ b/Wabbajack.Server/Services/NexusPermissionsUpdater.cs @@ -17,9 +17,7 @@ namespace Wabbajack.Server.Services private DiscordWebHook _discord; private SqlService _sql; - public static TimeSpan MaxSync = TimeSpan.FromHours(4); - - public NexusPermissionsUpdater(ILogger logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromSeconds(1)) + public NexusPermissionsUpdater(ILogger logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromMinutes(5)) { _discord = discord; _sql = sql; @@ -32,38 +30,33 @@ namespace Wabbajack.Server.Services var data = await _sql.ModListArchives(); var nexusArchives = data.Select(a => a.State).OfType().Select(d => (d.Game, d.ModID)) + .Where(g => g.Game.MetaData().NexusGameId != 0) .Distinct() .ToList(); _logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods"); - using var queue = new WorkQueue(1); + using var queue = new WorkQueue(); - var prev = await _sql.GetNexusPermissions(); + var prev = await _sql.GetHiddenNexusMods(); + _logger.LogInformation($"Found {prev.Count} hidden nexus mods to check"); - var lag = MaxSync / nexusArchives.Count * 2; - - - await nexusArchives.PMap(queue, async archive => + await prev.PMap(queue, async archive => { - _logger.LogInformation($"Checking permissions for {archive.Game} {archive.ModID}"); - var result = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID); - await _sql.SetNexusPermission(archive.Game, archive.ModID, result); + var (game, modID) = archive.Key; + _logger.LogInformation($"Checking permissions for {game} {modID}"); + var result = await HTMLInterface.GetUploadPermissions(game, modID); + await _sql.SetNexusPermission(game, modID, result); - if (prev.TryGetValue((archive.Game, archive.ModID), out var oldPermission)) + if (archive.Value != result) { - if (oldPermission != result) - { - await _discord.Send(Channel.Spam, - new DiscordMessage { - Content = $"Permissions status of {archive.Game} {archive.ModID} was {oldPermission} is now {result}" - }); - await _sql.PurgeNexusCache(archive.ModID); - await _quickSync.Notify(); - } + await _discord.Send(Channel.Ham, + new DiscordMessage { + Content = $"Permissions status of {game} {modID} was {archive.Value} is now {result}" + }); + await _sql.PurgeNexusCache(modID); + await _quickSync.Notify(); } - - await Task.Delay(lag); }); return 1; diff --git a/Wabbajack.Server/Services/NexusPoll.cs b/Wabbajack.Server/Services/NexusPoll.cs index b1e77953..d7d90950 100644 --- a/Wabbajack.Server/Services/NexusPoll.cs +++ b/Wabbajack.Server/Services/NexusPoll.cs @@ -18,13 +18,15 @@ namespace Wabbajack.Server.Services private AppSettings _settings; private GlobalInformation _globalInformation; private ILogger _logger; + private NexusKeyMaintainance _keys; - public NexusPoll(ILogger logger, AppSettings settings, SqlService service, GlobalInformation globalInformation) + public NexusPoll(ILogger logger, AppSettings settings, SqlService service, GlobalInformation globalInformation, NexusKeyMaintainance keys) { _sql = service; _settings = settings; _globalInformation = globalInformation; _logger = logger; + _keys = keys; } public async Task UpdateNexusCacheRSS() @@ -67,7 +69,7 @@ namespace Wabbajack.Server.Services { using var _ = _logger.BeginScope("Nexus Update via API"); _logger.Log(LogLevel.Information, "Starting Nexus Update via API"); - var api = await NexusApiClient.Get(); + var api = await _keys.GetClient(); var gameTasks = GameRegistry.Games.Values .Where(game => game.NexusName != null) diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs index 6cb4546c..40e3fb65 100644 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -55,7 +55,7 @@ namespace Wabbajack.Server.Services $"Building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}" }); - if (patch.Src.Archive.Hash == patch.Dest.Archive.Hash) + if (patch.Src.Archive.Hash == patch.Dest.Archive.Hash && patch.Src.Archive.State.PrimaryKeyString == patch.Dest.Archive.State.PrimaryKeyString) { await patch.Fail(_sql, "Hashes match"); continue; @@ -76,7 +76,7 @@ namespace Wabbajack.Server.Services await using var destStream = await destPath.OpenShared(); await using var sigStream = await sigFile.Path.Create(); await using var patchOutput = await patchFile.Path.Create(); - OctoDiff.Create(destStream, srcStream, sigStream, patchOutput); + OctoDiff.Create(destStream, srcStream, sigStream, patchOutput, new OctoDiff.ProgressReporter(TimeSpan.FromSeconds(1), (s, p) => _logger.LogInformation($"Patch Builder: {p} {s}"))); await patchOutput.DisposeAsync(); var size = patchFile.Path.Size; diff --git a/Wabbajack.Server/Services/QuickSync.cs b/Wabbajack.Server/Services/QuickSync.cs index 127644b7..3a772c21 100644 --- a/Wabbajack.Server/Services/QuickSync.cs +++ b/Wabbajack.Server/Services/QuickSync.cs @@ -3,6 +3,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Wabbajack.Common; namespace Wabbajack.Server.Services @@ -11,6 +12,12 @@ namespace Wabbajack.Server.Services { private Dictionary _syncs = new Dictionary(); private AsyncLock _lock = new AsyncLock(); + private ILogger _logger; + + public QuickSync(ILogger logger) + { + _logger = logger; + } public async Task GetToken() { @@ -36,18 +43,13 @@ namespace Wabbajack.Server.Services public async Task Notify() { + _logger.LogInformation($"Quicksync {typeof(T).Name}"); // Needs debugging - /* using var _ = await _lock.WaitAsync(); if (_syncs.TryGetValue(typeof(T), out var ct)) { ct.Cancel(); } - */ - } - - - } }