diff --git a/Wabbajack.Lib/Downloaders/NexusDownloader.cs b/Wabbajack.Lib/Downloaders/NexusDownloader.cs index d6b7fab9..9235e458 100644 --- a/Wabbajack.Lib/Downloaders/NexusDownloader.cs +++ b/Wabbajack.Lib/Downloaders/NexusDownloader.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Reactive; using System.Reactive.Linq; using System.Threading.Tasks; +using F23.StringSimilarity; using Newtonsoft.Json; using ReactiveUI; using Wabbajack.Common; @@ -246,14 +247,16 @@ namespace Wabbajack.Lib.Downloaders var mod = await client.GetModInfo(Game, ModID); var files = await client.GetModFiles(Game, ModID); var oldFile = files.files.FirstOrDefault(f => f.file_id == FileID); - var newFile = files.files.Where(f => f.category_name != null).OrderByDescending(f => f.uploaded_timestamp).FirstOrDefault(); + var nl = new Levenshtein(); + var newFile = files.files.Where(f => f.category_name != null) + .OrderBy(f => nl.Distance(oldFile.name.ToLowerInvariant(), f.name.ToLowerInvariant())).FirstOrDefault(); if (!mod.available || oldFile == default || newFile == default) { return default; } // Size is in KB - if (oldFile.size > 2_500_000 || newFile.size > 2_500_000 || oldFile.file_id == newFile.file_id) + if (oldFile.size > 4_500_000 || newFile.size > 4_500_000 || oldFile.file_id == newFile.file_id) { return default; } diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index dd48d9dd..067f9f91 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -13,6 +13,9 @@ 81.3.100 + + 3.1.0 + 6.1.2 diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index 69502283..2509536d 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -560,6 +560,8 @@ CREATE TABLE [dbo].[Patches]( [Finished] [datetime] NULL, [IsFailed] [tinyint] NULL, [FailMessage] [varchar](MAX) NULL, +[LastUsed] [datetime] NULL, +[Downloads] [bigint] NOT NULL DEFAULT 0, CONSTRAINT [PK_Patches] PRIMARY KEY CLUSTERED ( [SrcId] ASC, diff --git a/Wabbajack.Server/Controllers/ModUpgrade.cs b/Wabbajack.Server/Controllers/ModUpgrade.cs index 9ea9c8eb..711a9b15 100644 --- a/Wabbajack.Server/Controllers/ModUpgrade.cs +++ b/Wabbajack.Server/Controllers/ModUpgrade.cs @@ -15,13 +15,15 @@ namespace Wabbajack.BuildServer.Controllers private SqlService _sql; private DiscordWebHook _discord; private AppSettings _settings; + private QuickSync _quickSync; - public ModUpgrade(ILogger logger, SqlService sql, DiscordWebHook discord, AppSettings settings) + public ModUpgrade(ILogger logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync, AppSettings settings) { _logger = logger; _sql = sql; _discord = discord; _settings = settings; + _quickSync = quickSync; } [HttpPost] @@ -49,6 +51,7 @@ namespace Wabbajack.BuildServer.Controllers if (patch.PatchSize != 0) { _logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch Found"); + await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id); return Ok( $"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{Consts.ArchiveUpdatesCDNFolder}/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); @@ -57,6 +60,16 @@ namespace Wabbajack.BuildServer.Controllers return NotFound("Patch creation failed"); } + + if (!newDownload.DownloadFinished.HasValue) + { + await _quickSync.Notify(); + } + else + { + await _quickSync.Notify(); + } + _logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found is processing"); // Still processing return Accepted(); diff --git a/Wabbajack.Server/DataLayer/Patches.cs b/Wabbajack.Server/DataLayer/Patches.cs index e654dbd3..c694f71a 100644 --- a/Wabbajack.Server/DataLayer/Patches.cs +++ b/Wabbajack.Server/DataLayer/Patches.cs @@ -110,7 +110,10 @@ namespace Wabbajack.Server.DataLayer { await using var conn = await Open(); var patch = await conn.QueryFirstOrDefaultAsync<(Guid, Guid, long, DateTime?, bool?, string)>( - "SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE Finished is NULL"); + @"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage FROM dbo.Patches p + LEFT JOIN dbo.ArchiveDownloads src ON src.Id = p.SrcId + LEFT JOIN dbo.ArchiveDownloads dest ON dest.Id = p.DestId + WHERE p.Finished is NULL AND src.IsFailed = 0 AND dest.IsFailed = 0 "); if (patch == default) return default(Patch); @@ -144,5 +147,14 @@ namespace Wabbajack.Server.DataLayer } return results; } + + public async Task MarkPatchUsage(Guid srcId, Guid destId) + { + await using var conn = await Open(); + await conn.ExecuteAsync( + @"UPDATE dbo.Patches SET Downloads = Downloads + 1, LastUsed = GETUTCDATE() WHERE SrcId = @srcId AND DestID = @destId", + new {SrcId = srcId, DestId = destId}); + + } } } diff --git a/Wabbajack.Server/Services/AbstractService.cs b/Wabbajack.Server/Services/AbstractService.cs index 3c42904b..12e90f1f 100644 --- a/Wabbajack.Server/Services/AbstractService.cs +++ b/Wabbajack.Server/Services/AbstractService.cs @@ -16,12 +16,14 @@ namespace Wabbajack.Server.Services protected AppSettings _settings; private TimeSpan _delay; protected ILogger _logger; + protected QuickSync _quickSync; - public AbstractService(ILogger logger, AppSettings settings, TimeSpan delay) + public AbstractService(ILogger logger, AppSettings settings, QuickSync quickSync, TimeSpan delay) { _settings = settings; _delay = delay; _logger = logger; + _quickSync = quickSync; } public void Start() @@ -32,7 +34,7 @@ namespace Wabbajack.Server.Services { while (true) { - + await _quickSync.ResetToken(); try { await Execute(); @@ -42,9 +44,9 @@ namespace Wabbajack.Server.Services _logger.LogError(ex, "Running Service Loop"); } - await Task.Delay(_delay); + var token = await _quickSync.GetToken(); + await Task.Delay(_delay, token); } - }); } } diff --git a/Wabbajack.Server/Services/ArchiveDownloader.cs b/Wabbajack.Server/Services/ArchiveDownloader.cs index df39d2d2..f0c13dbe 100644 --- a/Wabbajack.Server/Services/ArchiveDownloader.cs +++ b/Wabbajack.Server/Services/ArchiveDownloader.cs @@ -16,11 +16,14 @@ namespace Wabbajack.Server.Services private SqlService _sql; private ArchiveMaintainer _archiveMaintainer; private NexusApiClient _nexusClient; + private DiscordWebHook _discord; - public ArchiveDownloader(ILogger logger, AppSettings settings, SqlService sql, ArchiveMaintainer archiveMaintainer) : base(logger, settings, TimeSpan.FromMinutes(10)) + public ArchiveDownloader(ILogger logger, AppSettings settings, SqlService sql, ArchiveMaintainer archiveMaintainer, DiscordWebHook discord, QuickSync quickSync) + : base(logger, settings, quickSync, TimeSpan.FromMinutes(10)) { _sql = sql; _archiveMaintainer = archiveMaintainer; + _discord = discord; } public override async Task Execute() @@ -58,6 +61,7 @@ namespace Wabbajack.Server.Services try { _logger.Log(LogLevel.Information, $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"); + await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Downloading {nextDownload.Archive.State.PrimaryKeyString}"}); await DownloadDispatcher.PrepareAll(new[] {nextDownload.Archive.State}); await using var tempPath = new TempFile(); @@ -86,17 +90,26 @@ namespace Wabbajack.Server.Services _logger.Log(LogLevel.Information, $"Finished Archiving {nextDownload.Archive.State.PrimaryKeyString}"); await nextDownload.Finish(_sql); + await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Finished downloading {nextDownload.Archive.State.PrimaryKeyString}"}); + } catch (Exception ex) { _logger.Log(LogLevel.Warning, $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}"); await nextDownload.Fail(_sql, ex.ToString()); + await _discord.Send(Channel.Spam, new DiscordMessage {Content = $"Error downloading {nextDownload.Archive.State.PrimaryKeyString}"}); } count++; } + if (count > 0) + { + // Wake the Patch builder up in case it needs to build a patch now + await _quickSync.Notify(); + } + return count; } } diff --git a/Wabbajack.Server/Services/DiscordWebHook.cs b/Wabbajack.Server/Services/DiscordWebHook.cs index 750e01db..2172b44d 100644 --- a/Wabbajack.Server/Services/DiscordWebHook.cs +++ b/Wabbajack.Server/Services/DiscordWebHook.cs @@ -23,7 +23,7 @@ namespace Wabbajack.Server.Services private ILogger _logger; private Random _random = new Random(); - public DiscordWebHook(ILogger logger, AppSettings settings) : base(logger, settings, TimeSpan.FromHours(1)) + public DiscordWebHook(ILogger logger, AppSettings settings, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromHours(1)) { _settings = settings; _logger = logger; diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index 840e92a3..48ed35a2 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -30,8 +30,8 @@ namespace Wabbajack.Server.Services new (ModListSummary Summary, DetailedStatus Detailed)[0]; - public ListValidator(ILogger logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives) - : base(logger, settings, TimeSpan.FromMinutes(5)) + public ListValidator(ILogger logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives, QuickSync quickSync) + : base(logger, settings, quickSync, TimeSpan.FromMinutes(5)) { _sql = sql; _discord = discord; @@ -56,7 +56,7 @@ namespace Wabbajack.Server.Services { var (_, result) = await ValidateArchive(data, archive); if (result == ArchiveStatus.InValid) - return await TryToHeal(data, archive); + return await TryToHeal(data, archive, metadata); return (archive, result); }); @@ -133,7 +133,7 @@ namespace Wabbajack.Server.Services } private AsyncLock _healLock = new AsyncLock(); - private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive) + private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive, ModlistMetadata modList) { using var _ = await _healLock.WaitAsync(); @@ -182,7 +182,7 @@ namespace Wabbajack.Server.Services await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload}); _logger.Log(LogLevel.Information, $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}"); - await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}" }); + await _discord.Send(Channel.Ham, new DiscordMessage { Content = $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`" }); await upgrade.NewFile.DisposeAsync(); diff --git a/Wabbajack.Server/Services/NexusKeyMaintainance.cs b/Wabbajack.Server/Services/NexusKeyMaintainance.cs index bf21b232..9b964fed 100644 --- a/Wabbajack.Server/Services/NexusKeyMaintainance.cs +++ b/Wabbajack.Server/Services/NexusKeyMaintainance.cs @@ -13,7 +13,7 @@ namespace Wabbajack.Server.Services { private SqlService _sql; - public NexusKeyMaintainance(ILogger logger, AppSettings settings, SqlService sql) : base(logger, settings, TimeSpan.FromHours(1)) + public NexusKeyMaintainance(ILogger logger, AppSettings settings, SqlService sql, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromHours(1)) { _sql = sql; } diff --git a/Wabbajack.Server/Services/NonNexusDownloadValidator.cs b/Wabbajack.Server/Services/NonNexusDownloadValidator.cs index 8e346c69..6b929e42 100644 --- a/Wabbajack.Server/Services/NonNexusDownloadValidator.cs +++ b/Wabbajack.Server/Services/NonNexusDownloadValidator.cs @@ -16,8 +16,8 @@ namespace Wabbajack.Server.Services { private SqlService _sql; - public NonNexusDownloadValidator(ILogger logger, AppSettings settings, SqlService sql) - : base(logger, settings, TimeSpan.FromHours(2)) + public NonNexusDownloadValidator(ILogger logger, AppSettings settings, SqlService sql, QuickSync quickSync) + : base(logger, settings, quickSync, TimeSpan.FromHours(2)) { _sql = sql; } diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs index b78a859c..07d10cc1 100644 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -22,7 +22,7 @@ namespace Wabbajack.Server.Services private ArchiveMaintainer _maintainer; public PatchBuilder(ILogger logger, SqlService sql, AppSettings settings, ArchiveMaintainer maintainer, - DiscordWebHook discordWebHook) : base(logger, settings, TimeSpan.FromMinutes(1)) + DiscordWebHook discordWebHook, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromMinutes(1)) { _discordWebHook = discordWebHook; _sql = sql; @@ -103,6 +103,12 @@ namespace Wabbajack.Server.Services } } + if (count > 0) + { + // Notify the List Validator that we may have more patches + await _quickSync.Notify(); + } + return count; } diff --git a/Wabbajack.Server/Services/QuickSync.cs b/Wabbajack.Server/Services/QuickSync.cs new file mode 100644 index 00000000..36a7a913 --- /dev/null +++ b/Wabbajack.Server/Services/QuickSync.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.Server.Services +{ + public class QuickSync + { + private Dictionary _syncs = new Dictionary(); + private AsyncLock _lock = new AsyncLock(); + + public async Task GetToken() + { + using var _ = await _lock.WaitAsync(); + if (_syncs.TryGetValue(typeof(T), out var result)) + { + return result.Token; + } + var token = new CancellationTokenSource(); + _syncs[typeof(T)] = token; + return token.Token; + } + + public async Task ResetToken() + { + using var _ = await _lock.WaitAsync(); + if (_syncs.TryGetValue(typeof(T), out var ct)) + { + ct.Cancel(); + } + _syncs[typeof(T)] = new CancellationTokenSource(); + } + + public async Task Notify() + { + using var _ = await _lock.WaitAsync(); + if (_syncs.TryGetValue(typeof(T), out var ct)) + { + ct.Cancel(); + } + } + + + + } +} diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index b6000912..5ec03847 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -55,6 +55,7 @@ namespace Wabbajack.Server }); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton();