From 7d58dbc1618b02b2cac075ccd19a7cc3e2af247f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 20 May 2020 06:18:47 -0600 Subject: [PATCH] List validator now tries to heal lists --- Wabbajack.Lib/Downloaders/HTTPDownloader.cs | 11 +++- Wabbajack.Server.Test/ModlistUpdater.cs | 2 +- .../DataLayer/ArchiveDownloads.cs | 22 ++++++- Wabbajack.Server/DataLayer/Patches.cs | 24 +++++++ Wabbajack.Server/Services/ListValidator.cs | 63 +++++++++++++++++-- Wabbajack.Server/Services/PatchBuilder.cs | 13 ++++ 6 files changed, 127 insertions(+), 8 deletions(-) diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index 1a104d5b..2836693d 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -217,9 +217,16 @@ TOP: var tmpFile = new TempFile(); var newArchive = new Archive(this) {Name = a.Name}; - - if (!await Download(newArchive, tmpFile.Path)) + + try + { + if (!await Download(newArchive, tmpFile.Path)) + return default; + } + catch (HttpRequestException) + { return default; + } newArchive.Hash = await tmpFile.Path.FileHashAsync(); newArchive.Size = tmpFile.Path.Size; diff --git a/Wabbajack.Server.Test/ModlistUpdater.cs b/Wabbajack.Server.Test/ModlistUpdater.cs index 40463b75..6a85d9cb 100644 --- a/Wabbajack.Server.Test/ModlistUpdater.cs +++ b/Wabbajack.Server.Test/ModlistUpdater.cs @@ -76,7 +76,7 @@ namespace Wabbajack.Server.Test Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_updates/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); } - [Fact] + [Fact] public async Task TestEndToEndArchiveUpdating() { var modLists = await MakeModList(); diff --git a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs index d45724d6..8a2b84d2 100644 --- a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs +++ b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs @@ -13,6 +13,26 @@ namespace Wabbajack.Server.DataLayer { public partial class SqlService { + public async Task AddKnownDownload(Archive a, DateTime downloadFinished) + { + await using var conn = await Open(); + var Id = Guid.NewGuid(); + await conn.ExecuteAsync( + "INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader, DownloadFinished, IsFailed) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader, @DownloadFinished, @IsFailed)", + new + { + Id = Id, + PrimaryKeyString = a.State.PrimaryKeyString, + Size = a.Size == 0 ? null : (long?)a.Size, + Hash = a.Hash == default ? null : (Hash?)a.Hash, + DownloadState = a.State, + Downloader = AbstractDownloadState.TypeToName[a.State.GetType()], + DownloadFinished = downloadFinished, + IsFailed = false + }); + return Id; + } + public async Task EnqueueDownload(Archive a) { await using var conn = await Open(); @@ -26,7 +46,7 @@ namespace Wabbajack.Server.DataLayer Size = a.Size == 0 ? null : (long?)a.Size, Hash = a.Hash == default ? null : (Hash?)a.Hash, DownloadState = a.State, - Downloader = AbstractDownloadState.TypeToName[a.State.GetType()] + Downloader = AbstractDownloadState.TypeToName[a.State.GetType()], }); return Id; } diff --git a/Wabbajack.Server/DataLayer/Patches.cs b/Wabbajack.Server/DataLayer/Patches.cs index f79ae4c4..dfc07b88 100644 --- a/Wabbajack.Server/DataLayer/Patches.cs +++ b/Wabbajack.Server/DataLayer/Patches.cs @@ -1,7 +1,10 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Dapper; using Wabbajack.Common; +using Wabbajack.Lib; using Wabbajack.Server.DTOs; namespace Wabbajack.Server.DataLayer @@ -121,5 +124,26 @@ namespace Wabbajack.Server.DataLayer FailMessage = patch.Item6 }; } + + public async Task> PatchesForSource(Guid sourceDownload) + { + await using var conn = await Open(); + var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>( + "SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId", new {SrcId = sourceDownload}); + + List results = new List(); + foreach (var (srcId, destId, patchSize, finished, isFinished, failMessage) in patches) + { + results.Add( new Patch { + Src = await GetArchiveDownload(srcId), + Dest = await GetArchiveDownload(destId), + PatchSize = patchSize, + Finished = finished, + IsFailed = isFinished, + FailMessage = failMessage + }); + } + return results; + } } } diff --git a/Wabbajack.Server/Services/ListValidator.cs b/Wabbajack.Server/Services/ListValidator.cs index afb2c751..98ccdc65 100644 --- a/Wabbajack.Server/Services/ListValidator.cs +++ b/Wabbajack.Server/Services/ListValidator.cs @@ -24,17 +24,19 @@ namespace Wabbajack.Server.Services private SqlService _sql; private DiscordWebHook _discord; private NexusKeyMaintainance _nexus; + private ArchiveMaintainer _archives; public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries { get; private set; } = new (ModListSummary Summary, DetailedStatus Detailed)[0]; - public ListValidator(ILogger logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus) - : base(logger, settings, TimeSpan.FromMinutes(10)) + public ListValidator(ILogger logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives) + : base(logger, settings, TimeSpan.FromMinutes(5)) { _sql = sql; _discord = discord; _nexus = nexus; + _archives = archives; } public override async Task Execute() @@ -53,7 +55,8 @@ namespace Wabbajack.Server.Services var archives = await modList.Archives.PMap(queue, async archive => { var (_, result) = await ValidateArchive(data, archive); - // TODO : auto-healing goes here + if (result == ArchiveStatus.InValid) + return await TryToHeal(data, archive); return (archive, result); }); @@ -128,7 +131,59 @@ namespace Wabbajack.Server.Services Summaries = results; return Summaries.Count(s => s.Summary.HasFailures); } - + + private AsyncLock _healLock = new AsyncLock(); + private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive) + { + using var _ = await _healLock.WaitAsync(); + + if (!(archive.State is IUpgradingState)) + return (archive, ArchiveStatus.InValid); + + var srcDownload = await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size); + if (srcDownload == null || srcDownload.IsFailed == true) + { + return (archive, ArchiveStatus.InValid); + } + + + var patches = await _sql.PatchesForSource(srcDownload.Id); + foreach (var patch in patches) + { + if (patch.Finished is null) + return (archive, ArchiveStatus.Updating); + + if (patch.IsFailed == true) + continue; + + var (_, status) = await ValidateArchive(data, patch.Dest.Archive); + if (status == ArchiveStatus.Valid) + return (archive, ArchiveStatus.Updated); + } + + + var upgradeTime = DateTime.UtcNow; + var upgrade = await (archive.State as IUpgradingState)?.FindUpgrade(archive); + if (upgrade == default) + { + return (archive, ArchiveStatus.InValid); + } + + await _archives.Ingest(upgrade.NewFile.Path); + + var id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime); + var destDownload = await _sql.GetArchiveDownload(id); + + await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload}); + + _logger.Log(LogLevel.Information, $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}"); + await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}" }); + + upgrade.NewFile.Dispose(); + + return (archive, ArchiveStatus.Updating); + } + private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(ValidationData data, Archive archive) { switch (archive.State) diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs index aa9961e8..e9c56470 100644 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -70,11 +70,24 @@ namespace Wabbajack.Server.Services var size = await ftpClient.GetFileSizeAsync(patchName); await patch.Finish(_sql, size); + await _discordWebHook.Send(Channel.Spam, + new DiscordMessage + { + Content = + $"Built {size.ToFileSizeString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}" + }); } catch (Exception ex) { _logger.LogError(ex, "Error while building patch"); await patch.Fail(_sql, ex.ToString()); + await _discordWebHook.Send(Channel.Spam, + new DiscordMessage + { + Content = + $"Failure building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}" + }); + } count++;