List validator now tries to heal lists

Wabbajack.Lib/Downloaders/HTTPDownloader.cs

@@ -218,8 +218,15 @@ TOP:
                 
                 var newArchive = new Archive(this) {Name = a.Name};
 
-                if (!await Download(newArchive, tmpFile.Path))
+                try
+                {
+                    if (!await Download(newArchive, tmpFile.Path))
+                        return default;
+                }
+                catch (HttpRequestException)
+                {
                     return default;
+                }
 
                 newArchive.Hash = await tmpFile.Path.FileHashAsync();
                 newArchive.Size = tmpFile.Path.Size;

Wabbajack.Server.Test/ModlistUpdater.cs

@@ -76,7 +76,7 @@ namespace Wabbajack.Server.Test
             Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_updates/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
         }
 
-                [Fact]
+        [Fact]
         public async Task TestEndToEndArchiveUpdating()
         {
             var modLists = await MakeModList();

Wabbajack.Server/DataLayer/ArchiveDownloads.cs

@@ -13,6 +13,26 @@ namespace Wabbajack.Server.DataLayer
 {
     public partial class SqlService
     {
+        public async Task<Guid> AddKnownDownload(Archive a, DateTime downloadFinished)
+        {
+            await using var conn = await Open();
+            var Id = Guid.NewGuid();
+            await conn.ExecuteAsync(
+                "INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader, DownloadFinished, IsFailed) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader, @DownloadFinished, @IsFailed)",
+                new
+                {
+                    Id = Id,
+                    PrimaryKeyString = a.State.PrimaryKeyString,
+                    Size = a.Size == 0 ? null : (long?)a.Size,
+                    Hash = a.Hash == default ? null : (Hash?)a.Hash,
+                    DownloadState = a.State,
+                    Downloader = AbstractDownloadState.TypeToName[a.State.GetType()],
+                    DownloadFinished = downloadFinished,
+                    IsFailed = false
+                });
+            return Id;
+        }
+        
         public async Task<Guid> EnqueueDownload(Archive a)
         {
             await using var conn = await Open();
@@ -26,7 +46,7 @@ namespace Wabbajack.Server.DataLayer
                     Size = a.Size == 0 ? null : (long?)a.Size,
                     Hash = a.Hash == default ? null : (Hash?)a.Hash,
                     DownloadState = a.State,
-                    Downloader = AbstractDownloadState.TypeToName[a.State.GetType()]
+                    Downloader = AbstractDownloadState.TypeToName[a.State.GetType()],
                 });
             return Id;
         }

Wabbajack.Server/DataLayer/Patches.cs

@@ -1,7 +1,10 @@
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using Dapper;
 using Wabbajack.Common;
+using Wabbajack.Lib;
 using Wabbajack.Server.DTOs;
 
 namespace Wabbajack.Server.DataLayer
@@ -121,5 +124,26 @@ namespace Wabbajack.Server.DataLayer
                 FailMessage = patch.Item6
             };
         }
+
+        public async Task<List<Patch>> PatchesForSource(Guid sourceDownload)
+        {
+            await using var conn = await Open();
+            var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
+                "SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId", new {SrcId = sourceDownload});
+
+            List<Patch> results = new List<Patch>();
+            foreach (var (srcId, destId, patchSize, finished, isFinished, failMessage) in patches)
+            {
+                results.Add( new Patch {
+                    Src = await GetArchiveDownload(srcId), 
+                    Dest = await GetArchiveDownload(destId),
+                    PatchSize = patchSize,
+                    Finished = finished,
+                    IsFailed = isFinished,
+                    FailMessage = failMessage
+                });
+            } 
+            return results;
+        }
     }
 }

Wabbajack.Server/Services/ListValidator.cs

@@ -24,17 +24,19 @@ namespace Wabbajack.Server.Services
         private SqlService _sql;
         private DiscordWebHook _discord;
         private NexusKeyMaintainance _nexus;
+        private ArchiveMaintainer _archives;
 
         public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries { get; private set; } =
             new (ModListSummary Summary, DetailedStatus Detailed)[0];
 
 
-        public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus) 
+        public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives) 
-            : base(logger, settings, TimeSpan.FromMinutes(10))
+            : base(logger, settings, TimeSpan.FromMinutes(5))
         {
             _sql = sql;
             _discord = discord;
             _nexus = nexus;
+            _archives = archives;
         }
 
         public override async Task<int> Execute()
@@ -53,7 +55,8 @@ namespace Wabbajack.Server.Services
                 var archives = await modList.Archives.PMap(queue, async archive =>
                 {
                     var (_, result) = await ValidateArchive(data, archive);
-                    // TODO : auto-healing goes here
+                    if (result == ArchiveStatus.InValid)
+                        return await TryToHeal(data, archive);
                     return (archive, result);
                 });
 
@@ -129,6 +132,58 @@ namespace Wabbajack.Server.Services
             return Summaries.Count(s => s.Summary.HasFailures);
         }
 
+        private AsyncLock _healLock = new AsyncLock();
+        private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive)
+        {
+            using var _ = await _healLock.WaitAsync();
+
+            if (!(archive.State is IUpgradingState))
+                return (archive, ArchiveStatus.InValid);
+            
+            var srcDownload = await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size);
+            if (srcDownload == null || srcDownload.IsFailed == true)
+            {
+                return (archive, ArchiveStatus.InValid);
+            }
+
+            
+            var patches = await _sql.PatchesForSource(srcDownload.Id);
+            foreach (var patch in patches)
+            {
+                if (patch.Finished is null)
+                    return (archive, ArchiveStatus.Updating);
+
+                if (patch.IsFailed == true)
+                    continue;
+                
+                var (_, status) = await ValidateArchive(data, patch.Dest.Archive);
+                if (status == ArchiveStatus.Valid)
+                    return (archive, ArchiveStatus.Updated);
+            }
+            
+            
+            var upgradeTime = DateTime.UtcNow;
+            var upgrade = await (archive.State as IUpgradingState)?.FindUpgrade(archive);
+            if (upgrade == default)
+            {
+                return (archive, ArchiveStatus.InValid);
+            }
+
+            await _archives.Ingest(upgrade.NewFile.Path);
+
+            var id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime);
+            var destDownload = await _sql.GetArchiveDownload(id);
+
+            await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload});
+            
+            _logger.Log(LogLevel.Information, $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}");
+            await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}" });
+
+            upgrade.NewFile.Dispose();
+
+            return (archive, ArchiveStatus.Updating);
+        }
+
         private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(ValidationData data, Archive archive)
         {
             switch (archive.State)

Wabbajack.Server/Services/PatchBuilder.cs

@@ -70,11 +70,24 @@ namespace Wabbajack.Server.Services
                     var size = await ftpClient.GetFileSizeAsync(patchName);
                     
                     await patch.Finish(_sql, size);
+                    await _discordWebHook.Send(Channel.Spam,
+                        new DiscordMessage
+                        {
+                            Content =
+                                $"Built {size.ToFileSizeString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
+                        });
                 }
                 catch (Exception ex)
                 {
                     _logger.LogError(ex, "Error while building patch");
                     await patch.Fail(_sql, ex.ToString());
+                    await _discordWebHook.Send(Channel.Spam,
+                        new DiscordMessage
+                        {
+                            Content =
+                                $"Failure building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
+                        });                    
+
                 }
 
                 count++;