Fixes for when authors create lists with broken download links

Wabbajack.Server.Test/ModListValidationTests.cs

@@ -38,12 +38,16 @@ namespace Wabbajack.BuildServer.Test
             Consts.ModlistMetadataURL = modlist.ToString();
             var sql = Fixture.GetService<SqlService>();
             var downloader = Fixture.GetService<ModListDownloader>();
-            await downloader.CheckForNewLists();
+            Assert.Equal(2, await downloader.CheckForNewLists());
 
             foreach (var list in ModListMetaData)
             {
                 Assert.True(await sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash));
             }
+            
+            // Nothing has changed so we shouldn't be downloading anything this time
+            Assert.Equal(0, await downloader.CheckForNewLists());
+
         }
         
         private async Task<Uri> MakeModList()
@@ -90,6 +94,22 @@ namespace Wabbajack.BuildServer.Test
                         MachineURL = "test_list",
                         Download = MakeURL("test_modlist.wabbajack")
                     }
+                },
+                new ModlistMetadata
+                {
+                    Official = true,
+                    Author = "Test Suite",
+                    Description = "A list with a broken hash",
+                    DownloadMetadata = new DownloadMetadata()
+                    {
+                        Hash = Hash.FromLong(42),
+                        Size = 42
+                    },
+                    Links = new ModlistMetadata.LinksObject
+                    {
+                        MachineURL = "broken_list",
+                        Download = MakeURL("test_modlist.wabbajack")
+                    }
                 }
             };
 

Wabbajack.Server.Test/sql/wabbajack_db.sql

@@ -323,6 +323,7 @@ CREATE TABLE [dbo].[ModLists](
  [Hash] [bigint] NOT NULL,
  [Metadata] [nvarchar](max) NOT NULL,
  [Modlist] [nvarchar](max) NOT NULL,
+ [BrokenDownload] [tinyint] NOT NULL,
  CONSTRAINT [PK_ModLists] PRIMARY KEY CLUSTERED
      (
       [MachineURL] ASC

Wabbajack.Server/DataLayer/ModLists.cs

@@ -9,7 +9,7 @@ namespace Wabbajack.Server.DataLayer
 {
     public partial class SqlService
     {
-        public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist)
+        public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist, bool brokenDownload)
         {
             await using var conn = await Open();
             await using var tran = await conn.BeginTransactionAsync();
@@ -18,13 +18,14 @@ namespace Wabbajack.Server.DataLayer
                 new {MachineUrl = metadata.Links.MachineURL}, tran);
 
             await conn.ExecuteAsync(
-                @"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList) VALUES (@MachineUrl, @Hash, @Metadata, @ModList)",
+                @"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList, BrokenDownload) VALUES (@MachineUrl, @Hash, @Metadata, @ModList, @BrokenDownload)",
                 new
                 {
                     MachineUrl = metadata.Links.MachineURL,
                     Hash = hash,
                     MetaData = metadata.ToJson(),
-                    ModList = modlist.ToJson()
+                    ModList = modlist.ToJson(),
+                    BrokenDownload = brokenDownload
                 }, tran);
             
             var entries = modlist.Archives.Select(a =>

Wabbajack.Server/Services/ArchiveMaintainer.cs

@@ -12,7 +12,7 @@ using File = System.IO.File;
 namespace Wabbajack.Server.Services
 {
     /// <summary>
-    /// Maintains a concurrent cache of all the files we've downloaded, indexed by Hash
+    /// Maintains a concurrent cache of all the files we've downloaded, indexed by Hash. 
     /// </summary>
     public class ArchiveMaintainer
     {

Wabbajack.Server/Services/ModListDownloader.cs

@@ -1,5 +1,6 @@
 using System;
 using System.IO.Compression;
+using System.Runtime.CompilerServices;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.Extensions.Logging;
@@ -38,6 +39,7 @@ namespace Wabbajack.Server.Services
                     {
                         try
                         {
+                            _logger.Log(LogLevel.Information, "Checking for updated mod lists");
                             await CheckForNewLists();
                         }
                         catch (Exception ex)
@@ -52,16 +54,20 @@ namespace Wabbajack.Server.Services
             }
         }
 
-        public async Task CheckForNewLists()
+        public async Task<int> CheckForNewLists()
         {
+            int downloaded = 0;
             var lists = await ModlistMetadata.LoadFromGithub();
             foreach (var list in lists)
             {
                 try
                 {
-                    if (_maintainer.HaveArchive(list.DownloadMetadata!.Hash))
+                    if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
                         continue;
 
+
+                    if (!_maintainer.HaveArchive(list.DownloadMetadata!.Hash))
+                    {
                         _logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
                         var tf = new TempFile();
                         var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
@@ -72,11 +78,23 @@ namespace Wabbajack.Server.Services
                             continue;
                         }
 
+                        downloaded += 1;
                         await state.Download(new Archive(state) {Name = $"{list.Links.MachineURL}.wabbajack"}, tf.Path);
-                    var modistPath = await _maintainer.Ingest(tf.Path);
+                        var hash = await tf.Path.FileHashAsync();
+                        if (hash != list.DownloadMetadata.Hash)
+                        {
+                            _logger.Log(LogLevel.Error,
+                                $"Downloaded modlist {list.Links.MachineURL} {list.DownloadMetadata.Hash} didn't match metadata hash of {hash}");
+                            await _sql.IngestModList(list.DownloadMetadata.Hash, list, new ModList(), true);
+                            continue;
+                        }
 
+                        await _maintainer.Ingest(tf.Path);
+                    }
+
+                    _maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath);
                     ModList modlist;
-                    await using (var fs = modistPath.OpenRead())
+                    await using (var fs = modlistPath.OpenRead())
                     using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
                     await using (var entry = zip.GetEntry("modlist")?.Open())
                     {
@@ -97,13 +115,15 @@ namespace Wabbajack.Server.Services
                         }
                     }
 
-                    await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist);
+                    await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist, false);
                 }
                 catch (Exception ex)
                 {
                     _logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
                 }
             }
+            _logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");
+            return downloaded;
         }
     }