Add List Validation back into the app

Wabbajack.Server.Test/ADBTest.cs

@@ -39,8 +39,7 @@ namespace Wabbajack.BuildServer.Test
         
         private async Task CreateSchema()
         {
-            Utils.Log("Creating Database");
-            //var conn = new SqlConnection("Data Source=localhost,1433;User ID=test;Password=test;MultipleActiveResultSets=true");
+            Utils.Log($"Creating Database {DBName}");
             await using var conn = new SqlConnection(CONN_STR);
 
             await conn.OpenAsync();
@@ -61,6 +60,7 @@ namespace Wabbajack.BuildServer.Test
             
             await new SqlCommand($"INSERT INTO dbo.ApiKeys (APIKey, Owner) VALUES ('{APIKey}', '{User}');", conn).ExecuteNonQueryAsync();
             _finishedSchema = true;
+            Utils.Log($"Finished creating database {DBName}");
         }
 
         private static IEnumerable<string> SplitSqlStatements(string sqlScript)

Wabbajack.Server.Test/ModListValidationTests.cs

@@ -9,6 +9,7 @@ using Wabbajack.Lib;
 using Wabbajack.Lib.Downloaders;
 using Wabbajack.Lib.ModListRegistry;
 using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DTOs;
 using Wabbajack.Server.Services;
 using Xunit;
 using Xunit.Abstractions;
@@ -50,6 +51,90 @@ namespace Wabbajack.BuildServer.Test
 
         }
         
+        [Fact]
+        public async Task CanValidateModLists()
+        {
+            var modlists = await MakeModList();
+            Consts.ModlistMetadataURL = modlists.ToString();
+            Utils.Log("Updating modlists");
+            await RevalidateLists(true);
+            
+            Utils.Log("Checking validated results");
+            var data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
+            Assert.NotNull(data);
+            Assert.Equal(0, data.ValidationSummary.Failed);
+            Assert.Equal(1, data.ValidationSummary.Passed);
+            
+            await CheckListFeeds(0, 1);
+
+            Utils.Log("Break List");
+            var archive = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
+            await archive.MoveToAsync(archive.WithExtension(new Extension(".moved")), true);
+
+            // We can revalidate but the non-nexus archives won't be checked yet since the list didn't change
+            await RevalidateLists(false);
+            
+            data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
+            Assert.NotNull(data);
+            Assert.Equal(0, data.ValidationSummary.Failed);
+            Assert.Equal(1, data.ValidationSummary.Passed);
+
+            // Run the non-nexus validator
+            await RevalidateLists(true);
+
+            data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
+            Assert.NotNull(data);
+            Assert.Equal(1, data.ValidationSummary.Failed);
+            Assert.Equal(0, data.ValidationSummary.Passed);
+            
+            await CheckListFeeds(1, 0);
+            
+            Utils.Log("Fix List");
+            await archive.WithExtension(new Extension(".moved")).MoveToAsync(archive, false);
+
+            await RevalidateLists(true);
+
+            data = (await ModlistMetadata.LoadFromGithub()).FirstOrDefault(l => l.Links.MachineURL == "test_list");
+            Assert.NotNull(data);
+            Assert.Equal(0, data.ValidationSummary.Failed);
+            Assert.Equal(1, data.ValidationSummary.Passed);
+            
+            await CheckListFeeds(0, 1);
+
+        }
+        
+        private async Task RevalidateLists(bool runNonNexus)
+        {
+            
+            var downloader = Fixture.GetService<ModListDownloader>();
+            await downloader.CheckForNewLists();
+
+            if (runNonNexus)
+            {
+                var nonNexus = Fixture.GetService<NonNexusDownloadValidator>();
+                await nonNexus.Execute();
+            }
+
+            var validator = Fixture.GetService<ListValidator>();
+            await validator.Execute();
+        }
+
+        private async Task CheckListFeeds(int failed, int passed)
+        {
+            var statusJson = await _client.GetJsonAsync<DetailedStatus>(MakeURL("lists/status/test_list.json"));
+            Assert.Equal(failed, statusJson.Archives.Count(a => a.IsFailing));
+            Assert.Equal(passed, statusJson.Archives.Count(a => !a.IsFailing));
+
+            
+            var statusHtml = await _client.GetHtmlAsync(MakeURL("lists/status/test_list.html"));
+            Assert.NotEmpty(statusHtml.DocumentNode.Descendants().Where(n => n.InnerHtml == $"Failed ({failed}):"));
+            Assert.NotEmpty(statusHtml.DocumentNode.Descendants().Where(n => n.InnerHtml == $"Passed ({passed}):"));
+            
+            var statusRss = await _client.GetHtmlAsync(MakeURL("lists/status/test_list/broken.rss"));
+            Assert.Equal(failed, statusRss.DocumentNode.SelectNodes("//item")?.Count ?? 0);
+        }
+
+        
         private async Task<Uri> MakeModList()
         {
             var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!");

Wabbajack.Server/Controllers/ListsStatus.cs

@@ -0,0 +1,130 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using Nettle;
+using Wabbajack.Common;
+using Wabbajack.Lib.ModListRegistry;
+using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DTOs;
+using Wabbajack.Server.Services;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [ApiController]
+    [Route("/lists")]
+    public class ListsStatus : ControllerBase
+    {
+        private ILogger<ListsStatus> _logger;
+        private ListValidator _validator;
+
+        public ListsStatus(ILogger<ListsStatus> logger, ListValidator validator)
+        {
+            _logger = logger;
+            _validator = validator;
+        }
+        
+        [HttpGet]
+        [Route("status.json")]
+        public async Task<IEnumerable<ModListSummary>> HandleGetLists()
+        {
+            return (_validator.Summaries).Select(d => d.Summary);
+        }
+
+        
+        private static readonly Func<object, string> HandleGetRssFeedTemplate = NettleEngine.GetCompiler().Compile(@"
+<?xml version=""1.0""?>
+<rss version=""2.0"">
+  <channel>
+    <title>{{lst.Name}} - Broken Mods</title>
+    <link>http://build.wabbajack.org/status/{{lst.Name}}.html</link>
+    <description>These are mods that are broken and need updating</description>
+    {{ each $.failed }}
+    <item>
+       <title>{{$.Archive.Name}} {{$.Archive.Hash}} {{$.Archive.State.PrimaryKeyString}}</title>
+       <link>{{$.Archive.Name}}</link>
+    </item>
+    {{/each}}
+  </channel>
+</rss>
+        ");
+
+        [HttpGet]
+        [Route("status/{Name}/broken.rss")]
+        public async Task<ContentResult> HandleGetRSSFeed(string Name)
+        {
+            var lst = await DetailedStatus(Name);
+            var response = HandleGetRssFeedTemplate(new
+            {
+                lst,
+                failed = lst.Archives.Where(a => a.IsFailing).ToList(),
+                passed = lst.Archives.Where(a => !a.IsFailing).ToList()
+            });
+            return new ContentResult
+            {
+                ContentType = "application/rss+xml",
+                StatusCode = (int) HttpStatusCode.OK,
+                Content = response
+            };
+        }
+        
+        private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
+            <html><body>
+                <h2>{{lst.Name}} - {{lst.Checked}} - {{ago}}min ago</h2>
+                <h3>Failed ({{failed.Count}}):</h3>
+                <ul>
+                {{each $.failed }}
+                <li>{{$.Archive.Name}}</li>
+                {{/each}}
+                </ul>
+                <h3>Passed ({{passed.Count}}):</h3>
+                <ul>
+                {{each $.passed }}
+                <li>{{$.Archive.Name}}</li>
+                {{/each}}
+                </ul>
+            </body></html>
+        ");
+
+        [HttpGet]
+        [Route("status/{Name}.html")]
+        public async Task<ContentResult> HandleGetListHtml(string Name)
+        {
+
+            var lst = await DetailedStatus(Name);
+            var response = HandleGetListTemplate(new
+            {
+                lst,
+                ago = (DateTime.UtcNow - lst.Checked).TotalMinutes,
+                failed = lst.Archives.Where(a => a.IsFailing).ToList(),
+                passed = lst.Archives.Where(a => !a.IsFailing).ToList()
+            });
+            return new ContentResult
+            {
+                ContentType = "text/html",
+                StatusCode = (int) HttpStatusCode.OK,
+                Content = response
+            };
+        }
+        
+        [HttpGet]
+        [Route("status/{Name}.json")]
+        public async Task<IActionResult> HandleGetListJson(string Name)
+        {
+            return Ok((await DetailedStatus(Name)).ToJson());
+        }
+        
+        private async Task<DetailedStatus> DetailedStatus(string Name)
+        {
+            return _validator.Summaries
+                .Select(d => d.Detailed)
+                .FirstOrDefault(d => d.MachineName == Name);
+        }
+
+
+        
+    }
+}

Wabbajack.Server/DTOs/ArchiveStatus.cs

@@ -0,0 +1,10 @@
+namespace Wabbajack.Server.DTOs
+{
+    enum ArchiveStatus
+    {
+        Valid,
+        InValid,
+        Updating,
+        Updated,
+    }
+}

Wabbajack.Server/DTOs/DetailedStatus.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using Wabbajack.Common.Serialization.Json;
+using Wabbajack.Lib;
+using Wabbajack.Lib.ModListRegistry;
+
+namespace Wabbajack.Server.DTOs
+{
+    [JsonName("DetailedStatus")]
+    public class DetailedStatus
+    {
+        public string Name { get; set; }
+        public DateTime Checked { get; set; } = DateTime.UtcNow;
+        public List<DetailedStatusItem> Archives { get; set; }
+        public DownloadMetadata DownloadMetaData { get; set; }
+        public bool HasFailures { get; set; }
+        public string MachineName { get; set; }
+    }
+
+    [JsonName("DetailedStatusItem")]
+    public class DetailedStatusItem
+    {
+        public bool IsFailing { get; set; }
+        public Archive Archive { get; set; }
+    }
+}

Wabbajack.Server/DTOs/ValidationData.cs

@@ -0,0 +1,14 @@
+using System.Collections.Generic;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.ModListRegistry;
+
+namespace Wabbajack.Server.DTOs
+{
+    public class ValidationData
+    {
+        public HashSet<(long Game, long ModId, long FileId)> NexusFiles { get; set; }
+        public Dictionary<(string PrimaryKeyString, Hash Hash), bool> ArchiveStatus { get; set; }
+        public List<(ModlistMetadata Metadata, ModList ModList)> ModLists { get; set; }
+    }
+}

Wabbajack.Server/DataLayer/NonNexusModlistArchives.cs

@@ -0,0 +1,47 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Dapper;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+
+namespace Wabbajack.Server.DataLayer
+{
+    public partial class SqlService
+    {
+        
+        public async Task<List<Archive>> GetNonNexusModlistArchives()
+        {
+            await using var conn = await Open();
+            var results = await conn.QueryAsync<(Hash Hash, long Size, string State)>(
+                @"SELECT Hash, Size, State FROM dbo.ModListArchives WHERE PrimaryKeyString NOT LIKE 'NexusDownloader+State|%'");
+            return results.Select(r => new Archive (r.State.FromJsonString<AbstractDownloadState>()) 
+            {
+                Size = r.Size,
+                Hash = r.Hash,
+                
+            }).ToList();}
+
+        public async Task UpdateNonNexusModlistArchivesStatus(IEnumerable<(Archive Archive, bool IsValid)> results)
+        {
+            await using var conn = await Open();
+            var trans = await conn.BeginTransactionAsync();
+            await conn.ExecuteAsync("DELETE FROM dbo.ModlistArchiveStatus;", transaction:trans);
+            
+            foreach (var itm in results.DistinctBy(itm => (itm.Archive.Hash, itm.Archive.State.PrimaryKeyString)))
+            {
+                await conn.ExecuteAsync(
+                    @"INSERT INTO dbo.ModlistArchiveStatus (PrimaryKeyStringHash, PrimaryKeyString, Hash, IsValid) 
+               VALUES (HASHBYTES('SHA2_256', @PrimaryKeyString), @PrimaryKeyString, @Hash, @IsValid)", new
+                    {
+                        PrimaryKeyString = itm.Archive.State.PrimaryKeyString,
+                        Hash = itm.Archive.Hash,
+                        IsValid = itm.IsValid
+                    }, trans);
+            }
+
+            await trans.CommitAsync();
+        }
+    }
+}

Wabbajack.Server/DataLayer/ValidationData.cs

@@ -0,0 +1,54 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Dapper;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.ModListRegistry;
+using Wabbajack.Server.DTOs;
+
+namespace Wabbajack.Server.DataLayer
+{
+    public partial class SqlService
+    {
+        public async Task<ValidationData> GetValidationData()
+        {
+            var nexusFiles = AllNexusFiles();
+            var archiveStatus = AllModListArchivesStatus();
+            var modLists = AllModLists();
+            return new ValidationData
+            {
+                NexusFiles = await nexusFiles,
+                ArchiveStatus = await archiveStatus,
+                ModLists = await modLists,
+            };
+        }
+        
+        public async Task<Dictionary<(string PrimaryKeyString, Hash Hash), bool>> AllModListArchivesStatus()
+        {
+            await using var conn = await Open();
+            var results =
+                await conn.QueryAsync<(string, Hash, bool)>(
+                    @"SELECT PrimaryKeyString, Hash, IsValid FROM dbo.ModListArchiveStatus");
+            return results.ToDictionary(v => (v.Item1, v.Item2), v => v.Item3);
+        }
+
+        public async Task<HashSet<(long NexusGameId, long ModId, long FileId)>> AllNexusFiles()
+        {
+            await using var conn = await Open();
+            var results = await conn.QueryAsync<(long, long, long)>(@"SELECT Game, ModId, p.file_id
+                                                      FROM [NexusModFiles] files
+                                                      CROSS APPLY
+                                                      OPENJSON(Data, '$.files') WITH (file_id bigint '$.file_id', category varchar(max) '$.category_name') p 
+                                                      WHERE p.category is not null");
+            return results.ToHashSet();
+        }
+        
+        public async Task<List<(ModlistMetadata, ModList)>> AllModLists()
+        {
+            await using var conn = await Open();
+            var results = await conn.QueryAsync<(string, string)>(@"SELECT Metadata, ModList FROM dbo.ModLists");
+            return results.Select(m => (m.Item1.FromJsonString<ModlistMetadata>(), m.Item2.FromJsonString<ModList>())).ToList();
+        }
+    }
+}

Wabbajack.Server/Services/AbstractService.cs

@@ -0,0 +1,64 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.Extensions.Logging;
+using Wabbajack.BuildServer;
+
+namespace Wabbajack.Server.Services
+{
+    public interface IStartable
+    {
+        public void Start();
+    }
+    
+    public abstract class AbstractService<TP, TR> : IStartable
+    {
+        protected AppSettings _settings;
+        private TimeSpan _delay;
+        protected ILogger<TP> _logger;
+
+        public AbstractService(ILogger<TP> logger, AppSettings settings, TimeSpan delay)
+        {
+            _settings = settings;
+            _delay = delay;
+            _logger = logger;
+        }
+
+        public void Start()
+        {
+            if (_settings.RunBackEndJobs)
+            {
+                Task.Run(async () =>
+                {
+                    while (true)
+                    {
+
+                        try
+                        {
+                            await Execute();
+                        }
+                        catch (Exception ex)
+                        {
+                            _logger.LogError(ex, "Running Service Loop");
+                        }
+
+                        await Task.Delay(_delay);
+                    }
+
+                });
+            }
+        }
+
+        public abstract Task<TR> Execute();
+    }
+    
+    public static class AbstractServiceExtensions 
+    {
+        public static void UseService<T>(this IApplicationBuilder b)
+        {
+            var poll = (IStartable)b.ApplicationServices.GetService(typeof(T));
+            poll.Start();
+        }
+    
+    }
+}

Wabbajack.Server/Services/ListValidator.cs

@@ -0,0 +1,108 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Logging;
+using RocksDbSharp;
+using Wabbajack.BuildServer;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.ModListRegistry;
+using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DTOs;
+
+namespace Wabbajack.Server.Services
+{
+    public class ListValidator : AbstractService<ListValidator, int>
+    {
+        private SqlService _sql;
+
+        public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries { get; private set; } =
+            new (ModListSummary Summary, DetailedStatus Detailed)[0];
+
+
+        public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql) 
+            : base(logger, settings, TimeSpan.FromMinutes(10))
+        {
+            _sql = sql;
+        }
+
+        public override async Task<int> Execute()
+        {
+            var data = await _sql.GetValidationData();
+            
+            using var queue = new WorkQueue();
+
+            var results = await data.ModLists.PMap(queue, async list =>
+            {
+                var (metadata, modList) = list;
+                var archives = await modList.Archives.PMap(queue, async archive =>
+                {
+                    var (_, result) = ValidateArchive(data, archive);
+                    // TODO : auto-healing goes here
+                    return (archive, result);
+                });
+
+                var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid);
+                var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated);
+                var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating);
+
+                var summary =  new ModListSummary
+                {
+                    Checked = DateTime.UtcNow,
+                    Failed = failedCount,
+                    Passed = passCount,
+                    Updating = updatingCount,
+                    MachineURL = metadata.Links.MachineURL,
+                    Name = metadata.Title,
+                };
+
+                var detailed = new DetailedStatus
+                {
+                    Name = metadata.Title,
+                    Checked = DateTime.UtcNow,
+                    DownloadMetaData = metadata.DownloadMetadata,
+                    HasFailures = failedCount > 0,
+                    MachineName = metadata.Links.MachineURL,
+                    Archives = archives.Select(a => new DetailedStatusItem
+                    {
+                        Archive = a.Item1, IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating
+                    }).ToList()
+                };
+
+                return (summary, detailed);
+            });
+            Summaries = results;
+            return Summaries.Count(s => s.Summary.HasFailures);
+        }
+        
+        private static (Archive archive, ArchiveStatus) ValidateArchive(ValidationData data, Archive archive)
+        {
+            switch (archive.State)
+            {
+                case GoogleDriveDownloader.State _:
+                    // Disabled for now due to GDrive rate-limiting the build server
+                    return (archive, ArchiveStatus.Valid);
+                case NexusDownloader.State nexusState when data.NexusFiles.Contains((
+                    nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)):
+                    return (archive, ArchiveStatus.Valid);
+                case NexusDownloader.State _:
+                    return (archive, ArchiveStatus.InValid);
+                case ManualDownloader.State _:
+                    return (archive, ArchiveStatus.Valid);
+                default:
+                {
+                    if (data.ArchiveStatus.TryGetValue((archive.State.PrimaryKeyString, archive.Hash),
+                        out bool isValid))
+                    {
+                        return isValid ? (archive, ArchiveStatus.Valid) : (archive, ArchiveStatus.InValid);
+                    }
+
+                    return (archive, ArchiveStatus.InValid);
+                }
+            }
+        }
+    }
+}

Wabbajack.Server/Services/NonNexusDownloadValidator.cs

@@ -0,0 +1,53 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Dapper;
+using Microsoft.Extensions.Logging;
+using Splat;
+using Wabbajack.BuildServer;
+using Wabbajack.Common;
+using Wabbajack.Server.DataLayer;
+using LogLevel = Microsoft.Extensions.Logging.LogLevel;
+
+namespace Wabbajack.Server.Services
+{
+    public class NonNexusDownloadValidator : AbstractService<NonNexusDownloadValidator, int>
+    {
+        private SqlService _sql;
+
+        public NonNexusDownloadValidator(ILogger<NonNexusDownloadValidator> logger, AppSettings settings, SqlService sql)
+            : base(logger, settings, TimeSpan.FromHours(2))
+        {
+            _sql = sql;
+        }
+
+        public override async Task<int> Execute()
+        {
+            var archives = await _sql.GetNonNexusModlistArchives();
+            _logger.Log(LogLevel.Information, "Validating {archives.Count} non-Nexus archives");
+            using var queue = new WorkQueue();
+            var results = await archives.PMap(queue, async archive =>
+            {
+                try
+                {
+                    var isValid = await archive.State.Verify(archive);
+                    return (Archive: archive, IsValid: isValid);
+                }
+                catch (Exception)
+                {
+                    return (Archive: archive, IsValid: false);
+                }
+
+            });
+
+            await _sql.UpdateNonNexusModlistArchivesStatus(results);
+            var failed = results.Count(r => !r.IsValid);
+            var passed = results.Count() - failed;
+            foreach(var (archive, _) in results.Where(f => f.IsValid))
+                _logger.Log(LogLevel.Warning, $"Validation failed for {archive.Name} from {archive.State.PrimaryKeyString}");
+            
+            _logger.Log(LogLevel.Information, $"Non-nexus validation completed {failed} out of {passed} failed");
+            return failed;
+        }
+    }
+}

Wabbajack.Server/Startup.cs

@@ -1,4 +1,5 @@
-using System.Diagnostics;
+using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
@@ -59,6 +60,9 @@ namespace Wabbajack.Server
             services.AddSingleton<NexusPoll>();
             services.AddSingleton<ArchiveMaintainer>();
             services.AddSingleton<ModListDownloader>();
+            services.AddSingleton<NonNexusDownloadValidator>();
+            services.AddSingleton<ListValidator>();
+            
             services.AddMvc();
             services.AddControllers()
                 .AddNewtonsoftJson(o =>
@@ -103,6 +107,9 @@ namespace Wabbajack.Server
             app.UseNexusPoll();
             app.UseArchiveMaintainer();
             app.UseModListDownloader();
+            
+            app.UseService<NonNexusDownloadValidator>();
+            app.UseService<ListValidator>();
 
             app.Use(next =>
             {