From 087875fba35cd16a1dad9fa4031eb28c73c84e50 Mon Sep 17 00:00:00 2001
From: Timothy Baldridge <tbaldridge@gmail.com>
Date: Tue, 12 May 2020 21:39:03 -0600
Subject: [PATCH] Add ModListdownloader and supporting tests

---
 .../ModListValidationTests.cs                 | 108 ++++++++++++++++
 Wabbajack.Server/DataLayer/ModLists.cs        |  62 +++++++++
 .../Services/ModListDownloader.cs             | 119 ++++++++++++++++++
 Wabbajack.Server/Services/NexusPoll.cs        |   1 -
 Wabbajack.Server/Startup.cs                   |   3 +
 5 files changed, 292 insertions(+), 1 deletion(-)
 create mode 100644 Wabbajack.Server.Test/ModListValidationTests.cs
 create mode 100644 Wabbajack.Server/DataLayer/ModLists.cs
 create mode 100644 Wabbajack.Server/Services/ModListDownloader.cs

diff --git a/Wabbajack.Server.Test/ModListValidationTests.cs b/Wabbajack.Server.Test/ModListValidationTests.cs
new file mode 100644
index 00000000..f64d5da8
--- /dev/null
+++ b/Wabbajack.Server.Test/ModListValidationTests.cs
@@ -0,0 +1,108 @@
+using System;
+using System.Collections.Generic;
+using System.IO.Compression;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.ModListRegistry;
+using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.Services;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Wabbajack.BuildServer.Test
+{
+    public class ModListValidationTests : ABuildServerSystemTest
+    {
+        public ModListValidationTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
+        {
+        }
+        
+        [Fact]
+        public async Task CanLoadMetadataFromTestServer()
+        {
+            var modlist = await MakeModList();
+            Consts.ModlistMetadataURL = modlist.ToString();
+            var data = await ModlistMetadata.LoadFromGithub();
+            Assert.Single(data);
+            Assert.Equal("test_list", data.First().Links.MachineURL);
+        }
+
+        [Fact]
+        public async Task CanIngestModLists()
+        {
+            var modlist = await MakeModList();
+            Consts.ModlistMetadataURL = modlist.ToString();
+            var sql = Fixture.GetService<SqlService>();
+            var downloader = Fixture.GetService<ModListDownloader>();
+            await downloader.CheckForNewLists();
+
+            foreach (var list in ModListMetaData)
+            {
+                Assert.True(await sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash));
+            }
+        }
+        
+        private async Task<Uri> MakeModList()
+        {
+            var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!");
+            var test_archive_path = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder);
+            await test_archive_path.WriteAllBytesAsync(archive_data);
+
+
+
+            ModListData = new ModList();
+            ModListData.Archives.Add(
+                new Archive(new HTTPDownloader.State(MakeURL("test_archive.txt")))
+                {
+                    Hash = await test_archive_path.FileHashAsync(),
+                    Name = "test_archive",
+                    Size = test_archive_path.Size,
+                });
+            
+            var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder);
+
+            await using (var fs = modListPath.Create())
+            {
+                using var za = new ZipArchive(fs, ZipArchiveMode.Create);
+                var entry = za.CreateEntry("modlist");
+                await using var es = entry.Open();
+                ModListData.ToJson(es);
+            }
+
+            ModListMetaData = new List<ModlistMetadata>
+            {
+                new ModlistMetadata
+                {
+                    Official = false,
+                    Author = "Test Suite",
+                    Description = "A test",
+                    DownloadMetadata = new DownloadMetadata
+                    {
+                        Hash = await modListPath.FileHashAsync(), 
+                        Size = modListPath.Size
+                    },
+                    Links = new ModlistMetadata.LinksObject
+                    {
+                        MachineURL = "test_list",
+                        Download = MakeURL("test_modlist.wabbajack")
+                    }
+                }
+            };
+
+            var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder);
+
+            ModListMetaData.ToJson(metadataPath);
+            
+            return new Uri(MakeURL("test_mod_list_metadata.json"));
+        }
+
+        public ModList ModListData { get; set; }
+
+        public List<ModlistMetadata> ModListMetaData { get; set; }
+
+    }
+}
diff --git a/Wabbajack.Server/DataLayer/ModLists.cs b/Wabbajack.Server/DataLayer/ModLists.cs
new file mode 100644
index 00000000..d73357e6
--- /dev/null
+++ b/Wabbajack.Server/DataLayer/ModLists.cs
@@ -0,0 +1,62 @@
+using System.Linq;
+using System.Threading.Tasks;
+using Dapper;
+using Wabbajack.Lib;
+using Wabbajack.Lib.ModListRegistry;
+using Wabbajack.Common;
+
+namespace Wabbajack.Server.DataLayer
+{
+    public partial class SqlService
+    {
+        public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist)
+        {
+            await using var conn = await Open();
+            await using var tran = await conn.BeginTransactionAsync();
+
+            await conn.ExecuteAsync(@"DELETE FROM dbo.ModLists Where MachineUrl = @MachineUrl",
+                new {MachineUrl = metadata.Links.MachineURL}, tran);
+
+            await conn.ExecuteAsync(
+                @"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList) VALUES (@MachineUrl, @Hash, @Metadata, @ModList)",
+                new
+                {
+                    MachineUrl = metadata.Links.MachineURL,
+                    Hash = hash,
+                    MetaData = metadata.ToJson(),
+                    ModList = modlist.ToJson()
+                }, tran);
+            
+            var entries = modlist.Archives.Select(a =>
+                new
+                {
+                    MachineUrl = metadata.Links.MachineURL,
+                    Hash = a.Hash,
+                    Size = a.Size,
+                    State = a.State.ToJson(),
+                    PrimaryKeyString = a.State.PrimaryKeyString
+                }).ToArray();
+
+            await conn.ExecuteAsync(@"DELETE FROM dbo.ModListArchives WHERE MachineURL = @machineURL",
+                new {MachineUrl = metadata.Links.MachineURL}, tran);
+            
+            foreach (var entry in entries)
+            {
+                await conn.ExecuteAsync(
+                    "INSERT INTO dbo.ModListArchives (MachineURL, Hash, Size, PrimaryKeyString, State) VALUES (@MachineURL, @Hash, @Size, @PrimaryKeyString, @State)",
+                    entry, tran);
+            }
+            
+            await tran.CommitAsync();
+        }
+        
+        public async Task<bool> HaveIndexedModlist(string machineUrl, Hash hash)
+        {
+            await using var conn = await Open();
+            var result = await conn.QueryFirstOrDefaultAsync<string>(
+                "SELECT MachineURL from dbo.Modlists WHERE MachineURL = @MachineUrl AND Hash = @Hash",
+                new {MachineUrl = machineUrl, Hash = hash});
+            return result != null;
+        }
+    }
+}
diff --git a/Wabbajack.Server/Services/ModListDownloader.cs b/Wabbajack.Server/Services/ModListDownloader.cs
new file mode 100644
index 00000000..07638860
--- /dev/null
+++ b/Wabbajack.Server/Services/ModListDownloader.cs
@@ -0,0 +1,119 @@
+using System;
+using System.IO.Compression;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+using Wabbajack.BuildServer;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.Downloaders;
+using Wabbajack.Lib.ModListRegistry;
+using Wabbajack.Server.DataLayer;
+
+namespace Wabbajack.Server.Services
+{
+    public class ModListDownloader
+    {
+        private ILogger<ModListDownloader> _logger;
+        private AppSettings _settings;
+        private ArchiveMaintainer _maintainer;
+        private SqlService _sql;
+
+        public ModListDownloader(ILogger<ModListDownloader> logger, AppSettings settings, ArchiveMaintainer maintainer, SqlService sql)
+        {
+            _logger = logger;
+            _settings = settings;
+            _maintainer = maintainer;
+            _sql = sql;
+        }
+
+        public void Start()
+        {
+            if (_settings.RunBackEndJobs)
+            {
+                Task.Run(async () =>
+                {
+                    while (true)
+                    {
+                        try
+                        {
+                            await CheckForNewLists();
+                        }
+                        catch (Exception ex)
+                        {
+                            _logger.LogError(ex, "Error checking list");
+                        }
+
+                        await Task.Delay(TimeSpan.FromMinutes(5));
+
+                    }
+                });
+            }
+        }
+
+        public async Task CheckForNewLists()
+        {
+            var lists = await ModlistMetadata.LoadFromGithub();
+            foreach (var list in lists)
+            {
+                try
+                {
+                    if (_maintainer.HaveArchive(list.DownloadMetadata!.Hash))
+                        continue;
+
+                    _logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
+                    var tf = new TempFile();
+                    var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
+                    if (state == null)
+                    {
+                        _logger.Log(LogLevel.Error,
+                            $"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}");
+                        continue;
+                    }
+
+                    await state.Download(new Archive(state) {Name = $"{list.Links.MachineURL}.wabbajack"}, tf.Path);
+                    var modistPath = await _maintainer.Ingest(tf.Path);
+                    
+                    ModList modlist;
+                    await using (var fs = modistPath.OpenRead())
+                    using (var zip = new ZipArchive(fs, ZipArchiveMode.Read))
+                    await using (var entry = zip.GetEntry("modlist")?.Open())
+                    {
+                        if (entry == null)
+                        {
+                            Utils.Log($"Bad Modlist {list.Links.MachineURL}");
+                            continue;
+                        }
+
+                        try
+                        {
+                            modlist = entry.FromJson<ModList>();
+                        }
+                        catch (JsonReaderException ex)
+                        {
+                            Utils.Log($"Bad JSON format for {list.Links.MachineURL}");
+                            continue;
+                        }
+                    }
+
+                    await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
+                }
+            }
+        }
+    }
+    
+    public static class ModListDownloaderExtensions 
+    {
+        public static void UseModListDownloader(this IApplicationBuilder b)
+        {
+            var poll = (ModListDownloader)b.ApplicationServices.GetService(typeof(ModListDownloader));
+            poll.Start();
+        }
+    
+    }
+}
diff --git a/Wabbajack.Server/Services/NexusPoll.cs b/Wabbajack.Server/Services/NexusPoll.cs
index 64c1f18e..d3691adc 100644
--- a/Wabbajack.Server/Services/NexusPoll.cs
+++ b/Wabbajack.Server/Services/NexusPoll.cs
@@ -170,6 +170,5 @@ namespace Wabbajack.Server.Services
             var poll = (NexusPoll)b.ApplicationServices.GetService(typeof(NexusPoll));
             poll.Start();
         }
-    
     }
 }
diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs
index e5bd00fc..61935204 100644
--- a/Wabbajack.Server/Startup.cs
+++ b/Wabbajack.Server/Startup.cs
@@ -58,6 +58,7 @@ namespace Wabbajack.Server
             services.AddSingleton<GlobalInformation>();
             services.AddSingleton<NexusPoll>();
             services.AddSingleton<ArchiveMaintainer>();
+            services.AddSingleton<ModListDownloader>();
             services.AddMvc();
             services.AddControllers()
                 .AddNewtonsoftJson(o =>
@@ -100,6 +101,8 @@ namespace Wabbajack.Server
             app.UseAuthentication();
             app.UseAuthorization();
             app.UseNexusPoll();
+            app.UseArchiveMaintainer();
+            app.UseModListDownloader();
 
             app.Use(next =>
             {