From 087875fba35cd16a1dad9fa4031eb28c73c84e50 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge <tbaldridge@gmail.com> Date: Tue, 12 May 2020 21:39:03 -0600 Subject: [PATCH] Add ModListdownloader and supporting tests --- .../ModListValidationTests.cs | 108 ++++++++++++++++ Wabbajack.Server/DataLayer/ModLists.cs | 62 +++++++++ .../Services/ModListDownloader.cs | 119 ++++++++++++++++++ Wabbajack.Server/Services/NexusPoll.cs | 1 - Wabbajack.Server/Startup.cs | 3 + 5 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 Wabbajack.Server.Test/ModListValidationTests.cs create mode 100644 Wabbajack.Server/DataLayer/ModLists.cs create mode 100644 Wabbajack.Server/Services/ModListDownloader.cs diff --git a/Wabbajack.Server.Test/ModListValidationTests.cs b/Wabbajack.Server.Test/ModListValidationTests.cs new file mode 100644 index 00000000..f64d5da8 --- /dev/null +++ b/Wabbajack.Server.Test/ModListValidationTests.cs @@ -0,0 +1,108 @@ +using System; +using System.Collections.Generic; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.ModListRegistry; +using Wabbajack.Server.DataLayer; +using Wabbajack.Server.Services; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.BuildServer.Test +{ + public class ModListValidationTests : ABuildServerSystemTest + { + public ModListValidationTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture) + { + } + + [Fact] + public async Task CanLoadMetadataFromTestServer() + { + var modlist = await MakeModList(); + Consts.ModlistMetadataURL = modlist.ToString(); + var data = await ModlistMetadata.LoadFromGithub(); + Assert.Single(data); + Assert.Equal("test_list", data.First().Links.MachineURL); + } + + [Fact] + public async Task CanIngestModLists() + { + var modlist = await MakeModList(); + Consts.ModlistMetadataURL = modlist.ToString(); + var sql = Fixture.GetService<SqlService>(); + var downloader = Fixture.GetService<ModListDownloader>(); + await downloader.CheckForNewLists(); + + foreach (var list in ModListMetaData) + { + Assert.True(await sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash)); + } + } + + private async Task<Uri> MakeModList() + { + var archive_data = Encoding.UTF8.GetBytes("Cheese for Everyone!"); + var test_archive_path = "test_archive.txt".RelativeTo(Fixture.ServerPublicFolder); + await test_archive_path.WriteAllBytesAsync(archive_data); + + + + ModListData = new ModList(); + ModListData.Archives.Add( + new Archive(new HTTPDownloader.State(MakeURL("test_archive.txt"))) + { + Hash = await test_archive_path.FileHashAsync(), + Name = "test_archive", + Size = test_archive_path.Size, + }); + + var modListPath = "test_modlist.wabbajack".RelativeTo(Fixture.ServerPublicFolder); + + await using (var fs = modListPath.Create()) + { + using var za = new ZipArchive(fs, ZipArchiveMode.Create); + var entry = za.CreateEntry("modlist"); + await using var es = entry.Open(); + ModListData.ToJson(es); + } + + ModListMetaData = new List<ModlistMetadata> + { + new ModlistMetadata + { + Official = false, + Author = "Test Suite", + Description = "A test", + DownloadMetadata = new DownloadMetadata + { + Hash = await modListPath.FileHashAsync(), + Size = modListPath.Size + }, + Links = new ModlistMetadata.LinksObject + { + MachineURL = "test_list", + Download = MakeURL("test_modlist.wabbajack") + } + } + }; + + var metadataPath = "test_mod_list_metadata.json".RelativeTo(Fixture.ServerPublicFolder); + + ModListMetaData.ToJson(metadataPath); + + return new Uri(MakeURL("test_mod_list_metadata.json")); + } + + public ModList ModListData { get; set; } + + public List<ModlistMetadata> ModListMetaData { get; set; } + + } +} diff --git a/Wabbajack.Server/DataLayer/ModLists.cs b/Wabbajack.Server/DataLayer/ModLists.cs new file mode 100644 index 00000000..d73357e6 --- /dev/null +++ b/Wabbajack.Server/DataLayer/ModLists.cs @@ -0,0 +1,62 @@ +using System.Linq; +using System.Threading.Tasks; +using Dapper; +using Wabbajack.Lib; +using Wabbajack.Lib.ModListRegistry; +using Wabbajack.Common; + +namespace Wabbajack.Server.DataLayer +{ + public partial class SqlService + { + public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist) + { + await using var conn = await Open(); + await using var tran = await conn.BeginTransactionAsync(); + + await conn.ExecuteAsync(@"DELETE FROM dbo.ModLists Where MachineUrl = @MachineUrl", + new {MachineUrl = metadata.Links.MachineURL}, tran); + + await conn.ExecuteAsync( + @"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList) VALUES (@MachineUrl, @Hash, @Metadata, @ModList)", + new + { + MachineUrl = metadata.Links.MachineURL, + Hash = hash, + MetaData = metadata.ToJson(), + ModList = modlist.ToJson() + }, tran); + + var entries = modlist.Archives.Select(a => + new + { + MachineUrl = metadata.Links.MachineURL, + Hash = a.Hash, + Size = a.Size, + State = a.State.ToJson(), + PrimaryKeyString = a.State.PrimaryKeyString + }).ToArray(); + + await conn.ExecuteAsync(@"DELETE FROM dbo.ModListArchives WHERE MachineURL = @machineURL", + new {MachineUrl = metadata.Links.MachineURL}, tran); + + foreach (var entry in entries) + { + await conn.ExecuteAsync( + "INSERT INTO dbo.ModListArchives (MachineURL, Hash, Size, PrimaryKeyString, State) VALUES (@MachineURL, @Hash, @Size, @PrimaryKeyString, @State)", + entry, tran); + } + + await tran.CommitAsync(); + } + + public async Task<bool> HaveIndexedModlist(string machineUrl, Hash hash) + { + await using var conn = await Open(); + var result = await conn.QueryFirstOrDefaultAsync<string>( + "SELECT MachineURL from dbo.Modlists WHERE MachineURL = @MachineUrl AND Hash = @Hash", + new {MachineUrl = machineUrl, Hash = hash}); + return result != null; + } + } +} diff --git a/Wabbajack.Server/Services/ModListDownloader.cs b/Wabbajack.Server/Services/ModListDownloader.cs new file mode 100644 index 00000000..07638860 --- /dev/null +++ b/Wabbajack.Server/Services/ModListDownloader.cs @@ -0,0 +1,119 @@ +using System; +using System.IO.Compression; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Wabbajack.BuildServer; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.ModListRegistry; +using Wabbajack.Server.DataLayer; + +namespace Wabbajack.Server.Services +{ + public class ModListDownloader + { + private ILogger<ModListDownloader> _logger; + private AppSettings _settings; + private ArchiveMaintainer _maintainer; + private SqlService _sql; + + public ModListDownloader(ILogger<ModListDownloader> logger, AppSettings settings, ArchiveMaintainer maintainer, SqlService sql) + { + _logger = logger; + _settings = settings; + _maintainer = maintainer; + _sql = sql; + } + + public void Start() + { + if (_settings.RunBackEndJobs) + { + Task.Run(async () => + { + while (true) + { + try + { + await CheckForNewLists(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error checking list"); + } + + await Task.Delay(TimeSpan.FromMinutes(5)); + + } + }); + } + } + + public async Task CheckForNewLists() + { + var lists = await ModlistMetadata.LoadFromGithub(); + foreach (var list in lists) + { + try + { + if (_maintainer.HaveArchive(list.DownloadMetadata!.Hash)) + continue; + + _logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}"); + var tf = new TempFile(); + var state = DownloadDispatcher.ResolveArchive(list.Links.Download); + if (state == null) + { + _logger.Log(LogLevel.Error, + $"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}"); + continue; + } + + await state.Download(new Archive(state) {Name = $"{list.Links.MachineURL}.wabbajack"}, tf.Path); + var modistPath = await _maintainer.Ingest(tf.Path); + + ModList modlist; + await using (var fs = modistPath.OpenRead()) + using (var zip = new ZipArchive(fs, ZipArchiveMode.Read)) + await using (var entry = zip.GetEntry("modlist")?.Open()) + { + if (entry == null) + { + Utils.Log($"Bad Modlist {list.Links.MachineURL}"); + continue; + } + + try + { + modlist = entry.FromJson<ModList>(); + } + catch (JsonReaderException ex) + { + Utils.Log($"Bad JSON format for {list.Links.MachineURL}"); + continue; + } + } + + await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}"); + } + } + } + } + + public static class ModListDownloaderExtensions + { + public static void UseModListDownloader(this IApplicationBuilder b) + { + var poll = (ModListDownloader)b.ApplicationServices.GetService(typeof(ModListDownloader)); + poll.Start(); + } + + } +} diff --git a/Wabbajack.Server/Services/NexusPoll.cs b/Wabbajack.Server/Services/NexusPoll.cs index 64c1f18e..d3691adc 100644 --- a/Wabbajack.Server/Services/NexusPoll.cs +++ b/Wabbajack.Server/Services/NexusPoll.cs @@ -170,6 +170,5 @@ namespace Wabbajack.Server.Services var poll = (NexusPoll)b.ApplicationServices.GetService(typeof(NexusPoll)); poll.Start(); } - } } diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index e5bd00fc..61935204 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -58,6 +58,7 @@ namespace Wabbajack.Server services.AddSingleton<GlobalInformation>(); services.AddSingleton<NexusPoll>(); services.AddSingleton<ArchiveMaintainer>(); + services.AddSingleton<ModListDownloader>(); services.AddMvc(); services.AddControllers() .AddNewtonsoftJson(o => @@ -100,6 +101,8 @@ namespace Wabbajack.Server app.UseAuthentication(); app.UseAuthorization(); app.UseNexusPoll(); + app.UseArchiveMaintainer(); + app.UseModListDownloader(); app.Use(next => {