From 8d8af953ac7b469e2ce10240c51878fc6d1598f9 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 13 May 2020 06:09:20 -0600 Subject: [PATCH 1/3] Fixes for when authors create lists with broken download links --- .../ModListValidationTests.cs | 22 ++++++++- Wabbajack.Server.Test/sql/wabbajack_db.sql | 1 + Wabbajack.Server/DataLayer/ModLists.cs | 7 +-- .../Services/ArchiveMaintainer.cs | 2 +- .../Services/ModListDownloader.cs | 48 +++++++++++++------ 5 files changed, 61 insertions(+), 19 deletions(-) diff --git a/Wabbajack.Server.Test/ModListValidationTests.cs b/Wabbajack.Server.Test/ModListValidationTests.cs index f64d5da8..ddddc54b 100644 --- a/Wabbajack.Server.Test/ModListValidationTests.cs +++ b/Wabbajack.Server.Test/ModListValidationTests.cs @@ -38,12 +38,16 @@ namespace Wabbajack.BuildServer.Test Consts.ModlistMetadataURL = modlist.ToString(); var sql = Fixture.GetService(); var downloader = Fixture.GetService(); - await downloader.CheckForNewLists(); + Assert.Equal(2, await downloader.CheckForNewLists()); foreach (var list in ModListMetaData) { Assert.True(await sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash)); } + + // Nothing has changed so we shouldn't be downloading anything this time + Assert.Equal(0, await downloader.CheckForNewLists()); + } private async Task MakeModList() @@ -90,6 +94,22 @@ namespace Wabbajack.BuildServer.Test MachineURL = "test_list", Download = MakeURL("test_modlist.wabbajack") } + }, + new ModlistMetadata + { + Official = true, + Author = "Test Suite", + Description = "A list with a broken hash", + DownloadMetadata = new DownloadMetadata() + { + Hash = Hash.FromLong(42), + Size = 42 + }, + Links = new ModlistMetadata.LinksObject + { + MachineURL = "broken_list", + Download = MakeURL("test_modlist.wabbajack") + } } }; diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index 2775b241..a51319a5 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -323,6 +323,7 @@ CREATE TABLE [dbo].[ModLists]( [Hash] [bigint] NOT NULL, [Metadata] [nvarchar](max) NOT NULL, [Modlist] [nvarchar](max) NOT NULL, + [BrokenDownload] [tinyint] NOT NULL, CONSTRAINT [PK_ModLists] PRIMARY KEY CLUSTERED ( [MachineURL] ASC diff --git a/Wabbajack.Server/DataLayer/ModLists.cs b/Wabbajack.Server/DataLayer/ModLists.cs index d73357e6..82e4b443 100644 --- a/Wabbajack.Server/DataLayer/ModLists.cs +++ b/Wabbajack.Server/DataLayer/ModLists.cs @@ -9,7 +9,7 @@ namespace Wabbajack.Server.DataLayer { public partial class SqlService { - public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist) + public async Task IngestModList(Hash hash, ModlistMetadata metadata, ModList modlist, bool brokenDownload) { await using var conn = await Open(); await using var tran = await conn.BeginTransactionAsync(); @@ -18,13 +18,14 @@ namespace Wabbajack.Server.DataLayer new {MachineUrl = metadata.Links.MachineURL}, tran); await conn.ExecuteAsync( - @"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList) VALUES (@MachineUrl, @Hash, @Metadata, @ModList)", + @"INSERT INTO dbo.ModLists (MachineUrl, Hash, Metadata, ModList, BrokenDownload) VALUES (@MachineUrl, @Hash, @Metadata, @ModList, @BrokenDownload)", new { MachineUrl = metadata.Links.MachineURL, Hash = hash, MetaData = metadata.ToJson(), - ModList = modlist.ToJson() + ModList = modlist.ToJson(), + BrokenDownload = brokenDownload }, tran); var entries = modlist.Archives.Select(a => diff --git a/Wabbajack.Server/Services/ArchiveMaintainer.cs b/Wabbajack.Server/Services/ArchiveMaintainer.cs index f6237461..632f752c 100644 --- a/Wabbajack.Server/Services/ArchiveMaintainer.cs +++ b/Wabbajack.Server/Services/ArchiveMaintainer.cs @@ -12,7 +12,7 @@ using File = System.IO.File; namespace Wabbajack.Server.Services { /// - /// Maintains a concurrent cache of all the files we've downloaded, indexed by Hash + /// Maintains a concurrent cache of all the files we've downloaded, indexed by Hash. /// public class ArchiveMaintainer { diff --git a/Wabbajack.Server/Services/ModListDownloader.cs b/Wabbajack.Server/Services/ModListDownloader.cs index 07638860..dc35ab5c 100644 --- a/Wabbajack.Server/Services/ModListDownloader.cs +++ b/Wabbajack.Server/Services/ModListDownloader.cs @@ -1,5 +1,6 @@ using System; using System.IO.Compression; +using System.Runtime.CompilerServices; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Logging; @@ -38,6 +39,7 @@ namespace Wabbajack.Server.Services { try { + _logger.Log(LogLevel.Information, "Checking for updated mod lists"); await CheckForNewLists(); } catch (Exception ex) @@ -52,31 +54,47 @@ namespace Wabbajack.Server.Services } } - public async Task CheckForNewLists() + public async Task CheckForNewLists() { + int downloaded = 0; var lists = await ModlistMetadata.LoadFromGithub(); foreach (var list in lists) { try { - if (_maintainer.HaveArchive(list.DownloadMetadata!.Hash)) + if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash)) continue; - _logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}"); - var tf = new TempFile(); - var state = DownloadDispatcher.ResolveArchive(list.Links.Download); - if (state == null) + + if (!_maintainer.HaveArchive(list.DownloadMetadata!.Hash)) { - _logger.Log(LogLevel.Error, - $"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}"); - continue; + _logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}"); + var tf = new TempFile(); + var state = DownloadDispatcher.ResolveArchive(list.Links.Download); + if (state == null) + { + _logger.Log(LogLevel.Error, + $"Now downloader found for list {list.Links.MachineURL} : {list.Links.Download}"); + continue; + } + + downloaded += 1; + await state.Download(new Archive(state) {Name = $"{list.Links.MachineURL}.wabbajack"}, tf.Path); + var hash = await tf.Path.FileHashAsync(); + if (hash != list.DownloadMetadata.Hash) + { + _logger.Log(LogLevel.Error, + $"Downloaded modlist {list.Links.MachineURL} {list.DownloadMetadata.Hash} didn't match metadata hash of {hash}"); + await _sql.IngestModList(list.DownloadMetadata.Hash, list, new ModList(), true); + continue; + } + + await _maintainer.Ingest(tf.Path); } - await state.Download(new Archive(state) {Name = $"{list.Links.MachineURL}.wabbajack"}, tf.Path); - var modistPath = await _maintainer.Ingest(tf.Path); - + _maintainer.TryGetPath(list.DownloadMetadata.Hash, out var modlistPath); ModList modlist; - await using (var fs = modistPath.OpenRead()) + await using (var fs = modlistPath.OpenRead()) using (var zip = new ZipArchive(fs, ZipArchiveMode.Read)) await using (var entry = zip.GetEntry("modlist")?.Open()) { @@ -97,13 +115,15 @@ namespace Wabbajack.Server.Services } } - await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist); + await _sql.IngestModList(list.DownloadMetadata!.Hash, list, modlist, false); } catch (Exception ex) { _logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}"); } } + _logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists"); + return downloaded; } } From aab0a7eda6cbfe8d89b28f4b447367e8f11499f4 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 13 May 2020 07:45:32 -0600 Subject: [PATCH 2/3] Fix broken test --- Wabbajack.Server.Test/ModListValidationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Wabbajack.Server.Test/ModListValidationTests.cs b/Wabbajack.Server.Test/ModListValidationTests.cs index ddddc54b..31f2dbdf 100644 --- a/Wabbajack.Server.Test/ModListValidationTests.cs +++ b/Wabbajack.Server.Test/ModListValidationTests.cs @@ -27,7 +27,7 @@ namespace Wabbajack.BuildServer.Test var modlist = await MakeModList(); Consts.ModlistMetadataURL = modlist.ToString(); var data = await ModlistMetadata.LoadFromGithub(); - Assert.Single(data); + Assert.Equal(2, data.Count); Assert.Equal("test_list", data.First().Links.MachineURL); } From 0207c7ac07312381776b1222afbf7f9fde91203d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 13 May 2020 08:06:18 -0600 Subject: [PATCH 3/3] Fix possible NPE --- Wabbajack.Lib/MO2Compiler.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 866b0674..29e36e73 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -88,7 +88,7 @@ namespace Wabbajack.Lib var otherProfilesPath = MO2ProfileDir.Combine("otherprofiles.txt"); SelectedProfiles = new HashSet(); if (otherProfilesPath.Exists) SelectedProfiles = (await otherProfilesPath.ReadAllLinesAsync()).ToHashSet(); - SelectedProfiles.Add(MO2Profile); + SelectedProfiles.Add(MO2Profile!); Info("Using Profiles: " + string.Join(", ", SelectedProfiles.OrderBy(p => p))); @@ -322,7 +322,7 @@ namespace Wabbajack.Lib Archives = SelectedArchives.ToList(), ModManager = ModManager.MO2, Directives = InstallDirectives, - Name = ModListName ?? MO2Profile, + Name = ModListName ?? MO2Profile!, Author = ModListAuthor ?? "", Description = ModListDescription ?? "", Readme = ModlistReadme ?? "",