From f80ae8b142e5cfca2f285ae1fa37b1de66aac08e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 1 Nov 2020 17:30:49 -0700 Subject: [PATCH 1/2] Updates to smooth out server update features --- Wabbajack.Server.Test/MirroredFilesTests.cs | 7 +++ Wabbajack.Server.Test/ModFileTests.cs | 21 +++++++- .../DataLayer/ArchiveDownloads.cs | 17 +++++-- Wabbajack.Server/DataLayer/MirroredFiles.cs | 48 +++++++++++++++++++ Wabbajack.Server/DataLayer/SqlService.cs | 3 ++ .../Services/MirrorQueueService.cs | 28 +++++++++++ Wabbajack.Server/Services/MirrorUploader.cs | 11 ++++- Wabbajack.Server/Startup.cs | 2 + Wabbajack.Test/EndToEndTests.cs | 2 +- Wabbajack.Test/TestUtils.cs | 8 ++-- 10 files changed, 135 insertions(+), 12 deletions(-) create mode 100644 Wabbajack.Server/Services/MirrorQueueService.cs diff --git a/Wabbajack.Server.Test/MirroredFilesTests.cs b/Wabbajack.Server.Test/MirroredFilesTests.cs index 227397d0..30727abd 100644 --- a/Wabbajack.Server.Test/MirroredFilesTests.cs +++ b/Wabbajack.Server.Test/MirroredFilesTests.cs @@ -50,6 +50,13 @@ namespace Wabbajack.Server.Test var file2 = new TempFile(); await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, file2.Path); } + + [Fact] + public async Task CanQueueFiles() + { + var service = Fixture.GetService(); + Assert.Equal(1, await service.Execute()); + } } } diff --git a/Wabbajack.Server.Test/ModFileTests.cs b/Wabbajack.Server.Test/ModFileTests.cs index 28de4e52..10f8b842 100644 --- a/Wabbajack.Server.Test/ModFileTests.cs +++ b/Wabbajack.Server.Test/ModFileTests.cs @@ -1,10 +1,12 @@ using System; +using System.Linq; using System.Threading.Tasks; using Wabbajack.BuildServer.Test; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DTOs; using Xunit; using Xunit.Abstractions; @@ -22,23 +24,38 @@ namespace Wabbajack.Server.Test public async Task CanGetDownloadStates() { var sql = Fixture.GetService(); + var hash = Hash.FromBase64("eSIyd+KOG3s="); var archive = new Archive(new WabbajackCDNDownloader.State(new Uri( "https://wabbajack.b-cdn.net/WABBAJACK_TEST_FILE.zip_a1a3e961-5c0b-4ccf-84b4-7aa437d9640d"))) { - Size = 20, Hash = Hash.FromBase64("eSIyd+KOG3s=") + Size = 20, Hash = hash }; await sql.EnqueueDownload(archive); + await sql.UpsertMirroredFile(new MirroredFile() + { + Created = DateTime.UtcNow, + Uploaded = DateTime.UtcNow, + Hash = hash, + Rationale = "Test File" + }); var dld = await sql.GetNextPendingDownload(); await dld.Finish(sql); var state = await ClientAPI.InferDownloadState(archive.Hash); - Assert.Equal(archive.State.GetMetaIniString(), state!.GetMetaIniString()); + var archives = await (await ClientAPI.GetClient()).GetJsonAsync( + $"{Consts.WabbajackBuildServerUri}mod_files/by_hash/{hash.ToHex()}"); + + Assert.True(archives.Length >= 2); + Assert.NotNull(archives.FirstOrDefault(a => a.State is WabbajackCDNDownloader.State)); + + await sql.DeleteMirroredFile(hash); + } } } diff --git a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs index 0af2f679..f988d750 100644 --- a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs +++ b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs @@ -241,10 +241,19 @@ namespace Wabbajack.Server.DataLayer var files = (await conn.QueryAsync<(long, Hash, AbstractDownloadState)>( @"SELECT Size, Hash, DownloadState from dbo.ArchiveDownloads WHERE Hash = @Hash AND IsFailed = 0 AND DownloadFinished IS NOT NULL ORDER BY DownloadFinished DESC", new {Hash = hash}) - ); - return files.Select(e => - new Archive(e.Item3) {Size = e.Item1, Hash = e.Item2} - ).ToArray(); + ).Select(e => + new Archive(e.Item3) {Size = e.Item1, Hash = e.Item2} + ).ToList(); + + if (await HaveMirror(hash) && files.Count > 0) + { + var ffile = files.First(); + var url = new Uri($"https://{(await _mirrorCreds).Username}.b-cdn.net/{hash.ToHex()}"); + files.Add(new Archive( + new WabbajackCDNDownloader.State(url)) {Hash = hash, Size = ffile.Size, Name = ffile.Name}); + } + + return files.ToArray(); } } } diff --git a/Wabbajack.Server/DataLayer/MirroredFiles.cs b/Wabbajack.Server/DataLayer/MirroredFiles.cs index 8d4b7265..9c3b5803 100644 --- a/Wabbajack.Server/DataLayer/MirroredFiles.cs +++ b/Wabbajack.Server/DataLayer/MirroredFiles.cs @@ -49,6 +49,13 @@ namespace Wabbajack.Server.DataLayer await trans.CommitAsync(); } + public async Task DeleteMirroredFile(Hash hash) + { + await using var conn = await Open(); + await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash", + new {Hash = hash}); + } + public async Task InsertAllNexusMirrors() { var permissions = (await GetNexusPermissions()).Where(p => p.Value == HTMLInterface.PermissionValue.Yes); @@ -82,5 +89,46 @@ namespace Wabbajack.Server.DataLayer return await conn.QueryFirstOrDefaultAsync("SELECT Hash FROM dbo.MirroredArchives WHERE Hash = @Hash", new {Hash = hash}) != default; } + + public async Task QueueMirroredFiles() + { + await using var conn = await Open(); + + await conn.ExecuteAsync(@" + + INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) + + SELECT hs.Hash, GETUTCDATE(), 'File has re-upload permissions on the Nexus' FROM + (SELECT DISTINCT ad.Hash FROM dbo.NexusModPermissions p + INNER JOIN GameMetadata md on md.NexusGameId = p.NexusGameID + INNER JOIN dbo.ArchiveDownloads ad on ad.PrimaryKeyString like 'NexusDownloader+State|'+md.WabbajackName+'|'+CAST(p.ModID as nvarchar)+'|%' + WHERE p.Permissions = 1 + AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives) + ) hs + + INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) + SELECT DISTINCT Hash, GETUTCDATE(), 'File is hosted on GitHub' + FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%github.com/%' + AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives) + + + INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) + SELECT DISTINCT Hash, GETUTCDATE(), 'File license allows uploading to any Non-nexus site' + FROM dbo.ArchiveDownloads ad WHERE PrimaryKeyString like '%enbdev.com/%' + AND ad.Hash not in (SELECT Hash from dbo.MirroredArchives) + + INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) + SELECT DISTINCT Hash, GETUTCDATE(), 'DynDOLOD file' /*, Name*/ + from dbo.ModListArchives mla WHERE Name like '%DynDoLOD%standalone%' + and Hash not in (select Hash from dbo.MirroredArchives) + + INSERT INTO dbo.MirroredArchives (Hash, Created, Rationale) + SELECT DISTINCT Hash, GETUTCDATE(), 'Distribution allowed by author' /*, Name*/ + from dbo.ModListArchives mla WHERE Name like '%particle%patch%' + and Hash not in (select Hash from dbo.MirroredArchives) + + + "); + } } } diff --git a/Wabbajack.Server/DataLayer/SqlService.cs b/Wabbajack.Server/DataLayer/SqlService.cs index 8965a682..a118b9a6 100644 --- a/Wabbajack.Server/DataLayer/SqlService.cs +++ b/Wabbajack.Server/DataLayer/SqlService.cs @@ -1,16 +1,19 @@ using System.Data.SqlClient; using System.Threading.Tasks; using Wabbajack.BuildServer; +using Wabbajack.Server.DTOs; namespace Wabbajack.Server.DataLayer { public partial class SqlService { private AppSettings _settings; + private Task _mirrorCreds; public SqlService(AppSettings settings) { _settings = settings; + _mirrorCreds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors); } diff --git a/Wabbajack.Server/Services/MirrorQueueService.cs b/Wabbajack.Server/Services/MirrorQueueService.cs new file mode 100644 index 00000000..a1b9d811 --- /dev/null +++ b/Wabbajack.Server/Services/MirrorQueueService.cs @@ -0,0 +1,28 @@ +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.BuildServer; +using Wabbajack.Server.DataLayer; + +namespace Wabbajack.Server.Services +{ + public class MirrorQueueService : AbstractService + { + private DiscordWebHook _discord; + private SqlService _sql; + + public MirrorQueueService(ILogger logger, AppSettings settings, QuickSync quickSync, + DiscordWebHook discordWebHook, SqlService sqlService) : + base(logger, settings, quickSync, TimeSpan.FromMinutes(5)) + { + _discord = discordWebHook; + _sql = sqlService; + } + + public override async Task Execute() + { + await _sql.QueueMirroredFiles(); + return 1; + } + } +} diff --git a/Wabbajack.Server/Services/MirrorUploader.cs b/Wabbajack.Server/Services/MirrorUploader.cs index d0c5f9b4..8fdf39d5 100644 --- a/Wabbajack.Server/Services/MirrorUploader.cs +++ b/Wabbajack.Server/Services/MirrorUploader.cs @@ -23,11 +23,14 @@ namespace Wabbajack.Server.Services { private SqlService _sql; private ArchiveMaintainer _archives; + private DiscordWebHook _discord; - public MirrorUploader(ILogger logger, AppSettings settings, SqlService sql, QuickSync quickSync, ArchiveMaintainer archives) : base(logger, settings, quickSync, TimeSpan.FromHours(1)) + public MirrorUploader(ILogger logger, AppSettings settings, SqlService sql, QuickSync quickSync, ArchiveMaintainer archives, DiscordWebHook discord) + : base(logger, settings, quickSync, TimeSpan.FromHours(1)) { _sql = sql; _archives = archives; + _discord = discord; } public override async Task Execute() @@ -61,6 +64,12 @@ namespace Wabbajack.Server.Services goto TOP; } + await _discord.Send(Channel.Spam, + new DiscordMessage + { + Content = $"Uploading {toUpload.Hash} - {toUpload.Created} because {toUpload.Rationale}" + }); + var definition = await Client.GenerateFileDefinition(queue, path, (s, percent) => { }); using (var client = await GetClient(creds)) diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 2856fd04..381f9eea 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -70,6 +70,7 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddMvc(); services.AddControllers() @@ -127,6 +128,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); + app.UseService(); app.Use(next => { diff --git a/Wabbajack.Test/EndToEndTests.cs b/Wabbajack.Test/EndToEndTests.cs index 9f702261..34ee1b49 100644 --- a/Wabbajack.Test/EndToEndTests.cs +++ b/Wabbajack.Test/EndToEndTests.cs @@ -63,7 +63,7 @@ namespace Wabbajack.Test DownloadAndInstall(Game.SkyrimSpecialEdition, 4783, "Frost Armor UNP"), DownloadAndInstall(Game.SkyrimSpecialEdition, 32359, "Frost Armor HDT"), DownloadAndInstall("https://github.com/ShikyoKira/Project-New-Reign---Nemesis-Main/releases/download/v0.84-beta/Nemesis.Unlimited.Behavior.Engine.v0.84-beta.rar", "Nemesis.Unlimited.Behavior.Engine.v0.84-beta.rar", "Nemesis"), - DownloadAndInstall(Game.Fallout4, 40136, "RAR test File")); + DownloadAndInstall(Game.Fallout4, 40136, "RAR test File")); // ShouldPullFrom Mirror // We're going to fully patch this mod from another source. await modfiles[3].Download.DeleteAsync(); diff --git a/Wabbajack.Test/TestUtils.cs b/Wabbajack.Test/TestUtils.cs index 7a9f4de6..e121b1d2 100644 --- a/Wabbajack.Test/TestUtils.cs +++ b/Wabbajack.Test/TestUtils.cs @@ -172,13 +172,13 @@ namespace Wabbajack.Test } await DownloadsPath.Combine(name + Consts.MetaFileExtension).WriteAllLinesAsync( - "[General]", - "manualURL=" - ); + "[General]", + "manualURL=" + ); return name; } - + public async Task VerifyInstalledFile(string mod, string file) { var src = SourcePath.Combine((string)Consts.MO2ModFolderName, mod, file); From 0c5913c70a9e5e56963702b1eb9fef59c97b9412 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 1 Nov 2020 21:06:18 -0700 Subject: [PATCH 2/2] Extractor fixes for broken Lucian introspection --- Wabbajack.Common/Util/TempFile.cs | 6 ++++++ .../FileExtractorTests.cs | 8 ++++---- .../VirtualFileSystemTests.cs | 18 ++++++++++++++++++ .../FileExtractor2/FileExtractor.cs | 1 + 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/Wabbajack.Common/Util/TempFile.cs b/Wabbajack.Common/Util/TempFile.cs index 5105dcea..11f74bcb 100644 --- a/Wabbajack.Common/Util/TempFile.cs +++ b/Wabbajack.Common/Util/TempFile.cs @@ -40,6 +40,12 @@ namespace Wabbajack.Common } this.DeleteAfter = deleteAfter; } + + public TempFile(Extension ext) + :this(new FileInfo((string)GetTempFilePath().WithExtension(ext))) + { + } + public async ValueTask DisposeAsync() { if (DeleteAfter && Path.Exists) diff --git a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs index ab0019dd..79283ec1 100644 --- a/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/FileExtractorTests.cs @@ -224,7 +224,7 @@ namespace Wabbajack.VirtualFileSystem.Test } - private static AbsolutePath _stagingFolder = ((RelativePath)"NexusDownloads").RelativeToEntryPoint(); + public static AbsolutePath StagingFolder = ((RelativePath)"NexusDownloads").RelativeToEntryPoint(); private static async Task DownloadMod(Game game, int mod) { @@ -235,9 +235,9 @@ namespace Wabbajack.VirtualFileSystem.Test return await DownloadNexusFile(game, mod, file); } - private static async Task DownloadNexusFile(Game game, int mod, NexusFileInfo file) + public static async Task DownloadNexusFile(Game game, int mod, NexusFileInfo file) { - var src = _stagingFolder.Combine(file.file_name); + var src = StagingFolder.Combine(file.file_name); if (src.Exists) return src; @@ -251,7 +251,7 @@ namespace Wabbajack.VirtualFileSystem.Test return src; } - private async Task DownloadMod(Game game, int mod, int fileId) + public static async Task DownloadMod(Game game, int mod, int fileId) { using var client = await NexusApiClient.Get(); var results = await client.GetModFiles(game, mod); diff --git a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs index 2444ab06..0cf95c29 100644 --- a/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs +++ b/Wabbajack.VirtualFileSystem.Test/VirtualFileSystemTests.cs @@ -5,6 +5,8 @@ using System.Linq; using System.Text; using System.Threading.Tasks; using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; using Xunit; using Xunit.Abstractions; @@ -186,6 +188,22 @@ namespace Wabbajack.VirtualFileSystem.Test } + + [Theory] + [InlineData(Game.SkyrimSpecialEdition, 20035, 130759)] // Lucian + public async Task CanAnalyzeMods(Game game, int modid, int fileId) + { + await using var tmpFolder = await TempFolder.Create(); + + var path = await FileExtractorTests.DownloadMod(game, modid, fileId); + await path.CopyToAsync(path.FileName.RelativeTo(tmpFolder.Dir)); + + var context = new Context(Queue); + await context.AddRoot(tmpFolder.Dir); + + Assert.True(context.Index.ByFullPath.Count >= 3); + } + private static async Task AddFile(AbsolutePath filename, string text) { filename.Parent.CreateDirectory(); diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs index 85eec1f8..1809e4d8 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs @@ -199,6 +199,7 @@ namespace Wabbajack.VirtualFileSystem .WithExtension(source.Extension)); await using var s = await sf.GetStream(); await spoolFile.Path.WriteAllAsync(s); + source = spoolFile.Path; } Utils.Log(new GenericInfo($"Extracting {(string)source.FileName}",