From c782b63c6cacb4fb853868d309f542a9dfd160b2 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 3 Aug 2020 20:41:29 -0600 Subject: [PATCH 1/6] WIP --- Wabbajack.CLI/Verbs/HashVariants.cs | 24 +++++++++++++++++++ Wabbajack.CLI/Verbs/ParseMeta.cs | 28 ++++++++++++++++++++++ Wabbajack.Server.Test/sql/wabbajack_db.sql | 19 +++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 Wabbajack.CLI/Verbs/HashVariants.cs create mode 100644 Wabbajack.CLI/Verbs/ParseMeta.cs diff --git a/Wabbajack.CLI/Verbs/HashVariants.cs b/Wabbajack.CLI/Verbs/HashVariants.cs new file mode 100644 index 00000000..9eb02d3d --- /dev/null +++ b/Wabbajack.CLI/Verbs/HashVariants.cs @@ -0,0 +1,24 @@ +using System; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("hash-variants", HelpText = "Print all the known variants (formats) of a hash")] + public class HashVariants : AVerb + { + [Option('i', "input", Required = true, HelpText = "Input Hash")] + public string Input { get; set; } = ""; + + protected override async Task Run() + { + var hash = Hash.Interpret(Input); + Console.WriteLine($"Base64: {hash.ToBase64()}"); + Console.WriteLine($"Hex: {hash.ToHex()}"); + Console.WriteLine($"Long: {(long)hash}"); + Console.WriteLine($"ULong (uncommon): {(ulong)hash}"); + return ExitCode.Ok; + } + } +} diff --git a/Wabbajack.CLI/Verbs/ParseMeta.cs b/Wabbajack.CLI/Verbs/ParseMeta.cs new file mode 100644 index 00000000..825e86e1 --- /dev/null +++ b/Wabbajack.CLI/Verbs/ParseMeta.cs @@ -0,0 +1,28 @@ +using System; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Lib.Downloaders; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("parse-meta", HelpText = "Parse a .meta file, figure out the download state and print it")] + public class ParseMeta : AVerb + { + + [Option('i', "input", Required = true, HelpText = "Input meta file to parse")] + public string Input { get; set; } = ""; + protected override async Task Run() + { + var meta = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(((AbsolutePath)Input).LoadIniFile()); + if (meta == null) + { + Console.WriteLine("Cannot resolve meta!"); + return ExitCode.Error; + } + + Console.WriteLine($"PrimaryKeyString : {meta.PrimaryKeyString}"); + return ExitCode.Ok; + } + } +} diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index 1bda2ece..f2add2d0 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -669,6 +669,25 @@ CONSTRAINT [PK_NexusModPermissions] PRIMARY KEY CLUSTERED ) ON [PRIMARY] GO +/****** Object: Table [dbo].[MirroredArchives] Script Date: 8/3/2020 8:39:33 PM ******/ +SET ANSI_NULLS ON +GO + +SET QUOTED_IDENTIFIER ON +GO + +CREATE TABLE [dbo].[MirroredArchives]( + [Hash] [bigint] NOT NULL, + [Created] [datetime] NOT NULL, + [Uploaded] [datetime] NULL, + [Rationale] [nvarchar](max) NOT NULL, + CONSTRAINT [PK_MirroredArchives] PRIMARY KEY CLUSTERED + ( + [Hash] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO + /****** Object: StoredProcedure [dbo].[MergeAllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/ From 81b819009da34dcc68e482b1d6a30bdf3c884143 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 3 Aug 2020 20:42:19 -0600 Subject: [PATCH 2/6] WIP --- Wabbajack.Common/Hash.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Wabbajack.Common/Hash.cs b/Wabbajack.Common/Hash.cs index 943a81d5..73900dc7 100644 --- a/Wabbajack.Common/Hash.cs +++ b/Wabbajack.Common/Hash.cs @@ -103,6 +103,16 @@ namespace Wabbajack.Common { return BitConverter.GetBytes(_code); } + + public static Hash Interpret(string input) + { + return input.Length switch + { + 16 => FromHex(input), + 12 when input.EndsWith('=') => FromBase64(input), + _ => FromLong(long.Parse(input)) + }; + } } public static partial class Utils From abe30fe8a37bb56febb08ee949f434d5069db546 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 3 Aug 2020 21:05:19 -0600 Subject: [PATCH 3/6] DTOs --- Wabbajack.Server/DTOs/MirroredFile.cs | 13 ++++ .../DataLayer/ArchiveDownloads.cs | 6 ++ Wabbajack.Server/DataLayer/MirroredFiles.cs | 71 +++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 Wabbajack.Server/DTOs/MirroredFile.cs create mode 100644 Wabbajack.Server/DataLayer/MirroredFiles.cs diff --git a/Wabbajack.Server/DTOs/MirroredFile.cs b/Wabbajack.Server/DTOs/MirroredFile.cs new file mode 100644 index 00000000..a6fafb24 --- /dev/null +++ b/Wabbajack.Server/DTOs/MirroredFile.cs @@ -0,0 +1,13 @@ +using System; +using Wabbajack.Common; + +namespace Wabbajack.Server.DTOs +{ + public class MirroredFile + { + public Hash Hash { get; set; } + public DateTime Created { get; set; } + public DateTime? Uploaded { get; set; } + public string Rationale { get; set; } + } +} diff --git a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs index 1f7c77dc..508a508f 100644 --- a/Wabbajack.Server/DataLayer/ArchiveDownloads.cs +++ b/Wabbajack.Server/DataLayer/ArchiveDownloads.cs @@ -56,6 +56,12 @@ namespace Wabbajack.Server.DataLayer await using var conn = await Open(); return (await conn.QueryAsync<(Hash, string)>("SELECT Hash, PrimaryKeyString FROM ArchiveDownloads")).ToHashSet(); } + + public async Task> GetAllArchiveDownloadStates() + { + await using var conn = await Open(); + return (await conn.QueryAsync<(Hash, AbstractDownloadState)>("SELECT Hash, DownloadState FROM ArchiveDownloads")).ToHashSet(); + } public async Task GetArchiveDownload(Guid id) diff --git a/Wabbajack.Server/DataLayer/MirroredFiles.cs b/Wabbajack.Server/DataLayer/MirroredFiles.cs new file mode 100644 index 00000000..72f47944 --- /dev/null +++ b/Wabbajack.Server/DataLayer/MirroredFiles.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AngleSharp.Common; +using Dapper; +using Wabbajack.Common; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.NexusApi; +using Wabbajack.Server.DTOs; + +namespace Wabbajack.Server.DataLayer +{ + public partial class SqlService + { + public async Task GetNextMirroredFile() + { + await using var conn = await Open(); + var results = await conn.QueryFirstOrDefaultAsync<(Hash, DateTime, DateTime, string)>( + "SELECT Hash, Created, Uploaded, Rationale from dbo.MirroredArchives WHERE Uploaded IS NULL"); + return new MirroredFile + { + Hash = results.Item1, Created = results.Item2, Uploaded = results.Item3, Rationale = results.Item4 + }; + } + + public async Task> GetAllMirroredHashes() + { + await using var conn = await Open(); + return (await conn.QueryAsync("SELECT Hash FROM dbo.MirroredArchives")).ToHashSet(); + } + + public async Task UpsertMirroredFile(MirroredFile file) + { + await using var conn = await Open(); + await using var trans = await conn.BeginTransactionAsync(); + + await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash", new {file.Hash}, trans); + await conn.ExecuteAsync( + "INSERT INTO dbo.MirroredArchives (Hash, Created, Updated, Rationale) VALUES (@Hash, @Created, @Updated, @Rationale)", + file, trans); + await trans.CommitAsync(); + } + + public async Task InsertAllNexusMirrors() + { + var permissions = (await GetNexusPermissions()).Where(p => p.Value == HTMLInterface.PermissionValue.Yes); + var downloads = (await GetAllArchiveDownloadStates()).Where(a => a.State is NexusDownloader.State).ToDictionary(a => + { + var nd = (NexusDownloader.State)a.State; + return (nd.Game, nd.ModID); + }, a => a.Hash); + + var existing = await GetAllMirroredHashes(); + + foreach (var (key, _) in permissions) + { + if (!downloads.TryGetValue(key, out var hash)) continue; + if (existing.Contains(hash)) continue; + + await UpsertMirroredFile(new MirroredFile + { + Hash = hash, + Created = DateTime.UtcNow, + Rationale = + $"Mod ({key.Item1} {key.Item2}) has allowed re-upload permissions on the Nexus" + }); + } + } + } +} From 7f892d26d7da6d18e7bf7bfe80f606c04f4fb1c5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 4 Aug 2020 18:34:09 -0600 Subject: [PATCH 4/6] Mirrored files, permissions, and download support --- Wabbajack.CLI/OptionsDefinition.cs | 4 +- Wabbajack.Common/Consts.cs | 1 + Wabbajack.Lib/ACompiler.cs | 4 + Wabbajack.Lib/AuthorApi/Client.cs | 2 +- .../Downloaders/DownloadDispatcher.cs | 28 ++++- .../ABuildServerSystemTest.cs | 1 + Wabbajack.Server.Test/MirroredFilesTests.cs | 55 +++++++++ .../Wabbajack.Server.Test.csproj | 2 +- Wabbajack.Server.Test/sql/wabbajack_db.sql | 41 +++++-- Wabbajack.Server/Controllers/AuthoredFiles.cs | 2 +- Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs | 18 ++- Wabbajack.Server/DTOs/MirroredFile.cs | 17 +++ Wabbajack.Server/DataLayer/MirroredFiles.cs | 18 ++- Wabbajack.Server/DataLayer/Nexus.cs | 33 ++++++ Wabbajack.Server/Services/AbstractService.cs | 13 ++ Wabbajack.Server/Services/MirrorUploader.cs | 111 ++++++++++++++++++ .../Services/NexusPermissionsUpdater.cs | 37 +++--- Wabbajack.Server/Services/PatchBuilder.cs | 2 +- Wabbajack.Server/Startup.cs | 2 + 19 files changed, 349 insertions(+), 42 deletions(-) create mode 100644 Wabbajack.Server.Test/MirroredFilesTests.cs create mode 100644 Wabbajack.Server/Services/MirrorUploader.cs diff --git a/Wabbajack.CLI/OptionsDefinition.cs b/Wabbajack.CLI/OptionsDefinition.cs index 9e98749d..204d6b9e 100644 --- a/Wabbajack.CLI/OptionsDefinition.cs +++ b/Wabbajack.CLI/OptionsDefinition.cs @@ -26,7 +26,9 @@ namespace Wabbajack.CLI typeof(InlinedFileReport), typeof(ExtractBSA), typeof(PurgeNexusCache), - typeof(ForceHealing) + typeof(ForceHealing), + typeof(HashVariants), + typeof(ParseMeta) }; } } diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 497f1123..8d9b681a 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -149,6 +149,7 @@ namespace Wabbajack.Common public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2; public static string ArchiveUpdatesCDNFolder = "archive_updates"; + public static Uri WabbajackMirror = new Uri("https://wabbajack-mirror.b-cdn.net"); } } diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index 46885cc8..705dabaf 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -150,6 +150,9 @@ namespace Wabbajack.Lib await using (var of = await ModListOutputFolder.Combine("modlist").Create()) ModList.ToJson(of); + await ModListOutputFolder.Combine("sig") + .WriteAllBytesAsync((await ModListOutputFolder.Combine("modlist").FileHashAsync()).ToArray()); + await ClientAPI.SendModListDefinition(ModList); await ModListOutputFile.DeleteAsync(); @@ -157,6 +160,7 @@ namespace Wabbajack.Lib await using (var fs = await ModListOutputFile.Create()) { using var za = new ZipArchive(fs, ZipArchiveMode.Create); + await ModListOutputFolder.EnumerateFiles() .DoProgress("Compressing ModList", async f => diff --git a/Wabbajack.Lib/AuthorApi/Client.cs b/Wabbajack.Lib/AuthorApi/Client.cs index a129ea89..bf2da6a0 100644 --- a/Wabbajack.Lib/AuthorApi/Client.cs +++ b/Wabbajack.Lib/AuthorApi/Client.cs @@ -39,7 +39,7 @@ namespace Wabbajack.Lib.AuthorApi } - public async Task GenerateFileDefinition(WorkQueue queue, AbsolutePath path, Action progressFn) + public static async Task GenerateFileDefinition(WorkQueue queue, AbsolutePath path, Action progressFn) { IEnumerable Blocks(AbsolutePath path) { diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index 6a0dacec..019721a2 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -93,8 +93,14 @@ namespace Wabbajack.Lib.Downloaders public static async Task DownloadWithPossibleUpgrade(Archive archive, AbsolutePath destination) { - var success = await Download(archive, destination); - if (success) + if (await Download(archive, destination)) + { + await destination.FileHashCachedAsync(); + return true; + } + + + if (await DownloadFromMirror(archive, destination)) { await destination.FileHashCachedAsync(); return true; @@ -147,6 +153,24 @@ namespace Wabbajack.Lib.Downloaders return true; } + private static async Task DownloadFromMirror(Archive archive, AbsolutePath destination) + { + try + { + var newArchive = + new Archive( + new WabbajackCDNDownloader.State(new Uri($"{Consts.WabbajackMirror}{archive.Hash.ToHex()}"))) + { + Hash = archive.Hash, Size = archive.Size, Name = archive.Name + }; + return await Download(newArchive, destination); + } + catch (Exception ex) + { + return false; + } + } + private static async Task Download(Archive archive, AbsolutePath destination) { try diff --git a/Wabbajack.Server.Test/ABuildServerSystemTest.cs b/Wabbajack.Server.Test/ABuildServerSystemTest.cs index 052c9e2d..7eadaa50 100644 --- a/Wabbajack.Server.Test/ABuildServerSystemTest.cs +++ b/Wabbajack.Server.Test/ABuildServerSystemTest.cs @@ -61,6 +61,7 @@ namespace Wabbajack.BuildServer.Test _token = new CancellationTokenSource(); _task = _host.RunAsync(_token.Token); Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080"); + Consts.WabbajackMirror = new Uri("https://wabbajack-test.b-cdn.net"); await "ServerWhitelist.yaml".RelativeTo(ServerPublicFolder).WriteAllTextAsync( "GoogleIDs:\nAllowedPrefixes:\n - http://localhost"); diff --git a/Wabbajack.Server.Test/MirroredFilesTests.cs b/Wabbajack.Server.Test/MirroredFilesTests.cs new file mode 100644 index 00000000..227397d0 --- /dev/null +++ b/Wabbajack.Server.Test/MirroredFilesTests.cs @@ -0,0 +1,55 @@ +using System; +using System.Threading.Tasks; +using Wabbajack.BuildServer.Test; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.Downloaders; +using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DTOs; +using Wabbajack.Server.Services; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.Server.Test +{ + public class MirroredFilesTests : ABuildServerSystemTest + { + public MirroredFilesTests(ITestOutputHelper output, SingletonAdaptor fixture) : base(output, fixture) + { + } + + [Fact] + public async Task CanUploadAndDownloadMirroredFiles() + { + var file = new TempFile(); + await file.Path.WriteAllBytesAsync(RandomData(1024 * 1024 * 6)); + var dataHash = await file.Path.FileHashAsync(); + + await Fixture.GetService().Ingest(file.Path); + Assert.True(Fixture.GetService().HaveArchive(dataHash)); + + var sql = Fixture.GetService(); + + await sql.UpsertMirroredFile(new MirroredFile + { + Created = DateTime.UtcNow, + Rationale = "Test File", + Hash = dataHash + }); + + var uploader = Fixture.GetService(); + Assert.Equal(1, await uploader.Execute()); + + + var archive = new Archive(new HTTPDownloader.State(MakeURL(dataHash.ToString()))) + { + Hash = dataHash, + Size = file.Path.Size + }; + + var file2 = new TempFile(); + await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, file2.Path); + } + + } +} diff --git a/Wabbajack.Server.Test/Wabbajack.Server.Test.csproj b/Wabbajack.Server.Test/Wabbajack.Server.Test.csproj index e3ec849a..ad63aadd 100644 --- a/Wabbajack.Server.Test/Wabbajack.Server.Test.csproj +++ b/Wabbajack.Server.Test/Wabbajack.Server.Test.csproj @@ -2,7 +2,7 @@ netcoreapp3.1 - + win10-x64 false diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index f2add2d0..5833e1fb 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -677,17 +677,42 @@ SET QUOTED_IDENTIFIER ON GO CREATE TABLE [dbo].[MirroredArchives]( - [Hash] [bigint] NOT NULL, - [Created] [datetime] NOT NULL, - [Uploaded] [datetime] NULL, - [Rationale] [nvarchar](max) NOT NULL, - CONSTRAINT [PK_MirroredArchives] PRIMARY KEY CLUSTERED - ( - [Hash] ASC - )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] + [Hash] [bigint] NOT NULL, + [Created] [datetime] NOT NULL, + [Uploaded] [datetime] NULL, + [Rationale] [nvarchar](max) NOT NULL, + [FailMessage] [nvarchar](max) NULL, + CONSTRAINT [PK_MirroredArchives] PRIMARY KEY CLUSTERED + ( + [Hash] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] GO +/****** Object: Table [dbo].[GameMetadata] Script Date: 8/3/2020 8:39:33 PM ******/ +SET ANSI_NULLS ON +GO + +SET QUOTED_IDENTIFIER ON +GO + +CREATE TABLE [dbo].[GameMetadata]( + [NexusGameId] [bigint] NULL, + [WabbajackName] [nvarchar](50) NOT NULL, + CONSTRAINT [PK_GameMetadata] PRIMARY KEY CLUSTERED + ( + [WabbajackName] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) +GO + +CREATE NONCLUSTERED INDEX [IDX_GameAndName-20200804-164236] ON [dbo].[GameMetadata] + ( + [NexusGameId] ASC, + [WabbajackName] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +GO + /****** Object: StoredProcedure [dbo].[MergeAllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/ diff --git a/Wabbajack.Server/Controllers/AuthoredFiles.cs b/Wabbajack.Server/Controllers/AuthoredFiles.cs index 7c51107c..0f99e791 100644 --- a/Wabbajack.Server/Controllers/AuthoredFiles.cs +++ b/Wabbajack.Server/Controllers/AuthoredFiles.cs @@ -106,7 +106,7 @@ namespace Wabbajack.BuildServer.Controllers private async Task GetBunnyCdnFtpClient() { - var info = await Utils.FromEncryptedJson("bunny-cdn-ftp-info"); + var info = await BunnyCdnFtpInfo.GetCreds(StorageSpace.AuthoredFiles); var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)}; await client.ConnectAsync(); return client; diff --git a/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs b/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs index 132c62b8..652e9612 100644 --- a/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs +++ b/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs @@ -1,9 +1,25 @@ -namespace Wabbajack.Server.DTOs +using System.Collections.Generic; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.Server.DTOs { + public enum StorageSpace + { + AuthoredFiles, + Patches, + Mirrors + } + public class BunnyCdnFtpInfo { public string Username { get; set; } public string Password { get; set; } public string Hostname { get; set; } + + public static async Task GetCreds(StorageSpace space) + { + return (await Utils.FromEncryptedJson>("bunnycdn"))[space.ToString()]; + } } } diff --git a/Wabbajack.Server/DTOs/MirroredFile.cs b/Wabbajack.Server/DTOs/MirroredFile.cs index a6fafb24..a34c451d 100644 --- a/Wabbajack.Server/DTOs/MirroredFile.cs +++ b/Wabbajack.Server/DTOs/MirroredFile.cs @@ -1,5 +1,7 @@ using System; +using System.Threading.Tasks; using Wabbajack.Common; +using Wabbajack.Server.DataLayer; namespace Wabbajack.Server.DTOs { @@ -9,5 +11,20 @@ namespace Wabbajack.Server.DTOs public DateTime Created { get; set; } public DateTime? Uploaded { get; set; } public string Rationale { get; set; } + + public string FailMessage { get; set; } + + public async Task Finish(SqlService sql) + { + Uploaded = DateTime.UtcNow; + await sql.UpsertMirroredFile(this); + } + + public async Task Fail(SqlService sql, string message) + { + Uploaded = DateTime.UtcNow; + FailMessage = message; + await sql.UpsertMirroredFile(this); + } } } diff --git a/Wabbajack.Server/DataLayer/MirroredFiles.cs b/Wabbajack.Server/DataLayer/MirroredFiles.cs index 72f47944..c13e2d10 100644 --- a/Wabbajack.Server/DataLayer/MirroredFiles.cs +++ b/Wabbajack.Server/DataLayer/MirroredFiles.cs @@ -16,11 +16,12 @@ namespace Wabbajack.Server.DataLayer public async Task GetNextMirroredFile() { await using var conn = await Open(); - var results = await conn.QueryFirstOrDefaultAsync<(Hash, DateTime, DateTime, string)>( - "SELECT Hash, Created, Uploaded, Rationale from dbo.MirroredArchives WHERE Uploaded IS NULL"); + var result = await conn.QueryFirstOrDefaultAsync<(Hash, DateTime, DateTime, string, string)>( + "SELECT Hash, Created, Uploaded, Rationale, FailMessage from dbo.MirroredArchives WHERE Uploaded IS NULL"); + if (result == default) return null; return new MirroredFile { - Hash = results.Item1, Created = results.Item2, Uploaded = results.Item3, Rationale = results.Item4 + Hash = result.Item1, Created = result.Item2, Uploaded = result.Item3, Rationale = result.Item4, FailMessage = result.Item5 }; } @@ -37,8 +38,15 @@ namespace Wabbajack.Server.DataLayer await conn.ExecuteAsync("DELETE FROM dbo.MirroredArchives WHERE Hash = @Hash", new {file.Hash}, trans); await conn.ExecuteAsync( - "INSERT INTO dbo.MirroredArchives (Hash, Created, Updated, Rationale) VALUES (@Hash, @Created, @Updated, @Rationale)", - file, trans); + "INSERT INTO dbo.MirroredArchives (Hash, Created, Uploaded, Rationale, FailMessage) VALUES (@Hash, @Created, @Uploaded, @Rationale, @FailMessage)", + new + { + Hash = file.Hash, + Created = file.Created, + Uploaded = file.Uploaded, + Rationale = file.Rationale, + FailMessage = file.FailMessage + }, trans); await trans.CommitAsync(); } diff --git a/Wabbajack.Server/DataLayer/Nexus.cs b/Wabbajack.Server/DataLayer/Nexus.cs index 55f1d11d..20e26645 100644 --- a/Wabbajack.Server/DataLayer/Nexus.cs +++ b/Wabbajack.Server/DataLayer/Nexus.cs @@ -146,5 +146,38 @@ namespace Wabbajack.Server.DataLayer await tx.CommitAsync(); } + + public async Task UpdateGameMetadata() + { + await using var conn = await Open(); + var existing = (await conn.QueryAsync("SELECT WabbajackName FROM dbo.GameMetadata")).ToHashSet(); + + var missing = GameRegistry.Games.Values.Where(g => !existing.Contains(g.Game.ToString())).ToList(); + foreach (var add in missing.Where(g => g.NexusGameId != 0)) + { + await conn.ExecuteAsync( + "INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, WabbajackName)", + new {NexusGameId = add.NexusGameId, WabbajackName = add.ToString()}); + } + } + + public async Task SetNexusPermission(Game game, long modId, HTMLInterface.PermissionValue perm) + { + await using var conn = await Open(); + var tx = await conn.BeginTransactionAsync(); + + await conn.ExecuteAsync("DELETE FROM NexusModPermissions WHERE GameID = @GameID AND ModID = @ModID", new + { + GameID = game.MetaData().NexusGameId, + ModID = modId + }, + transaction:tx); + + await conn.ExecuteAsync( + "INSERT INTO NexusModPermissions (NexusGameID, ModID, Permissions) VALUES (@NexusGameID, @ModID, @Permissions)", + new {NexusGameID = game.MetaData().NexusGameId, ModID = modId, Permissions = (int)perm}, tx); + + await tx.CommitAsync(); + } } } diff --git a/Wabbajack.Server/Services/AbstractService.cs b/Wabbajack.Server/Services/AbstractService.cs index 5cbe06f4..1998c266 100644 --- a/Wabbajack.Server/Services/AbstractService.cs +++ b/Wabbajack.Server/Services/AbstractService.cs @@ -17,6 +17,7 @@ namespace Wabbajack.Server.Services private TimeSpan _delay; protected ILogger _logger; protected QuickSync _quickSync; + private bool _isSetup; public AbstractService(ILogger logger, AppSettings settings, QuickSync quickSync, TimeSpan delay) { @@ -24,14 +25,26 @@ namespace Wabbajack.Server.Services _delay = delay; _logger = logger; _quickSync = quickSync; + + _isSetup = false; + } + + public virtual async Task Setup() + { + } public void Start() { + if (_settings.RunBackEndJobs) { Task.Run(async () => { + await Setup(); + _isSetup = true; + + while (true) { await _quickSync.ResetToken(); diff --git a/Wabbajack.Server/Services/MirrorUploader.cs b/Wabbajack.Server/Services/MirrorUploader.cs new file mode 100644 index 00000000..12e94aec --- /dev/null +++ b/Wabbajack.Server/Services/MirrorUploader.cs @@ -0,0 +1,111 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Net; +using System.Threading.Tasks; +using FluentFTP; +using Microsoft.Extensions.Logging; +using Wabbajack.BuildServer; +using Wabbajack.BuildServer.Controllers; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.AuthorApi; +using Wabbajack.Lib.FileUploader; +using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DTOs; + +namespace Wabbajack.Server.Services +{ + public class MirrorUploader : AbstractService + { + private SqlService _sql; + private ArchiveMaintainer _archives; + + public MirrorUploader(ILogger logger, AppSettings settings, SqlService sql, QuickSync quickSync, ArchiveMaintainer archives) : base(logger, settings, quickSync, TimeSpan.FromHours(1)) + { + _sql = sql; + _archives = archives; + } + + public override async Task Execute() + { + + int uploaded = 0; + TOP: + var toUpload = await _sql.GetNextMirroredFile(); + if (toUpload == default) return uploaded; + uploaded += 1; + + try + { + using var queue = new WorkQueue(); + if (_archives.TryGetPath(toUpload.Hash, out var path)) + { + _logger.LogInformation($"Uploading mirror file {toUpload.Hash} {path.Size.FileSizeToString()}"); + + var definition = await Client.GenerateFileDefinition(queue, path, (s, percent) => { }); + + var creds = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Mirrors); + using (var client = await GetClient(creds)) + { + await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}"); + await client.CreateDirectoryAsync($"{definition.Hash.ToHex()}/parts"); + } + + string MakePath(long idx) + { + return $"{definition.Hash.ToHex()}/parts/{idx}"; + } + + await definition.Parts.PMap(queue, async part => + { + _logger.LogInformation($"Uploading mirror part ({part.Index}/{definition.Parts.Length})"); + var name = MakePath(part.Index); + var buffer = new byte[part.Size]; + await using (var fs = await path.OpenShared()) + { + fs.Position = part.Offset; + await fs.ReadAsync(buffer); + } + + using var client = await GetClient(creds); + await client.UploadAsync(new MemoryStream(buffer), name); + }); + + using (var client = await GetClient(creds)) + { + _logger.LogInformation($"Finishing mirror upload"); + await using var ms = new MemoryStream(); + await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true)) + { + definition.ToJson(gz); + } + + ms.Position = 0; + await client.UploadAsync(ms, $"{definition.Hash.ToHex()}/definition.json.gz"); + } + + await toUpload.Finish(_sql); + } + else + { + await toUpload.Fail(_sql, "Archive not found"); + } + } + catch (Exception ex) + { + _logger.LogInformation($"{toUpload.Created} {toUpload.Uploaded}"); + _logger.LogError(ex, "Error uploading"); + await toUpload.Fail(_sql, ex.ToString()); + } + goto TOP; + } + + private static async Task GetClient(BunnyCdnFtpInfo creds) + { + var ftpClient = new FtpClient(creds.Hostname, new NetworkCredential(creds.Username, creds.Password)); + await ftpClient.ConnectAsync(); + return ftpClient; + } + } +} diff --git a/Wabbajack.Server/Services/NexusPermissionsUpdater.cs b/Wabbajack.Server/Services/NexusPermissionsUpdater.cs index 59268655..b85c28bb 100644 --- a/Wabbajack.Server/Services/NexusPermissionsUpdater.cs +++ b/Wabbajack.Server/Services/NexusPermissionsUpdater.cs @@ -24,6 +24,8 @@ namespace Wabbajack.Server.Services public override async Task Execute() { + await _sql.UpdateGameMetadata(); + var permissions = await _sql.GetNexusPermissions(); var data = await _sql.ModListArchives(); @@ -33,38 +35,31 @@ namespace Wabbajack.Server.Services _logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods"); - using var queue = new WorkQueue(); + using var queue = new WorkQueue(2); - var results = await nexusArchives.PMap(queue, async archive => - { - var permissions = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID); - return (archive.Game, archive.ModID, permissions); - }); + var prev = await _sql.GetNexusPermissions(); - var updated = 0; - foreach (var result in results) + await nexusArchives.PMap(queue, async archive => { - if (permissions.TryGetValue((result.Game, result.ModID), out var oldPermission)) + var result = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID); + await _sql.SetNexusPermission(archive.Game, archive.ModID, result); + + if (prev.TryGetValue((archive.Game, archive.ModID), out var oldPermission)) { - if (oldPermission != result.permissions) + if (oldPermission != result) { await _discord.Send(Channel.Spam, new DiscordMessage { - Content = $"Permissions status of {result.Game} {result.ModID} was {oldPermission} is now {result.permissions} " + Content = $"Permissions status of {archive.Game} {archive.ModID} was {oldPermission} is now {result}" }); - await _sql.PurgeNexusCache(result.ModID); - updated += 1; + await _sql.PurgeNexusCache(archive.ModID); + await _quickSync.Notify(); } } - } + }); - await _sql.SetNexusPermissions(results); - - if (updated > 0) - await _quickSync.Notify(); - - - return updated; + return 1; } + } } diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs index af2d4cd7..0d9f29e6 100644 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -230,7 +230,7 @@ namespace Wabbajack.Server.Services private async Task GetBunnyCdnFtpClient() { - var info = await Utils.FromEncryptedJson("bunny-cdn-ftp-info"); + var info = await BunnyCdnFtpInfo.GetCreds(StorageSpace.Patches); var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)}; await client.ConnectAsync(); return client; diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 27b74615..d7354a6e 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -69,6 +69,7 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddMvc(); services.AddControllers() @@ -125,6 +126,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); + app.UseService(); app.Use(next => { From b9a3f9a3c4b91bd0129a38fef4d3920fc7ed830c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 5 Aug 2020 14:53:19 -0600 Subject: [PATCH 5/6] Fixes and logging for mirrored files and associated server changes --- Wabbajack.Common/Consts.cs | 1 - Wabbajack.Server/Controllers/ModUpgrade.cs | 5 ++++- Wabbajack.Server/DataLayer/Nexus.cs | 4 ++-- Wabbajack.Server/Services/AbstractService.cs | 1 + .../Services/NexusPermissionsUpdater.cs | 16 ++++++++++++---- Wabbajack.Server/Services/PatchBuilder.cs | 7 ++----- 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 8d9b681a..3150c51b 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -148,7 +148,6 @@ namespace Wabbajack.Common public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2; - public static string ArchiveUpdatesCDNFolder = "archive_updates"; public static Uri WabbajackMirror = new Uri("https://wabbajack-mirror.b-cdn.net"); } diff --git a/Wabbajack.Server/Controllers/ModUpgrade.cs b/Wabbajack.Server/Controllers/ModUpgrade.cs index 835776a8..f26c24c2 100644 --- a/Wabbajack.Server/Controllers/ModUpgrade.cs +++ b/Wabbajack.Server/Controllers/ModUpgrade.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DTOs; using Wabbajack.Server.Services; namespace Wabbajack.BuildServer.Controllers @@ -20,6 +21,7 @@ namespace Wabbajack.BuildServer.Controllers private DiscordWebHook _discord; private AppSettings _settings; private QuickSync _quickSync; + private Task _creds; public ModUpgrade(ILogger logger, SqlService sql, DiscordWebHook discord, QuickSync quickSync, AppSettings settings) { @@ -28,6 +30,7 @@ namespace Wabbajack.BuildServer.Controllers _discord = discord; _settings = settings; _quickSync = quickSync; + _creds = BunnyCdnFtpInfo.GetCreds(StorageSpace.Patches); } [HttpPost] @@ -93,7 +96,7 @@ namespace Wabbajack.BuildServer.Controllers await _sql.MarkPatchUsage(oldDownload.Id, newDownload.Id); return Ok( - $"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{Consts.ArchiveUpdatesCDNFolder}/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); + $"https://{(await _creds).Username}.b-cdn.net/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); } _logger.Log(LogLevel.Information, $"Upgrade requested from {oldDownload.Archive.Hash} to {newDownload.Archive.Hash} patch found but was failed"); diff --git a/Wabbajack.Server/DataLayer/Nexus.cs b/Wabbajack.Server/DataLayer/Nexus.cs index 20e26645..487ca3a5 100644 --- a/Wabbajack.Server/DataLayer/Nexus.cs +++ b/Wabbajack.Server/DataLayer/Nexus.cs @@ -156,8 +156,8 @@ namespace Wabbajack.Server.DataLayer foreach (var add in missing.Where(g => g.NexusGameId != 0)) { await conn.ExecuteAsync( - "INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, WabbajackName)", - new {NexusGameId = add.NexusGameId, WabbajackName = add.ToString()}); + "INSERT INTO dbo.GameMetaData (NexusGameID, WabbajackName) VALUES (@NexusGameId, @WabbajackName)", + new {NexusGameId = add.NexusGameId, WabbajackName = add.Game.ToString()}); } } diff --git a/Wabbajack.Server/Services/AbstractService.cs b/Wabbajack.Server/Services/AbstractService.cs index 1998c266..f54b9d6a 100644 --- a/Wabbajack.Server/Services/AbstractService.cs +++ b/Wabbajack.Server/Services/AbstractService.cs @@ -50,6 +50,7 @@ namespace Wabbajack.Server.Services await _quickSync.ResetToken(); try { + _logger.LogInformation($"Running: {GetType().Name}"); await Execute(); } catch (Exception ex) diff --git a/Wabbajack.Server/Services/NexusPermissionsUpdater.cs b/Wabbajack.Server/Services/NexusPermissionsUpdater.cs index b85c28bb..6190c1ae 100644 --- a/Wabbajack.Server/Services/NexusPermissionsUpdater.cs +++ b/Wabbajack.Server/Services/NexusPermissionsUpdater.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Wabbajack.BuildServer; @@ -15,8 +16,10 @@ namespace Wabbajack.Server.Services { private DiscordWebHook _discord; private SqlService _sql; + + public static TimeSpan MaxSync = TimeSpan.FromHours(4); - public NexusPermissionsUpdater(ILogger logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromHours(4)) + public NexusPermissionsUpdater(ILogger logger, AppSettings settings, QuickSync quickSync, DiscordWebHook discord, SqlService sql) : base(logger, settings, quickSync, TimeSpan.FromSeconds(1)) { _discord = discord; _sql = sql; @@ -26,8 +29,7 @@ namespace Wabbajack.Server.Services { await _sql.UpdateGameMetadata(); - var permissions = await _sql.GetNexusPermissions(); - + var data = await _sql.ModListArchives(); var nexusArchives = data.Select(a => a.State).OfType().Select(d => (d.Game, d.ModID)) .Distinct() @@ -35,12 +37,16 @@ namespace Wabbajack.Server.Services _logger.LogInformation($"Starting nexus permissions updates for {nexusArchives.Count} mods"); - using var queue = new WorkQueue(2); + using var queue = new WorkQueue(1); var prev = await _sql.GetNexusPermissions(); + var lag = MaxSync / nexusArchives.Count * 2; + + await nexusArchives.PMap(queue, async archive => { + _logger.LogInformation($"Checking permissions for {archive.Game} {archive.ModID}"); var result = await HTMLInterface.GetUploadPermissions(archive.Game, archive.ModID); await _sql.SetNexusPermission(archive.Game, archive.ModID, result); @@ -56,6 +62,8 @@ namespace Wabbajack.Server.Services await _quickSync.Notify(); } } + + await Task.Delay(lag); }); return 1; diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs index 0d9f29e6..b73fe9f4 100644 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -125,7 +125,7 @@ namespace Wabbajack.Server.Services private static string PatchName(Hash oldHash, Hash newHash) { - return $"{Consts.ArchiveUpdatesCDNFolder}\\{oldHash.ToHex()}_{newHash.ToHex()}"; + return $"\\{oldHash.ToHex()}_{newHash.ToHex()}"; } private async Task CleanupOldPatches() @@ -155,7 +155,7 @@ namespace Wabbajack.Server.Services if (pendingPatch != default) break; } - var files = await client.GetListingAsync($"{Consts.ArchiveUpdatesCDNFolder}\\"); + var files = await client.GetListingAsync($"\\"); _logger.LogInformation($"Found {files.Length} on the CDN"); var sqlFiles = await _sql.AllPatchHashes(); @@ -206,9 +206,6 @@ namespace Wabbajack.Server.Services $"Uploading {patchFile.Size.ToFileSizeString()} patch file to CDN"); using var client = await GetBunnyCdnFtpClient(); - if (!await client.DirectoryExistsAsync(Consts.ArchiveUpdatesCDNFolder)) - await client.CreateDirectoryAsync(Consts.ArchiveUpdatesCDNFolder); - await client.UploadFileAsync((string)patchFile, patchName, FtpRemoteExists.Overwrite); return; } From fd820f03fbb91b076adb2f078715aa0b8948703e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 5 Aug 2020 17:04:54 -0600 Subject: [PATCH 6/6] Fix broken patch test --- Wabbajack.Server.Test/ModlistUpdater.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Wabbajack.Server.Test/ModlistUpdater.cs b/Wabbajack.Server.Test/ModlistUpdater.cs index 58eb2193..fbcd9810 100644 --- a/Wabbajack.Server.Test/ModlistUpdater.cs +++ b/Wabbajack.Server.Test/ModlistUpdater.cs @@ -70,7 +70,7 @@ namespace Wabbajack.Server.Test await Assert.ThrowsAsync(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); Assert.True(await patcher.Execute() > 1); - Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_updates/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); + Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); } [Fact]