From 12395dbf00d2be49e946c444b6d281947397db67 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 4 Mar 2021 05:50:40 -0700 Subject: [PATCH 1/2] Few server fixes --- Wabbajack.Server/ApiKeyAuthorizationHandler.cs | 1 + Wabbajack.Server/Services/NonNexusDownloadValidator.cs | 2 +- Wabbajack.Server/Startup.cs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Wabbajack.Server/ApiKeyAuthorizationHandler.cs b/Wabbajack.Server/ApiKeyAuthorizationHandler.cs index e66f1a5c..aeb4e7b6 100644 --- a/Wabbajack.Server/ApiKeyAuthorizationHandler.cs +++ b/Wabbajack.Server/ApiKeyAuthorizationHandler.cs @@ -54,6 +54,7 @@ namespace Wabbajack.BuildServer //await LogRequest(metricsKey); if (metricsKey != default) { + await _keyCache.AddKey(metricsKey); if (await _sql.IsTarKey(metricsKey)) { await _sql.IngestMetric(new Metric diff --git a/Wabbajack.Server/Services/NonNexusDownloadValidator.cs b/Wabbajack.Server/Services/NonNexusDownloadValidator.cs index f7c0729c..f41e9767 100644 --- a/Wabbajack.Server/Services/NonNexusDownloadValidator.cs +++ b/Wabbajack.Server/Services/NonNexusDownloadValidator.cs @@ -41,7 +41,7 @@ namespace Wabbajack.Server.Services bool isValid = false; switch (archive.State) { - case WabbajackCDNDownloader.State _: + //case WabbajackCDNDownloader.State _: //case GoogleDriveDownloader.State _: // Let's try validating Google again 2/10/2021 case ManualDownloader.State _: case ModDBDownloader.State _: diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index 57f82378..f92a898e 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -144,7 +144,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); - app.UseService(); + //app.UseService(); app.UseService(); app.Use(next => From 1266ca63be6f83b1dc94334a2516c3fd630f367e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 5 Mar 2021 20:54:04 -0700 Subject: [PATCH 2/2] Fix a ton of server test bugs and rework code to work with the new fileserver --- Wabbajack.Server.Test/MirroredFilesTests.cs | 3 ++- Wabbajack.Server.Test/ModListValidationTests.cs | 2 +- Wabbajack.Server/Controllers/AuthoredFiles.cs | 15 ++++++++++++++- Wabbajack.Server/Services/MirrorUploader.cs | 7 +++++-- Wabbajack.Server/Services/PatchBuilder.cs | 2 +- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Wabbajack.Server.Test/MirroredFilesTests.cs b/Wabbajack.Server.Test/MirroredFilesTests.cs index 2b840c82..e6986776 100644 --- a/Wabbajack.Server.Test/MirroredFilesTests.cs +++ b/Wabbajack.Server.Test/MirroredFilesTests.cs @@ -48,8 +48,9 @@ namespace Wabbajack.Server.Test Size = file.Path.Size }; - var file2 = new TempFile(); + await using var file2 = new TempFile(); await DownloadDispatcher.DownloadWithPossibleUpgrade(archive, file2.Path); + Assert.Equal(dataHash!.Value, await file2.Path.FileHashAsync()); } [Fact] diff --git a/Wabbajack.Server.Test/ModListValidationTests.cs b/Wabbajack.Server.Test/ModListValidationTests.cs index 1234856a..69a28185 100644 --- a/Wabbajack.Server.Test/ModListValidationTests.cs +++ b/Wabbajack.Server.Test/ModListValidationTests.cs @@ -29,7 +29,7 @@ namespace Wabbajack.BuildServer.Test var modlist = await MakeModList("CanLoadMetadataFromTestServer.txt"); Consts.ModlistMetadataURL = modlist.ToString(); var data = await ModlistMetadata.LoadFromGithub(); - Assert.Equal(2, data.Count); + Assert.Equal(3, data.Count); Assert.Equal("test_list", data.First().Links.MachineURL); } diff --git a/Wabbajack.Server/Controllers/AuthoredFiles.cs b/Wabbajack.Server/Controllers/AuthoredFiles.cs index e6783c1e..51e9a742 100644 --- a/Wabbajack.Server/Controllers/AuthoredFiles.cs +++ b/Wabbajack.Server/Controllers/AuthoredFiles.cs @@ -79,6 +79,12 @@ namespace Wabbajack.BuildServer.Controllers _logger.Log(LogLevel.Information, $"Creating File upload {definition.OriginalFileName}"); definition = await _sql.CreateAuthoredFile(definition, user); + + using (var client = await GetBunnyCdnFtpClient()) + { + await client.CreateDirectoryAsync($"{definition.MungedName}"); + await client.CreateDirectoryAsync($"{definition.MungedName}/parts"); + } await _discord.Send(Channel.Ham, new DiscordMessage() {Content = $"{user} has started uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})"}); @@ -123,7 +129,14 @@ namespace Wabbajack.BuildServer.Controllers private async Task UploadAsync(Stream stream, string path) { using var client = await GetBunnyCdnFtpClient(); - await client.UploadAsync(stream, path); + try + { + await client.UploadAsync(stream, path); + } + catch (Exception ex) + { + throw ex; + } } [HttpDelete] diff --git a/Wabbajack.Server/Services/MirrorUploader.cs b/Wabbajack.Server/Services/MirrorUploader.cs index 8fdf39d5..0db04f40 100644 --- a/Wabbajack.Server/Services/MirrorUploader.cs +++ b/Wabbajack.Server/Services/MirrorUploader.cs @@ -86,7 +86,7 @@ namespace Wabbajack.Server.Services await definition.Parts.PMap(queue, async part => { _logger.LogInformation($"Uploading mirror part ({part.Index}/{definition.Parts.Length})"); - var name = MakePath(part.Index); + var buffer = new byte[part.Size]; await using (var fs = await path.OpenShared()) { @@ -95,7 +95,9 @@ namespace Wabbajack.Server.Services } using var client = await GetClient(creds); + var name = MakePath(part.Index); await client.UploadAsync(new MemoryStream(buffer), name); + }); using (var client = await GetClient(creds)) @@ -108,7 +110,8 @@ namespace Wabbajack.Server.Services } ms.Position = 0; - await client.UploadAsync(ms, $"{definition.Hash.ToHex()}/definition.json.gz"); + var remoteName = $"{definition.Hash.ToHex()}/definition.json.gz"; + await client.UploadAsync(ms, remoteName); } await toUpload.Finish(_sql); diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs index 20513046..3a2add4e 100644 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -125,7 +125,7 @@ namespace Wabbajack.Server.Services private static string PatchName(Hash oldHash, Hash newHash) { - return $"\\{oldHash.ToHex()}_{newHash.ToHex()}"; + return $"{oldHash.ToHex()}_{newHash.ToHex()}"; } private async Task CleanupOldPatches()