From e053136e25932ae12928e5f6786a117001187c4f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 9 May 2020 16:16:16 -0600 Subject: [PATCH] Rewrite the Authored file routines. --- Wabbajack.Common/Consts.cs | 3 + Wabbajack.Common/Http/Client.cs | 4 +- Wabbajack.Lib/AuthorApi/CDNFileDefinition.cs | 26 ++++ Wabbajack.Lib/AuthorApi/Client.cs | 136 +++++++++++++++++ .../Downloaders/DownloadDispatcher.cs | 4 +- .../UrlDownloaders/WabbajackCDNInfluencer.cs | 13 ++ .../Downloaders/WabbajackCDNDownloader.cs | 100 ++++++++++++ Wabbajack.Lib/FileUploader/AuthorAPI.cs | 96 +----------- .../ABuildServerSystemTest.cs | 4 +- Wabbajack.Server.Test/ADBTest.cs | 2 +- Wabbajack.Server.Test/AuthoredFilesTests.cs | 38 +++++ Wabbajack.Server.Test/LoginTests.cs | 39 +++++ Wabbajack.Server.Test/sql/wabbajack_db.sql | 14 ++ .../ApiKeyAuthorizationHandler.cs | 99 ++++++++++++ Wabbajack.Server/AppSettings.cs | 3 +- Wabbajack.Server/Controllers/AuthoredFiles.cs | 143 ++++++++++++++++++ Wabbajack.Server/Controllers/UploadedFiles.cs | 11 ++ Wabbajack.Server/Controllers/Users.cs | 51 +++++++ Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs | 9 ++ Wabbajack.Server/DataLayer/ApiKeys.cs | 48 ++++++ Wabbajack.Server/DataLayer/AuthoredFiles.cs | 62 ++++++++ Wabbajack.Server/DataLayer/Mappers.cs | 87 +++++++++++ Wabbajack.Server/Services/NexusPoll.cs | 2 + Wabbajack.Server/Startup.cs | 7 + Wabbajack.Server/appsettings.json | 3 +- .../View Models/Settings/AuthorFilesVM.cs | 11 +- 26 files changed, 909 insertions(+), 106 deletions(-) create mode 100644 Wabbajack.Lib/AuthorApi/CDNFileDefinition.cs create mode 100644 Wabbajack.Lib/AuthorApi/Client.cs create mode 100644 Wabbajack.Lib/Downloaders/UrlDownloaders/WabbajackCDNInfluencer.cs create mode 100644 Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs create mode 100644 Wabbajack.Server.Test/AuthoredFilesTests.cs create mode 100644 Wabbajack.Server.Test/LoginTests.cs create mode 100644 Wabbajack.Server/ApiKeyAuthorizationHandler.cs create mode 100644 Wabbajack.Server/Controllers/AuthoredFiles.cs create mode 100644 Wabbajack.Server/Controllers/UploadedFiles.cs create mode 100644 Wabbajack.Server/Controllers/Users.cs create mode 100644 Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs create mode 100644 Wabbajack.Server/DataLayer/ApiKeys.cs create mode 100644 Wabbajack.Server/DataLayer/AuthoredFiles.cs create mode 100644 Wabbajack.Server/DataLayer/Mappers.cs diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index dc3825c1..13827c09 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -142,5 +142,8 @@ namespace Wabbajack.Common public static string AuthorAPIKeyFile = "author-api-key.txt"; public static Uri WabbajackOrg = new Uri("https://www.wabbajack.org/"); + + public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2; + } } diff --git a/Wabbajack.Common/Http/Client.cs b/Wabbajack.Common/Http/Client.cs index 3a7bfb0a..e4a7f8a7 100644 --- a/Wabbajack.Common/Http/Client.cs +++ b/Wabbajack.Common/Http/Client.cs @@ -74,7 +74,9 @@ namespace Wabbajack.Common.Http try { var response = await ClientFactory.Client.SendAsync(msg, responseHeadersRead); - return response; + if (response.IsSuccessStatusCode) return response; + + throw new HttpRequestException($"Http Exception {response.StatusCode} - {response.ReasonPhrase} - {msg.RequestUri}");; } catch (Exception) { diff --git a/Wabbajack.Lib/AuthorApi/CDNFileDefinition.cs b/Wabbajack.Lib/AuthorApi/CDNFileDefinition.cs new file mode 100644 index 00000000..11a39e6b --- /dev/null +++ b/Wabbajack.Lib/AuthorApi/CDNFileDefinition.cs @@ -0,0 +1,26 @@ +using Wabbajack.Common; +using Wabbajack.Common.Serialization.Json; + +namespace Wabbajack.Lib.AuthorApi +{ + [JsonName("CDNFileDefinition")] + public class CDNFileDefinition + { + public string? Author { get; set; } + public RelativePath OriginalFileName { get; set; } + public long Size { get; set; } + public Hash Hash { get; set; } + public CDNFilePartDefinition[] Parts { get; set; } = { }; + public string? ServerAssignedUniqueId { get; set; } + public string MungedName => $"{OriginalFileName}_{ServerAssignedUniqueId!}"; + } + + [JsonName("CDNFilePartDefinition")] + public class CDNFilePartDefinition + { + public long Size { get; set; } + public long Offset { get; set; } + public Hash Hash { get; set; } + public long Index { get; set; } + } +} diff --git a/Wabbajack.Lib/AuthorApi/Client.cs b/Wabbajack.Lib/AuthorApi/Client.cs new file mode 100644 index 00000000..e467a3b6 --- /dev/null +++ b/Wabbajack.Lib/AuthorApi/Client.cs @@ -0,0 +1,136 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Drawing; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using Wabbajack.Common; + +namespace Wabbajack.Lib.AuthorApi +{ + public class Client + { + public static async Task Create(string? apiKey = null) + { + var client = await GetAuthorizedClient(apiKey); + return new Client(client); + } + + private Client(Common.Http.Client client) + { + _client = client; + } + + public static async Task GetAuthorizedClient(string? apiKey = null) + { + var client = new Common.Http.Client(); + client.Headers.Add(("X-API-KEY", await GetAPIKey(apiKey))); + return client; + } + + public static string? ApiKeyOverride = null; + private Common.Http.Client _client; + + public static async ValueTask GetAPIKey(string? apiKey = null) + { + return apiKey ?? (await Consts.LocalAppDataPath.Combine(Consts.AuthorAPIKeyFile).ReadAllTextAsync()).Trim(); + } + + + public async Task GenerateFileDefinition(WorkQueue queue, AbsolutePath path, Action progressFn) + { + IEnumerable Blocks(AbsolutePath path) + { + var size = path.Size; + for (long block = 0; block * Consts.UPLOADED_FILE_BLOCK_SIZE < size; block ++) + yield return new CDNFilePartDefinition + { + Index = block, + Size = Math.Min(Consts.UPLOADED_FILE_BLOCK_SIZE, size - block * Consts.UPLOADED_FILE_BLOCK_SIZE), + Offset = block * Consts.UPLOADED_FILE_BLOCK_SIZE + }; + } + + var parts = Blocks(path).ToArray(); + var definition = new CDNFileDefinition + { + OriginalFileName = path.FileName, + Size = path.Size, + Hash = await path.FileHashCachedAsync(), + Parts = await parts.PMap(queue, async part => + { + progressFn("Hashing file parts", Percent.FactoryPutInRange(part.Index, parts.Length)); + var buffer = new byte[part.Size]; + await using (var fs = path.OpenShared()) + { + fs.Position = part.Offset; + await fs.ReadAsync(buffer); + } + part.Hash = buffer.xxHash(); + return part; + }) + }; + + return definition; + } + + public async Task UploadFile(WorkQueue queue, AbsolutePath path, Action progressFn) + { + var definition = await GenerateFileDefinition(queue, path, progressFn); + + using (var result = await _client.PutAsync($"{Consts.WabbajackBuildServerUri}authored_files/create", + new StringContent(definition.ToJson()))) + { + progressFn("Starting upload", Percent.Zero); + definition.ServerAssignedUniqueId = await result.Content.ReadAsStringAsync(); + } + + var results = await definition.Parts.PMap(queue, async part => + { + progressFn("Uploading Part", Percent.FactoryPutInRange(part.Index, definition.Parts.Length)); + var buffer = new byte[part.Size]; + await using (var fs = path.OpenShared()) + { + fs.Position = part.Offset; + await fs.ReadAsync(buffer); + } + + int retries = 0; + while (true) + { + try + { + using var putResult = await _client.PutAsync( + $"{Consts.WabbajackBuildServerUri}authored_files/{definition.ServerAssignedUniqueId}/part/{part.Index}", + new ByteArrayContent(buffer)); + var hash = Hash.FromBase64(await putResult.Content.ReadAsStringAsync()); + if (hash != part.Hash) + throw new InvalidDataException("Hashes don't match"); + return hash; + } + catch (Exception ex) + { + Utils.Log("Failure uploading part"); + Utils.Log(ex.ToString()); + if (retries <= 4) + { + retries++; + continue; + } + Utils.ErrorThrow(ex); + } + } + }); + + progressFn("Finalizing upload", Percent.Zero); + using (var result = await _client.PutAsync($"{Consts.WabbajackBuildServerUri}authored_files/{definition.ServerAssignedUniqueId}/finish", + new StringContent(definition.ToJson()))) + { + progressFn("Finished", Percent.One); + return new Uri(await result.Content.ReadAsStringAsync()); + } + } + } +} diff --git a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs index 5cffafda..bdd3eb9a 100644 --- a/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs +++ b/Wabbajack.Lib/Downloaders/DownloadDispatcher.cs @@ -25,6 +25,7 @@ namespace Wabbajack.Lib.Downloaders new BethesdaNetDownloader(), new TESAllianceDownloader(), new YouTubeDownloader(), + new WabbajackCDNDownloader(), new HTTPDownloader(), new ManualDownloader(), }; @@ -32,7 +33,8 @@ namespace Wabbajack.Lib.Downloaders public static readonly List Inferencers = new List() { new BethesdaNetInferencer(), - new YoutubeInferencer() + new YoutubeInferencer(), + new WabbajackCDNInfluencer() }; private static readonly Dictionary IndexedDownloaders; diff --git a/Wabbajack.Lib/Downloaders/UrlDownloaders/WabbajackCDNInfluencer.cs b/Wabbajack.Lib/Downloaders/UrlDownloaders/WabbajackCDNInfluencer.cs new file mode 100644 index 00000000..599ddbbf --- /dev/null +++ b/Wabbajack.Lib/Downloaders/UrlDownloaders/WabbajackCDNInfluencer.cs @@ -0,0 +1,13 @@ +using System; +using System.Threading.Tasks; + +namespace Wabbajack.Lib.Downloaders.UrlDownloaders +{ + public class WabbajackCDNInfluencer : IUrlInferencer + { + public async Task Infer(Uri uri) + { + return WabbajackCDNDownloader.StateFromUrl(uri); + } + } +} diff --git a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs new file mode 100644 index 00000000..d0a8720b --- /dev/null +++ b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs @@ -0,0 +1,100 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Common.Serialization.Json; +using Wabbajack.Lib.AuthorApi; +using Wabbajack.Lib.Downloaders.UrlDownloaders; +using Wabbajack.Lib.Exceptions; +using Wabbajack.Lib.Validation; + +namespace Wabbajack.Lib.Downloaders +{ + public class WabbajackCDNDownloader : IDownloader + { + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode = false) + { + var url = (Uri)DownloaderUtils.GetDirectURL(archiveINI); + return url == null ? null : StateFromUrl(url); + } + + public async Task Prepare() + { + } + + + public static AbstractDownloadState? StateFromUrl(Uri url) + { + if (url.Host == "wabbajacktest.b-cdn.net" || url.Host == "wabbajack.b-cdn.net") + { + return new State(url); + } + return null; + } + + [JsonName("WabbajackCDNDownloader+State")] + public class State : AbstractDownloadState + { + public Uri Url { get; set; } + public State(Uri url) + { + Url = url; + } + + public override object[] PrimaryKey => new object[] {Url}; + public override bool IsWhitelisted(ServerWhitelist whitelist) + { + return true; + } + + public override async Task Download(Archive a, AbsolutePath destination) + { + var definition = await GetDefinition(); + await using var fs = destination.Create(); + var client = new Common.Http.Client(); + await definition.Parts.DoProgress($"Downloading {a.Name}", async part => + { + fs.Position = part.Offset; + using var response = await client.GetAsync($"{Url}/parts/{part.Index}"); + if (!response.IsSuccessStatusCode) + throw new HttpException((int)response.StatusCode, response.ReasonPhrase); + await response.Content.CopyToAsync(fs); + }); + return true; + } + + public override async Task Verify(Archive archive) + { + var definition = await GetDefinition(); + return true; + } + + private async Task GetDefinition() + { + var client = new Common.Http.Client(); + using var data = await client.GetAsync(Url + "/definition.json.gz"); + await using var gz = new GZipStream(await data.Content.ReadAsStreamAsync(), CompressionMode.Decompress); + return gz.FromJson(); + } + + public override IDownloader GetDownloader() + { + return DownloadDispatcher.GetInstance(); + } + + public override string? GetManifestURL(Archive a) + { + return Url.ToString(); + } + + public override string[] GetMetaIni() + { + return new[] {"[General]", $"directURL={Url}"}; + } + } + + + } +} diff --git a/Wabbajack.Lib/FileUploader/AuthorAPI.cs b/Wabbajack.Lib/FileUploader/AuthorAPI.cs index 25fd0796..e2b36c0e 100644 --- a/Wabbajack.Lib/FileUploader/AuthorAPI.cs +++ b/Wabbajack.Lib/FileUploader/AuthorAPI.cs @@ -28,101 +28,7 @@ namespace Wabbajack.Lib.FileUploader if (ApiKeyOverride != null) return ApiKeyOverride; return apiKey ?? (await Consts.LocalAppDataPath.Combine(Consts.AuthorAPIKeyFile).ReadAllTextAsync()).Trim(); } - - public static Uri UploadURL => new Uri($"{Consts.WabbajackBuildServerUri}upload_file"); - public static long BLOCK_SIZE = (long)1024 * 1024 * 2; - public static int MAX_CONNECTIONS = 8; - public static Task UploadFile(AbsolutePath filename, Action progressFn, string? apikey = null) - { - var tcs = new TaskCompletionSource(); - Task.Run(async () => - { - var client = await GetAuthorizedClient(apikey); - - var fsize = filename.Size; - var hashTask = filename.FileHashAsync(); - - Utils.Log($"{UploadURL}/{filename.FileName.ToString()}/start"); - using var response = await client.PutAsync($"{UploadURL}/{filename.FileName.ToString()}/start", new StringContent("")); - if (!response.IsSuccessStatusCode) - { - Utils.Log("Error starting upload"); - Utils.Log(await response.Content.ReadAsStringAsync()); - tcs.SetException(new Exception($"Start Error: {response.StatusCode} {response.ReasonPhrase}")); - return; - } - - IEnumerable Blocks(long fsize) - { - for (long block = 0; block * BLOCK_SIZE < fsize; block ++) - yield return block; - } - - var key = await response.Content.ReadAsStringAsync(); - long sent = 0; - using (var iqueue = new WorkQueue(MAX_CONNECTIONS)) - { - iqueue.Report("Starting Upload", Percent.One); - await Blocks(fsize) - .PMap(iqueue, async blockIdx => - { - if (tcs.Task.IsFaulted) return; - var blockOffset = blockIdx * BLOCK_SIZE; - var blockSize = blockOffset + BLOCK_SIZE > fsize - ? fsize - blockOffset - : BLOCK_SIZE; - Interlocked.Add(ref sent, blockSize); - progressFn((double)sent / fsize); - - var data = new byte[blockSize]; - await using (var fs = filename.OpenRead()) - { - fs.Position = blockOffset; - await fs.ReadAsync(data, 0, data.Length); - } - - - var offsetResponse = await client.PutAsync(UploadURL + $"/{key}/data/{blockOffset}", - new ByteArrayContent(data)); - - if (!offsetResponse.IsSuccessStatusCode) - { - Utils.Log(await offsetResponse.Content.ReadAsStringAsync()); - tcs.SetException(new Exception($"Put Error: {offsetResponse.StatusCode} {offsetResponse.ReasonPhrase}")); - return; - } - - var val = long.Parse(await offsetResponse.Content.ReadAsStringAsync()); - if (val != blockOffset + data.Length) - { - tcs.SetResult($"Sync Error {val} vs {blockOffset + data.Length} Offset {blockOffset} Size {data.Length}"); - tcs.SetException(new Exception($"Sync Error {val} vs {blockOffset + data.Length}")); - } - }); - } - - if (!tcs.Task.IsFaulted) - { - progressFn(1.0); - var hash = (await hashTask).ToHex(); - using var finalResponse = await client.PutAsync(UploadURL + $"/{key}/finish/{hash}", new StringContent("")); - if (finalResponse.IsSuccessStatusCode) - tcs.SetResult(await finalResponse.Content.ReadAsStringAsync()); - else - { - Utils.Log("Finalization Error: "); - Utils.Log(await finalResponse.Content.ReadAsStringAsync()); - tcs.SetException(new Exception( - $"Finalization Error: {finalResponse.StatusCode} {finalResponse.ReasonPhrase}")); - } - } - - progressFn(0.0); - - }); - return tcs.Task; - } - + public static async Task GetAuthorizedClient(string? apiKey = null) { var client = new Common.Http.Client(); diff --git a/Wabbajack.Server.Test/ABuildServerSystemTest.cs b/Wabbajack.Server.Test/ABuildServerSystemTest.cs index 66a82282..253e8d0c 100644 --- a/Wabbajack.Server.Test/ABuildServerSystemTest.cs +++ b/Wabbajack.Server.Test/ABuildServerSystemTest.cs @@ -161,9 +161,9 @@ namespace Wabbajack.BuildServer.Test } - protected byte[] RandomData() + protected byte[] RandomData(long? size = null) { - var arr = new byte[_random.Next(1024)]; + var arr = new byte[size ?? _random.Next(1024)]; _random.NextBytes(arr); return arr; } diff --git a/Wabbajack.Server.Test/ADBTest.cs b/Wabbajack.Server.Test/ADBTest.cs index 453f93e4..c5942b01 100644 --- a/Wabbajack.Server.Test/ADBTest.cs +++ b/Wabbajack.Server.Test/ADBTest.cs @@ -26,7 +26,7 @@ namespace Wabbajack.BuildServer.Test { DBName = "test_db" + Guid.NewGuid().ToString().Replace("-", "_"); User = Guid.NewGuid().ToString().Replace("-", ""); - //APIKey = SqlService.NewAPIKey(); + APIKey = SqlService.NewAPIKey(); } public string APIKey { get; } diff --git a/Wabbajack.Server.Test/AuthoredFilesTests.cs b/Wabbajack.Server.Test/AuthoredFilesTests.cs new file mode 100644 index 00000000..74086a00 --- /dev/null +++ b/Wabbajack.Server.Test/AuthoredFilesTests.cs @@ -0,0 +1,38 @@ +using System; +using System.Threading.Tasks; +using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.AuthorApi; +using Wabbajack.Lib.Downloaders; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.BuildServer.Test +{ + public class AuthoredFilesTests : ABuildServerSystemTest + { + public AuthoredFilesTests(ITestOutputHelper output, SingletonAdaptor fixture) : base(output, fixture) + { + } + + [Fact] + public async Task CanUploadDownloadAndDeleteAuthoredFiles() + { + using var file = new TempFile(); + await file.Path.WriteAllBytesAsync(RandomData(Consts.UPLOADED_FILE_BLOCK_SIZE * 4 + Consts.UPLOADED_FILE_BLOCK_SIZE / 3)); + var originalHash = await file.Path.FileHashAsync(); + + var client = await Client.Create(Fixture.APIKey); + using var queue = new WorkQueue(2); + var uri = await client.UploadFile(queue, file.Path, (s, percent) => Utils.Log($"({percent}) {s}")); + + var state = await DownloadDispatcher.Infer(uri); + Assert.IsType(state); + + await state.Download(new Archive(state) {Name = (string)file.Path.FileName}, file.Path); + Assert.Equal(originalHash, await file.Path.FileHashAsync()); + + } + + } +} diff --git a/Wabbajack.Server.Test/LoginTests.cs b/Wabbajack.Server.Test/LoginTests.cs new file mode 100644 index 00000000..e04676f3 --- /dev/null +++ b/Wabbajack.Server.Test/LoginTests.cs @@ -0,0 +1,39 @@ +using System; +using System.Threading.Tasks; +using Wabbajack.Common; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.BuildServer.Test +{ + public class LoginTests : ABuildServerSystemTest + { + public LoginTests(ITestOutputHelper output, SingletonAdaptor fixture) : base(output, fixture) + { + } + + [Fact] + public async Task CanCreateLogins() + { + var newUserName = Guid.NewGuid().ToString(); + + var newKey = await _authedClient.GetStringAsync(MakeURL($"users/add/{newUserName}")); + + Assert.NotEmpty(newKey); + Assert.NotNull(newKey); + Assert.NotEqual(newKey, Fixture.APIKey); + + + var done = await _authedClient.GetStringAsync(MakeURL("users/export")); + Assert.Equal("done", done); + + foreach (var (userName, apiKey) in new[] {(newUserName, newKey), (Fixture.User, Fixture.APIKey)}) + { + var exported = await Fixture.ServerTempFolder.Combine("exported_users", userName, Consts.AuthorAPIKeyFile) + .ReadAllTextAsync(); + Assert.Equal(exported, apiKey); + + } + } + } +} diff --git a/Wabbajack.Server.Test/sql/wabbajack_db.sql b/Wabbajack.Server.Test/sql/wabbajack_db.sql index 1f6315b3..86a1511d 100644 --- a/Wabbajack.Server.Test/sql/wabbajack_db.sql +++ b/Wabbajack.Server.Test/sql/wabbajack_db.sql @@ -403,6 +403,20 @@ CREATE TABLE [dbo].[Metrics]( )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] GO + +/****** Object: Table [dbo].[AuthoredFiles] Script Date: 5/9/2020 2:22:00 PM ******/ +CREATE TABLE [dbo].[AuthoredFiles]( + [ServerAssignedUniqueId] [uniqueidentifier] NOT NULL, + [LastTouched] [datetime] NOT NULL, + [CDNFileDefinition] [nvarchar](max) NOT NULL, + [Finalized] [datetime] NULL, + CONSTRAINT [PK_AuthoredFiles] PRIMARY KEY CLUSTERED + ( + [ServerAssignedUniqueId] ASC + )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] +GO + /****** Uploaded Files [UploadedFiles] *************/ CREATE TABLE [dbo].[UploadedFiles]( diff --git a/Wabbajack.Server/ApiKeyAuthorizationHandler.cs b/Wabbajack.Server/ApiKeyAuthorizationHandler.cs new file mode 100644 index 00000000..325ee6c3 --- /dev/null +++ b/Wabbajack.Server/ApiKeyAuthorizationHandler.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Claims; +using System.Text.Encodings.Web; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using Wabbajack.Server.DataLayer; + + +namespace Wabbajack.BuildServer +{ + + public class ApiKeyAuthenticationOptions : AuthenticationSchemeOptions + { + public const string DefaultScheme = "API Key"; + public string Scheme => DefaultScheme; + public string AuthenticationType = DefaultScheme; + } + + public class ApiKeyAuthenticationHandler : AuthenticationHandler + { + private const string ProblemDetailsContentType = "application/problem+json"; + private readonly SqlService _sql; + private const string ApiKeyHeaderName = "X-Api-Key"; + + public ApiKeyAuthenticationHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder, + ISystemClock clock, + SqlService db) : base(options, logger, encoder, clock) + { + _sql = db; + } + + protected override async Task HandleAuthenticateAsync() + { + if (!Request.Headers.TryGetValue(ApiKeyHeaderName, out var apiKeyHeaderValues)) + { + return AuthenticateResult.NoResult(); + } + + var providedApiKey = apiKeyHeaderValues.FirstOrDefault(); + + if (apiKeyHeaderValues.Count == 0 || string.IsNullOrWhiteSpace(providedApiKey)) + { + return AuthenticateResult.NoResult(); + } + + var owner = await _sql.LoginByApiKey(providedApiKey); + + if (owner != null) + { + var claims = new List {new Claim(ClaimTypes.Name, owner)}; + + /* + claims.AddRange(existingApiKey.Roles.Select(role => new Claim(ClaimTypes.Role, role))); + */ + + var identity = new ClaimsIdentity(claims, Options.AuthenticationType); + var identities = new List {identity}; + var principal = new ClaimsPrincipal(identities); + var ticket = new AuthenticationTicket(principal, Options.Scheme); + + return AuthenticateResult.Success(ticket); + } + + return AuthenticateResult.Fail("Invalid API Key provided."); + } + + protected override async Task HandleChallengeAsync(AuthenticationProperties properties) + { + Response.StatusCode = 401; + Response.ContentType = ProblemDetailsContentType; + await Response.WriteAsync("Unauthorized"); + } + + protected override async Task HandleForbiddenAsync(AuthenticationProperties properties) + { + Response.StatusCode = 403; + Response.ContentType = ProblemDetailsContentType; + await Response.WriteAsync("forbidden"); + } + } + + public static class ApiKeyAuthorizationHandlerExtensions + { + public static AuthenticationBuilder AddApiKeySupport(this AuthenticationBuilder authenticationBuilder, Action options) + { + return authenticationBuilder.AddScheme(ApiKeyAuthenticationOptions.DefaultScheme, options); + } + + } +} diff --git a/Wabbajack.Server/AppSettings.cs b/Wabbajack.Server/AppSettings.cs index 85766005..f651dd96 100644 --- a/Wabbajack.Server/AppSettings.cs +++ b/Wabbajack.Server/AppSettings.cs @@ -25,8 +25,7 @@ namespace Wabbajack.BuildServer public bool RunFrontEndJobs { get; set; } public bool RunBackEndJobs { get; set; } - public string BunnyCDN_User { get; set; } - public string BunnyCDN_Password { get; set; } + public string BunnyCDN_StorageZone { get; set; } public string SqlConnection { get; set; } public int MaxJobs { get; set; } = 2; diff --git a/Wabbajack.Server/Controllers/AuthoredFiles.cs b/Wabbajack.Server/Controllers/AuthoredFiles.cs new file mode 100644 index 00000000..c9c4b811 --- /dev/null +++ b/Wabbajack.Server/Controllers/AuthoredFiles.cs @@ -0,0 +1,143 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Net; +using System.Security.Claims; +using System.Threading.Tasks; +using FluentFTP; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using SharpCompress.Compressors.LZMA; +using Wabbajack.Common; +using Wabbajack.Lib.AuthorApi; +using Wabbajack.Server.DataLayer; +using Wabbajack.Server.DTOs; + +namespace Wabbajack.BuildServer.Controllers +{ + [Route("/authored_files")] + public class AuthoredFiles : ControllerBase + { + private SqlService _sql; + private ILogger _logger; + private AppSettings _settings; + + public AuthoredFiles(ILogger logger, SqlService sql, AppSettings settings) + { + _sql = sql; + _logger = logger; + _settings = settings; + } + + [HttpPut] + [Route("{serverAssignedUniqueId}/part/{index}")] + public async Task UploadFilePart(string serverAssignedUniqueId, long index) + { + var user = User.FindFirstValue(ClaimTypes.Name); + var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId); + if (definition.Author != user) + return Forbid("File Id does not match authorized user"); + _logger.Log(LogLevel.Information, $"Uploading File part {definition.OriginalFileName} - ({index} / {definition.Parts.Length})"); + + await _sql.TouchAuthoredFile(definition); + var part = definition.Parts[index]; + + await using var ms = new MemoryStream(); + await Request.Body.CopyToLimitAsync(ms, part.Size); + ms.Position = 0; + if (ms.Length != part.Size) + return BadRequest($"Couldn't read enough data for part {part.Size} vs {ms.Length}"); + + var hash = ms.xxHash(); + if (hash != part.Hash) + return BadRequest($"Hashes don't match for index {index}. Sizes ({ms.Length} vs {part.Size}). Hashes ({hash} vs {part.Hash}"); + + ms.Position = 0; + await UploadAsync(ms, $"{definition.MungedName}/parts/{index}"); + return Ok(part.Hash.ToBase64()); + } + + [HttpPut] + [Route("create")] + public async Task CreateUpload() + { + var user = User.FindFirstValue(ClaimTypes.Name); + + var data = await Request.Body.ReadAllTextAsync(); + var definition = data.FromJsonString(); + + _logger.Log(LogLevel.Information, $"Creating File upload {definition.OriginalFileName}"); + + definition = await _sql.CreateAuthoredFile(definition, user); + + return Ok(definition.ServerAssignedUniqueId); + } + + [HttpPut] + [Route("{serverAssignedUniqueId}/finish")] + public async Task CreateUpload(string serverAssignedUniqueId) + { + var user = User.FindFirstValue(ClaimTypes.Name); + var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId); + if (definition.Author != user) + return Forbid("File Id does not match authorized user"); + _logger.Log(LogLevel.Information, $"Finalizing file upload {definition.OriginalFileName}"); + + await _sql.Finalize(definition); + + await using var ms = new MemoryStream(); + await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true)) + { + definition.ToJson(gz); + } + ms.Position = 0; + await UploadAsync(ms, $"{definition.MungedName}/definition.json.gz"); + + return Ok($"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{definition.MungedName}"); + } + + private async Task GetBunnyCdnFtpClient() + { + var info = Utils.FromEncryptedJson("bunny-cdn-ftp-info"); + var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)}; + await client.ConnectAsync(); + return client; + } + + private async Task UploadAsync(Stream stream, string path) + { + using var client = await GetBunnyCdnFtpClient(); + await client.UploadAsync(stream, path); + } + + [HttpDelete] + [Route("{serverAssignedUniqueId}")] + public async Task DeleteUpload(string serverAssignedUniqueId) + { + var user = User.FindFirstValue(ClaimTypes.Name); + var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId); + if (definition.Author != user) + return Forbid("File Id does not match authorized user"); + _logger.Log(LogLevel.Information, $"Finalizing file upload {definition.OriginalFileName}"); + + await DeleteFolderOrSilentlyFail($"{definition.MungedName}"); + + await _sql.DeleteFileDefinition(definition); + return Ok(); + } + + private async Task DeleteFolderOrSilentlyFail(string path) + { + try + { + using var client = await GetBunnyCdnFtpClient(); + await client.DeleteDirectoryAsync(path); + } + catch (Exception) + { + _logger.Log(LogLevel.Information, $"Delete failed for {path}"); + } + } + + } +} diff --git a/Wabbajack.Server/Controllers/UploadedFiles.cs b/Wabbajack.Server/Controllers/UploadedFiles.cs new file mode 100644 index 00000000..ae52e256 --- /dev/null +++ b/Wabbajack.Server/Controllers/UploadedFiles.cs @@ -0,0 +1,11 @@ +using Microsoft.AspNetCore.Mvc; + +namespace Wabbajack.BuildServer.Controllers +{ + + [ApiController] + public class UploadedFiles + { + + } +} diff --git a/Wabbajack.Server/Controllers/Users.cs b/Wabbajack.Server/Controllers/Users.cs new file mode 100644 index 00000000..a4089623 --- /dev/null +++ b/Wabbajack.Server/Controllers/Users.cs @@ -0,0 +1,51 @@ +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Wabbajack.Common; +using Wabbajack.Server.DataLayer; + +namespace Wabbajack.BuildServer.Controllers +{ + [Authorize] + [Route("/users")] + public class Users : ControllerBase + { + private AppSettings _settings; + private ILogger _logger; + private SqlService _sql; + + public Users(ILogger logger, SqlService sql, AppSettings settings) + { + _settings = settings; + _logger = logger; + _sql = sql; + } + + [HttpGet] + [Route("add/{Name}")] + public async Task AddUser(string Name) + { + return await _sql.AddLogin(Name); + } + + [HttpGet] + [Route("export")] + public async Task Export() + { + var mainFolder = _settings.TempPath.Combine("exported_users"); + mainFolder.CreateDirectory(); + + foreach (var (owner, key) in await _sql.GetAllUserKeys()) + { + var folder = mainFolder.Combine(owner); + folder.CreateDirectory(); + await folder.Combine(Consts.AuthorAPIKeyFile).WriteAllTextAsync(key); + } + + return "done"; + } + + } + +} diff --git a/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs b/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs new file mode 100644 index 00000000..132c62b8 --- /dev/null +++ b/Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs @@ -0,0 +1,9 @@ +namespace Wabbajack.Server.DTOs +{ + public class BunnyCdnFtpInfo + { + public string Username { get; set; } + public string Password { get; set; } + public string Hostname { get; set; } + } +} diff --git a/Wabbajack.Server/DataLayer/ApiKeys.cs b/Wabbajack.Server/DataLayer/ApiKeys.cs new file mode 100644 index 00000000..b177a14f --- /dev/null +++ b/Wabbajack.Server/DataLayer/ApiKeys.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Dapper; +using Wabbajack.Common; + +namespace Wabbajack.Server.DataLayer +{ + public partial class SqlService + { + public async Task LoginByApiKey(string key) + { + await using var conn = await Open(); + var result = await conn.QueryAsync(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @ApiKey", + new {ApiKey = key}); + return result.FirstOrDefault(); + } + + public async Task AddLogin(string name) + { + var key = NewAPIKey(); + await using var conn = await Open(); + + + await conn.ExecuteAsync("INSERT INTO dbo.ApiKeys (Owner, ApiKey) VALUES (@Owner, @ApiKey)", + new {Owner = name, ApiKey = key}); + return key; + } + + + public static string NewAPIKey() + { + var arr = new byte[128]; + new Random().NextBytes(arr); + return arr.ToHex(); + } + + public async Task> GetAllUserKeys() + { + await using var conn = await Open(); + var result = await conn.QueryAsync<(string Owner, string Key)>("SELECT Owner, ApiKey FROM dbo.ApiKeys"); + return result; + } + + + } +} diff --git a/Wabbajack.Server/DataLayer/AuthoredFiles.cs b/Wabbajack.Server/DataLayer/AuthoredFiles.cs new file mode 100644 index 00000000..134e9057 --- /dev/null +++ b/Wabbajack.Server/DataLayer/AuthoredFiles.cs @@ -0,0 +1,62 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Dapper; +using Wabbajack.Common; +using Wabbajack.Lib.AuthorApi; + +namespace Wabbajack.Server.DataLayer +{ + public partial class SqlService + { + public async Task TouchAuthoredFile(CDNFileDefinition definition) + { + await using var conn = await Open(); + await conn.ExecuteAsync("UPDATE AuthoredFiles SET LastTouched = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid", + new { + Uid = definition.ServerAssignedUniqueId + }); + } + + public async Task CreateAuthoredFile(CDNFileDefinition definition, string login) + { + definition.Author = login; + var uid = Guid.NewGuid().ToString(); + await using var conn = await Open(); + definition.ServerAssignedUniqueId = uid; + await conn.ExecuteAsync("INSERT INTO dbo.AuthoredFiles (ServerAssignedUniqueId, LastTouched, CDNFileDefinition) VALUES (@Uid, GETUTCDATE(), @CdnFile)", + new { + Uid = uid, + CdnFile = definition + }); + return definition; + } + + public async Task Finalize(CDNFileDefinition definition) + { + await using var conn = await Open(); + await conn.ExecuteAsync("UPDATE AuthoredFiles SET LastTouched = GETUTCDATE(), Finalized = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid", + new { + Uid = definition.ServerAssignedUniqueId + }); + } + + public async Task GetCDNFileDefinition(string serverAssignedUniqueId) + { + await using var conn = await Open(); + return (await conn.QueryAsync( + "SELECT CDNFileDefinition FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid", + new {Uid = serverAssignedUniqueId})).First(); + } + + public async Task DeleteFileDefinition(CDNFileDefinition definition) + { + await using var conn = await Open(); + return (await conn.QueryAsync( + "DELETE FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid", + new {Uid = definition.ServerAssignedUniqueId})).First(); + } + + + } +} diff --git a/Wabbajack.Server/DataLayer/Mappers.cs b/Wabbajack.Server/DataLayer/Mappers.cs new file mode 100644 index 00000000..0de6dc9b --- /dev/null +++ b/Wabbajack.Server/DataLayer/Mappers.cs @@ -0,0 +1,87 @@ +using System; +using System.Data; +using Dapper; +using Wabbajack.Common; +using Wabbajack.Lib.AuthorApi; +using Wabbajack.Lib.Downloaders; + +namespace Wabbajack.Server.DataLayer +{ + public partial class SqlService + { + static SqlService() + { + SqlMapper.AddTypeHandler(new HashMapper()); + SqlMapper.AddTypeHandler(new RelativePathMapper()); + SqlMapper.AddTypeHandler(new JsonMapper()); + SqlMapper.AddTypeHandler(new JsonMapper()); + SqlMapper.AddTypeHandler(new VersionMapper()); + SqlMapper.AddTypeHandler(new GameMapper()); + } + + class JsonMapper : SqlMapper.TypeHandler + { + public override void SetValue(IDbDataParameter parameter, T value) + { + parameter.Value = value.ToJson(); + } + + public override T Parse(object value) + { + return ((string)value).FromJsonString(); + } + } + + class RelativePathMapper : SqlMapper.TypeHandler + { + public override void SetValue(IDbDataParameter parameter, RelativePath value) + { + parameter.Value = value.ToJson(); + } + + public override RelativePath Parse(object value) + { + return (RelativePath)(string)value; + } + } + + class HashMapper : SqlMapper.TypeHandler + { + public override void SetValue(IDbDataParameter parameter, Hash value) + { + parameter.Value = (long)value; + } + + public override Hash Parse(object value) + { + return Hash.FromLong((long)value); + } + } + + class VersionMapper : SqlMapper.TypeHandler + { + public override void SetValue(IDbDataParameter parameter, Version value) + { + parameter.Value = value.ToString(); + } + + public override Version Parse(object value) + { + return Version.Parse((string)value); + } + } + + class GameMapper : SqlMapper.TypeHandler + { + public override void SetValue(IDbDataParameter parameter, Game value) + { + parameter.Value = value.ToString(); + } + + public override Game Parse(object value) + { + return GameRegistry.GetByFuzzyName((string)value).Game; + } + } + } +} diff --git a/Wabbajack.Server/Services/NexusPoll.cs b/Wabbajack.Server/Services/NexusPoll.cs index 5cab96e6..64c1f18e 100644 --- a/Wabbajack.Server/Services/NexusPoll.cs +++ b/Wabbajack.Server/Services/NexusPoll.cs @@ -120,6 +120,8 @@ namespace Wabbajack.Server.Services }); _logger.Log(LogLevel.Information, $"Purged {purged.Sum()} cache entries"); + _globalInformation.LastNexusSyncUTC = DateTime.UtcNow; + } public void Start() diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index e82ed858..ffb66314 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -40,6 +40,13 @@ namespace Wabbajack.Server c.SwaggerDoc("v1", new OpenApiInfo {Title = "Wabbajack Build API", Version = "v1"}); }); + services.AddAuthentication(options => + { + options.DefaultAuthenticateScheme = ApiKeyAuthenticationOptions.DefaultScheme; + options.DefaultChallengeScheme = ApiKeyAuthenticationOptions.DefaultScheme; + }) + .AddApiKeySupport(options => {}); + services.Configure(x => { x.ValueLengthLimit = int.MaxValue; diff --git a/Wabbajack.Server/appsettings.json b/Wabbajack.Server/appsettings.json index 28170196..6ef56c22 100644 --- a/Wabbajack.Server/appsettings.json +++ b/Wabbajack.Server/appsettings.json @@ -14,8 +14,7 @@ "JobScheduler": false, "RunFrontEndJobs": true, "RunBackEndJobs": false, - "BunnyCDN_User": "wabbajackcdn", - "BunnyCDN_Password": "XXXX", + "BunnyCDN_StorageZone": "wabbajacktest", "SQLConnection": "Data Source=.\\SQLEXPRESS;Integrated Security=True;Initial Catalog=wabbajack_prod;MultipleActiveResultSets=true" }, "AllowedHosts": "*" diff --git a/Wabbajack/View Models/Settings/AuthorFilesVM.cs b/Wabbajack/View Models/Settings/AuthorFilesVM.cs index ff9032fc..09869587 100644 --- a/Wabbajack/View Models/Settings/AuthorFilesVM.cs +++ b/Wabbajack/View Models/Settings/AuthorFilesVM.cs @@ -6,6 +6,7 @@ using System.Windows.Input; using ReactiveUI; using ReactiveUI.Fody.Helpers; using Wabbajack.Common; +using Wabbajack.Lib.AuthorApi; using Wabbajack.Lib.FileUploader; namespace Wabbajack @@ -49,8 +50,14 @@ namespace Wabbajack _isUploading.OnNext(true); try { - FinalUrl = await AuthorAPI.UploadFile(Picker.TargetPath, - progress => UploadProgress = progress); + using var queue = new WorkQueue(); + var result = await (await Client.Create()).UploadFile(queue, Picker.TargetPath, + (msg, progress) => + { + FinalUrl = msg; + UploadProgress = (double)progress; + }); + FinalUrl = result.ToString(); } catch (Exception ex) {