From 160ac8a4c35dafae54dbccd3a4fdf14632994f87 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 30 Mar 2020 14:38:46 -0600 Subject: [PATCH] Fixes for author APIs --- .../ABuildServerSystemTest.cs | 6 ++ .../UploadedFilesTest.cs | 25 ++++++++- Wabbajack.BuildServer/AppSettings.cs | 2 + .../Controllers/UploadedFiles.cs | 23 +++++--- .../Models/Jobs/UploadToCDN.cs | 5 ++ Wabbajack.Common/Consts.cs | 1 + Wabbajack.Common/Paths.cs | 5 ++ Wabbajack.Common/Utils.cs | 8 +++ Wabbajack.Lib/FileUploader/AuthorAPI.cs | 56 ++++++++++--------- .../View Models/Settings/AuthorFilesVM.cs | 2 +- 10 files changed, 96 insertions(+), 37 deletions(-) diff --git a/Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs b/Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs index 782f3629..a8aae303 100644 --- a/Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs +++ b/Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs @@ -31,10 +31,13 @@ namespace Wabbajack.BuildServer.Test $"WabbajackSettings:ArchiveDir={"archives".RelativeTo(AbsolutePath.EntryPoint)}", $"WabbajackSettings:TempFolder={ServerTempFolder}", $"WabbajackSettings:SQLConnection={PublicConnStr}", + $"WabbajackSettings:BunnyCDN_User=TEST", + $"WabbajackSettings:BunnyCDN_Password=TEST", }, true); _host = builder.Build(); _token = new CancellationTokenSource(); _task = _host.RunAsync(_token.Token); + Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080"); } public void Dispose() @@ -52,16 +55,19 @@ namespace Wabbajack.BuildServer.Test private readonly IDisposable _unsubMsgs; private readonly IDisposable _unsubErr; protected Client _authedClient; + protected WorkQueue _queue; public ABuildServerSystemTest(ITestOutputHelper output, BuildServerFixture fixture) : base(output) { + Filters.Clear(); _unsubMsgs = Utils.LogMessages.OfType().Subscribe(onNext: msg => XunitContext.WriteLine(msg.ShortDescription)); _unsubErr = Utils.LogMessages.OfType().Subscribe(msg => XunitContext.WriteLine("ERROR: User intervention required: " + msg.ShortDescription)); _client = new Client(); _authedClient = new Client(); _authedClient.Headers.Add(("x-api-key", fixture.APIKey)); + _queue = new WorkQueue(); Fixture = fixture; } diff --git a/Wabbajack.BuildServer.Test/UploadedFilesTest.cs b/Wabbajack.BuildServer.Test/UploadedFilesTest.cs index ad89e0bd..f71da927 100644 --- a/Wabbajack.BuildServer.Test/UploadedFilesTest.cs +++ b/Wabbajack.BuildServer.Test/UploadedFilesTest.cs @@ -4,6 +4,8 @@ using System.Threading.Tasks; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; using Wabbajack.Common; +using Wabbajack.Lib; +using Wabbajack.Lib.FileUploader; using Xunit; using Xunit.Abstractions; using Xunit.Priority; @@ -33,7 +35,7 @@ namespace Wabbajack.BuildServer.Test } [Fact, Priority(1)] - public async Task CanLoadUploadedFiles() + public async Task CanListMyUploadedFiles() { var result = (await _authedClient.GetStringAsync(MakeURL("uploaded_files/list"))).FromJSONString(); Utils.Log("Loaded: " + result); @@ -47,5 +49,26 @@ namespace Wabbajack.BuildServer.Test Assert.DoesNotContain("file3-17b3e918-8409-48e6-b7ff-6af858bfd1ba.zip", result); } + [Fact] + public async Task CanUploadFilesUsingClientApi() + { + using (var file = new TempFile()) + { + var data = new byte[1024 * 1024 * 8 * 4]; + await using (var fs = file.Path.Create()) + { + await fs.WriteAsync(data); + } + + Utils.Log($"Uploading {file.Path.Size.ToFileSizeString()} file"); + var result = await AuthorAPI.UploadFile(file.Path, + progress => Utils.Log($"Uploading : {progress * 100}%"), Fixture.APIKey); + + Utils.Log($"Result {result}"); + Assert.StartsWith("https://wabbajackpush.b-cdn.net/" +(string)file.Path.FileNameWithoutExtension, result); + } + + } + } } diff --git a/Wabbajack.BuildServer/AppSettings.cs b/Wabbajack.BuildServer/AppSettings.cs index b37e5bd0..9037b977 100644 --- a/Wabbajack.BuildServer/AppSettings.cs +++ b/Wabbajack.BuildServer/AppSettings.cs @@ -14,6 +14,8 @@ namespace Wabbajack.BuildServer public AbsolutePath ArchiveDir { get; set; } public string TempFolder { get; set; } + + public AbsolutePath TempPath => (AbsolutePath)TempFolder; public bool JobScheduler { get; set; } public bool JobRunner { get; set; } diff --git a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs index f981e51b..999251d1 100644 --- a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs @@ -49,8 +49,8 @@ namespace Wabbajack.BuildServer.Controllers var key = Encoding.UTF8.GetBytes($"{Path.GetFileNameWithoutExtension(Name)}|{guid.ToString()}|{Path.GetExtension(Name)}").ToHex(); _writeLocks.GetOrAdd(key, new AsyncLock()); - - System.IO.File.Create(Path.Combine("public", "tmp_files", key)).Close(); + + await using var fs = _settings.TempPath.Combine(key).Create(); Utils.Log($"Starting Ingest for {key}"); return Ok(key); } @@ -62,20 +62,24 @@ namespace Wabbajack.BuildServer.Controllers { if (!Key.All(a => HexChars.Contains(a))) return BadRequest("NOT A VALID FILENAME"); - Utils.Log($"Writing at position {Offset} in ingest file {Key}"); var ms = new MemoryStream(); await Request.Body.CopyToAsync(ms); ms.Position = 0; - + + Utils.Log($"Writing {ms.Length} at position {Offset} in ingest file {Key}"); + long position; using (var _ = await _writeLocks[Key].Wait()) - await using (var file = System.IO.File.Open(Path.Combine("public", "tmp_files", Key), FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) { + await using var file = _settings.TempPath.Combine(Key).WriteShared(); file.Position = Offset; await ms.CopyToAsync(file); - position = file.Position; + position = Offset + ms.Length; } + + Utils.Log($"Wrote {ms.Length} as position {Offset} result {position}"); + return Ok(position); } @@ -132,7 +136,8 @@ namespace Wabbajack.BuildServer.Controllers var originalName = $"{parts[0]}{parts[2]}"; var finalPath = "public".RelativeTo(AbsolutePath.EntryPoint).Combine("files", finalName); - "public".RelativeTo(AbsolutePath.EntryPoint).MoveTo(finalPath); + _settings.TempPath.Combine(Key).MoveTo(finalPath); + var hash = await finalPath.FileHashAsync(); if (expectedHash != hash) @@ -151,8 +156,8 @@ namespace Wabbajack.BuildServer.Controllers Size = finalPath.Size, CDNName = "wabbajackpush" }; - await Db.UploadedFiles.InsertOneAsync(record); - await Db.Jobs.InsertOneAsync(new Job + await SQL.AddUploadedFile(record); + await SQL.EnqueueJob(new Job { Priority = Job.JobPriority.High, Payload = new UploadToCDN {FileId = record.Id} }); diff --git a/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs b/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs index a64488af..a54ef2d4 100644 --- a/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs +++ b/Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs @@ -27,6 +27,11 @@ namespace Wabbajack.BuildServer.Models.Jobs int retries = 0; TOP: var file = await db.UploadedFiles.AsQueryable().Where(f => f.Id == FileId).FirstOrDefaultAsync(); + + if (settings.BunnyCDN_User == "TEST" && settings.BunnyCDN_Password == "TEST") + { + return JobResult.Success(); + } using (var client = new FtpClient("storage.bunnycdn.com")) { diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 7986ff25..5fb5ff9a 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -107,6 +107,7 @@ namespace Wabbajack.Common public static string WabbajackCacheLocation = "http://build.wabbajack.org/nexus_api_cache/"; public static string WabbajackCacheHostname = "build.wabbajack.org"; + public static Uri WabbajackBuildServerUri = new Uri("https://build.wabbajack.org"); public static int WabbajackCachePort = 80; public static int MaxHTTPRetries = 4; public static RelativePath MO2ModFolderName = (RelativePath)"mods"; diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index 1893d470..be7442a7 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -384,6 +384,11 @@ namespace Wabbajack.Common { return File.Open(_path, FileMode.Open, FileAccess.Read); } + + public FileStream WriteShared() + { + return File.Open(_path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite); + } } public struct RelativePath : IPath, IEquatable, IComparable diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index e2d98ab1..daeaf9e0 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -1137,6 +1137,14 @@ namespace Wabbajack.Common random.NextBytes(bytes); return bytes.ToHex(); } + + public static byte[] RandomData(int size) + { + var random = new Random(); + byte[] bytes = new byte[size]; + random.NextBytes(bytes); + return bytes; + } public static async Task CopyFileAsync(string src, string dest) { diff --git a/Wabbajack.Lib/FileUploader/AuthorAPI.cs b/Wabbajack.Lib/FileUploader/AuthorAPI.cs index 72818cba..866355d9 100644 --- a/Wabbajack.Lib/FileUploader/AuthorAPI.cs +++ b/Wabbajack.Lib/FileUploader/AuthorAPI.cs @@ -21,24 +21,25 @@ namespace Wabbajack.Lib.FileUploader { public static IObservable HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable("author-api-key.txt"); - public static async Task GetAPIKey() + public static async Task GetAPIKey(string apiKey = null) { - return (await Consts.LocalAppDataPath.Combine("author-api-key.txt").ReadAllTextAsync()).Trim(); + return apiKey ?? (await Consts.LocalAppDataPath.Combine("author-api-key.txt").ReadAllTextAsync()).Trim(); } - public static readonly Uri UploadURL = new Uri("https://build.wabbajack.org/upload_file"); + public static Uri UploadURL => new Uri($"{Consts.WabbajackBuildServerUri}upload_file"); public static long BLOCK_SIZE = (long)1024 * 1024 * 2; public static int MAX_CONNECTIONS = 8; - public static Task UploadFile(WorkQueue queue, AbsolutePath filename, Action progressFn) + public static Task UploadFile(AbsolutePath filename, Action progressFn, string apikey=null) { var tcs = new TaskCompletionSource(); Task.Run(async () => { - var client = await GetAuthorizedClient(); + var client = await GetAuthorizedClient(apikey); var fsize = filename.Size; - var hash_task = filename.FileHashAsync(); + var hashTask = filename.FileHashAsync(); + Utils.Log($"{UploadURL}/{filename.FileName.ToString()}/start"); var response = await client.PutAsync($"{UploadURL}/{filename.FileName.ToString()}/start", new StringContent("")); if (!response.IsSuccessStatusCode) { @@ -58,36 +59,39 @@ namespace Wabbajack.Lib.FileUploader { iqueue.Report("Starting Upload", Percent.One); await Blocks(fsize) - .PMap(iqueue, async block_idx => + .PMap(iqueue, async blockIdx => { if (tcs.Task.IsFaulted) return; - var block_offset = block_idx * BLOCK_SIZE; - var block_size = block_offset + BLOCK_SIZE > fsize - ? fsize - block_offset + var blockOffset = blockIdx * BLOCK_SIZE; + var blockSize = blockOffset + BLOCK_SIZE > fsize + ? fsize - blockOffset : BLOCK_SIZE; - Interlocked.Add(ref sent, block_size); + Interlocked.Add(ref sent, blockSize); progressFn((double)sent / fsize); - await using var fs = filename.OpenRead(); - fs.Position = block_offset; - var data = new byte[block_size]; - await fs.ReadAsync(data, 0, data.Length); + var data = new byte[blockSize]; + await using (var fs = filename.OpenRead()) + { + fs.Position = blockOffset; + await fs.ReadAsync(data, 0, data.Length); + } - - response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}", + + var offsetResponse = await client.PutAsync(UploadURL + $"/{key}/data/{blockOffset}", new ByteArrayContent(data)); - if (!response.IsSuccessStatusCode) + if (!offsetResponse.IsSuccessStatusCode) { - tcs.SetException(new Exception($"Put Error: {response.StatusCode} {response.ReasonPhrase}")); + Utils.Log(await offsetResponse.Content.ReadAsStringAsync()); + tcs.SetException(new Exception($"Put Error: {offsetResponse.StatusCode} {offsetResponse.ReasonPhrase}")); return; } - var val = long.Parse(await response.Content.ReadAsStringAsync()); - if (val != block_offset + data.Length) + var val = long.Parse(await offsetResponse.Content.ReadAsStringAsync()); + if (val != blockOffset + data.Length) { - tcs.SetResult($"Sync Error {val} vs {block_offset + data.Length}"); - tcs.SetException(new Exception($"Sync Error {val} vs {block_offset + data.Length}")); + tcs.SetResult($"Sync Error {val} vs {blockOffset + data.Length} Offset {blockOffset} Size {data.Length}"); + tcs.SetException(new Exception($"Sync Error {val} vs {blockOffset + data.Length}")); } }); } @@ -95,7 +99,7 @@ namespace Wabbajack.Lib.FileUploader if (!tcs.Task.IsFaulted) { progressFn(1.0); - var hash = (await hash_task).ToHex(); + var hash = (await hashTask).ToHex(); response = await client.PutAsync(UploadURL + $"/{key}/finish/{hash}", new StringContent("")); if (response.IsSuccessStatusCode) tcs.SetResult(await response.Content.ReadAsStringAsync()); @@ -109,10 +113,10 @@ namespace Wabbajack.Lib.FileUploader return tcs.Task; } - public static async Task GetAuthorizedClient() + public static async Task GetAuthorizedClient(string apiKey=null) { var client = new Common.Http.Client(); - client.Headers.Add(("X-API-KEY", await GetAPIKey())); + client.Headers.Add(("X-API-KEY", await GetAPIKey(apiKey))); return client; } diff --git a/Wabbajack/View Models/Settings/AuthorFilesVM.cs b/Wabbajack/View Models/Settings/AuthorFilesVM.cs index e2b5bddf..71b0c91e 100644 --- a/Wabbajack/View Models/Settings/AuthorFilesVM.cs +++ b/Wabbajack/View Models/Settings/AuthorFilesVM.cs @@ -59,7 +59,7 @@ namespace Wabbajack _isUploading.OnNext(true); try { - FinalUrl = await AuthorAPI.UploadFile(Queue, Picker.TargetPath, + FinalUrl = await AuthorAPI.UploadFile(Picker.TargetPath, progress => UploadProgress = progress); } catch (Exception ex)