Fixes for author APIs

Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs

@@ -31,10 +31,13 @@ namespace Wabbajack.BuildServer.Test
                     $"WabbajackSettings:ArchiveDir={"archives".RelativeTo(AbsolutePath.EntryPoint)}",
                     $"WabbajackSettings:TempFolder={ServerTempFolder}",
                     $"WabbajackSettings:SQLConnection={PublicConnStr}",
+                    $"WabbajackSettings:BunnyCDN_User=TEST",
+                    $"WabbajackSettings:BunnyCDN_Password=TEST",
                 }, true);
             _host = builder.Build();
             _token = new CancellationTokenSource();
             _task = _host.RunAsync(_token.Token);
+            Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080");
         }
 
         public void Dispose()
@@ -52,16 +55,19 @@ namespace Wabbajack.BuildServer.Test
         private readonly IDisposable _unsubMsgs;
         private readonly IDisposable _unsubErr;
         protected Client _authedClient;
+        protected WorkQueue _queue;
 
 
         public ABuildServerSystemTest(ITestOutputHelper output, BuildServerFixture fixture) : base(output)
         {
+            Filters.Clear();
             _unsubMsgs = Utils.LogMessages.OfType<IInfo>().Subscribe(onNext: msg => XunitContext.WriteLine(msg.ShortDescription));
             _unsubErr = Utils.LogMessages.OfType<IUserIntervention>().Subscribe(msg =>
                 XunitContext.WriteLine("ERROR: User intervention required: " + msg.ShortDescription));
             _client = new Client();
             _authedClient = new Client();
             _authedClient.Headers.Add(("x-api-key", fixture.APIKey));
+            _queue = new WorkQueue();
             Fixture = fixture;
         }
 

Wabbajack.BuildServer.Test/UploadedFilesTest.cs

@@ -4,6 +4,8 @@ using System.Threading.Tasks;
 using Wabbajack.BuildServer.Model.Models;
 using Wabbajack.BuildServer.Models;
 using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.FileUploader;
 using Xunit;
 using Xunit.Abstractions;
 using Xunit.Priority;
@@ -33,7 +35,7 @@ namespace Wabbajack.BuildServer.Test
         }
         
         [Fact, Priority(1)]
-        public async Task CanLoadUploadedFiles()
+        public async Task CanListMyUploadedFiles()
         {
             var result = (await _authedClient.GetStringAsync(MakeURL("uploaded_files/list"))).FromJSONString<string[]>();
             Utils.Log("Loaded: " + result);
@@ -47,5 +49,26 @@ namespace Wabbajack.BuildServer.Test
             Assert.DoesNotContain("file3-17b3e918-8409-48e6-b7ff-6af858bfd1ba.zip", result);
         }
 
+        [Fact]
+        public async Task CanUploadFilesUsingClientApi()
+        {
+            using (var file = new TempFile())
+            {
+                var data = new byte[1024 * 1024 * 8 * 4];
+                await using (var fs = file.Path.Create())
+                {
+                    await fs.WriteAsync(data);
+                }
+
+                Utils.Log($"Uploading {file.Path.Size.ToFileSizeString()} file");
+                var result = await AuthorAPI.UploadFile(file.Path,
+                    progress => Utils.Log($"Uploading : {progress * 100}%"), Fixture.APIKey);
+
+                Utils.Log($"Result {result}");
+                Assert.StartsWith("https://wabbajackpush.b-cdn.net/" +(string)file.Path.FileNameWithoutExtension, result);
+            }
+
+        }
+
     }
 }

Wabbajack.BuildServer/AppSettings.cs

@@ -15,6 +15,8 @@ namespace Wabbajack.BuildServer
         
         public string TempFolder { get; set; }
 
+        public AbsolutePath TempPath => (AbsolutePath)TempFolder;
+        
         public bool JobScheduler { get; set; }
         public bool JobRunner { get; set; }
         

Wabbajack.BuildServer/Controllers/UploadedFiles.cs

@@ -50,7 +50,7 @@ namespace Wabbajack.BuildServer.Controllers
             
             _writeLocks.GetOrAdd(key, new AsyncLock());
 
-            System.IO.File.Create(Path.Combine("public", "tmp_files", key)).Close();
+            await using var fs = _settings.TempPath.Combine(key).Create();
             Utils.Log($"Starting Ingest for {key}");
             return Ok(key);
         }
@@ -62,20 +62,24 @@ namespace Wabbajack.BuildServer.Controllers
         {
             if (!Key.All(a => HexChars.Contains(a)))
                 return BadRequest("NOT A VALID FILENAME");
-            Utils.Log($"Writing at position {Offset} in ingest file {Key}");
             
             var ms = new MemoryStream();
             await Request.Body.CopyToAsync(ms);
             ms.Position = 0;
 
+            Utils.Log($"Writing {ms.Length} at position {Offset} in ingest file {Key}");
+
             long position;
             using (var _ = await _writeLocks[Key].Wait())
-            await using (var file = System.IO.File.Open(Path.Combine("public", "tmp_files", Key), FileMode.Open, FileAccess.Write, FileShare.ReadWrite))
             {
+                await using var file = _settings.TempPath.Combine(Key).WriteShared();
                 file.Position = Offset;
                 await ms.CopyToAsync(file);
-                position = file.Position;
+                position = Offset + ms.Length;
             }
+
+            Utils.Log($"Wrote {ms.Length} as position {Offset} result {position}");
+
             return Ok(position);
         }
 
@@ -132,7 +136,8 @@ namespace Wabbajack.BuildServer.Controllers
             var originalName = $"{parts[0]}{parts[2]}";
 
             var finalPath = "public".RelativeTo(AbsolutePath.EntryPoint).Combine("files", finalName);
-            "public".RelativeTo(AbsolutePath.EntryPoint).MoveTo(finalPath);
+            _settings.TempPath.Combine(Key).MoveTo(finalPath);
+            
             var hash = await finalPath.FileHashAsync();
 
             if (expectedHash != hash)
@@ -151,8 +156,8 @@ namespace Wabbajack.BuildServer.Controllers
                 Size = finalPath.Size,
                 CDNName = "wabbajackpush"
             };
-            await Db.UploadedFiles.InsertOneAsync(record);
+            await SQL.AddUploadedFile(record);
-            await Db.Jobs.InsertOneAsync(new Job
+            await SQL.EnqueueJob(new Job
             {
                 Priority = Job.JobPriority.High, Payload = new UploadToCDN {FileId = record.Id}
             });

Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs

@@ -28,6 +28,11 @@ namespace Wabbajack.BuildServer.Models.Jobs
             TOP:
             var file = await db.UploadedFiles.AsQueryable().Where(f => f.Id == FileId).FirstOrDefaultAsync();
 
+            if (settings.BunnyCDN_User == "TEST" && settings.BunnyCDN_Password == "TEST")
+            {
+                return JobResult.Success();
+            }
+            
             using (var client = new FtpClient("storage.bunnycdn.com"))
             {
                 client.Credentials = new NetworkCredential(settings.BunnyCDN_User, settings.BunnyCDN_Password);

Wabbajack.Common/Consts.cs

@@ -107,6 +107,7 @@ namespace Wabbajack.Common
         public static string WabbajackCacheLocation = "http://build.wabbajack.org/nexus_api_cache/";
 
         public static string WabbajackCacheHostname = "build.wabbajack.org";
+        public static Uri WabbajackBuildServerUri = new Uri("https://build.wabbajack.org");
         public static int WabbajackCachePort = 80;
         public static int MaxHTTPRetries = 4;
         public static RelativePath MO2ModFolderName = (RelativePath)"mods";

Wabbajack.Common/Paths.cs

@@ -384,6 +384,11 @@ namespace Wabbajack.Common
         {
             return File.Open(_path, FileMode.Open, FileAccess.Read);
         }
+
+        public FileStream WriteShared()
+        {
+            return File.Open(_path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite);
+        }
     }
 
     public struct RelativePath : IPath, IEquatable<RelativePath>, IComparable<RelativePath>

Wabbajack.Common/Utils.cs

@@ -1138,6 +1138,14 @@ namespace Wabbajack.Common
             return bytes.ToHex();
         }
         
+        public static byte[] RandomData(int size)
+        {
+            var random = new Random();
+            byte[] bytes = new byte[size];
+            random.NextBytes(bytes);
+            return bytes;
+        }
+
         public static async Task CopyFileAsync(string src, string dest)
         {
             await using var s = File.OpenRead(src);

Wabbajack.Lib/FileUploader/AuthorAPI.cs

@@ -21,24 +21,25 @@ namespace Wabbajack.Lib.FileUploader
     {
         public static IObservable<bool> HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable("author-api-key.txt");
 
-        public static async Task<string> GetAPIKey()
+        public static async Task<string> GetAPIKey(string apiKey = null)
         {
-            return (await Consts.LocalAppDataPath.Combine("author-api-key.txt").ReadAllTextAsync()).Trim();
+            return apiKey ?? (await Consts.LocalAppDataPath.Combine("author-api-key.txt").ReadAllTextAsync()).Trim();
         }
         
-        public static readonly Uri UploadURL = new Uri("https://build.wabbajack.org/upload_file");
+        public static Uri UploadURL => new Uri($"{Consts.WabbajackBuildServerUri}upload_file");
         public static long BLOCK_SIZE = (long)1024 * 1024 * 2;
         public static int MAX_CONNECTIONS = 8;
-        public static Task<string> UploadFile(WorkQueue queue, AbsolutePath filename, Action<double> progressFn)
+        public static Task<string> UploadFile(AbsolutePath filename, Action<double> progressFn, string apikey=null)
         {
             var tcs = new TaskCompletionSource<string>();
             Task.Run(async () =>
             {
-                var client = await GetAuthorizedClient();
+                var client = await GetAuthorizedClient(apikey);
 
                 var fsize = filename.Size;
-                var hash_task = filename.FileHashAsync();
+                var hashTask = filename.FileHashAsync();
 
+                Utils.Log($"{UploadURL}/{filename.FileName.ToString()}/start");
                 var response = await client.PutAsync($"{UploadURL}/{filename.FileName.ToString()}/start", new StringContent(""));
                 if (!response.IsSuccessStatusCode)
                 {
@@ -58,36 +59,39 @@ namespace Wabbajack.Lib.FileUploader
                 {
                     iqueue.Report("Starting Upload", Percent.One);
                 await Blocks(fsize)
-                    .PMap(iqueue, async block_idx =>
+                    .PMap(iqueue, async blockIdx =>
                     {
                         if (tcs.Task.IsFaulted) return;
-                        var block_offset = block_idx * BLOCK_SIZE;
+                        var blockOffset = blockIdx * BLOCK_SIZE;
-                        var block_size = block_offset + BLOCK_SIZE > fsize
+                        var blockSize = blockOffset + BLOCK_SIZE > fsize
-                            ? fsize - block_offset
+                            ? fsize - blockOffset
                             : BLOCK_SIZE;
-                        Interlocked.Add(ref sent, block_size);
+                        Interlocked.Add(ref sent, blockSize);
                         progressFn((double)sent / fsize);
 
-                        await using var fs = filename.OpenRead();
+                        var data = new byte[blockSize];
-                        fs.Position = block_offset;
+                        await using (var fs = filename.OpenRead())
-                        var data = new byte[block_size];
+                        {
+                            fs.Position = blockOffset;
                             await fs.ReadAsync(data, 0, data.Length);
+                        }
 
 
-                        response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}",
+                        var offsetResponse = await client.PutAsync(UploadURL + $"/{key}/data/{blockOffset}",
                             new ByteArrayContent(data));
 
-                        if (!response.IsSuccessStatusCode)
+                        if (!offsetResponse.IsSuccessStatusCode)
                         {
-                            tcs.SetException(new Exception($"Put Error: {response.StatusCode} {response.ReasonPhrase}"));
+                            Utils.Log(await offsetResponse.Content.ReadAsStringAsync());
+                            tcs.SetException(new Exception($"Put Error: {offsetResponse.StatusCode} {offsetResponse.ReasonPhrase}"));
                             return;
                         }
 
-                        var val = long.Parse(await response.Content.ReadAsStringAsync());
+                        var val = long.Parse(await offsetResponse.Content.ReadAsStringAsync());
-                        if (val != block_offset + data.Length)
+                        if (val != blockOffset + data.Length)
                         {
-                            tcs.SetResult($"Sync Error {val} vs {block_offset + data.Length}");
+                            tcs.SetResult($"Sync Error {val} vs {blockOffset + data.Length} Offset {blockOffset} Size {data.Length}");
-                            tcs.SetException(new Exception($"Sync Error {val} vs {block_offset + data.Length}"));
+                            tcs.SetException(new Exception($"Sync Error {val} vs {blockOffset + data.Length}"));
                         }
                     });
                 }
@@ -95,7 +99,7 @@ namespace Wabbajack.Lib.FileUploader
                 if (!tcs.Task.IsFaulted)
                 {
                     progressFn(1.0);
-                    var hash = (await hash_task).ToHex();
+                    var hash = (await hashTask).ToHex();
                     response = await client.PutAsync(UploadURL + $"/{key}/finish/{hash}", new StringContent(""));
                     if (response.IsSuccessStatusCode)
                         tcs.SetResult(await response.Content.ReadAsStringAsync());
@@ -109,10 +113,10 @@ namespace Wabbajack.Lib.FileUploader
             return tcs.Task;
         }
 
-        public static async Task<Common.Http.Client> GetAuthorizedClient()
+        public static async Task<Common.Http.Client> GetAuthorizedClient(string apiKey=null)
         {
             var client = new Common.Http.Client();
-            client.Headers.Add(("X-API-KEY", await GetAPIKey()));
+            client.Headers.Add(("X-API-KEY", await GetAPIKey(apiKey)));
             return client;
         }
         

Wabbajack/View Models/Settings/AuthorFilesVM.cs

@@ -59,7 +59,7 @@ namespace Wabbajack
                 _isUploading.OnNext(true);
                 try
                 {
-                    FinalUrl = await AuthorAPI.UploadFile(Queue, Picker.TargetPath,
+                    FinalUrl = await AuthorAPI.UploadFile(Picker.TargetPath,
                         progress => UploadProgress = progress);
                 }
                 catch (Exception ex)