Rewrite the Authored file routines.

Wabbajack.Common/Consts.cs

@@ -142,5 +142,8 @@ namespace Wabbajack.Common
         public static string AuthorAPIKeyFile = "author-api-key.txt";
 
         public static Uri WabbajackOrg = new Uri("https://www.wabbajack.org/");
+        
+        public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2;
+
     }
 }

Wabbajack.Common/Http/Client.cs

@@ -74,7 +74,9 @@ namespace Wabbajack.Common.Http
             try
             {
                 var response = await ClientFactory.Client.SendAsync(msg, responseHeadersRead);
-                return response;
+                if (response.IsSuccessStatusCode) return response;
+
+                throw new HttpRequestException($"Http Exception {response.StatusCode} - {response.ReasonPhrase} - {msg.RequestUri}");;
             }
             catch (Exception)
             {

Wabbajack.Lib/AuthorApi/CDNFileDefinition.cs

@@ -0,0 +1,26 @@
+using Wabbajack.Common;
+using Wabbajack.Common.Serialization.Json;
+
+namespace Wabbajack.Lib.AuthorApi
+{
+    [JsonName("CDNFileDefinition")]
+    public class CDNFileDefinition
+    {
+        public string? Author { get; set; }
+        public RelativePath OriginalFileName { get; set; }
+        public long Size { get; set; }
+        public Hash Hash { get; set; }
+        public CDNFilePartDefinition[] Parts { get; set; } = { };
+        public string? ServerAssignedUniqueId { get; set; }
+        public string MungedName => $"{OriginalFileName}_{ServerAssignedUniqueId!}";
+    }
+
+    [JsonName("CDNFilePartDefinition")]
+    public class CDNFilePartDefinition
+    {
+        public long Size { get; set; }
+        public long Offset { get; set; }
+        public Hash Hash { get; set; }
+        public long Index { get; set; }
+    }
+}

Wabbajack.Lib/AuthorApi/Client.cs

@@ -0,0 +1,136 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Drawing;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+
+namespace Wabbajack.Lib.AuthorApi
+{
+    public class Client
+    {
+        public static async Task<Client> Create(string? apiKey = null)
+        {
+            var client = await GetAuthorizedClient(apiKey);
+            return new Client(client); 
+        }
+
+        private Client(Common.Http.Client client)
+        {
+            _client = client;
+        }
+        
+        public static async Task<Common.Http.Client> GetAuthorizedClient(string? apiKey = null)
+        {
+            var client = new Common.Http.Client();
+            client.Headers.Add(("X-API-KEY", await GetAPIKey(apiKey)));
+            return client;
+        }
+
+        public static string? ApiKeyOverride = null;
+        private Common.Http.Client _client;
+
+        public static async ValueTask<string> GetAPIKey(string? apiKey = null)
+        {
+            return apiKey ?? (await Consts.LocalAppDataPath.Combine(Consts.AuthorAPIKeyFile).ReadAllTextAsync()).Trim();
+        }
+
+
+        public async Task<CDNFileDefinition> GenerateFileDefinition(WorkQueue queue, AbsolutePath path, Action<string, Percent> progressFn)
+        {
+            IEnumerable<CDNFilePartDefinition> Blocks(AbsolutePath path)
+            {
+                var size = path.Size;
+                for (long block = 0; block * Consts.UPLOADED_FILE_BLOCK_SIZE < size; block ++)
+                    yield return new CDNFilePartDefinition
+                    {
+                        Index = block,
+                        Size = Math.Min(Consts.UPLOADED_FILE_BLOCK_SIZE, size - block * Consts.UPLOADED_FILE_BLOCK_SIZE),
+                        Offset = block * Consts.UPLOADED_FILE_BLOCK_SIZE
+                    };
+            }
+            
+            var parts = Blocks(path).ToArray();
+            var definition = new CDNFileDefinition
+            {
+                OriginalFileName = path.FileName, 
+                Size = path.Size, 
+                Hash = await path.FileHashCachedAsync(),
+                Parts = await parts.PMap(queue, async part =>
+                {
+                    progressFn("Hashing file parts", Percent.FactoryPutInRange(part.Index, parts.Length));
+                    var buffer = new byte[part.Size];
+                    await using (var fs = path.OpenShared())
+                    {
+                        fs.Position = part.Offset;
+                        await fs.ReadAsync(buffer);
+                    }
+                    part.Hash = buffer.xxHash();
+                    return part;
+                })
+            };
+
+            return definition;
+        }
+
+        public async Task<Uri> UploadFile(WorkQueue queue, AbsolutePath path, Action<string, Percent> progressFn)
+        {
+            var definition = await GenerateFileDefinition(queue, path, progressFn);
+
+            using (var result = await _client.PutAsync($"{Consts.WabbajackBuildServerUri}authored_files/create",
+                new StringContent(definition.ToJson())))
+            {
+                progressFn("Starting upload", Percent.Zero);
+                definition.ServerAssignedUniqueId = await result.Content.ReadAsStringAsync();
+            }
+
+            var results = await definition.Parts.PMap(queue, async part =>
+            {
+                progressFn("Uploading Part", Percent.FactoryPutInRange(part.Index, definition.Parts.Length));
+                var buffer = new byte[part.Size];
+                await using (var fs = path.OpenShared())
+                {
+                    fs.Position = part.Offset;
+                    await fs.ReadAsync(buffer);
+                }
+
+                int retries = 0;
+                while (true)
+                {
+                    try
+                    {
+                        using var putResult = await _client.PutAsync(
+                            $"{Consts.WabbajackBuildServerUri}authored_files/{definition.ServerAssignedUniqueId}/part/{part.Index}",
+                            new ByteArrayContent(buffer));
+                        var hash = Hash.FromBase64(await putResult.Content.ReadAsStringAsync());
+                        if (hash != part.Hash)
+                            throw new InvalidDataException("Hashes don't match");
+                        return hash;
+                    }
+                    catch (Exception ex)
+                    {
+                        Utils.Log("Failure uploading part");
+                        Utils.Log(ex.ToString());
+                        if (retries <= 4)
+                        {
+                            retries++;
+                            continue;
+                        }
+                        Utils.ErrorThrow(ex);
+                    }
+                }
+            });
+            
+            progressFn("Finalizing upload", Percent.Zero);
+            using (var result = await _client.PutAsync($"{Consts.WabbajackBuildServerUri}authored_files/{definition.ServerAssignedUniqueId}/finish",
+                new StringContent(definition.ToJson())))
+            {
+                progressFn("Finished", Percent.One);
+                return new Uri(await result.Content.ReadAsStringAsync());
+            }
+        }
+    }
+}

Wabbajack.Lib/Downloaders/DownloadDispatcher.cs

@@ -25,6 +25,7 @@ namespace Wabbajack.Lib.Downloaders
             new BethesdaNetDownloader(),
             new TESAllianceDownloader(),
             new YouTubeDownloader(),
+            new WabbajackCDNDownloader(),
             new HTTPDownloader(),
             new ManualDownloader(),
         };
@@ -32,7 +33,8 @@ namespace Wabbajack.Lib.Downloaders
         public static readonly List<IUrlInferencer> Inferencers = new List<IUrlInferencer>()
         {
             new BethesdaNetInferencer(),
-            new YoutubeInferencer()
+            new YoutubeInferencer(),
+            new WabbajackCDNInfluencer()
         };
 
         private static readonly Dictionary<Type, IDownloader> IndexedDownloaders;

Wabbajack.Lib/Downloaders/UrlDownloaders/WabbajackCDNInfluencer.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Threading.Tasks;
+
+namespace Wabbajack.Lib.Downloaders.UrlDownloaders
+{
+    public class WabbajackCDNInfluencer : IUrlInferencer
+    {
+        public async Task<AbstractDownloadState?> Infer(Uri uri)
+        {
+            return WabbajackCDNDownloader.StateFromUrl(uri);
+        }
+    }
+}

Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs

@@ -0,0 +1,100 @@
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.Common.Serialization.Json;
+using Wabbajack.Lib.AuthorApi;
+using Wabbajack.Lib.Downloaders.UrlDownloaders;
+using Wabbajack.Lib.Exceptions;
+using Wabbajack.Lib.Validation;
+
+namespace Wabbajack.Lib.Downloaders
+{
+    public class WabbajackCDNDownloader : IDownloader
+    {
+        public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode = false)
+        {
+            var url = (Uri)DownloaderUtils.GetDirectURL(archiveINI);
+            return url == null ? null : StateFromUrl(url);
+        }
+
+        public async Task Prepare()
+        {
+        }
+
+       
+        public static AbstractDownloadState? StateFromUrl(Uri url)
+        {
+            if (url.Host == "wabbajacktest.b-cdn.net" || url.Host == "wabbajack.b-cdn.net")
+            {
+                return new State(url);
+            }
+            return null;
+        }
+
+        [JsonName("WabbajackCDNDownloader+State")]
+        public class State : AbstractDownloadState
+        {
+            public Uri Url { get; set; }
+            public State(Uri url)
+            {
+                Url = url;
+            }
+
+            public override object[] PrimaryKey => new object[] {Url};
+            public override bool IsWhitelisted(ServerWhitelist whitelist)
+            {
+                return true;
+            }
+
+            public override async Task<bool> Download(Archive a, AbsolutePath destination)
+            {
+                var definition = await GetDefinition();
+                await using var fs = destination.Create();
+                var client = new Common.Http.Client();
+                await definition.Parts.DoProgress($"Downloading {a.Name}", async part =>
+                {
+                    fs.Position = part.Offset;
+                    using var response = await client.GetAsync($"{Url}/parts/{part.Index}");
+                    if (!response.IsSuccessStatusCode)
+                        throw new HttpException((int)response.StatusCode, response.ReasonPhrase);
+                    await response.Content.CopyToAsync(fs);
+                });
+                return true;
+            }
+
+            public override async Task<bool> Verify(Archive archive)
+            {
+                var definition = await GetDefinition();
+                return true;
+            }
+
+            private async Task<CDNFileDefinition> GetDefinition()
+            {
+                var client = new Common.Http.Client();
+                using var data = await client.GetAsync(Url + "/definition.json.gz");
+                await using var gz = new GZipStream(await data.Content.ReadAsStreamAsync(), CompressionMode.Decompress);
+                return gz.FromJson<CDNFileDefinition>();
+            }
+
+            public override IDownloader GetDownloader()
+            {
+                return DownloadDispatcher.GetInstance<WabbajackCDNDownloader>();
+            }
+
+            public override string? GetManifestURL(Archive a)
+            {
+                return Url.ToString();
+            }
+
+            public override string[] GetMetaIni()
+            {
+                return new[] {"[General]", $"directURL={Url}"};
+            }
+        }
+
+
+    }
+}

Wabbajack.Lib/FileUploader/AuthorAPI.cs

@@ -29,100 +29,6 @@ namespace Wabbajack.Lib.FileUploader
             return apiKey ?? (await Consts.LocalAppDataPath.Combine(Consts.AuthorAPIKeyFile).ReadAllTextAsync()).Trim();
         }
  
-        public static Uri UploadURL => new Uri($"{Consts.WabbajackBuildServerUri}upload_file");
-        public static long BLOCK_SIZE = (long)1024 * 1024 * 2;
-        public static int MAX_CONNECTIONS = 8;
-        public static Task<string> UploadFile(AbsolutePath filename, Action<double> progressFn, string? apikey = null)
-        {
-            var tcs = new TaskCompletionSource<string>();
-            Task.Run(async () =>
-            {
-                var client = await GetAuthorizedClient(apikey);
-
-                var fsize = filename.Size;
-                var hashTask = filename.FileHashAsync();
-
-                Utils.Log($"{UploadURL}/{filename.FileName.ToString()}/start");
-                using var response = await client.PutAsync($"{UploadURL}/{filename.FileName.ToString()}/start", new StringContent(""));
-                if (!response.IsSuccessStatusCode)
-                {
-                    Utils.Log("Error starting upload");
-                    Utils.Log(await response.Content.ReadAsStringAsync());
-                    tcs.SetException(new Exception($"Start Error: {response.StatusCode} {response.ReasonPhrase}"));
-                    return;
-                }
-
-                IEnumerable<long> Blocks(long fsize)
-                {
-                    for (long block = 0; block * BLOCK_SIZE < fsize; block ++)
-                        yield return block;
-                }
-
-                var key = await response.Content.ReadAsStringAsync();
-                long sent = 0;
-                using (var iqueue = new WorkQueue(MAX_CONNECTIONS))
-                {
-                    iqueue.Report("Starting Upload", Percent.One);
-                await Blocks(fsize)
-                    .PMap(iqueue, async blockIdx =>
-                    {
-                        if (tcs.Task.IsFaulted) return;
-                        var blockOffset = blockIdx * BLOCK_SIZE;
-                        var blockSize = blockOffset + BLOCK_SIZE > fsize
-                            ? fsize - blockOffset
-                            : BLOCK_SIZE;
-                        Interlocked.Add(ref sent, blockSize);
-                        progressFn((double)sent / fsize);
-
-                        var data = new byte[blockSize];
-                        await using (var fs = filename.OpenRead())
-                        {
-                            fs.Position = blockOffset;
-                            await fs.ReadAsync(data, 0, data.Length);
-                        }
-
-
-                        var offsetResponse = await client.PutAsync(UploadURL + $"/{key}/data/{blockOffset}",
-                            new ByteArrayContent(data));
-
-                        if (!offsetResponse.IsSuccessStatusCode)
-                        {
-                            Utils.Log(await offsetResponse.Content.ReadAsStringAsync());
-                            tcs.SetException(new Exception($"Put Error: {offsetResponse.StatusCode} {offsetResponse.ReasonPhrase}"));
-                            return;
-                        }
-
-                        var val = long.Parse(await offsetResponse.Content.ReadAsStringAsync());
-                        if (val != blockOffset + data.Length)
-                        {
-                            tcs.SetResult($"Sync Error {val} vs {blockOffset + data.Length} Offset {blockOffset} Size {data.Length}");
-                            tcs.SetException(new Exception($"Sync Error {val} vs {blockOffset + data.Length}"));
-                        }
-                    });
-                }
-
-                if (!tcs.Task.IsFaulted)
-                {
-                    progressFn(1.0);
-                    var hash = (await hashTask).ToHex();
-                    using var finalResponse = await client.PutAsync(UploadURL + $"/{key}/finish/{hash}", new StringContent(""));
-                    if (finalResponse.IsSuccessStatusCode)
-                        tcs.SetResult(await finalResponse.Content.ReadAsStringAsync());
-                    else
-                    {
-                        Utils.Log("Finalization Error: ");
-                        Utils.Log(await finalResponse.Content.ReadAsStringAsync());
-                        tcs.SetException(new Exception(
-                            $"Finalization Error: {finalResponse.StatusCode} {finalResponse.ReasonPhrase}"));
-                    }
-                }
-
-                progressFn(0.0);
-
-            });
-            return tcs.Task;
-        }
-
         public static async Task<Common.Http.Client> GetAuthorizedClient(string? apiKey = null)
         {
             var client = new Common.Http.Client();

Wabbajack.Server.Test/ABuildServerSystemTest.cs

@@ -161,9 +161,9 @@ namespace Wabbajack.BuildServer.Test
         }
 
 
-        protected byte[] RandomData()
+        protected byte[] RandomData(long? size = null)
         {
-            var arr = new byte[_random.Next(1024)];
+            var arr = new byte[size ?? _random.Next(1024)];
             _random.NextBytes(arr);
             return arr;
         }

Wabbajack.Server.Test/ADBTest.cs

@@ -26,7 +26,7 @@ namespace Wabbajack.BuildServer.Test
         {
             DBName = "test_db" + Guid.NewGuid().ToString().Replace("-", "_");
             User = Guid.NewGuid().ToString().Replace("-", "");
-            //APIKey = SqlService.NewAPIKey();
+            APIKey = SqlService.NewAPIKey();
         }
 
         public string APIKey { get; }

Wabbajack.Server.Test/AuthoredFilesTests.cs

@@ -0,0 +1,38 @@
+using System;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Wabbajack.Lib.AuthorApi;
+using Wabbajack.Lib.Downloaders;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Wabbajack.BuildServer.Test
+{
+    public class AuthoredFilesTests : ABuildServerSystemTest
+    {
+        public AuthoredFilesTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
+        {
+        }
+
+        [Fact]
+        public async Task CanUploadDownloadAndDeleteAuthoredFiles()
+        {
+            using var file = new TempFile();
+            await file.Path.WriteAllBytesAsync(RandomData(Consts.UPLOADED_FILE_BLOCK_SIZE * 4 + Consts.UPLOADED_FILE_BLOCK_SIZE / 3));
+            var originalHash = await file.Path.FileHashAsync();
+
+            var client = await Client.Create(Fixture.APIKey);
+            using var queue = new WorkQueue(2);
+            var uri = await client.UploadFile(queue, file.Path, (s, percent) => Utils.Log($"({percent}) {s}"));
+
+            var state = await DownloadDispatcher.Infer(uri);
+            Assert.IsType<WabbajackCDNDownloader.State>(state);
+
+            await state.Download(new Archive(state) {Name = (string)file.Path.FileName}, file.Path);
+            Assert.Equal(originalHash, await file.Path.FileHashAsync());
+
+        }
+
+    }
+}

Wabbajack.Server.Test/LoginTests.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Threading.Tasks;
+using Wabbajack.Common;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Wabbajack.BuildServer.Test
+{
+    public class LoginTests : ABuildServerSystemTest
+    {
+        public LoginTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
+        {
+        }
+
+        [Fact]
+        public async Task CanCreateLogins()
+        {
+            var newUserName = Guid.NewGuid().ToString();
+
+            var newKey = await _authedClient.GetStringAsync(MakeURL($"users/add/{newUserName}"));
+            
+            Assert.NotEmpty(newKey);
+            Assert.NotNull(newKey);
+            Assert.NotEqual(newKey, Fixture.APIKey);
+
+
+            var done = await _authedClient.GetStringAsync(MakeURL("users/export"));
+            Assert.Equal("done", done);
+            
+            foreach (var (userName, apiKey) in new[] {(newUserName, newKey), (Fixture.User, Fixture.APIKey)})
+            {
+                var exported = await Fixture.ServerTempFolder.Combine("exported_users", userName, Consts.AuthorAPIKeyFile)
+                    .ReadAllTextAsync();
+                Assert.Equal(exported, apiKey);
+
+            }
+        }
+    }
+}

Wabbajack.Server.Test/sql/wabbajack_db.sql

@@ -403,6 +403,20 @@ CREATE TABLE [dbo].[Metrics](
                                         )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
 ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
 GO
+
+/****** Object:  Table [dbo].[AuthoredFiles]    Script Date: 5/9/2020 2:22:00 PM ******/
+CREATE TABLE [dbo].[AuthoredFiles](
+                                      [ServerAssignedUniqueId] [uniqueidentifier] NOT NULL,
+                                      [LastTouched] [datetime] NOT NULL,
+                                      [CDNFileDefinition] [nvarchar](max) NOT NULL,
+                                      [Finalized] [datetime] NULL,
+                                      CONSTRAINT [PK_AuthoredFiles] PRIMARY KEY CLUSTERED
+                                          (
+                                           [ServerAssignedUniqueId] ASC
+                                              )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
+GO
+
 /****** Uploaded Files [UploadedFiles] *************/
 
 CREATE TABLE [dbo].[UploadedFiles](

Wabbajack.Server/ApiKeyAuthorizationHandler.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Security.Claims;
+using System.Text.Encodings.Web;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authentication;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Newtonsoft.Json;
+using Wabbajack.Server.DataLayer;
+
+
+namespace Wabbajack.BuildServer
+{
+
+    public class ApiKeyAuthenticationOptions : AuthenticationSchemeOptions
+    {
+        public const string DefaultScheme = "API Key";
+        public string Scheme => DefaultScheme;
+        public string AuthenticationType = DefaultScheme;
+    }
+
+    public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthenticationOptions>
+    {
+        private const string ProblemDetailsContentType = "application/problem+json";
+        private readonly SqlService _sql;
+        private const string ApiKeyHeaderName = "X-Api-Key";
+
+        public ApiKeyAuthenticationHandler(
+            IOptionsMonitor<ApiKeyAuthenticationOptions> options,
+            ILoggerFactory logger,
+            UrlEncoder encoder,
+            ISystemClock clock,
+            SqlService db) : base(options, logger, encoder, clock)
+        {
+            _sql = db;
+        }
+
+        protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
+        {
+            if (!Request.Headers.TryGetValue(ApiKeyHeaderName, out var apiKeyHeaderValues))
+            {
+                return AuthenticateResult.NoResult();
+            }
+
+            var providedApiKey = apiKeyHeaderValues.FirstOrDefault();
+
+            if (apiKeyHeaderValues.Count == 0 || string.IsNullOrWhiteSpace(providedApiKey))
+            {
+                return AuthenticateResult.NoResult();
+            }
+
+            var owner = await _sql.LoginByApiKey(providedApiKey);
+
+            if (owner != null)
+            {
+                var claims = new List<Claim> {new Claim(ClaimTypes.Name, owner)};
+
+                /*
+                claims.AddRange(existingApiKey.Roles.Select(role => new Claim(ClaimTypes.Role, role)));
+                */
+
+                var identity = new ClaimsIdentity(claims, Options.AuthenticationType);
+                var identities = new List<ClaimsIdentity> {identity};
+                var principal = new ClaimsPrincipal(identities);
+                var ticket = new AuthenticationTicket(principal, Options.Scheme);
+
+                return AuthenticateResult.Success(ticket);
+            }
+
+            return AuthenticateResult.Fail("Invalid API Key provided.");
+        }
+
+        protected override async Task HandleChallengeAsync(AuthenticationProperties properties)
+        {
+            Response.StatusCode = 401;
+            Response.ContentType = ProblemDetailsContentType;
+            await Response.WriteAsync("Unauthorized");
+        }
+
+        protected override async Task HandleForbiddenAsync(AuthenticationProperties properties)
+        {
+            Response.StatusCode = 403;
+            Response.ContentType = ProblemDetailsContentType;
+            await Response.WriteAsync("forbidden");
+        }
+    }
+
+    public static class ApiKeyAuthorizationHandlerExtensions
+    {
+        public static AuthenticationBuilder AddApiKeySupport(this AuthenticationBuilder authenticationBuilder, Action<ApiKeyAuthenticationOptions> options)
+        {
+            return authenticationBuilder.AddScheme<ApiKeyAuthenticationOptions, ApiKeyAuthenticationHandler>(ApiKeyAuthenticationOptions.DefaultScheme, options);
+        }
+
+    }
+}

Wabbajack.Server/AppSettings.cs

@@ -25,8 +25,7 @@ namespace Wabbajack.BuildServer
         public bool RunFrontEndJobs { get; set; }
         public bool RunBackEndJobs { get; set; }
         
-        public string BunnyCDN_User { get; set; }
+        public string BunnyCDN_StorageZone { get; set; }
-        public string BunnyCDN_Password { get; set; }
         public string SqlConnection { get; set; }
 
         public int MaxJobs { get; set; } = 2;

Wabbajack.Server/Controllers/AuthoredFiles.cs

@@ -0,0 +1,143 @@
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Net;
+using System.Security.Claims;
+using System.Threading.Tasks;
+using FluentFTP;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using SharpCompress.Compressors.LZMA;
+using Wabbajack.Common;
+using Wabbajack.Lib.AuthorApi;
+using Wabbajack.Server.DataLayer;
+using Wabbajack.Server.DTOs;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [Route("/authored_files")]
+    public class AuthoredFiles : ControllerBase
+    {
+        private SqlService _sql;
+        private ILogger<AuthoredFiles> _logger;
+        private AppSettings _settings;
+
+        public AuthoredFiles(ILogger<AuthoredFiles> logger, SqlService sql, AppSettings settings)
+        {
+            _sql = sql;
+            _logger = logger;
+            _settings = settings;
+        }
+
+        [HttpPut]
+        [Route("{serverAssignedUniqueId}/part/{index}")]
+        public async Task<IActionResult> UploadFilePart(string serverAssignedUniqueId, long index)
+        {
+            var user = User.FindFirstValue(ClaimTypes.Name);
+            var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId);
+            if (definition.Author != user)
+                return Forbid("File Id does not match authorized user");
+            _logger.Log(LogLevel.Information, $"Uploading File part {definition.OriginalFileName} - ({index} / {definition.Parts.Length})");
+
+            await _sql.TouchAuthoredFile(definition);
+            var part = definition.Parts[index];
+
+            await using var ms = new MemoryStream();
+            await Request.Body.CopyToLimitAsync(ms, part.Size);
+            ms.Position = 0;
+            if (ms.Length != part.Size)
+                return BadRequest($"Couldn't read enough data for part {part.Size} vs {ms.Length}");
+
+            var hash = ms.xxHash();
+            if (hash != part.Hash)
+                return BadRequest($"Hashes don't match for index {index}. Sizes ({ms.Length} vs {part.Size}). Hashes ({hash} vs {part.Hash}");
+
+            ms.Position = 0;
+            await UploadAsync(ms, $"{definition.MungedName}/parts/{index}");
+            return Ok(part.Hash.ToBase64());
+        }
+        
+        [HttpPut]
+        [Route("create")]
+        public async Task<IActionResult> CreateUpload()
+        {
+            var user = User.FindFirstValue(ClaimTypes.Name);
+
+            var data = await Request.Body.ReadAllTextAsync();
+            var definition = data.FromJsonString<CDNFileDefinition>();
+            
+            _logger.Log(LogLevel.Information, $"Creating File upload {definition.OriginalFileName}");
+
+            definition = await _sql.CreateAuthoredFile(definition, user);
+            
+            return Ok(definition.ServerAssignedUniqueId);
+        }
+        
+        [HttpPut]
+        [Route("{serverAssignedUniqueId}/finish")]
+        public async Task<IActionResult> CreateUpload(string serverAssignedUniqueId)
+        {
+            var user = User.FindFirstValue(ClaimTypes.Name);
+            var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId);
+            if (definition.Author != user)
+                return Forbid("File Id does not match authorized user");
+            _logger.Log(LogLevel.Information, $"Finalizing file upload {definition.OriginalFileName}");
+
+            await _sql.Finalize(definition);
+            
+            await using var ms = new MemoryStream();
+            await using (var gz = new GZipStream(ms, CompressionLevel.Optimal, true))
+            {
+                definition.ToJson(gz);
+            }
+            ms.Position = 0;
+            await UploadAsync(ms, $"{definition.MungedName}/definition.json.gz");
+            
+            return Ok($"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{definition.MungedName}");
+        }
+
+        private async Task<FtpClient> GetBunnyCdnFtpClient()
+        {
+            var info = Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");
+            var client = new FtpClient(info.Hostname) {Credentials = new NetworkCredential(info.Username, info.Password)};
+            await client.ConnectAsync();
+            return client;
+        }
+
+        private async Task UploadAsync(Stream stream, string path)
+        {
+            using var client = await GetBunnyCdnFtpClient();
+            await client.UploadAsync(stream, path);
+        }
+
+        [HttpDelete]
+        [Route("{serverAssignedUniqueId}")]
+        public async Task<IActionResult> DeleteUpload(string serverAssignedUniqueId)
+        {
+            var user = User.FindFirstValue(ClaimTypes.Name);
+            var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId);
+            if (definition.Author != user)
+                return Forbid("File Id does not match authorized user");
+            _logger.Log(LogLevel.Information, $"Finalizing file upload {definition.OriginalFileName}");
+
+            await DeleteFolderOrSilentlyFail($"{definition.MungedName}");
+
+            await _sql.DeleteFileDefinition(definition);
+            return Ok();
+        }
+
+        private async Task DeleteFolderOrSilentlyFail(string path)
+        {
+            try
+            {
+                using var client = await GetBunnyCdnFtpClient();
+                await client.DeleteDirectoryAsync(path);
+            }
+            catch (Exception)
+            {
+                _logger.Log(LogLevel.Information, $"Delete failed for {path}");
+            }
+        }
+
+    }
+}

Wabbajack.Server/Controllers/UploadedFiles.cs

@@ -0,0 +1,11 @@
+using Microsoft.AspNetCore.Mvc;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    
+    [ApiController]
+    public class UploadedFiles
+    {
+        
+    }
+}

Wabbajack.Server/Controllers/Users.cs

@@ -0,0 +1,51 @@
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using Wabbajack.Common;
+using Wabbajack.Server.DataLayer;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    [Authorize]
+    [Route("/users")]
+    public class Users : ControllerBase
+    {
+        private AppSettings _settings;
+        private ILogger<Users> _logger;
+        private SqlService _sql;
+
+        public Users(ILogger<Users> logger, SqlService sql, AppSettings settings)
+        {
+            _settings = settings;
+            _logger = logger;
+            _sql = sql;
+        }
+        
+        [HttpGet]
+        [Route("add/{Name}")]
+        public async Task<string> AddUser(string Name)
+        {
+            return await _sql.AddLogin(Name);
+        }
+
+        [HttpGet]
+        [Route("export")]
+        public async Task<string> Export()
+        {
+            var mainFolder = _settings.TempPath.Combine("exported_users");
+            mainFolder.CreateDirectory();
+
+            foreach (var (owner, key) in await _sql.GetAllUserKeys())
+            {
+                var folder = mainFolder.Combine(owner);
+                folder.CreateDirectory();
+                await folder.Combine(Consts.AuthorAPIKeyFile).WriteAllTextAsync(key);
+            }
+
+            return "done";
+        }
+
+    }
+
+}

Wabbajack.Server/DTOs/BunnyCdnFtpInfo.cs

@@ -0,0 +1,9 @@
+namespace Wabbajack.Server.DTOs
+{
+    public class BunnyCdnFtpInfo
+    {
+        public string Username { get; set; }
+        public string Password { get; set; }
+        public string Hostname { get; set; }
+    }
+}

Wabbajack.Server/DataLayer/ApiKeys.cs

@@ -0,0 +1,48 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Dapper;
+using Wabbajack.Common;
+
+namespace Wabbajack.Server.DataLayer
+{
+    public partial class SqlService
+    {
+        public async Task<string> LoginByApiKey(string key)
+        {
+            await using var conn = await Open();
+            var result = await conn.QueryAsync<string>(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @ApiKey",
+                new {ApiKey = key});
+            return result.FirstOrDefault();
+        }
+        
+        public async Task<string> AddLogin(string name)
+        {
+            var key = NewAPIKey();
+            await using var conn = await Open();
+
+
+            await conn.ExecuteAsync("INSERT INTO dbo.ApiKeys (Owner, ApiKey) VALUES (@Owner, @ApiKey)",
+                new {Owner = name, ApiKey = key});
+            return key;
+        }
+        
+                
+        public static string NewAPIKey()
+        {
+            var arr = new byte[128];
+            new Random().NextBytes(arr);
+            return arr.ToHex();
+        }
+        
+        public async Task<IEnumerable<(string Owner, string Key)>> GetAllUserKeys()
+        {
+            await using var conn = await Open();
+            var result = await conn.QueryAsync<(string Owner, string Key)>("SELECT Owner, ApiKey FROM dbo.ApiKeys");
+            return result;
+        }
+
+
+    }
+}

Wabbajack.Server/DataLayer/AuthoredFiles.cs

@@ -0,0 +1,62 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Dapper;
+using Wabbajack.Common;
+using Wabbajack.Lib.AuthorApi;
+
+namespace Wabbajack.Server.DataLayer
+{
+    public partial class SqlService
+    {
+        public async Task TouchAuthoredFile(CDNFileDefinition definition)
+        {
+            await using var conn = await Open();
+            await conn.ExecuteAsync("UPDATE AuthoredFiles SET LastTouched = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid",
+                new {
+                    Uid = definition.ServerAssignedUniqueId
+                });
+        }
+
+        public async Task<CDNFileDefinition> CreateAuthoredFile(CDNFileDefinition definition, string login)
+        {
+            definition.Author = login;
+            var uid = Guid.NewGuid().ToString();
+            await using var conn = await Open();
+            definition.ServerAssignedUniqueId = uid;
+            await conn.ExecuteAsync("INSERT INTO dbo.AuthoredFiles (ServerAssignedUniqueId, LastTouched, CDNFileDefinition) VALUES (@Uid, GETUTCDATE(), @CdnFile)",
+                new {
+                    Uid = uid,
+                    CdnFile = definition
+                });
+            return definition;
+        }
+
+        public async Task Finalize(CDNFileDefinition definition)
+        {
+            await using var conn = await Open();
+            await conn.ExecuteAsync("UPDATE AuthoredFiles SET LastTouched = GETUTCDATE(), Finalized = GETUTCDATE() WHERE ServerAssignedUniqueId = @Uid",
+                new {
+                    Uid = definition.ServerAssignedUniqueId
+                });
+        }
+
+        public async Task<CDNFileDefinition> GetCDNFileDefinition(string serverAssignedUniqueId)
+        {
+            await using var conn = await Open();
+            return (await conn.QueryAsync<CDNFileDefinition>(
+                "SELECT CDNFileDefinition FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid",
+                new {Uid = serverAssignedUniqueId})).First();
+        }
+        
+        public async Task<CDNFileDefinition> DeleteFileDefinition(CDNFileDefinition definition)
+        {
+            await using var conn = await Open();
+            return (await conn.QueryAsync<CDNFileDefinition>(
+                "DELETE FROM dbo.AuthoredFiles WHERE ServerAssignedUniqueID = @Uid",
+                new {Uid = definition.ServerAssignedUniqueId})).First();
+        }
+
+        
+    }
+}

Wabbajack.Server/DataLayer/Mappers.cs

@@ -0,0 +1,87 @@
+using System;
+using System.Data;
+using Dapper;
+using Wabbajack.Common;
+using Wabbajack.Lib.AuthorApi;
+using Wabbajack.Lib.Downloaders;
+
+namespace Wabbajack.Server.DataLayer
+{
+    public partial class SqlService
+    {
+        static SqlService()
+        {
+            SqlMapper.AddTypeHandler(new HashMapper());
+            SqlMapper.AddTypeHandler(new RelativePathMapper());
+            SqlMapper.AddTypeHandler(new JsonMapper<AbstractDownloadState>());
+            SqlMapper.AddTypeHandler(new JsonMapper<CDNFileDefinition>());
+            SqlMapper.AddTypeHandler(new VersionMapper());
+            SqlMapper.AddTypeHandler(new GameMapper());
+        }
+
+        class JsonMapper<T> : SqlMapper.TypeHandler<T>
+        {
+            public override void SetValue(IDbDataParameter parameter, T value)
+            {
+                parameter.Value = value.ToJson();
+            }
+
+            public override T Parse(object value)
+            {
+                return ((string)value).FromJsonString<T>();
+            }
+        }
+
+        class RelativePathMapper : SqlMapper.TypeHandler<RelativePath>
+        {
+            public override void SetValue(IDbDataParameter parameter, RelativePath value)
+            {
+                parameter.Value = value.ToJson();
+            }
+
+            public override RelativePath Parse(object value)
+            {
+                return (RelativePath)(string)value;
+            }
+        }
+        
+        class HashMapper : SqlMapper.TypeHandler<Hash>
+        {
+            public override void SetValue(IDbDataParameter parameter, Hash value)
+            {
+                parameter.Value = (long)value;
+            }
+
+            public override Hash Parse(object value)
+            {
+                return Hash.FromLong((long)value);
+            }
+        }
+        
+        class VersionMapper : SqlMapper.TypeHandler<Version>
+        {
+            public override void SetValue(IDbDataParameter parameter, Version value)
+            {
+                parameter.Value = value.ToString();
+            }
+
+            public override Version Parse(object value)
+            {
+                return Version.Parse((string)value);
+            }
+        }
+        
+        class GameMapper : SqlMapper.TypeHandler<Game>
+        {
+            public override void SetValue(IDbDataParameter parameter, Game value)
+            {
+                parameter.Value = value.ToString();
+            }
+
+            public override Game Parse(object value)
+            {
+                return GameRegistry.GetByFuzzyName((string)value).Game;
+            }
+        }
+    }
+}

Wabbajack.Server/Services/NexusPoll.cs

@@ -120,6 +120,8 @@ namespace Wabbajack.Server.Services
             });
 
             _logger.Log(LogLevel.Information, $"Purged {purged.Sum()} cache entries");
+            _globalInformation.LastNexusSyncUTC = DateTime.UtcNow;
+
         }
 
         public void Start()

Wabbajack.Server/Startup.cs

@@ -40,6 +40,13 @@ namespace Wabbajack.Server
                 c.SwaggerDoc("v1", new OpenApiInfo {Title = "Wabbajack Build API", Version = "v1"});
             });
             
+            services.AddAuthentication(options =>
+                {
+                    options.DefaultAuthenticateScheme = ApiKeyAuthenticationOptions.DefaultScheme;
+                    options.DefaultChallengeScheme = ApiKeyAuthenticationOptions.DefaultScheme;
+                })
+                .AddApiKeySupport(options => {});
+            
             services.Configure<FormOptions>(x =>
             {
                 x.ValueLengthLimit = int.MaxValue;

Wabbajack.Server/appsettings.json

@@ -14,8 +14,7 @@
     "JobScheduler": false,
     "RunFrontEndJobs": true,
     "RunBackEndJobs": false,
-    "BunnyCDN_User": "wabbajackcdn",
+    "BunnyCDN_StorageZone": "wabbajacktest",
-    "BunnyCDN_Password": "XXXX",
     "SQLConnection": "Data Source=.\\SQLEXPRESS;Integrated Security=True;Initial Catalog=wabbajack_prod;MultipleActiveResultSets=true"
   },
   "AllowedHosts": "*"

Wabbajack/View Models/Settings/AuthorFilesVM.cs

@@ -6,6 +6,7 @@ using System.Windows.Input;
 using ReactiveUI;
 using ReactiveUI.Fody.Helpers;
 using Wabbajack.Common;
+using Wabbajack.Lib.AuthorApi;
 using Wabbajack.Lib.FileUploader;
 
 namespace Wabbajack
@@ -49,8 +50,14 @@ namespace Wabbajack
                 _isUploading.OnNext(true);
                 try
                 {
-                    FinalUrl = await AuthorAPI.UploadFile(Picker.TargetPath,
+                    using var queue = new WorkQueue();
-                        progress => UploadProgress = progress);
+                    var result = await (await Client.Create()).UploadFile(queue, Picker.TargetPath,
+                        (msg, progress) =>
+                        {
+                            FinalUrl = msg;
+                            UploadProgress = (double)progress;
+                        });
+                    FinalUrl = result.ToString();
                 }
                 catch (Exception ex)
                 {