Start on file upload support

Wabbajack.BuildServer/ApiKeyAuthenticationHandler.cs

@@ -0,0 +1,87 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Security.Claims;
+using System.Text.Encodings.Web;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authentication;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Newtonsoft.Json;
+using Wabbajack.BuildServer.Models;
+
+namespace Wabbajack.BuildServer
+{
+
+    public class ApiKeyAuthenticationOptions : AuthenticationSchemeOptions
+    {
+        public const string DefaultScheme = "API Key";
+        public string Scheme => DefaultScheme;
+        public string AuthenticationType = DefaultScheme;
+    }
+
+    public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthenticationOptions>
+    {
+        private const string ProblemDetailsContentType = "application/problem+json";
+        private readonly DBContext _db;
+        private const string ApiKeyHeaderName = "X-Api-Key";
+
+        public ApiKeyAuthenticationHandler(
+            IOptionsMonitor<ApiKeyAuthenticationOptions> options,
+            ILoggerFactory logger,
+            UrlEncoder encoder,
+            ISystemClock clock,
+            DBContext db) : base(options, logger, encoder, clock)
+        {
+            _db = db;
+        }
+
+        protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
+        {
+            if (!Request.Headers.TryGetValue(ApiKeyHeaderName, out var apiKeyHeaderValues))
+            {
+                return AuthenticateResult.NoResult();
+            }
+
+            var providedApiKey = apiKeyHeaderValues.FirstOrDefault();
+
+            if (apiKeyHeaderValues.Count == 0 || string.IsNullOrWhiteSpace(providedApiKey))
+            {
+                return AuthenticateResult.NoResult();
+            }
+
+            var existingApiKey = await ApiKey.Get(_db, providedApiKey);
+
+            if (existingApiKey != null)
+            {
+                var claims = new List<Claim> {new Claim(ClaimTypes.Name, existingApiKey.Owner)};
+
+                claims.AddRange(existingApiKey.Roles.Select(role => new Claim(ClaimTypes.Role, role)));
+
+                var identity = new ClaimsIdentity(claims, Options.AuthenticationType);
+                var identities = new List<ClaimsIdentity> {identity};
+                var principal = new ClaimsPrincipal(identities);
+                var ticket = new AuthenticationTicket(principal, Options.Scheme);
+
+                return AuthenticateResult.Success(ticket);
+            }
+
+            return AuthenticateResult.Fail("Invalid API Key provided.");
+        }
+
+        protected override async Task HandleChallengeAsync(AuthenticationProperties properties)
+        {
+            Response.StatusCode = 401;
+            Response.ContentType = ProblemDetailsContentType;
+            await Response.WriteAsync("Unauthorized");
+        }
+
+        protected override async Task HandleForbiddenAsync(AuthenticationProperties properties)
+        {
+            Response.StatusCode = 403;
+            Response.ContentType = ProblemDetailsContentType;
+            await Response.WriteAsync("forbidden");
+        }
+    }
+}

Wabbajack.BuildServer/AppSettings.cs

@@ -11,5 +11,7 @@ namespace Wabbajack.BuildServer
         
         public string DownloadDir { get; set; }
         public string ArchiveDir { get; set; }
+        
+        public bool MinimalMode { get; set; }
     }
 }

Wabbajack.BuildServer/Controllers/Heartbeat.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Linq;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using Wabbajack.BuildServer.Models;
@@ -27,5 +28,13 @@ namespace Wabbajack.BuildServer.Controllers
         {
             return DateTime.Now - _startTime;
         }
+
+        [HttpGet("only-authenticated")]
+        [Authorize]
+        public IActionResult OnlyAuthenticated()
+        {
+            var message = $"Hello from {nameof(OnlyAuthenticated)}";
+            return new ObjectResult(message);
+        }
     }
 }

Wabbajack.BuildServer/Controllers/Jobs.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using MongoDB.Driver;
@@ -10,6 +11,7 @@ using Wabbajack.BuildServer.Models.JobQueue;
 
 namespace Wabbajack.BuildServer.Controllers
 {
+    [Authorize]
     [ApiController]
     [Route("/jobs")]
     public class Jobs : AControllerBase<Jobs>

Wabbajack.BuildServer/Controllers/UploadedFiles.cs

@@ -0,0 +1,74 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Security.Claims;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Identity;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+using Nettle;
+using Wabbajack.BuildServer.Models;
+using Wabbajack.Common;
+
+namespace Wabbajack.BuildServer.Controllers
+{
+    public class UploadedFiles : AControllerBase<UploadedFiles>
+    {
+        public UploadedFiles(ILogger<UploadedFiles> logger, DBContext db) : base(logger, db)
+        {
+  
+        }
+
+        [HttpPost]
+        [Authorize]
+        [Route("upload_file")]
+        public async Task<IActionResult> UploadFile(IList<IFormFile> files)
+        {
+            var user = User.FindFirstValue(ClaimTypes.Name);
+            foreach (var file in files) 
+                await UploadedFile.Ingest(Db, file, user);
+
+            return Ok();
+        }
+        
+        private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
+            <html><body>
+                <table>
+                {{each $.files }}
+                <tr><td><a href='{{$.Link}}'>{{$.Name}}</a></td><td>{{$.Size}}</td><td>{{$.Date}}</td><td>{{$.Uploader}}</td></tr>
+                {{/each}}
+                </table>
+            </body></html>
+        ");
+
+        [HttpGet]
+        [Route("uploaded_files")]
+        public async Task<ContentResult> UploadedFilesGet()
+        {
+            var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync();
+            var response = HandleGetListTemplate(new
+            {
+                files = files.Select(file => new
+                {
+                    Link = file.Uri,
+                    Size = file.Size.ToFileSizeString(),
+                    file.Name,
+                    Date = file.UploadDate,
+                    file.Uploader
+                })
+                
+            });
+            return new ContentResult
+            {
+                ContentType = "text/html",
+                StatusCode = (int) HttpStatusCode.OK,
+                Content = response
+            };
+        }
+    }
+}

Wabbajack.BuildServer/Extensions.cs

@@ -1,12 +1,15 @@
 using System;
+using System.IO;
 using System.Linq;
 using System.Linq.Expressions;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.Extensions.Logging;
 using MongoDB.Driver;
 using MongoDB.Driver.Linq;
 using Wabbajack.Common;
+using File = Alphaleonis.Win32.Filesystem.File;
 
 namespace Wabbajack.BuildServer
 {
@@ -24,5 +27,22 @@ namespace Wabbajack.BuildServer
 
             manager.StartJobRunners();
         }
+        
+        public static async Task CopyFileAsync(string sourcePath, string destinationPath)
+        {
+            using (Stream source = File.OpenRead(sourcePath))
+            {
+                using(Stream destination = File.Create(destinationPath))
+                {
+                    await source.CopyToAsync(destination);
+                }
+            }
+        }
+
+        
+        public static AuthenticationBuilder AddApiKeySupport(this AuthenticationBuilder authenticationBuilder, Action<ApiKeyAuthenticationOptions> options)
+        {
+            return authenticationBuilder.AddScheme<ApiKeyAuthenticationOptions, ApiKeyAuthenticationHandler>(ApiKeyAuthenticationOptions.DefaultScheme, options);
+        }
     }
 }

Wabbajack.BuildServer/GraphQL/Query.cs

@@ -54,6 +54,13 @@ namespace Wabbajack.BuildServer.GraphQL
                     return data;
                 });
             
+            FieldAsync<ListGraphType<UploadedFileType>>("uploadedFiles",
+                resolve: async context =>
+                {
+                    var data = await db.UploadedFiles.AsQueryable().ToListAsync();
+                    return data;
+                });
+
             FieldAsync<ListGraphType<MetricResultType>>("dailyUniqueMetrics",
                 arguments: new QueryArguments(
                     new QueryArgument<MetricEnum> {Name = "metric_type", Description = "The grouping of metric data to query"}

Wabbajack.BuildServer/GraphQL/UploadedFileType.cs

@@ -0,0 +1,22 @@
+using GraphQL.Types;
+using Wabbajack.BuildServer.Models;
+
+namespace Wabbajack.BuildServer.GraphQL
+{
+    public class UploadedFileType : ObjectGraphType<UploadedFile>
+    {
+        public UploadedFileType()
+        {
+            Name = "UploadedFile";
+            Description = "A file uploaded for hosting on Wabbajack's static file hosting";
+            Field(x => x.Id, type: typeof(IdGraphType)).Description("Unique Id of the Job");
+            Field(x => x.Name).Description("Non-unique name of the file");
+            Field(x => x.MungedName, type: typeof(IdGraphType)).Description("Unique file name");
+            Field(x => x.UploadDate, type: typeof(DateGraphType)).Description("Date of the file upload");
+            Field(x => x.Uploader, type: typeof(IdGraphType)).Description("Uploader of the file");
+            Field(x => x.Uri, type: typeof(UriGraphType)).Description("URI of the file");
+            Field(x => x.Hash).Description("xxHash64 of the file");
+            Field(x => x.Size).Description("Size of the file");
+        }
+    }
+}

Wabbajack.BuildServer/JobManager.cs

@@ -26,6 +26,7 @@ namespace Wabbajack.BuildServer
 
         public void StartJobRunners()
         {
+            if (Settings.MinimalMode) return;
             for (var idx = 0; idx < 2; idx++)
             {
                 Task.Run(async () =>
@@ -67,6 +68,7 @@ namespace Wabbajack.BuildServer
         
         public async Task JobScheduler()
         {
+            if (Settings.MinimalMode) return;
             Utils.LogMessages.Subscribe(msg => Logger.Log(LogLevel.Information, msg.ToString()));
             while (true)
             {

Wabbajack.BuildServer/Models/ApiKey.cs

@@ -0,0 +1,23 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using MongoDB.Driver;
+using MongoDB.Driver.Linq;
+
+namespace Wabbajack.BuildServer.Models
+{
+    public class ApiKey
+    {
+        public string Id { get; set; }
+        
+        public string Key { get; set; }
+        public string Owner { get; set; }
+        
+        public List<string> CanUploadLists { get; set; }
+        public List<string> Roles { get; set; }
+
+        public static async Task<ApiKey> Get(DBContext db, string key)
+        {
+            return await db.ApiKeys.AsQueryable().Where(k => k.Key == key).FirstOrDefaultAsync();
+        }
+    }
+}

Wabbajack.BuildServer/Models/DBContext.cs

@@ -28,10 +28,12 @@ namespace Wabbajack.BuildServer.Models
         public IMongoCollection<IndexedFile> IndexedFiles => Client.GetCollection<IndexedFile>(_settings.Collections["IndexedFiles"]);
         public IMongoCollection<NexusCacheData<List<NexusUpdateEntry>>> NexusUpdates => Client.GetCollection<NexusCacheData<List<NexusUpdateEntry>>>(_settings.Collections["NexusUpdates"]);
         
+        public IMongoCollection<ApiKey> ApiKeys => Client.GetCollection<ApiKey>(_settings.Collections["ApiKeys"]);        
+        public IMongoCollection<UploadedFile> UploadedFiles => Client.GetCollection<UploadedFile>(_settings.Collections["UploadedFiles"]);
+
         public IMongoCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>> NexusModFiles =>
             Client.GetCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>>(
                 _settings.Collections["NexusModFiles"]);
-
         private IMongoDatabase Client => new MongoClient($"mongodb://{_settings.Host}").GetDatabase(_settings.Database);
     }
     public class Settings

Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs

@@ -26,7 +26,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
                 var whitelists = new ValidateModlist(queue);
                 await whitelists.LoadListsFromGithub();
                 
-                foreach (var list in modlists)
+                foreach (var list in modlists.Skip(8).Take(1))
                 {
                     try
                     {

Wabbajack.BuildServer/Models/UploadedFile.cs

@@ -0,0 +1,39 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http;
+using MongoDB.Bson.Serialization.Attributes;
+using Wabbajack.Common;
+using Path = Alphaleonis.Win32.Filesystem.Path;
+
+namespace Wabbajack.BuildServer.Models
+{
+    public class UploadedFile
+    {
+        public string Id { get; set; }
+        public string Name { get; set; }
+        public long Size { get; set; }
+        public string Hash { get; set; }
+        public string Uploader { get; set; }
+        public DateTime UploadDate { get; set; } = DateTime.UtcNow;
+
+        [BsonIgnore]
+        public string MungedName => $"{Path.GetFileNameWithoutExtension(Name)}-{Id}-{Path.GetExtension(Name)}";
+
+        [BsonIgnore] public object Uri => $"https://static.wabbajack.org/files/{MungedName}";
+
+        public static async Task<UploadedFile> Ingest(DBContext db, IFormFile src, string uploader)
+        {
+            var record = new UploadedFile {Uploader = uploader, Name = src.FileName, Id = Guid.NewGuid().ToString()};
+            var dest_path =
+                $@"public\\files\\{Path.GetFileNameWithoutExtension(src.FileName)}-{record.Id}{Path.GetExtension(src.FileName)}";
+
+            using (var stream = File.OpenWrite(dest_path)) 
+                await src.CopyToAsync(stream);
+            record.Size = new FileInfo(dest_path).Length;
+            record.Hash = await dest_path.FileHashAsync();
+            await db.UploadedFiles.InsertOneAsync(record);
+            return record;
+        }
+    }
+}

Wabbajack.BuildServer/Program.cs

@@ -21,7 +21,11 @@ namespace Wabbajack.BuildServer
                 .ConfigureWebHostDefaults(webBuilder =>
                 {
                     webBuilder.UseUrls("http://*:5000");
-                    webBuilder.UseStartup<Startup>();
+                    webBuilder.UseStartup<Startup>()
+                        .UseKestrel(options =>
+                        {
+                            options.Limits.MaxRequestBodySize = null;
+                        });
                 });
     }
 }

Wabbajack.BuildServer/Startup.cs

@@ -12,6 +12,12 @@ using Microsoft.AspNetCore.HttpsPolicy;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Authentication.AzureAD.UI;
+using Microsoft.AspNetCore.Authentication.Cookies;
+using Microsoft.AspNetCore.Authentication.OpenIdConnect;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http.Features;
+using Microsoft.AspNetCore.Mvc.Authorization;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
@@ -46,6 +52,19 @@ namespace Wabbajack.BuildServer
                 c.SwaggerDoc("v1", new OpenApiInfo {Title = "Wabbajack Build API", Version = "v1"});
             });
             
+            services.AddAuthentication(options =>
+                {
+                    options.DefaultAuthenticateScheme = ApiKeyAuthenticationOptions.DefaultScheme;
+                    options.DefaultChallengeScheme = ApiKeyAuthenticationOptions.DefaultScheme;
+                })
+                .AddApiKeySupport(options => {});
+
+            services.Configure<FormOptions>(x =>
+            {
+                x.ValueLengthLimit = int.MaxValue;
+                x.MultipartBodyLengthLimit = int.MaxValue;
+            });
+
             services.AddSingleton<DBContext>();
             services.AddSingleton<JobManager>();
             services.AddSingleton<AppSettings>();
@@ -69,6 +88,7 @@ namespace Wabbajack.BuildServer
                 app.UseDeveloperExceptionPage();
             }
 
+
             app.UseGraphiQl();
             app.UseDeveloperExceptionPage();
             app.UseStaticFiles();

Wabbajack.BuildServer/Wabbajack.BuildServer.csproj

@@ -13,6 +13,7 @@
         <PackageReference Include="graphiql" Version="1.2.0" />
         <PackageReference Include="GraphQL" Version="3.0.0-preview-1352" />
         <PackageReference Include="Microsoft.AspNetCore.Authentication.AzureAD.UI" Version="3.1.0" />
+        <PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
         <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.0" />
         <PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
         <PackageReference Include="Microsoft.OpenApi" Version="1.1.4" />
@@ -74,4 +75,8 @@
       <Content Remove="swiftshader\**" />
     </ItemGroup>
 
+    <ItemGroup>
+      <Folder Include="public\files" />
+    </ItemGroup>
+
 </Project>

Wabbajack.BuildServer/appsettings.json

@@ -24,12 +24,15 @@
       "JobQueue": "job_queue",
       "DownloadStates": "download_states",
       "IndexedFiles": "indexed_files",
-      "Metrics": "metrics"
+      "Metrics": "metrics",
+      "ApiKeys": "api_keys",
+      "UploadedFiles": "uploaded_files"
     }
   },
   "WabbajackSettings": {
     "DownloadDir": "c:\\tmp\\downloads",
-    "ArchiveDir": "c:\\archives"
+    "ArchiveDir": "c:\\archives",
+    "MinimalMode": true
   },
   "AllowedHosts": "*"
 }

Wabbajack.Lib/ModListRegistry/ModListMetadata.cs

@@ -4,6 +4,7 @@ using System.Drawing;
 using System.Linq;
 using System.Net.Http;
 using System.Threading.Tasks;
+using MongoDB.Bson.Serialization.Attributes;
 using Newtonsoft.Json;
 using Wabbajack.Common;
 using File = System.IO.File;
@@ -39,6 +40,7 @@ namespace Wabbajack.Lib.ModListRegistry
         [JsonIgnore] 
         public ModlistSummary ValidationSummary { get; set; } = new ModlistSummary();
 
+        [BsonIgnoreExtraElements]
         public class LinksObject
         {
             [JsonProperty("image")]

Wabbajack.Lib/Wabbajack.Lib.csproj

@@ -39,6 +39,9 @@
       <PackageReference Include="ModuleInit.Fody">
         <Version>2.1.0</Version>
       </PackageReference>
+      <PackageReference Include="MongoDB.Bson">
+        <Version>2.10.0</Version>
+      </PackageReference>
       <PackageReference Include="ReactiveUI">
         <Version>11.1.6</Version>
       </PackageReference>