diff --git a/Compression.BSA.Test/Compression.BSA.Test.csproj b/Compression.BSA.Test/Compression.BSA.Test.csproj index 69d444aa..122ede28 100644 --- a/Compression.BSA.Test/Compression.BSA.Test.csproj +++ b/Compression.BSA.Test/Compression.BSA.Test.csproj @@ -33,8 +33,8 @@ 4 CS1998 CS4014 - true - true + true + true pdbonly @@ -45,8 +45,8 @@ 4 CS1998 CS4014 - true - true + true + true true @@ -57,8 +57,8 @@ 7.3 prompt MinimumRecommendedRules.ruleset - true - true + true + true bin\x64\Release\ @@ -69,8 +69,8 @@ 7.3 prompt MinimumRecommendedRules.ruleset - true - true + true + true @@ -104,10 +104,10 @@ 2.2.6 - 2.0.0 + 2.1.0-beta2 - 2.0.0 + 2.1.0-beta2 diff --git a/Wabbajack.BuildServer/ApiKeyAuthenticationHandler.cs b/Wabbajack.BuildServer/ApiKeyAuthenticationHandler.cs new file mode 100644 index 00000000..1c54073f --- /dev/null +++ b/Wabbajack.BuildServer/ApiKeyAuthenticationHandler.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Claims; +using System.Text.Encodings.Web; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using Wabbajack.BuildServer.Models; + +namespace Wabbajack.BuildServer +{ + + public class ApiKeyAuthenticationOptions : AuthenticationSchemeOptions + { + public const string DefaultScheme = "API Key"; + public string Scheme => DefaultScheme; + public string AuthenticationType = DefaultScheme; + } + + public class ApiKeyAuthenticationHandler : AuthenticationHandler + { + private const string ProblemDetailsContentType = "application/problem+json"; + private readonly DBContext _db; + private const string ApiKeyHeaderName = "X-Api-Key"; + + public ApiKeyAuthenticationHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder, + ISystemClock clock, + DBContext db) : base(options, logger, encoder, clock) + { + _db = db; + } + + protected override async Task HandleAuthenticateAsync() + { + if (!Request.Headers.TryGetValue(ApiKeyHeaderName, out var apiKeyHeaderValues)) + { + return AuthenticateResult.NoResult(); + } + + var providedApiKey = apiKeyHeaderValues.FirstOrDefault(); + + if (apiKeyHeaderValues.Count == 0 || string.IsNullOrWhiteSpace(providedApiKey)) + { + return AuthenticateResult.NoResult(); + } + + var existingApiKey = await ApiKey.Get(_db, providedApiKey); + + if (existingApiKey != null) + { + var claims = new List {new Claim(ClaimTypes.Name, existingApiKey.Owner)}; + + claims.AddRange(existingApiKey.Roles.Select(role => new Claim(ClaimTypes.Role, role))); + + var identity = new ClaimsIdentity(claims, Options.AuthenticationType); + var identities = new List {identity}; + var principal = new ClaimsPrincipal(identities); + var ticket = new AuthenticationTicket(principal, Options.Scheme); + + return AuthenticateResult.Success(ticket); + } + + return AuthenticateResult.Fail("Invalid API Key provided."); + } + + protected override async Task HandleChallengeAsync(AuthenticationProperties properties) + { + Response.StatusCode = 401; + Response.ContentType = ProblemDetailsContentType; + await Response.WriteAsync("Unauthorized"); + } + + protected override async Task HandleForbiddenAsync(AuthenticationProperties properties) + { + Response.StatusCode = 403; + Response.ContentType = ProblemDetailsContentType; + await Response.WriteAsync("forbidden"); + } + } +} diff --git a/Wabbajack.BuildServer/AppSettings.cs b/Wabbajack.BuildServer/AppSettings.cs index fc579416..5470265e 100644 --- a/Wabbajack.BuildServer/AppSettings.cs +++ b/Wabbajack.BuildServer/AppSettings.cs @@ -11,5 +11,10 @@ namespace Wabbajack.BuildServer public string DownloadDir { get; set; } public string ArchiveDir { get; set; } + + public bool MinimalMode { get; set; } + + public bool RunFrontEndJobs { get; set; } + public bool RunBackEndJobs { get; set; } } } diff --git a/Wabbajack.BuildServer/Controllers/AControllerBase.cs b/Wabbajack.BuildServer/Controllers/AControllerBase.cs index 34f95c0c..3a1b7cd0 100644 --- a/Wabbajack.BuildServer/Controllers/AControllerBase.cs +++ b/Wabbajack.BuildServer/Controllers/AControllerBase.cs @@ -1,6 +1,13 @@ -using Microsoft.AspNetCore.Mvc; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using GraphQL; +using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; +using MongoDB.Driver; using Wabbajack.BuildServer.Models; +using Wabbajack.Common; namespace Wabbajack.BuildServer.Controllers { @@ -15,5 +22,7 @@ namespace Wabbajack.BuildServer.Controllers Db = db; Logger = logger; } + + } } diff --git a/Wabbajack.BuildServer/Controllers/Heartbeat.cs b/Wabbajack.BuildServer/Controllers/Heartbeat.cs index 3fcd4a56..73c12362 100644 --- a/Wabbajack.BuildServer/Controllers/Heartbeat.cs +++ b/Wabbajack.BuildServer/Controllers/Heartbeat.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Wabbajack.BuildServer.Models; @@ -27,5 +28,13 @@ namespace Wabbajack.BuildServer.Controllers { return DateTime.Now - _startTime; } + + [HttpGet("only-authenticated")] + [Authorize] + public IActionResult OnlyAuthenticated() + { + var message = $"Hello from {nameof(OnlyAuthenticated)}"; + return new ObjectResult(message); + } } } diff --git a/Wabbajack.BuildServer/Controllers/Jobs.cs b/Wabbajack.BuildServer/Controllers/Jobs.cs index 4079aada..098f1772 100644 --- a/Wabbajack.BuildServer/Controllers/Jobs.cs +++ b/Wabbajack.BuildServer/Controllers/Jobs.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using MongoDB.Driver; @@ -10,6 +11,7 @@ using Wabbajack.BuildServer.Models.JobQueue; namespace Wabbajack.BuildServer.Controllers { + [Authorize] [ApiController] [Route("/jobs")] public class Jobs : AControllerBase diff --git a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs new file mode 100644 index 00000000..89ae6061 --- /dev/null +++ b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Security.Claims; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Identity; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using Nettle; +using Wabbajack.BuildServer.Models; +using Wabbajack.Common; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.BuildServer.Controllers +{ + public class UploadedFiles : AControllerBase + { + public UploadedFiles(ILogger logger, DBContext db) : base(logger, db) + { + + } + + [HttpPut] + [Route("upload_file/{Name}/start")] + public async Task UploadFileStreaming(string Name) + { + var guid = Guid.NewGuid(); + var key = Encoding.UTF8.GetBytes($"{Path.GetFileNameWithoutExtension(Name)}|{guid.ToString()}|{Path.GetExtension(Name)}").ToHex(); + System.IO.File.Create(Path.Combine("public", "files", key)).Close(); + Utils.Log($"Starting Ingest for {key}"); + return Ok(key); + } + + static private HashSet HexChars = new HashSet("abcdef1234567890"); + [HttpPut] + [Route("upload_file/{Key}/data/{Offset}")] + public async Task UploadFilePart(string Key, long Offset) + { + if (!Key.All(a => HexChars.Contains(a))) + return BadRequest("NOT A VALID FILENAME"); + Utils.Log($"Writing at position {Offset} in ingest file {Key}"); + await using (var file = System.IO.File.Open(Path.Combine("public", "files", Key), FileMode.Open, FileAccess.Write)) + { + file.Position = Offset; + await Request.Body.CopyToAsync(file); + return Ok(file.Position.ToString()); + } + } + + [HttpPut] + [Route("upload_file/{Key}/finish")] + public async Task UploadFileFinish(string Key) + { + var user = User.FindFirstValue(ClaimTypes.Name); + if (!Key.All(a => HexChars.Contains(a))) + return BadRequest("NOT A VALID FILENAME"); + var parts = Encoding.UTF8.GetString(Key.FromHex()).Split('|'); + var final_name = $"{parts[0]}-{parts[1]}{parts[2]}"; + var original_name = $"{parts[0]}{parts[2]}"; + + var final_path = Path.Combine("public", "files", final_name); + System.IO.File.Move(Path.Combine("public", "files", Key), final_path); + var hash = await final_path.FileHashAsync(); + + var record = new UploadedFile + { + Id = parts[1], + Hash = hash, + Name = original_name, + Uploader = user, + Size = new FileInfo(final_path).Length + }; + await Db.UploadedFiles.InsertOneAsync(record); + return Ok(record.Uri); + } + + + private static readonly Func HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@" + + + {{each $.files }} + + {{/each}} +
{{$.Name}}{{$.Size}}{{$.Date}}{{$.Uploader}}
+ + "); + + [HttpGet] + [Route("uploaded_files")] + public async Task UploadedFilesGet() + { + var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync(); + var response = HandleGetListTemplate(new + { + files = files.Select(file => new + { + Link = file.Uri, + Size = file.Size.ToFileSizeString(), + file.Name, + Date = file.UploadDate, + file.Uploader + }) + + }); + return new ContentResult + { + ContentType = "text/html", + StatusCode = (int) HttpStatusCode.OK, + Content = response + }; + } + + + } +} diff --git a/Wabbajack.BuildServer/Controllers/Users.cs b/Wabbajack.BuildServer/Controllers/Users.cs new file mode 100644 index 00000000..a1e0c115 --- /dev/null +++ b/Wabbajack.BuildServer/Controllers/Users.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using Wabbajack.BuildServer.Models; +using Wabbajack.Common; + +namespace Wabbajack.BuildServer.Controllers +{ + [Authorize] + [Route("/users")] + public class Users : AControllerBase + { + public Users(ILogger logger, DBContext db) : base(logger, db) + { + } + + [HttpGet] + [Route("add/{Name}")] + public async Task AddUser(string Name) + { + var user = new ApiKey(); + var arr = new byte[128]; + new Random().NextBytes(arr); + user.Owner = Name; + user.Key = arr.ToHex(); + user.Id = Guid.NewGuid().ToString(); + user.Roles = new List(); + user.CanUploadLists = new List(); + + await Db.ApiKeys.InsertOneAsync(user); + + return user.Id; + } + + [HttpGet] + [Route("export")] + public async Task Export() + { + if (!Directory.Exists("exported_users")) + Directory.CreateDirectory("exported_users"); + + foreach (var user in await Db.ApiKeys.AsQueryable().ToListAsync()) + { + Directory.CreateDirectory(Path.Combine("exported_users", user.Owner)); + Alphaleonis.Win32.Filesystem.File.WriteAllText(Path.Combine("exported_users", user.Owner, "author-api-key.txt"), user.Key); + } + + return "done"; + } + + } + +} diff --git a/Wabbajack.BuildServer/Extensions.cs b/Wabbajack.BuildServer/Extensions.cs index a5c5d550..aa8026ec 100644 --- a/Wabbajack.BuildServer/Extensions.cs +++ b/Wabbajack.BuildServer/Extensions.cs @@ -1,12 +1,15 @@ using System; +using System.IO; using System.Linq; using System.Linq.Expressions; using System.Threading.Tasks; +using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Logging; using MongoDB.Driver; using MongoDB.Driver.Linq; using Wabbajack.Common; +using File = Alphaleonis.Win32.Filesystem.File; namespace Wabbajack.BuildServer { @@ -24,5 +27,22 @@ namespace Wabbajack.BuildServer manager.StartJobRunners(); } + + public static async Task CopyFileAsync(string sourcePath, string destinationPath) + { + using (Stream source = File.OpenRead(sourcePath)) + { + using(Stream destination = File.Create(destinationPath)) + { + await source.CopyToAsync(destination); + } + } + } + + + public static AuthenticationBuilder AddApiKeySupport(this AuthenticationBuilder authenticationBuilder, Action options) + { + return authenticationBuilder.AddScheme(ApiKeyAuthenticationOptions.DefaultScheme, options); + } } } diff --git a/Wabbajack.BuildServer/GraphQL/Query.cs b/Wabbajack.BuildServer/GraphQL/Query.cs index 5b7582be..b61a9815 100644 --- a/Wabbajack.BuildServer/GraphQL/Query.cs +++ b/Wabbajack.BuildServer/GraphQL/Query.cs @@ -53,6 +53,13 @@ namespace Wabbajack.BuildServer.GraphQL var data = await db.Jobs.AsQueryable().Where(j => j.Id == id).ToListAsync(); return data; }); + + FieldAsync>("uploadedFiles", + resolve: async context => + { + var data = await db.UploadedFiles.AsQueryable().ToListAsync(); + return data; + }); FieldAsync>("dailyUniqueMetrics", arguments: new QueryArguments( diff --git a/Wabbajack.BuildServer/GraphQL/UploadedFileType.cs b/Wabbajack.BuildServer/GraphQL/UploadedFileType.cs new file mode 100644 index 00000000..8399beff --- /dev/null +++ b/Wabbajack.BuildServer/GraphQL/UploadedFileType.cs @@ -0,0 +1,22 @@ +using GraphQL.Types; +using Wabbajack.BuildServer.Models; + +namespace Wabbajack.BuildServer.GraphQL +{ + public class UploadedFileType : ObjectGraphType + { + public UploadedFileType() + { + Name = "UploadedFile"; + Description = "A file uploaded for hosting on Wabbajack's static file hosting"; + Field(x => x.Id, type: typeof(IdGraphType)).Description("Unique Id of the Job"); + Field(x => x.Name).Description("Non-unique name of the file"); + Field(x => x.MungedName, type: typeof(IdGraphType)).Description("Unique file name"); + Field(x => x.UploadDate, type: typeof(DateGraphType)).Description("Date of the file upload"); + Field(x => x.Uploader, type: typeof(IdGraphType)).Description("Uploader of the file"); + Field(x => x.Uri, type: typeof(UriGraphType)).Description("URI of the file"); + Field(x => x.Hash).Description("xxHash64 of the file"); + Field(x => x.Size).Description("Size of the file"); + } + } +} diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index 6bd1ce36..26508d3f 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using MongoDB.Driver; using MongoDB.Driver.Linq; +using Nettle; using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.Jobs; @@ -26,6 +27,7 @@ namespace Wabbajack.BuildServer public void StartJobRunners() { + if (Settings.MinimalMode) return; for (var idx = 0; idx < 2; idx++) { Task.Run(async () => @@ -67,6 +69,7 @@ namespace Wabbajack.BuildServer public async Task JobScheduler() { + if (Settings.MinimalMode) return; Utils.LogMessages.Subscribe(msg => Logger.Log(LogLevel.Information, msg.ToString())); while (true) { @@ -104,6 +107,8 @@ namespace Wabbajack.BuildServer private async Task ScheduledJob(TimeSpan span, Job.JobPriority priority) where T : AJobPayload, new() { + if (!Settings.RunBackEndJobs && typeof(T).ImplementsInterface(typeof(IBackEndJob))) return; + if (!Settings.RunFrontEndJobs && typeof(T).ImplementsInterface(typeof(IFrontEndJob))) return; try { var jobs = await Db.Jobs.AsQueryable() diff --git a/Wabbajack.BuildServer/Models/ApiKey.cs b/Wabbajack.BuildServer/Models/ApiKey.cs new file mode 100644 index 00000000..2fe63b7a --- /dev/null +++ b/Wabbajack.BuildServer/Models/ApiKey.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using MongoDB.Driver; +using MongoDB.Driver.Linq; + +namespace Wabbajack.BuildServer.Models +{ + public class ApiKey + { + public string Id { get; set; } + + public string Key { get; set; } + public string Owner { get; set; } + + public List CanUploadLists { get; set; } + public List Roles { get; set; } + + public static async Task Get(DBContext db, string key) + { + return await db.ApiKeys.AsQueryable().Where(k => k.Key == key).FirstOrDefaultAsync(); + } + } +} diff --git a/Wabbajack.BuildServer/Models/DBContext.cs b/Wabbajack.BuildServer/Models/DBContext.cs index 128825c9..d3fb92f3 100644 --- a/Wabbajack.BuildServer/Models/DBContext.cs +++ b/Wabbajack.BuildServer/Models/DBContext.cs @@ -27,11 +27,13 @@ namespace Wabbajack.BuildServer.Models public IMongoCollection Metrics => Client.GetCollection(_settings.Collections["Metrics"]); public IMongoCollection IndexedFiles => Client.GetCollection(_settings.Collections["IndexedFiles"]); public IMongoCollection>> NexusUpdates => Client.GetCollection>>(_settings.Collections["NexusUpdates"]); + + public IMongoCollection ApiKeys => Client.GetCollection(_settings.Collections["ApiKeys"]); + public IMongoCollection UploadedFiles => Client.GetCollection(_settings.Collections["UploadedFiles"]); public IMongoCollection> NexusModFiles => Client.GetCollection>( _settings.Collections["NexusModFiles"]); - private IMongoDatabase Client => new MongoClient($"mongodb://{_settings.Host}").GetDatabase(_settings.Database); } public class Settings diff --git a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs index 109d3452..b0413386 100644 --- a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs +++ b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllArchives.cs @@ -12,7 +12,7 @@ using Wabbajack.Lib.ModListRegistry; namespace Wabbajack.BuildServer.Models.Jobs { - public class EnqueueAllArchives : AJobPayload + public class EnqueueAllArchives : AJobPayload, IBackEndJob { public override string Description => "Add missing modlist archives to indexer"; public override async Task Execute(DBContext db, AppSettings settings) diff --git a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs index 8fde6e50..2a67d83c 100644 --- a/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs +++ b/Wabbajack.BuildServer/Models/Jobs/EnqueueAllGameFiles.cs @@ -12,7 +12,7 @@ using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.BuildServer.Models.Jobs { - public class EnqueueAllGameFiles : AJobPayload + public class EnqueueAllGameFiles : AJobPayload, IBackEndJob { public override string Description { get => $"Enqueue all game files for indexing"; } public override async Task Execute(DBContext db, AppSettings settings) diff --git a/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs b/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs index eaabf2e9..49f3ec96 100644 --- a/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs +++ b/Wabbajack.BuildServer/Models/Jobs/EnqueueRecentFiles.cs @@ -13,7 +13,7 @@ using Wabbajack.Lib.NexusApi; namespace Wabbajack.BuildServer.Models.Jobs { - public class EnqueueRecentFiles : AJobPayload + public class EnqueueRecentFiles : AJobPayload, IFrontEndJob { public override string Description => "Enqueue the past days worth of mods for indexing"; diff --git a/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs b/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs index 4677dc2c..ac9134e5 100644 --- a/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/GetNexusUpdatesJob.cs @@ -10,7 +10,7 @@ using Newtonsoft.Json; namespace Wabbajack.BuildServer.Models.Jobs { - public class GetNexusUpdatesJob : AJobPayload + public class GetNexusUpdatesJob : AJobPayload, IFrontEndJob { public override string Description => "Poll the Nexus for updated mods, and clean any references to those mods"; diff --git a/Wabbajack.BuildServer/Models/Jobs/IBackEndJob.cs b/Wabbajack.BuildServer/Models/Jobs/IBackEndJob.cs new file mode 100644 index 00000000..7dfa3059 --- /dev/null +++ b/Wabbajack.BuildServer/Models/Jobs/IBackEndJob.cs @@ -0,0 +1,7 @@ +namespace Wabbajack.BuildServer.Models.Jobs +{ + public interface IBackEndJob + { + + } +} diff --git a/Wabbajack.BuildServer/Models/Jobs/IFrontEndJob.cs b/Wabbajack.BuildServer/Models/Jobs/IFrontEndJob.cs new file mode 100644 index 00000000..e8bf13a8 --- /dev/null +++ b/Wabbajack.BuildServer/Models/Jobs/IFrontEndJob.cs @@ -0,0 +1,7 @@ +namespace Wabbajack.BuildServer.Models.Jobs +{ + public interface IFrontEndJob + { + + } +} diff --git a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs index 003a8eeb..91982179 100644 --- a/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs +++ b/Wabbajack.BuildServer/Models/Jobs/IndexJob.cs @@ -16,7 +16,7 @@ using Wabbajack.VirtualFileSystem; namespace Wabbajack.BuildServer.Models.Jobs { - public class IndexJob : AJobPayload + public class IndexJob : AJobPayload, IBackEndJob { public Archive Archive { get; set; } public override string Description => $"Index ${Archive.State.PrimaryKeyString} and save the download/file state"; diff --git a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs index 43549101..809f7f5d 100644 --- a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs +++ b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs @@ -12,7 +12,7 @@ using File = Alphaleonis.Win32.Filesystem.File; namespace Wabbajack.BuildServer.Models.Jobs { - public class UpdateModLists : AJobPayload + public class UpdateModLists : AJobPayload, IFrontEndJob { public override string Description => "Validate curated modlists"; public override async Task Execute(DBContext db, AppSettings settings) diff --git a/Wabbajack.BuildServer/Models/Metric.cs b/Wabbajack.BuildServer/Models/Metric.cs index 61355831..796af29e 100644 --- a/Wabbajack.BuildServer/Models/Metric.cs +++ b/Wabbajack.BuildServer/Models/Metric.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; using System.Threading.Tasks; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; @@ -24,6 +25,7 @@ namespace Wabbajack.BuildServer.Models public static async Task> Report(DBContext db, string grouping) { + var regex = new Regex("\\d+\\."); var data = await db.Metrics.AsQueryable() .Where(m => m.MetricsKey != null) .Where(m => m.Action == grouping) @@ -40,7 +42,7 @@ namespace Wabbajack.BuildServer.Models var results = data .Where(d => !Guid.TryParse(d.Subject, out var _)) - .GroupBy(d => d.Subject) + .GroupBy(d => regex.Split(d.Subject).First()) .Select(by_series => { var by_day = by_series.GroupBy(d => d.Timestamp.ToString("yyyy-MM-dd")) diff --git a/Wabbajack.BuildServer/Models/UploadedFile.cs b/Wabbajack.BuildServer/Models/UploadedFile.cs new file mode 100644 index 00000000..42b5c6bd --- /dev/null +++ b/Wabbajack.BuildServer/Models/UploadedFile.cs @@ -0,0 +1,25 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using MongoDB.Bson.Serialization.Attributes; +using Wabbajack.Common; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.BuildServer.Models +{ + public class UploadedFile + { + public string Id { get; set; } + public string Name { get; set; } + public long Size { get; set; } + public string Hash { get; set; } + public string Uploader { get; set; } + public DateTime UploadDate { get; set; } = DateTime.UtcNow; + + [BsonIgnore] + public string MungedName => $"{Path.GetFileNameWithoutExtension(Name)}-{Id}{Path.GetExtension(Name)}"; + + [BsonIgnore] public object Uri => $"https://wabbajack.b-cdn.net/{MungedName}"; + } +} diff --git a/Wabbajack.BuildServer/Program.cs b/Wabbajack.BuildServer/Program.cs index f38d2517..ccff441b 100644 --- a/Wabbajack.BuildServer/Program.cs +++ b/Wabbajack.BuildServer/Program.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net; +using System.Security.Cryptography.X509Certificates; using System.Threading.Tasks; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; @@ -20,8 +22,22 @@ namespace Wabbajack.BuildServer Host.CreateDefaultBuilder(args) .ConfigureWebHostDefaults(webBuilder => { - webBuilder.UseUrls("http://*:5000"); - webBuilder.UseStartup(); + webBuilder.UseStartup() + .UseKestrel(options => + { + options.Listen(IPAddress.Any, 80); + options.Listen(IPAddress.Any, 443, listenOptions => + { + using (var store = new X509Store(StoreName.My)) + { + store.Open(OpenFlags.ReadOnly); + var cert = store.Certificates.Find(X509FindType.FindBySubjectName, "build.wabbajack.org", true)[0]; + listenOptions.UseHttps(cert); + + } + }); + options.Limits.MaxRequestBodySize = null; + }); }); } } diff --git a/Wabbajack.BuildServer/Startup.cs b/Wabbajack.BuildServer/Startup.cs index fc7f46d7..843c5575 100644 --- a/Wabbajack.BuildServer/Startup.cs +++ b/Wabbajack.BuildServer/Startup.cs @@ -11,7 +11,11 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.HttpsPolicy; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Authentication; -using Microsoft.AspNetCore.Authentication.AzureAD.UI; +using Microsoft.AspNetCore.Authentication.Cookies; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.Mvc.Authorization; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -22,6 +26,7 @@ using Swashbuckle.AspNetCore.Swagger; using Wabbajack.BuildServer.Controllers; using Wabbajack.BuildServer.Models; using Microsoft.AspNetCore.Mvc.NewtonsoftJson; +using Microsoft.AspNetCore.StaticFiles; using Wabbajack.BuildServer.Controllers; using Microsoft.Extensions.FileProviders; using Directory = System.IO.Directory; @@ -45,6 +50,19 @@ namespace Wabbajack.BuildServer { c.SwaggerDoc("v1", new OpenApiInfo {Title = "Wabbajack Build API", Version = "v1"}); }); + + services.AddAuthentication(options => + { + options.DefaultAuthenticateScheme = ApiKeyAuthenticationOptions.DefaultScheme; + options.DefaultChallengeScheme = ApiKeyAuthenticationOptions.DefaultScheme; + }) + .AddApiKeySupport(options => {}); + + services.Configure(x => + { + x.ValueLengthLimit = int.MaxValue; + x.MultipartBodyLengthLimit = int.MaxValue; + }); services.AddSingleton(); services.AddSingleton(); @@ -69,10 +87,16 @@ namespace Wabbajack.BuildServer app.UseDeveloperExceptionPage(); } + app.UseHttpsRedirection(); app.UseGraphiQl(); app.UseDeveloperExceptionPage(); + + var provider = new FileExtensionContentTypeProvider(); + provider.Mappings[".rar"] = "application/x-rar-compressed"; + provider.Mappings[".7z"] = "application/x-7z-compressed"; + provider.Mappings[".zip"] = "application/zip"; + provider.Mappings[".wabbajack"] = "application/zip"; app.UseStaticFiles(); - //app.UseHttpsRedirection(); app.UseSwagger(); app.UseSwaggerUI(c => @@ -88,7 +112,8 @@ namespace Wabbajack.BuildServer app.UseFileServer(new FileServerOptions { FileProvider = new PhysicalFileProvider( - Path.Combine(Directory.GetCurrentDirectory(), "public")) + Path.Combine(Directory.GetCurrentDirectory(), "public")), + StaticFileOptions = {ServeUnknownFileTypes = true}, }); app.UseEndpoints(endpoints => diff --git a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj index 08034efd..d70fac15 100644 --- a/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj +++ b/Wabbajack.BuildServer/Wabbajack.BuildServer.csproj @@ -12,12 +12,14 @@ - + + + - - + + @@ -74,4 +76,8 @@ + + + + diff --git a/Wabbajack.BuildServer/appsettings.json b/Wabbajack.BuildServer/appsettings.json index 9e7d3582..96c6a575 100644 --- a/Wabbajack.BuildServer/appsettings.json +++ b/Wabbajack.BuildServer/appsettings.json @@ -24,12 +24,17 @@ "JobQueue": "job_queue", "DownloadStates": "download_states", "IndexedFiles": "indexed_files", - "Metrics": "metrics" + "Metrics": "metrics", + "ApiKeys": "api_keys", + "UploadedFiles": "uploaded_files" } }, "WabbajackSettings": { "DownloadDir": "c:\\tmp\\downloads", - "ArchiveDir": "c:\\archives" + "ArchiveDir": "c:\\archives", + "MinimalMode": true, + "RunFrontEndJobs": true, + "RunBackEndJobs": true }, "AllowedHosts": "*" } diff --git a/Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj b/Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj index 3eebfd89..2166bcc6 100644 --- a/Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj +++ b/Wabbajack.Common.CSP/Wabbajack.Common.CSP.csproj @@ -6,7 +6,7 @@ - + diff --git a/Wabbajack.Common/StatusFileStream.cs b/Wabbajack.Common/StatusFileStream.cs index 69a61909..0d566c4e 100644 --- a/Wabbajack.Common/StatusFileStream.cs +++ b/Wabbajack.Common/StatusFileStream.cs @@ -6,9 +6,11 @@ namespace Wabbajack.Common { private string _message; private Stream _inner; + private WorkQueue _queue; - public StatusFileStream(Stream fs, string message) + public StatusFileStream(Stream fs, string message, WorkQueue queue = null) { + _queue = queue; _inner = fs; _message = message; } @@ -36,7 +38,14 @@ namespace Wabbajack.Common private void UpdateStatus() { - if (_inner.Length != 0) + if (_inner.Length == 0) + { + return; + } + + if (_queue != null) + _queue.Report(_message, (int) (_inner.Position * 100 / _inner.Length)); + else Utils.Status(_message, (int) (_inner.Position * 100 / _inner.Length)); } diff --git a/Wabbajack.Common/Wabbajack.Common.csproj b/Wabbajack.Common/Wabbajack.Common.csproj index 84015ee0..90da7065 100644 --- a/Wabbajack.Common/Wabbajack.Common.csproj +++ b/Wabbajack.Common/Wabbajack.Common.csproj @@ -24,7 +24,7 @@ - + diff --git a/Wabbajack.Lib/FileUploader/AuthorAPI.cs b/Wabbajack.Lib/FileUploader/AuthorAPI.cs new file mode 100644 index 00000000..90cfad9c --- /dev/null +++ b/Wabbajack.Lib/FileUploader/AuthorAPI.cs @@ -0,0 +1,88 @@ +using System; +using System.Net.Http; +using System.Reactive.Linq; +using System.Threading.Tasks; +using Alphaleonis.Win32.Filesystem; +using Wabbajack.Common; + +namespace Wabbajack.Lib.FileUploader +{ + public class AuthorAPI + { + public static IObservable HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable("author-api-key"); + + public static IObservable AuthorAPIKey => HaveAuthorAPIKey.Where(h => h) + .Select(_ => File.ReadAllText(Path.Combine(Consts.LocalAppDataPath, "author-api-key"))); + + + public static string GetAPIKey() + { + return File.ReadAllText(Path.Combine(Consts.LocalAppDataPath, "author-api-key.txt")).Trim(); + } + public static bool HasAPIKey => File.Exists(Path.Combine(Consts.LocalAppDataPath, "author-api-key.txt")); + + + public static readonly Uri UploadURL = new Uri("https://build.wabbajack.org/upload_file"); + public static long BLOCK_SIZE = (long)1024 * 1024 * 8; + public static Task UploadFile(WorkQueue queue, string filename) + { + var tcs = new TaskCompletionSource(); + queue.QueueTask(async () => + { + using (var stream = + new StatusFileStream(File.OpenRead(filename), $"Uploading {Path.GetFileName(filename)}", queue)) + { + var client = new HttpClient(); + client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey()); + var response = await client.PutAsync(UploadURL+$"/{Path.GetFileName(filename)}/start", new StringContent("")); + if (!response.IsSuccessStatusCode) + { + tcs.SetResult("FAILED"); + return; + } + + var key = await response.Content.ReadAsStringAsync(); + + var data = new byte[BLOCK_SIZE]; + while (stream.Position < stream.Length) + { + var old_offset = stream.Position; + + var new_size = Math.Min(stream.Length - stream.Position, BLOCK_SIZE); + + if (new_size != data.Length) + data = new byte[new_size]; + + await stream.ReadAsync(data, 0, data.Length); + + response = await client.PutAsync(UploadURL + $"/{key}/data/{old_offset}", + new ByteArrayContent(data)); + + if (!response.IsSuccessStatusCode) + { + tcs.SetResult("FAILED"); + return; + } + + var val = long.Parse(await response.Content.ReadAsStringAsync()); + if (val != old_offset + data.Length) + { + tcs.SetResult("Sync Error"); + return; + } + + + } + + response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent("")); + if (response.IsSuccessStatusCode) + tcs.SetResult(await response.Content.ReadAsStringAsync()); + else + tcs.SetResult("FAILED"); + } + }); + return tcs.Task; + } + + } +} diff --git a/Wabbajack.Lib/GraphQL/DTOs/UploadedFile.cs b/Wabbajack.Lib/GraphQL/DTOs/UploadedFile.cs new file mode 100644 index 00000000..f1d54b23 --- /dev/null +++ b/Wabbajack.Lib/GraphQL/DTOs/UploadedFile.cs @@ -0,0 +1,16 @@ +using System; + +namespace Wabbajack.Lib.GraphQL.DTOs +{ + public class UploadedFile + { + public string Id { get; set; } + public string Name { get; set; } + public string MungedName { get; set; } + public DateTime UploadDate { get; set; } + public string Uploader { get; set; } + public Uri Uri { get; set; } + public string Hash { get; set; } + public long Size { get; set; } + } +} diff --git a/Wabbajack.Lib/GraphQL/GraphQLService.cs b/Wabbajack.Lib/GraphQL/GraphQLService.cs new file mode 100644 index 00000000..148e3284 --- /dev/null +++ b/Wabbajack.Lib/GraphQL/GraphQLService.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading.Tasks; +using GraphQL.Client; +using GraphQL.Client.Http; +using GraphQL.Common.Request; +using Wabbajack.Common; +using Wabbajack.Lib.FileUploader; +using Wabbajack.Lib.GraphQL.DTOs; +using Path = Alphaleonis.Win32.Filesystem.Path; + +namespace Wabbajack.Lib.GraphQL +{ + public class GraphQLService + { + public static readonly Uri BaseURL = new Uri("https://build.wabbajack.org/graphql"); + + public static async Task> GetUploadedFiles() + { + var client = new GraphQLHttpClient(BaseURL); + var query = new GraphQLRequest + { + Query = @" + query uploadedFilesQuery { + uploadedFiles { + id + name + hash + uri + uploader + uploadDate + } + }" + }; + var result = await client.SendQueryAsync(query); + return result.GetDataFieldAs>("uploadedFiles"); + } + + } +} diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index f9cb3d70..31332ef4 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -4,6 +4,7 @@ using System.Drawing; using System.Linq; using System.Net.Http; using System.Threading.Tasks; +using MongoDB.Bson.Serialization.Attributes; using Newtonsoft.Json; using Wabbajack.Common; using File = System.IO.File; @@ -39,6 +40,7 @@ namespace Wabbajack.Lib.ModListRegistry [JsonIgnore] public ModlistSummary ValidationSummary { get; set; } = new ModlistSummary(); + [BsonIgnoreExtraElements] public class LinksObject { [JsonProperty("image")] diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index ba6bc79f..d72ededa 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -27,6 +27,9 @@ 2.2.2.1 + + 2.0.0-alpha.3 + 1.11.17 @@ -39,11 +42,17 @@ 2.1.0 + + 2.10.0 + + + 2.10.1 + - 11.1.6 + 11.1.11 - 11.1.6 + 11.1.11 0.24.0 diff --git a/Wabbajack.Test/Wabbajack.Test.csproj b/Wabbajack.Test/Wabbajack.Test.csproj index f8674b88..87deb314 100644 --- a/Wabbajack.Test/Wabbajack.Test.csproj +++ b/Wabbajack.Test/Wabbajack.Test.csproj @@ -179,16 +179,16 @@ 75.1.143 - 2.0.0 + 2.1.0-beta2 - 2.0.0 + 2.1.0-beta2 12.0.3 - 11.1.6 + 11.1.11 4.3.2 diff --git a/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj index 9df4add4..5791589c 100644 --- a/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj +++ b/Wabbajack.VirtualFileSystem.Test/Wabbajack.VirtualFileSystem.Test.csproj @@ -33,8 +33,8 @@ 4 CS1998 CS4014 - true - true + true + true pdbonly @@ -45,8 +45,8 @@ 4 CS1998 CS4014 - true - true + true + true true @@ -57,8 +57,8 @@ 7.3 prompt MinimumRecommendedRules.ruleset - true - true + true + true bin\x64\Release\ @@ -69,8 +69,8 @@ 7.3 prompt MinimumRecommendedRules.ruleset - true - true + true + true @@ -100,10 +100,10 @@ 2.2.6 - 2.0.0 + 2.1.0-beta2 - 2.0.0 + 2.1.0-beta2 4.3.2 diff --git a/Wabbajack/View Models/Settings/AuthorFilesVM.cs b/Wabbajack/View Models/Settings/AuthorFilesVM.cs new file mode 100644 index 00000000..e0f58c24 --- /dev/null +++ b/Wabbajack/View Models/Settings/AuthorFilesVM.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Reactive.Disposables; +using System.Reactive.Linq; +using System.Reactive.Subjects; +using System.Threading.Tasks; + +using System.Windows; +using Alphaleonis.Win32.Filesystem; +using Microsoft.WindowsAPICodePack.Shell.PropertySystem; +using ReactiveUI; +using ReactiveUI.Fody.Helpers; +using Wabbajack.Common; +using Wabbajack.Lib.FileUploader; +using Wabbajack.Lib.GraphQL; +using Wabbajack.Lib.GraphQL.DTOs; +using File = System.IO.File; + +namespace Wabbajack +{ + public class AuthorFilesVM : BackNavigatingVM + { + public Visibility IsVisible { get; } + + [Reactive] + public string SelectedFile { get; set; } + + public IReactiveCommand SelectFile { get; } + public IReactiveCommand Upload { get; } + + [Reactive] + public double UploadProgress { get; set; } + + private WorkQueue Queue = new WorkQueue(1); + + public AuthorFilesVM(SettingsVM vm) : base(vm.MWVM) + { + var sub = new Subject(); + Queue.Status.Select(s => (double)s.ProgressPercent).Subscribe(v => + { + UploadProgress = v; + }); + IsVisible = AuthorAPI.HasAPIKey ? Visibility.Visible : Visibility.Collapsed; + + SelectFile = ReactiveCommand.Create(() => + { + var fod = UIUtils.OpenFileDialog("*|*"); + if (fod != null) + SelectedFile = fod; + }); + + Upload = ReactiveCommand.Create(async () => + { + SelectedFile = await AuthorAPI.UploadFile(Queue, SelectedFile); + }); + } + } +} diff --git a/Wabbajack/View Models/Settings/SettingsVM.cs b/Wabbajack/View Models/Settings/SettingsVM.cs index 67ef7e9e..4175f7f2 100644 --- a/Wabbajack/View Models/Settings/SettingsVM.cs +++ b/Wabbajack/View Models/Settings/SettingsVM.cs @@ -14,12 +14,16 @@ namespace Wabbajack public LoginManagerVM Login { get; } public PerformanceSettings Performance { get; } + public AuthorFilesVM AuthorFile { get; } + public SettingsVM(MainWindowVM mainWindowVM) : base(mainWindowVM) { MWVM = mainWindowVM; Login = new LoginManagerVM(this); Performance = mainWindowVM.Settings.Performance; + AuthorFile = new AuthorFilesVM(this); } + } } diff --git a/Wabbajack/Views/Settings/AuthorFilesView.xaml b/Wabbajack/Views/Settings/AuthorFilesView.xaml new file mode 100644 index 00000000..0228eee6 --- /dev/null +++ b/Wabbajack/Views/Settings/AuthorFilesView.xaml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/Wabbajack/Views/Settings/AuthorFilesView.xaml.cs b/Wabbajack/Views/Settings/AuthorFilesView.xaml.cs new file mode 100644 index 00000000..18a61a88 --- /dev/null +++ b/Wabbajack/Views/Settings/AuthorFilesView.xaml.cs @@ -0,0 +1,14 @@ +using System.Windows.Controls; +using ReactiveUI; + +namespace Wabbajack +{ + public partial class AuthorFilesView : ReactiveUserControl + { + public AuthorFilesView() + { + InitializeComponent(); + } + } +} + diff --git a/Wabbajack/Views/Settings/SettingsView.xaml b/Wabbajack/Views/Settings/SettingsView.xaml index e738772f..76b43e9f 100644 --- a/Wabbajack/Views/Settings/SettingsView.xaml +++ b/Wabbajack/Views/Settings/SettingsView.xaml @@ -47,6 +47,7 @@ + diff --git a/Wabbajack/Wabbajack.csproj b/Wabbajack/Wabbajack.csproj index 39fc3f56..bec54fb2 100644 --- a/Wabbajack/Wabbajack.csproj +++ b/Wabbajack/Wabbajack.csproj @@ -173,6 +173,7 @@ Designer + ModListTileView.xaml @@ -184,6 +185,9 @@ + + AuthorFilesView.xaml + LoginItemView.xaml @@ -308,6 +312,7 @@ Designer MSBuild:Compile + Designer MSBuild:Compile @@ -561,16 +566,16 @@ 1.0.2 - 2.4.4 + 3.0.0-alpha.128 - 11.1.6 + 11.1.11 - 11.1.6 + 11.1.11 - 11.1.6 + 11.1.11 0.24.0