Merge pull request #401 from wabbajack-tools/file-upload

File upload and hosting for WJ
This commit is contained in:
Timothy Baldridge 2020-01-19 15:56:03 -07:00 committed by GitHub
commit e30dcf2358
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
44 changed files with 811 additions and 52 deletions

View File

@ -33,8 +33,8 @@
<WarningLevel>4</WarningLevel>
<NoWarn>CS1998</NoWarn>
<WarningsAsErrors>CS4014</WarningsAsErrors>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
@ -45,8 +45,8 @@
<WarningLevel>4</WarningLevel>
<NoWarn>CS1998</NoWarn>
<WarningsAsErrors>CS4014</WarningsAsErrors>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
@ -57,8 +57,8 @@
<LangVersion>7.3</LangVersion>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
<OutputPath>bin\x64\Release\</OutputPath>
@ -69,8 +69,8 @@
<LangVersion>7.3</LangVersion>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
@ -104,10 +104,10 @@
<Version>2.2.6</Version>
</PackageReference>
<PackageReference Include="MSTest.TestAdapter">
<Version>2.0.0</Version>
<Version>2.1.0-beta2</Version>
</PackageReference>
<PackageReference Include="MSTest.TestFramework">
<Version>2.0.0</Version>
<Version>2.1.0-beta2</Version>
</PackageReference>
</ItemGroup>
<Import Project="$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets" Condition="Exists('$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets')" />

View File

@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Claims;
using System.Text.Encodings.Web;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using Wabbajack.BuildServer.Models;
namespace Wabbajack.BuildServer
{
public class ApiKeyAuthenticationOptions : AuthenticationSchemeOptions
{
public const string DefaultScheme = "API Key";
public string Scheme => DefaultScheme;
public string AuthenticationType = DefaultScheme;
}
public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthenticationOptions>
{
private const string ProblemDetailsContentType = "application/problem+json";
private readonly DBContext _db;
private const string ApiKeyHeaderName = "X-Api-Key";
public ApiKeyAuthenticationHandler(
IOptionsMonitor<ApiKeyAuthenticationOptions> options,
ILoggerFactory logger,
UrlEncoder encoder,
ISystemClock clock,
DBContext db) : base(options, logger, encoder, clock)
{
_db = db;
}
protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
{
if (!Request.Headers.TryGetValue(ApiKeyHeaderName, out var apiKeyHeaderValues))
{
return AuthenticateResult.NoResult();
}
var providedApiKey = apiKeyHeaderValues.FirstOrDefault();
if (apiKeyHeaderValues.Count == 0 || string.IsNullOrWhiteSpace(providedApiKey))
{
return AuthenticateResult.NoResult();
}
var existingApiKey = await ApiKey.Get(_db, providedApiKey);
if (existingApiKey != null)
{
var claims = new List<Claim> {new Claim(ClaimTypes.Name, existingApiKey.Owner)};
claims.AddRange(existingApiKey.Roles.Select(role => new Claim(ClaimTypes.Role, role)));
var identity = new ClaimsIdentity(claims, Options.AuthenticationType);
var identities = new List<ClaimsIdentity> {identity};
var principal = new ClaimsPrincipal(identities);
var ticket = new AuthenticationTicket(principal, Options.Scheme);
return AuthenticateResult.Success(ticket);
}
return AuthenticateResult.Fail("Invalid API Key provided.");
}
protected override async Task HandleChallengeAsync(AuthenticationProperties properties)
{
Response.StatusCode = 401;
Response.ContentType = ProblemDetailsContentType;
await Response.WriteAsync("Unauthorized");
}
protected override async Task HandleForbiddenAsync(AuthenticationProperties properties)
{
Response.StatusCode = 403;
Response.ContentType = ProblemDetailsContentType;
await Response.WriteAsync("forbidden");
}
}
}

View File

@ -11,5 +11,10 @@ namespace Wabbajack.BuildServer
public string DownloadDir { get; set; }
public string ArchiveDir { get; set; }
public bool MinimalMode { get; set; }
public bool RunFrontEndJobs { get; set; }
public bool RunBackEndJobs { get; set; }
}
}

View File

@ -1,6 +1,13 @@
using Microsoft.AspNetCore.Mvc;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using GraphQL;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
namespace Wabbajack.BuildServer.Controllers
{
@ -15,5 +22,7 @@ namespace Wabbajack.BuildServer.Controllers
Db = db;
Logger = logger;
}
}
}

View File

@ -1,6 +1,7 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer.Models;
@ -27,5 +28,13 @@ namespace Wabbajack.BuildServer.Controllers
{
return DateTime.Now - _startTime;
}
[HttpGet("only-authenticated")]
[Authorize]
public IActionResult OnlyAuthenticated()
{
var message = $"Hello from {nameof(OnlyAuthenticated)}";
return new ObjectResult(message);
}
}
}

View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
@ -10,6 +11,7 @@ using Wabbajack.BuildServer.Models.JobQueue;
namespace Wabbajack.BuildServer.Controllers
{
[Authorize]
[ApiController]
[Route("/jobs")]
public class Jobs : AControllerBase<Jobs>

View File

@ -0,0 +1,122 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Security.Claims;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Identity;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nettle;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.BuildServer.Controllers
{
public class UploadedFiles : AControllerBase<UploadedFiles>
{
public UploadedFiles(ILogger<UploadedFiles> logger, DBContext db) : base(logger, db)
{
}
[HttpPut]
[Route("upload_file/{Name}/start")]
public async Task<IActionResult> UploadFileStreaming(string Name)
{
var guid = Guid.NewGuid();
var key = Encoding.UTF8.GetBytes($"{Path.GetFileNameWithoutExtension(Name)}|{guid.ToString()}|{Path.GetExtension(Name)}").ToHex();
System.IO.File.Create(Path.Combine("public", "files", key)).Close();
Utils.Log($"Starting Ingest for {key}");
return Ok(key);
}
static private HashSet<char> HexChars = new HashSet<char>("abcdef1234567890");
[HttpPut]
[Route("upload_file/{Key}/data/{Offset}")]
public async Task<IActionResult> UploadFilePart(string Key, long Offset)
{
if (!Key.All(a => HexChars.Contains(a)))
return BadRequest("NOT A VALID FILENAME");
Utils.Log($"Writing at position {Offset} in ingest file {Key}");
await using (var file = System.IO.File.Open(Path.Combine("public", "files", Key), FileMode.Open, FileAccess.Write))
{
file.Position = Offset;
await Request.Body.CopyToAsync(file);
return Ok(file.Position.ToString());
}
}
[HttpPut]
[Route("upload_file/{Key}/finish")]
public async Task<IActionResult> UploadFileFinish(string Key)
{
var user = User.FindFirstValue(ClaimTypes.Name);
if (!Key.All(a => HexChars.Contains(a)))
return BadRequest("NOT A VALID FILENAME");
var parts = Encoding.UTF8.GetString(Key.FromHex()).Split('|');
var final_name = $"{parts[0]}-{parts[1]}{parts[2]}";
var original_name = $"{parts[0]}{parts[2]}";
var final_path = Path.Combine("public", "files", final_name);
System.IO.File.Move(Path.Combine("public", "files", Key), final_path);
var hash = await final_path.FileHashAsync();
var record = new UploadedFile
{
Id = parts[1],
Hash = hash,
Name = original_name,
Uploader = user,
Size = new FileInfo(final_path).Length
};
await Db.UploadedFiles.InsertOneAsync(record);
return Ok(record.Uri);
}
private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
<html><body>
<table>
{{each $.files }}
<tr><td><a href='{{$.Link}}'>{{$.Name}}</a></td><td>{{$.Size}}</td><td>{{$.Date}}</td><td>{{$.Uploader}}</td></tr>
{{/each}}
</table>
</body></html>
");
[HttpGet]
[Route("uploaded_files")]
public async Task<ContentResult> UploadedFilesGet()
{
var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync();
var response = HandleGetListTemplate(new
{
files = files.Select(file => new
{
Link = file.Uri,
Size = file.Size.ToFileSizeString(),
file.Name,
Date = file.UploadDate,
file.Uploader
})
});
return new ContentResult
{
ContentType = "text/html",
StatusCode = (int) HttpStatusCode.OK,
Content = response
};
}
}
}

View File

@ -0,0 +1,58 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
namespace Wabbajack.BuildServer.Controllers
{
[Authorize]
[Route("/users")]
public class Users : AControllerBase<Users>
{
public Users(ILogger<Users> logger, DBContext db) : base(logger, db)
{
}
[HttpGet]
[Route("add/{Name}")]
public async Task<string> AddUser(string Name)
{
var user = new ApiKey();
var arr = new byte[128];
new Random().NextBytes(arr);
user.Owner = Name;
user.Key = arr.ToHex();
user.Id = Guid.NewGuid().ToString();
user.Roles = new List<string>();
user.CanUploadLists = new List<string>();
await Db.ApiKeys.InsertOneAsync(user);
return user.Id;
}
[HttpGet]
[Route("export")]
public async Task<string> Export()
{
if (!Directory.Exists("exported_users"))
Directory.CreateDirectory("exported_users");
foreach (var user in await Db.ApiKeys.AsQueryable().ToListAsync())
{
Directory.CreateDirectory(Path.Combine("exported_users", user.Owner));
Alphaleonis.Win32.Filesystem.File.WriteAllText(Path.Combine("exported_users", user.Owner, "author-api-key.txt"), user.Key);
}
return "done";
}
}
}

View File

@ -1,12 +1,15 @@
using System;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Wabbajack.Common;
using File = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.BuildServer
{
@ -24,5 +27,22 @@ namespace Wabbajack.BuildServer
manager.StartJobRunners();
}
public static async Task CopyFileAsync(string sourcePath, string destinationPath)
{
using (Stream source = File.OpenRead(sourcePath))
{
using(Stream destination = File.Create(destinationPath))
{
await source.CopyToAsync(destination);
}
}
}
public static AuthenticationBuilder AddApiKeySupport(this AuthenticationBuilder authenticationBuilder, Action<ApiKeyAuthenticationOptions> options)
{
return authenticationBuilder.AddScheme<ApiKeyAuthenticationOptions, ApiKeyAuthenticationHandler>(ApiKeyAuthenticationOptions.DefaultScheme, options);
}
}
}

View File

@ -53,6 +53,13 @@ namespace Wabbajack.BuildServer.GraphQL
var data = await db.Jobs.AsQueryable().Where(j => j.Id == id).ToListAsync();
return data;
});
FieldAsync<ListGraphType<UploadedFileType>>("uploadedFiles",
resolve: async context =>
{
var data = await db.UploadedFiles.AsQueryable().ToListAsync();
return data;
});
FieldAsync<ListGraphType<MetricResultType>>("dailyUniqueMetrics",
arguments: new QueryArguments(

View File

@ -0,0 +1,22 @@
using GraphQL.Types;
using Wabbajack.BuildServer.Models;
namespace Wabbajack.BuildServer.GraphQL
{
public class UploadedFileType : ObjectGraphType<UploadedFile>
{
public UploadedFileType()
{
Name = "UploadedFile";
Description = "A file uploaded for hosting on Wabbajack's static file hosting";
Field(x => x.Id, type: typeof(IdGraphType)).Description("Unique Id of the Job");
Field(x => x.Name).Description("Non-unique name of the file");
Field(x => x.MungedName, type: typeof(IdGraphType)).Description("Unique file name");
Field(x => x.UploadDate, type: typeof(DateGraphType)).Description("Date of the file upload");
Field(x => x.Uploader, type: typeof(IdGraphType)).Description("Uploader of the file");
Field(x => x.Uri, type: typeof(UriGraphType)).Description("URI of the file");
Field(x => x.Hash).Description("xxHash64 of the file");
Field(x => x.Size).Description("Size of the file");
}
}
}

View File

@ -4,6 +4,7 @@ using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nettle;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
@ -26,6 +27,7 @@ namespace Wabbajack.BuildServer
public void StartJobRunners()
{
if (Settings.MinimalMode) return;
for (var idx = 0; idx < 2; idx++)
{
Task.Run(async () =>
@ -67,6 +69,7 @@ namespace Wabbajack.BuildServer
public async Task JobScheduler()
{
if (Settings.MinimalMode) return;
Utils.LogMessages.Subscribe(msg => Logger.Log(LogLevel.Information, msg.ToString()));
while (true)
{
@ -104,6 +107,8 @@ namespace Wabbajack.BuildServer
private async Task ScheduledJob<T>(TimeSpan span, Job.JobPriority priority) where T : AJobPayload, new()
{
if (!Settings.RunBackEndJobs && typeof(T).ImplementsInterface(typeof(IBackEndJob))) return;
if (!Settings.RunFrontEndJobs && typeof(T).ImplementsInterface(typeof(IFrontEndJob))) return;
try
{
var jobs = await Db.Jobs.AsQueryable()

View File

@ -0,0 +1,23 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
namespace Wabbajack.BuildServer.Models
{
public class ApiKey
{
public string Id { get; set; }
public string Key { get; set; }
public string Owner { get; set; }
public List<string> CanUploadLists { get; set; }
public List<string> Roles { get; set; }
public static async Task<ApiKey> Get(DBContext db, string key)
{
return await db.ApiKeys.AsQueryable().Where(k => k.Key == key).FirstOrDefaultAsync();
}
}
}

View File

@ -27,11 +27,13 @@ namespace Wabbajack.BuildServer.Models
public IMongoCollection<Metric> Metrics => Client.GetCollection<Metric>(_settings.Collections["Metrics"]);
public IMongoCollection<IndexedFile> IndexedFiles => Client.GetCollection<IndexedFile>(_settings.Collections["IndexedFiles"]);
public IMongoCollection<NexusCacheData<List<NexusUpdateEntry>>> NexusUpdates => Client.GetCollection<NexusCacheData<List<NexusUpdateEntry>>>(_settings.Collections["NexusUpdates"]);
public IMongoCollection<ApiKey> ApiKeys => Client.GetCollection<ApiKey>(_settings.Collections["ApiKeys"]);
public IMongoCollection<UploadedFile> UploadedFiles => Client.GetCollection<UploadedFile>(_settings.Collections["UploadedFiles"]);
public IMongoCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>> NexusModFiles =>
Client.GetCollection<NexusCacheData<NexusApiClient.GetModFilesResponse>>(
_settings.Collections["NexusModFiles"]);
private IMongoDatabase Client => new MongoClient($"mongodb://{_settings.Host}").GetDatabase(_settings.Database);
}
public class Settings

View File

@ -12,7 +12,7 @@ using Wabbajack.Lib.ModListRegistry;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class EnqueueAllArchives : AJobPayload
public class EnqueueAllArchives : AJobPayload, IBackEndJob
{
public override string Description => "Add missing modlist archives to indexer";
public override async Task<JobResult> Execute(DBContext db, AppSettings settings)

View File

@ -12,7 +12,7 @@ using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class EnqueueAllGameFiles : AJobPayload
public class EnqueueAllGameFiles : AJobPayload, IBackEndJob
{
public override string Description { get => $"Enqueue all game files for indexing"; }
public override async Task<JobResult> Execute(DBContext db, AppSettings settings)

View File

@ -13,7 +13,7 @@ using Wabbajack.Lib.NexusApi;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class EnqueueRecentFiles : AJobPayload
public class EnqueueRecentFiles : AJobPayload, IFrontEndJob
{
public override string Description => "Enqueue the past days worth of mods for indexing";

View File

@ -10,7 +10,7 @@ using Newtonsoft.Json;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class GetNexusUpdatesJob : AJobPayload
public class GetNexusUpdatesJob : AJobPayload, IFrontEndJob
{
public override string Description => "Poll the Nexus for updated mods, and clean any references to those mods";

View File

@ -0,0 +1,7 @@
namespace Wabbajack.BuildServer.Models.Jobs
{
public interface IBackEndJob
{
}
}

View File

@ -0,0 +1,7 @@
namespace Wabbajack.BuildServer.Models.Jobs
{
public interface IFrontEndJob
{
}
}

View File

@ -16,7 +16,7 @@ using Wabbajack.VirtualFileSystem;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class IndexJob : AJobPayload
public class IndexJob : AJobPayload, IBackEndJob
{
public Archive Archive { get; set; }
public override string Description => $"Index ${Archive.State.PrimaryKeyString} and save the download/file state";

View File

@ -12,7 +12,7 @@ using File = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.BuildServer.Models.Jobs
{
public class UpdateModLists : AJobPayload
public class UpdateModLists : AJobPayload, IFrontEndJob
{
public override string Description => "Validate curated modlists";
public override async Task<JobResult> Execute(DBContext db, AppSettings settings)

View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
@ -24,6 +25,7 @@ namespace Wabbajack.BuildServer.Models
public static async Task<IEnumerable<MetricResult>> Report(DBContext db, string grouping)
{
var regex = new Regex("\\d+\\.");
var data = await db.Metrics.AsQueryable()
.Where(m => m.MetricsKey != null)
.Where(m => m.Action == grouping)
@ -40,7 +42,7 @@ namespace Wabbajack.BuildServer.Models
var results = data
.Where(d => !Guid.TryParse(d.Subject, out var _))
.GroupBy(d => d.Subject)
.GroupBy(d => regex.Split(d.Subject).First())
.Select(by_series =>
{
var by_day = by_series.GroupBy(d => d.Timestamp.ToString("yyyy-MM-dd"))

View File

@ -0,0 +1,25 @@
using System;
using System.IO;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using MongoDB.Bson.Serialization.Attributes;
using Wabbajack.Common;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.BuildServer.Models
{
public class UploadedFile
{
public string Id { get; set; }
public string Name { get; set; }
public long Size { get; set; }
public string Hash { get; set; }
public string Uploader { get; set; }
public DateTime UploadDate { get; set; } = DateTime.UtcNow;
[BsonIgnore]
public string MungedName => $"{Path.GetFileNameWithoutExtension(Name)}-{Id}{Path.GetExtension(Name)}";
[BsonIgnore] public object Uri => $"https://wabbajack.b-cdn.net/{MungedName}";
}
}

View File

@ -1,6 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Security.Cryptography.X509Certificates;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Configuration;
@ -20,8 +22,22 @@ namespace Wabbajack.BuildServer
Host.CreateDefaultBuilder(args)
.ConfigureWebHostDefaults(webBuilder =>
{
webBuilder.UseUrls("http://*:5000");
webBuilder.UseStartup<Startup>();
webBuilder.UseStartup<Startup>()
.UseKestrel(options =>
{
options.Listen(IPAddress.Any, 80);
options.Listen(IPAddress.Any, 443, listenOptions =>
{
using (var store = new X509Store(StoreName.My))
{
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Find(X509FindType.FindBySubjectName, "build.wabbajack.org", true)[0];
listenOptions.UseHttps(cert);
}
});
options.Limits.MaxRequestBodySize = null;
});
});
}
}

View File

@ -11,7 +11,11 @@ using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.HttpsPolicy;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authentication.AzureAD.UI;
using Microsoft.AspNetCore.Authentication.Cookies;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Http.Features;
using Microsoft.AspNetCore.Mvc.Authorization;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
@ -22,6 +26,7 @@ using Swashbuckle.AspNetCore.Swagger;
using Wabbajack.BuildServer.Controllers;
using Wabbajack.BuildServer.Models;
using Microsoft.AspNetCore.Mvc.NewtonsoftJson;
using Microsoft.AspNetCore.StaticFiles;
using Wabbajack.BuildServer.Controllers;
using Microsoft.Extensions.FileProviders;
using Directory = System.IO.Directory;
@ -45,6 +50,19 @@ namespace Wabbajack.BuildServer
{
c.SwaggerDoc("v1", new OpenApiInfo {Title = "Wabbajack Build API", Version = "v1"});
});
services.AddAuthentication(options =>
{
options.DefaultAuthenticateScheme = ApiKeyAuthenticationOptions.DefaultScheme;
options.DefaultChallengeScheme = ApiKeyAuthenticationOptions.DefaultScheme;
})
.AddApiKeySupport(options => {});
services.Configure<FormOptions>(x =>
{
x.ValueLengthLimit = int.MaxValue;
x.MultipartBodyLengthLimit = int.MaxValue;
});
services.AddSingleton<DBContext>();
services.AddSingleton<JobManager>();
@ -69,10 +87,16 @@ namespace Wabbajack.BuildServer
app.UseDeveloperExceptionPage();
}
app.UseHttpsRedirection();
app.UseGraphiQl();
app.UseDeveloperExceptionPage();
var provider = new FileExtensionContentTypeProvider();
provider.Mappings[".rar"] = "application/x-rar-compressed";
provider.Mappings[".7z"] = "application/x-7z-compressed";
provider.Mappings[".zip"] = "application/zip";
provider.Mappings[".wabbajack"] = "application/zip";
app.UseStaticFiles();
//app.UseHttpsRedirection();
app.UseSwagger();
app.UseSwaggerUI(c =>
@ -88,7 +112,8 @@ namespace Wabbajack.BuildServer
app.UseFileServer(new FileServerOptions
{
FileProvider = new PhysicalFileProvider(
Path.Combine(Directory.GetCurrentDirectory(), "public"))
Path.Combine(Directory.GetCurrentDirectory(), "public")),
StaticFileOptions = {ServeUnknownFileTypes = true},
});
app.UseEndpoints(endpoints =>

View File

@ -12,12 +12,14 @@
<ItemGroup>
<PackageReference Include="graphiql" Version="1.2.0" />
<PackageReference Include="GraphQL" Version="3.0.0-preview-1352" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.AzureAD.UI" Version="3.1.0" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.1" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.0" />
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Microsoft.OpenApi" Version="1.1.4" />
<PackageReference Include="MongoDB.Driver" Version="2.10.0" />
<PackageReference Include="MongoDB.Driver.Core" Version="2.10.0" />
<PackageReference Include="MongoDB.Driver" Version="2.10.1" />
<PackageReference Include="MongoDB.Driver.Core" Version="2.10.1" />
<PackageReference Include="Nettle" Version="1.3.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.0.0" />
</ItemGroup>
@ -74,4 +76,8 @@
<Content Remove="swiftshader\**" />
</ItemGroup>
<ItemGroup>
<Folder Include="public\files" />
</ItemGroup>
</Project>

View File

@ -24,12 +24,17 @@
"JobQueue": "job_queue",
"DownloadStates": "download_states",
"IndexedFiles": "indexed_files",
"Metrics": "metrics"
"Metrics": "metrics",
"ApiKeys": "api_keys",
"UploadedFiles": "uploaded_files"
}
},
"WabbajackSettings": {
"DownloadDir": "c:\\tmp\\downloads",
"ArchiveDir": "c:\\archives"
"ArchiveDir": "c:\\archives",
"MinimalMode": true,
"RunFrontEndJobs": true,
"RunBackEndJobs": true
},
"AllowedHosts": "*"
}

View File

@ -6,7 +6,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.Reactive" Version="4.3.2" />
<PackageReference Include="System.Threading.Tasks.Extensions" Version="4.5.3" />
<PackageReference Include="System.Threading.Tasks.Extensions" Version="4.6.0-preview.18571.3" />
</ItemGroup>
<ItemGroup>
<Folder Include="Properties\" />

View File

@ -6,9 +6,11 @@ namespace Wabbajack.Common
{
private string _message;
private Stream _inner;
private WorkQueue _queue;
public StatusFileStream(Stream fs, string message)
public StatusFileStream(Stream fs, string message, WorkQueue queue = null)
{
_queue = queue;
_inner = fs;
_message = message;
}
@ -36,7 +38,14 @@ namespace Wabbajack.Common
private void UpdateStatus()
{
if (_inner.Length != 0)
if (_inner.Length == 0)
{
return;
}
if (_queue != null)
_queue.Report(_message, (int) (_inner.Position * 100 / _inner.Length));
else
Utils.Status(_message, (int) (_inner.Position * 100 / _inner.Length));
}

View File

@ -24,7 +24,7 @@
<PackageReference Include="ini-parser-netstandard" Version="2.5.2" />
<PackageReference Include="Microsoft.Win32.Registry" Version="4.7.0" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="ReactiveUI" Version="11.1.6" />
<PackageReference Include="ReactiveUI" Version="11.1.11" />
<PackageReference Include="SharpZipLib" Version="1.2.0" />
<PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
<PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.7.0" />

View File

@ -0,0 +1,88 @@
using System;
using System.Net.Http;
using System.Reactive.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
namespace Wabbajack.Lib.FileUploader
{
public class AuthorAPI
{
public static IObservable<bool> HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable("author-api-key");
public static IObservable<string> AuthorAPIKey => HaveAuthorAPIKey.Where(h => h)
.Select(_ => File.ReadAllText(Path.Combine(Consts.LocalAppDataPath, "author-api-key")));
public static string GetAPIKey()
{
return File.ReadAllText(Path.Combine(Consts.LocalAppDataPath, "author-api-key.txt")).Trim();
}
public static bool HasAPIKey => File.Exists(Path.Combine(Consts.LocalAppDataPath, "author-api-key.txt"));
public static readonly Uri UploadURL = new Uri("https://build.wabbajack.org/upload_file");
public static long BLOCK_SIZE = (long)1024 * 1024 * 8;
public static Task<string> UploadFile(WorkQueue queue, string filename)
{
var tcs = new TaskCompletionSource<string>();
queue.QueueTask(async () =>
{
using (var stream =
new StatusFileStream(File.OpenRead(filename), $"Uploading {Path.GetFileName(filename)}", queue))
{
var client = new HttpClient();
client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey());
var response = await client.PutAsync(UploadURL+$"/{Path.GetFileName(filename)}/start", new StringContent(""));
if (!response.IsSuccessStatusCode)
{
tcs.SetResult("FAILED");
return;
}
var key = await response.Content.ReadAsStringAsync();
var data = new byte[BLOCK_SIZE];
while (stream.Position < stream.Length)
{
var old_offset = stream.Position;
var new_size = Math.Min(stream.Length - stream.Position, BLOCK_SIZE);
if (new_size != data.Length)
data = new byte[new_size];
await stream.ReadAsync(data, 0, data.Length);
response = await client.PutAsync(UploadURL + $"/{key}/data/{old_offset}",
new ByteArrayContent(data));
if (!response.IsSuccessStatusCode)
{
tcs.SetResult("FAILED");
return;
}
var val = long.Parse(await response.Content.ReadAsStringAsync());
if (val != old_offset + data.Length)
{
tcs.SetResult("Sync Error");
return;
}
}
response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent(""));
if (response.IsSuccessStatusCode)
tcs.SetResult(await response.Content.ReadAsStringAsync());
else
tcs.SetResult("FAILED");
}
});
return tcs.Task;
}
}
}

View File

@ -0,0 +1,16 @@
using System;
namespace Wabbajack.Lib.GraphQL.DTOs
{
public class UploadedFile
{
public string Id { get; set; }
public string Name { get; set; }
public string MungedName { get; set; }
public DateTime UploadDate { get; set; }
public string Uploader { get; set; }
public Uri Uri { get; set; }
public string Hash { get; set; }
public long Size { get; set; }
}
}

View File

@ -0,0 +1,44 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading.Tasks;
using GraphQL.Client;
using GraphQL.Client.Http;
using GraphQL.Common.Request;
using Wabbajack.Common;
using Wabbajack.Lib.FileUploader;
using Wabbajack.Lib.GraphQL.DTOs;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Lib.GraphQL
{
public class GraphQLService
{
public static readonly Uri BaseURL = new Uri("https://build.wabbajack.org/graphql");
public static async Task<List<UploadedFile>> GetUploadedFiles()
{
var client = new GraphQLHttpClient(BaseURL);
var query = new GraphQLRequest
{
Query = @"
query uploadedFilesQuery {
uploadedFiles {
id
name
hash
uri
uploader
uploadDate
}
}"
};
var result = await client.SendQueryAsync(query);
return result.GetDataFieldAs<List<UploadedFile>>("uploadedFiles");
}
}
}

View File

@ -4,6 +4,7 @@ using System.Drawing;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using MongoDB.Bson.Serialization.Attributes;
using Newtonsoft.Json;
using Wabbajack.Common;
using File = System.IO.File;
@ -39,6 +40,7 @@ namespace Wabbajack.Lib.ModListRegistry
[JsonIgnore]
public ModlistSummary ValidationSummary { get; set; } = new ModlistSummary();
[BsonIgnoreExtraElements]
public class LinksObject
{
[JsonProperty("image")]

View File

@ -27,6 +27,9 @@
<PackageReference Include="Genbox.AlphaFS">
<Version>2.2.2.1</Version>
</PackageReference>
<PackageReference Include="GraphQL.Client">
<Version>2.0.0-alpha.3</Version>
</PackageReference>
<PackageReference Include="HtmlAgilityPack">
<Version>1.11.17</Version>
</PackageReference>
@ -39,11 +42,17 @@
<PackageReference Include="ModuleInit.Fody">
<Version>2.1.0</Version>
</PackageReference>
<PackageReference Include="MongoDB.Bson">
<Version>2.10.0</Version>
</PackageReference>
<PackageReference Include="MongoDB.Bson">
<Version>2.10.1</Version>
</PackageReference>
<PackageReference Include="ReactiveUI">
<Version>11.1.6</Version>
<Version>11.1.11</Version>
</PackageReference>
<PackageReference Include="ReactiveUI.Fody">
<Version>11.1.6</Version>
<Version>11.1.11</Version>
</PackageReference>
<PackageReference Include="SharpCompress">
<Version>0.24.0</Version>

View File

@ -179,16 +179,16 @@
<Version>75.1.143</Version>
</PackageReference>
<PackageReference Include="MSTest.TestAdapter">
<Version>2.0.0</Version>
<Version>2.1.0-beta2</Version>
</PackageReference>
<PackageReference Include="MSTest.TestFramework">
<Version>2.0.0</Version>
<Version>2.1.0-beta2</Version>
</PackageReference>
<PackageReference Include="Newtonsoft.Json">
<Version>12.0.3</Version>
</PackageReference>
<PackageReference Include="ReactiveUI">
<Version>11.1.6</Version>
<Version>11.1.11</Version>
</PackageReference>
<PackageReference Include="System.Reactive">
<Version>4.3.2</Version>

View File

@ -33,8 +33,8 @@
<WarningLevel>4</WarningLevel>
<NoWarn>CS1998</NoWarn>
<WarningsAsErrors>CS4014</WarningsAsErrors>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
@ -45,8 +45,8 @@
<WarningLevel>4</WarningLevel>
<NoWarn>CS1998</NoWarn>
<WarningsAsErrors>CS4014</WarningsAsErrors>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
@ -57,8 +57,8 @@
<LangVersion>7.3</LangVersion>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
<OutputPath>bin\x64\Release\</OutputPath>
@ -69,8 +69,8 @@
<LangVersion>7.3</LangVersion>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
@ -100,10 +100,10 @@
<Version>2.2.6</Version>
</PackageReference>
<PackageReference Include="MSTest.TestAdapter">
<Version>2.0.0</Version>
<Version>2.1.0-beta2</Version>
</PackageReference>
<PackageReference Include="MSTest.TestFramework">
<Version>2.0.0</Version>
<Version>2.1.0-beta2</Version>
</PackageReference>
<PackageReference Include="System.Reactive">
<Version>4.3.2</Version>

View File

@ -0,0 +1,58 @@
using System;
using System.Collections.Generic;
using System.Reactive.Disposables;
using System.Reactive.Linq;
using System.Reactive.Subjects;
using System.Threading.Tasks;
using System.Windows;
using Alphaleonis.Win32.Filesystem;
using Microsoft.WindowsAPICodePack.Shell.PropertySystem;
using ReactiveUI;
using ReactiveUI.Fody.Helpers;
using Wabbajack.Common;
using Wabbajack.Lib.FileUploader;
using Wabbajack.Lib.GraphQL;
using Wabbajack.Lib.GraphQL.DTOs;
using File = System.IO.File;
namespace Wabbajack
{
public class AuthorFilesVM : BackNavigatingVM
{
public Visibility IsVisible { get; }
[Reactive]
public string SelectedFile { get; set; }
public IReactiveCommand SelectFile { get; }
public IReactiveCommand Upload { get; }
[Reactive]
public double UploadProgress { get; set; }
private WorkQueue Queue = new WorkQueue(1);
public AuthorFilesVM(SettingsVM vm) : base(vm.MWVM)
{
var sub = new Subject<double>();
Queue.Status.Select(s => (double)s.ProgressPercent).Subscribe(v =>
{
UploadProgress = v;
});
IsVisible = AuthorAPI.HasAPIKey ? Visibility.Visible : Visibility.Collapsed;
SelectFile = ReactiveCommand.Create(() =>
{
var fod = UIUtils.OpenFileDialog("*|*");
if (fod != null)
SelectedFile = fod;
});
Upload = ReactiveCommand.Create(async () =>
{
SelectedFile = await AuthorAPI.UploadFile(Queue, SelectedFile);
});
}
}
}

View File

@ -14,12 +14,16 @@ namespace Wabbajack
public LoginManagerVM Login { get; }
public PerformanceSettings Performance { get; }
public AuthorFilesVM AuthorFile { get; }
public SettingsVM(MainWindowVM mainWindowVM)
: base(mainWindowVM)
{
MWVM = mainWindowVM;
Login = new LoginManagerVM(this);
Performance = mainWindowVM.Settings.Performance;
AuthorFile = new AuthorFilesVM(this);
}
}
}

View File

@ -0,0 +1,45 @@
<rxui:ReactiveUserControl
x:Class="Wabbajack.AuthorFilesView"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:local="clr-namespace:Wabbajack"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:rxui="http://reactiveui.net"
d:DesignHeight="450"
d:DesignWidth="800"
x:TypeArguments="local:AuthorFilesVM"
mc:Ignorable="d">
<Border
Visibility="{Binding AuthorFile.IsVisible}"
x:Name="AuthorFiles"
Margin="5"
Background="{StaticResource BackgroundBrush}"
BorderBrush="{StaticResource ButtonNormalBorder}"
BorderThickness="1">
<Grid Margin="10">
<Grid.RowDefinitions>
<RowDefinition></RowDefinition>
<RowDefinition></RowDefinition>
<RowDefinition></RowDefinition>
<RowDefinition></RowDefinition>
</Grid.RowDefinitions>
<Grid.ColumnDefinitions>
<ColumnDefinition Width="300"></ColumnDefinition>
<ColumnDefinition Width="300"></ColumnDefinition>
</Grid.ColumnDefinitions>
<TextBlock
Grid.ColumnSpan="2"
Margin="5,0"
FontFamily="Lucida Sans"
FontSize="20"
FontWeight="Bold"
Text="File Uploader" />
<TextBlock Margin="5" Grid.Row="1" Grid.ColumnSpan="2" Text="{Binding AuthorFile.SelectedFile}"></TextBlock>
<ProgressBar Margin="5" Grid.Row="2" Grid.ColumnSpan="2" Value="{Binding AuthorFile.UploadProgress, Mode=OneWay}" Minimum="0" Maximum="1"></ProgressBar>
<Button Margin="5" Grid.Row="3" Grid.Column="0" Command="{Binding AuthorFile.SelectFile, Mode=OneTime}">Select</Button>
<Button Margin="5" Grid.Row="3" Grid.Column="1" Command="{Binding AuthorFile.Upload}">Upload</Button>
</Grid>
</Border>
</rxui:ReactiveUserControl>

View File

@ -0,0 +1,14 @@
using System.Windows.Controls;
using ReactiveUI;
namespace Wabbajack
{
public partial class AuthorFilesView : ReactiveUserControl<AuthorFilesVM>
{
public AuthorFilesView()
{
InitializeComponent();
}
}
}

View File

@ -47,6 +47,7 @@
<WrapPanel>
<local:LoginSettingsView x:Name="LoginView" />
<local:PerformanceSettingsView x:Name="PerformanceView" />
<local:AuthorFilesView x:Name="AuthorFilesView"></local:AuthorFilesView>
</WrapPanel>
</ScrollViewer>
</Grid>

View File

@ -173,6 +173,7 @@
<SubType>Designer</SubType>
</ApplicationDefinition>
<Compile Include="Extensions\DynamicDataExt.cs" />
<Compile Include="View Models\Settings\AuthorFilesVM.cs" />
<Compile Include="Views\ModListTileView.xaml.cs">
<DependentUpon>ModListTileView.xaml</DependentUpon>
</Compile>
@ -184,6 +185,9 @@
<Compile Include="Converters\ConverterRegistration.cs" />
<Compile Include="Extensions\IViewForExt.cs" />
<Compile Include="View Models\Interfaces\ICpuStatusVM.cs" />
<Compile Include="Views\Settings\AuthorFilesView.xaml.cs">
<DependentUpon>AuthorFilesView.xaml</DependentUpon>
</Compile>
<Compile Include="Views\Settings\LoginItemView.xaml.cs">
<DependentUpon>LoginItemView.xaml</DependentUpon>
</Compile>
@ -308,6 +312,7 @@
<SubType>Designer</SubType>
<Generator>MSBuild:Compile</Generator>
</Page>
<Page Include="Views\Settings\AuthorFilesView.xaml" />
<Page Include="Views\Settings\LoginItemView.xaml">
<SubType>Designer</SubType>
<Generator>MSBuild:Compile</Generator>
@ -561,16 +566,16 @@
<Version>1.0.2</Version>
</PackageReference>
<PackageReference Include="protobuf-net">
<Version>2.4.4</Version>
<Version>3.0.0-alpha.128</Version>
</PackageReference>
<PackageReference Include="ReactiveUI.Events.WPF">
<Version>11.1.6</Version>
<Version>11.1.11</Version>
</PackageReference>
<PackageReference Include="ReactiveUI.Fody">
<Version>11.1.6</Version>
<Version>11.1.11</Version>
</PackageReference>
<PackageReference Include="ReactiveUI.WPF">
<Version>11.1.6</Version>
<Version>11.1.11</Version>
</PackageReference>
<PackageReference Include="SharpCompress">
<Version>0.24.0</Version>