Uploaded files and API keys are now stored in SQL, still working on adding more tests

This commit is contained in:
Timothy Baldridge 2020-03-29 21:47:35 -06:00
parent 90434f23fa
commit fcb589a442
18 changed files with 341 additions and 55 deletions

View File

@ -0,0 +1,83 @@
using System;
using System.Net.Http;
using System.Reactive.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Wabbajack.Common;
using Wabbajack.Common.Http;
using Wabbajack.Common.StatusFeed;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class BuildServerFixture : ADBTest, IDisposable
{
private IHost _host;
private CancellationTokenSource _token;
private Task _task;
public readonly TempFolder _severTempFolder = new TempFolder();
public AbsolutePath ServerTempFolder => _severTempFolder.Dir;
public BuildServerFixture()
{
var builder = Program.CreateHostBuilder(
new[]
{
$"WabbajackSettings:DownloadDir={"tmp".RelativeTo(AbsolutePath.EntryPoint)}",
$"WabbajackSettings:ArchiveDir={"archives".RelativeTo(AbsolutePath.EntryPoint)}",
$"WabbajackSettings:TempDir={ServerTempFolder}",
$"WabbajackSettings:SQLConnection={PublicConnStr}",
}, true);
_host = builder.Build();
_token = new CancellationTokenSource();
_task = _host.RunAsync(_token.Token);
}
public void Dispose()
{
if (!_token.IsCancellationRequested)
_token.Cancel();
_task.Wait();
_severTempFolder.DisposeAsync().AsTask().Wait();
}
}
public class ABuildServerSystemTest : XunitContextBase, IClassFixture<BuildServerFixture>
{
protected readonly Client _client;
private readonly IDisposable _unsubMsgs;
private readonly IDisposable _unsubErr;
protected Client _authedClient;
public ABuildServerSystemTest(ITestOutputHelper output, BuildServerFixture fixture) : base(output)
{
_unsubMsgs = Utils.LogMessages.OfType<IInfo>().Subscribe(onNext: msg => XunitContext.WriteLine(msg.ShortDescription));
_unsubErr = Utils.LogMessages.OfType<IUserIntervention>().Subscribe(msg =>
XunitContext.WriteLine("ERROR: User intervention required: " + msg.ShortDescription));
_client = new Client();
_authedClient = new Client();
_authedClient.Headers.Add(("x-api-key", fixture.APIKey));
Fixture = fixture;
}
public BuildServerFixture Fixture { get; set; }
protected string MakeURL(string path)
{
return "http://localhost:8080/" + path;
}
public override void Dispose()
{
base.Dispose();
_unsubMsgs.Dispose();
_unsubErr.Dispose();
}
}
}

View File

@ -7,36 +7,32 @@ using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Controllers;
using Wabbajack.Common;
using Wabbajack.BuildServer.Model.Models;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class ABuildServerTest : IAsyncLifetime
public class ADBTest : IAsyncLifetime
{
private static string CONN_STR = @"Data Source=.\SQLEXPRESS;Integrated Security=True;";
public string PublicConnStr => CONN_STR + $";Initial Catalog={DBName}";
private AppSettings _appSettings;
protected SqlService _sqlService;
private bool _finishedSchema;
private string DBName { get; }
public ABuildServerTest(ITestOutputHelper helper)
public ADBTest()
{
TestContext = helper;
DBName = "test_db" + Guid.NewGuid().ToString().Replace("-", "_");
_appSettings = MakeAppSettings();
_sqlService = new SqlService(_appSettings);
User = Guid.NewGuid().ToString().Replace("-", "");
APIKey = Users.NewAPIKey();
}
private AppSettings MakeAppSettings()
{
return new AppSettings
{
SqlConnection = CONN_STR + $"Initial Catalog={DBName}"
};
}
public ITestOutputHelper TestContext { get;}
public string APIKey { get; }
public string User { get; }
public async Task InitializeAsync()
{
@ -45,7 +41,7 @@ namespace Wabbajack.BuildServer.Test
private async Task CreateSchema()
{
TestContext.WriteLine("Creating Database");
Utils.Log("Creating Database");
//var conn = new SqlConnection("Data Source=localhost,1433;User ID=test;Password=test;MultipleActiveResultSets=true");
await using var conn = new SqlConnection(CONN_STR);
@ -60,8 +56,12 @@ namespace Wabbajack.BuildServer.Test
foreach (var statement in SplitSqlStatements(schemaString))
{
await new SqlCommand(statement, conn).ExecuteNonQueryAsync();
}
await new SqlCommand($"USE {DBName}", conn).ExecuteNonQueryAsync();
await new SqlCommand($"INSERT INTO dbo.ApiKeys (APIKey, Owner) VALUES ('{APIKey}', '{User}');", conn).ExecuteNonQueryAsync();
_finishedSchema = true;
}
private static IEnumerable<string> SplitSqlStatements(string sqlScript)
@ -82,7 +82,9 @@ namespace Wabbajack.BuildServer.Test
async Task IAsyncLifetime.DisposeAsync()
{
TestContext.WriteLine("Deleting Database");
// Don't delete it if the setup failed, so we can debug the issue
if (!_finishedSchema) return;
Utils.Log("Deleting Database");
await using var conn = new SqlConnection(CONN_STR);
await conn.OpenAsync();

View File

@ -0,0 +1,31 @@
using System;
using System.Threading.Tasks;
using Wabbajack.Common;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class BasicServerTests : ABuildServerSystemTest
{
public BasicServerTests(ITestOutputHelper output, BuildServerFixture fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanGetHeartbeat()
{
var heartbeat = (await _client.GetStringAsync(MakeURL("heartbeat"))).FromJSONString<string>();
Assert.True(TimeSpan.Parse(heartbeat) > TimeSpan.Zero);
}
[Fact]
public async Task CanContactAuthedEndpoint()
{
var logs = await _authedClient.GetStringAsync(MakeURL("heartbeat/logs"));
Assert.NotEmpty(logs);
}
}
}

View File

@ -10,7 +10,7 @@ using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class BasicTest : ABuildServerTest
public class BasicTest : ADBTest
{
[Fact]
public async Task CanEneuqueAndGetJobs()
@ -40,7 +40,7 @@ namespace Wabbajack.BuildServer.Test
}
}
public BasicTest(ITestOutputHelper helper) : base(helper)
public BasicTest()
{

View File

@ -0,0 +1,31 @@
using System;
using System.Net.Http;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.Common;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.BuildServer.Test
{
public class UploadedFilesTest : ABuildServerSystemTest
{
public UploadedFilesTest(ITestOutputHelper helper, BuildServerFixture fixture) : base(helper, fixture)
{
}
[Fact]
public async Task CanIngestMongoDBExports()
{
@"sql\uploaded_files_ingest.json".RelativeTo(AbsolutePath.EntryPoint).CopyTo(Fixture.ServerTempFolder.Combine("uploaded_files_ingest.json"));
using var response = await _authedClient.GetAsync(MakeURL("ingest/uploaded_files/uploaded_files_ingest.json"));
var result = await response.Content.ReadAsStringAsync();
Utils.Log("Loaded: " + result);
Assert.Equal("251", result);
}
}
}

View File

@ -8,10 +8,11 @@
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />
<PackageReference Include="xunit" Version="2.4.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.0" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
<PackageReference Include="coverlet.collector" Version="1.0.1" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.1" />
<PackageReference Include="XunitContext" Version="1.9.0" />
</ItemGroup>
<ItemGroup>
@ -22,4 +23,10 @@
<ProjectReference Include="..\Wabbajack.BuildServer\Wabbajack.BuildServer.csproj" />
</ItemGroup>
<ItemGroup>
<None Update="sql\uploaded_files_ingest.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

View File

@ -229,6 +229,15 @@ CREATE TABLE [dbo].[ArchiveContent](
[PathHash] AS (CONVERT([binary](32),hashbytes('SHA2_256',[Path]))) PERSISTED NOT NULL
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
CREATE STATISTICS [Child_Parent_Stat] ON [dbo].[ArchiveContent]([Child], [Parent])
GO
CREATE CLUSTERED INDEX [Child_Parent_IDX] ON [dbo].[ArchiveContent]
(
[Child] ASC,
[Parent] ASC
)WITH (SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF) ON [PRIMARY]
GO
/****** Object: Table [dbo].[AllFilesInArchive] Script Date: 3/28/2020 4:58:59 PM ******/
SET ANSI_NULLS ON
GO
@ -278,6 +287,40 @@ CREATE TABLE [dbo].[Metrics](
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
/****** Uploaded Files [UploadedFiles] *************/
CREATE TABLE [dbo].[UploadedFiles](
[Id] [uniqueidentifier] NOT NULL,
[Name] [nvarchar](max) NOT NULL,
[Size] [bigint] NOT NULL,
[UploadedBy] [nvarchar](40) NOT NULL,
[Hash] [bigint] NOT NULL,
[UploadDate] [datetime] NOT NULL,
[CDNName] [nvarchar](max) NULL,
CONSTRAINT [PK_UploadedFiles] PRIMARY KEY CLUSTERED
(
[Id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
/****** API Keys [ApiKeys] ********/
CREATE TABLE [dbo].[ApiKeys](
[APIKey] [nvarchar](260) NOT NULL,
[Owner] [nvarchar](40) NOT NULL,
CONSTRAINT [PK_ApiKeys] PRIMARY KEY CLUSTERED
(
[APIKey] ASC,
[Owner] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY]
GO
CREATE UNIQUE NONCLUSTERED INDEX [ByAPIKey] ON [dbo].[ApiKeys]
(
[APIKey] ASC
)
INCLUDE([Owner]) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
GO
/****** Object: Index [IX_Child] Script Date: 3/28/2020 4:58:59 PM ******/
CREATE NONCLUSTERED INDEX [IX_Child] ON [dbo].[AllFilesInArchive]
(

View File

@ -9,6 +9,7 @@ using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
namespace Wabbajack.BuildServer
@ -24,7 +25,7 @@ namespace Wabbajack.BuildServer
public class ApiKeyAuthenticationHandler : AuthenticationHandler<ApiKeyAuthenticationOptions>
{
private const string ProblemDetailsContentType = "application/problem+json";
private readonly DBContext _db;
private readonly SqlService _db;
private const string ApiKeyHeaderName = "X-Api-Key";
public ApiKeyAuthenticationHandler(
@ -32,7 +33,7 @@ namespace Wabbajack.BuildServer
ILoggerFactory logger,
UrlEncoder encoder,
ISystemClock clock,
DBContext db) : base(options, logger, encoder, clock)
SqlService db) : base(options, logger, encoder, clock)
{
_db = db;
}
@ -51,13 +52,15 @@ namespace Wabbajack.BuildServer
return AuthenticateResult.NoResult();
}
var existingApiKey = await ApiKey.Get(_db, providedApiKey);
var owner = await _db.LoginByAPIKey(providedApiKey);
if (existingApiKey != null)
if (owner != null)
{
var claims = new List<Claim> {new Claim(ClaimTypes.Name, existingApiKey.Owner)};
var claims = new List<Claim> {new Claim(ClaimTypes.Name, owner)};
/*
claims.AddRange(existingApiKey.Roles.Select(role => new Claim(ClaimTypes.Role, role)));
*/
var identity = new ClaimsIdentity(claims, Options.AuthenticationType);
var identities = new List<ClaimsIdentity> {identity};

View File

@ -9,15 +9,12 @@ namespace Wabbajack.BuildServer
{
config.Bind("WabbajackSettings", this);
}
public AppSettings()
{
}
public AbsolutePath DownloadDir { get; set; }
public AbsolutePath ArchiveDir { get; set; }
public string TempFolder { get; set; }
public bool JobScheduler { get; set; }
public bool JobRunner { get; set; }

View File

@ -16,6 +16,8 @@ using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nettle;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Org.BouncyCastle.Crypto.Engines;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
@ -142,7 +144,7 @@ namespace Wabbajack.BuildServer.Controllers
_writeLocks.TryRemove(Key, out var _);
var record = new UploadedFile
{
Id = parts[1],
Id = Guid.Parse(parts[1]),
Hash = hash,
Name = originalName,
Uploader = user,
@ -235,8 +237,42 @@ namespace Wabbajack.BuildServer.Controllers
return Ok($"Deleted {name}");
return NotFound(name);
}
[HttpGet]
[Route("ingest/uploaded_files/{name}")]
[Authorize]
public async Task<IActionResult> IngestMongoDB(string name)
{
var fullPath = name.RelativeTo((AbsolutePath)_settings.TempFolder);
await using var fs = fullPath.OpenRead();
var files = new List<UploadedFile>();
using var rdr = new JsonTextReader(new StreamReader(fs)) {SupportMultipleContent = true};
while (await rdr.ReadAsync())
{
dynamic obj = await JObject.LoadAsync(rdr);
var uf = new UploadedFile
{
Id = Guid.Parse((string)obj._id),
Name = obj.Name,
Size = long.Parse((string)(obj.Size["$numberLong"] ?? obj.Size["$numberInt"])),
Hash = Hash.FromBase64((string)obj.Hash),
Uploader = obj.Uploader,
UploadDate = long.Parse(((string)obj.UploadDate["$date"]["$numberLong"]).Substring(0, 10)).AsUnixTime(),
CDNName = obj.CDNName
};
files.Add(uf);
await SQL.AddUploadedFile(uf);
}
return Ok(files.Count);
}
}
}

View File

@ -25,10 +25,7 @@ namespace Wabbajack.BuildServer.Controllers
public async Task<string> AddUser(string Name)
{
var user = new ApiKey();
var arr = new byte[128];
new Random().NextBytes(arr);
user.Owner = Name;
user.Key = arr.ToHex();
user.Key = NewAPIKey();
user.Id = Guid.NewGuid().ToString();
user.Roles = new List<string>();
user.CanUploadLists = new List<string>();
@ -54,6 +51,12 @@ namespace Wabbajack.BuildServer.Controllers
return "done";
}
public static string NewAPIKey()
{
var arr = new byte[128];
new Random().NextBytes(arr);
return arr.ToHex();
}
}
}

View File

@ -20,7 +20,7 @@ namespace Wabbajack.BuildServer.Models.Jobs
{
public override string Description => $"Push an uploaded file ({FileId}) to the CDN";
public string FileId { get; set; }
public Guid FileId { get; set; }
public override async Task<JobResult> Execute(DBContext db, SqlService sql, AppSettings settings)
{

View File

@ -29,6 +29,7 @@ namespace Wabbajack.BuildServer.Model.Models
private async Task<SqlConnection> Open()
{
var conn = new SqlConnection(_settings.SqlConnection);
Utils.Log("CONN : " + _settings.SqlConnection);
await conn.OpenAsync();
return conn;
}
@ -166,6 +167,17 @@ namespace Wabbajack.BuildServer.Model.Models
.ToList();
}
#region UserRoutines
public async Task<string> LoginByAPIKey(string key)
{
await using var conn = await Open();
var result = await conn.QueryAsync<string>(@"SELECT Owner as Id FROM dbo.ApiKeys WHERE ApiKey = @Key",
new {Key = key});
return result.FirstOrDefault();
}
#endregion
#region JobRoutines
@ -247,5 +259,23 @@ namespace Wabbajack.BuildServer.Model.Models
#endregion
public async Task AddUploadedFile(UploadedFile uf)
{
await using var conn = await Open();
await conn.ExecuteAsync(
"INSERT INTO dbo.UploadedFiles (Id, Name, Size, UploadedBy, Hash, UploadDate, CDNName) VALUES " +
"(@Id, @Name, @Size, @UploadedBy, @Hash, @UploadDate, @CDNName)",
new
{
Id = uf.Id.ToString(),
Name = uf.Name,
Size = uf.Size,
UploadedBy = uf.Uploader,
Hash = (long)uf.Hash,
UploadDate = uf.UploadDate,
CDNName = uf.CDNName
});
}
}
}

View File

@ -10,7 +10,7 @@ namespace Wabbajack.BuildServer.Models
{
public class UploadedFile
{
public string Id { get; set; }
public Guid Id { get; set; }
public string Name { get; set; }
public long Size { get; set; }
public Hash Hash { get; set; }

View File

@ -15,27 +15,31 @@ namespace Wabbajack.BuildServer
{
public static void Main(string[] args)
{
CreateHostBuilder(args).Build().Run();
CreateHostBuilder(args, false).Build().Run();
}
public static IHostBuilder CreateHostBuilder(string[] args) =>
public static IHostBuilder CreateHostBuilder(string[] args, bool testMode) =>
Host.CreateDefaultBuilder(args)
.ConfigureWebHostDefaults(webBuilder =>
{
webBuilder.UseStartup<Startup>()
.UseKestrel(options =>
{
options.Listen(IPAddress.Any, 80);
options.Listen(IPAddress.Any, 443, listenOptions =>
options.Listen(IPAddress.Any, testMode ? 8080 : 80);
if (!testMode)
{
using (var store = new X509Store(StoreName.My))
options.Listen(IPAddress.Any, 443, listenOptions =>
{
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Find(X509FindType.FindBySubjectName, "build.wabbajack.org", true)[0];
listenOptions.UseHttps(cert);
}
});
using (var store = new X509Store(StoreName.My))
{
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Find(X509FindType.FindBySubjectName,
"build.wabbajack.org", true)[0];
listenOptions.UseHttps(cert);
}
});
}
options.Limits.MaxRequestBodySize = null;
});
});

View File

@ -36,6 +36,12 @@ using Directory = System.IO.Directory;
namespace Wabbajack.BuildServer
{
public class TestStartup : Startup
{
public TestStartup(IConfiguration configuration) : base(configuration)
{
}
}
public class Startup
{
public Startup(IConfiguration configuration)
@ -90,7 +96,9 @@ namespace Wabbajack.BuildServer
app.UseDeveloperExceptionPage();
}
app.UseHttpsRedirection();
if (!(this is TestStartup))
app.UseHttpsRedirection();
app.UseGraphiQl();
app.UseDeveloperExceptionPage();

View File

@ -13,7 +13,7 @@
}
},
"MongoDB": {
"Host": "internal.test.mongodb",
"Host": "foo.bar.baz",
"Database": "wabbajack",
"Collections": {
"NexusModInfos": "nexus_mod_infos",
@ -32,13 +32,14 @@
"WabbajackSettings": {
"DownloadDir": "c:\\tmp\\downloads",
"ArchiveDir": "w:\\archives",
"TempFolder": "c:\\tmp",
"JobRunner": true,
"JobScheduler": false,
"RunFrontEndJobs": true,
"RunBackEndJobs": true,
"BunnyCDN_User": "wabbajackcdn",
"BunnyCDN_Password": "XXXX",
"SQLConnection": "Data Source=192.168.3.1,1433;Initial Catalog=wabbajack_dev;User ID=wabbajack;Password=wabbajack;MultipleActiveResultSets=true"
"SQLConnection": "Data Source=_,1433;Initial Catalog=wabbajack_dev;User ID=wabbajack;Password=wabbajack;MultipleActiveResultSets=true"
},
"AllowedHosts": "*"
}

View File

@ -52,6 +52,13 @@ namespace Wabbajack.Common.Http
private async Task<string> SendStringAsync(HttpRequestMessage request)
{
using var result = await SendAsync(request);
if (!result.IsSuccessStatusCode)
{
Utils.Log("Internal Error");
Utils.Log(await result.Content.ReadAsStringAsync());
throw new Exception(
$"Bad HTTP request {result.StatusCode} {result.ReasonPhrase} - {request.RequestUri}");
}
return await result.Content.ReadAsStringAsync();
}