Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes for author APIs

Browse Source
This commit is contained in:
Timothy Baldridge 2020-03-30 14:38:46 -06:00
parent 8b9210eeb0
commit 160ac8a4c3
10 changed files with 96 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Wabbajack.BuildServer.Test/ABuildServerSystemTest.cs
View File

@ -31,10 +31,13 @@ namespace Wabbajack.BuildServer.Test
$"WabbajackSettings:ArchiveDir={"archives".RelativeTo(AbsolutePath.EntryPoint)}",
$"WabbajackSettings:TempFolder={ServerTempFolder}",
$"WabbajackSettings:SQLConnection={PublicConnStr}",
$"WabbajackSettings:BunnyCDN_User=TEST",
$"WabbajackSettings:BunnyCDN_Password=TEST",
}, true);
_host = builder.Build();
_token = new CancellationTokenSource();
_task = _host.RunAsync(_token.Token);
Consts.WabbajackBuildServerUri = new Uri("http://localhost:8080");
}
public void Dispose()
@ -52,16 +55,19 @@ namespace Wabbajack.BuildServer.Test
private readonly IDisposable _unsubMsgs;
private readonly IDisposable _unsubErr;
protected Client _authedClient;
protected WorkQueue _queue;
public ABuildServerSystemTest(ITestOutputHelper output, BuildServerFixture fixture) : base(output)
{
Filters.Clear();
_unsubMsgs = Utils.LogMessages.OfType<IInfo>().Subscribe(onNext: msg => XunitContext.WriteLine(msg.ShortDescription));
_unsubErr = Utils.LogMessages.OfType<IUserIntervention>().Subscribe(msg =>
XunitContext.WriteLine("ERROR: User intervention required: " + msg.ShortDescription));
_client = new Client();
_authedClient = new Client();
_authedClient.Headers.Add(("x-api-key", fixture.APIKey));
_queue = new WorkQueue();
Fixture = fixture;
}

25
Wabbajack.BuildServer.Test/UploadedFilesTest.cs
View File

@ -4,6 +4,8 @@ using System.Threading.Tasks;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.FileUploader;
using Xunit;
using Xunit.Abstractions;
using Xunit.Priority;
@ -33,7 +35,7 @@ namespace Wabbajack.BuildServer.Test
}
[Fact, Priority(1)]
public async Task CanLoadUploadedFiles()
public async Task CanListMyUploadedFiles()
{
var result = (await _authedClient.GetStringAsync(MakeURL("uploaded_files/list"))).FromJSONString<string[]>();
Utils.Log("Loaded: " + result);
@ -47,5 +49,26 @@ namespace Wabbajack.BuildServer.Test
Assert.DoesNotContain("file3-17b3e918-8409-48e6-b7ff-6af858bfd1ba.zip", result);
}
[Fact]
public async Task CanUploadFilesUsingClientApi()
{
using (var file = new TempFile())
{
var data = new byte[1024 * 1024 * 8 * 4];
await using (var fs = file.Path.Create())
{
await fs.WriteAsync(data);
}
Utils.Log($"Uploading {file.Path.Size.ToFileSizeString()} file");
var result = await AuthorAPI.UploadFile(file.Path,
progress => Utils.Log($"Uploading : {progress * 100}%"), Fixture.APIKey);
Utils.Log($"Result {result}");
Assert.StartsWith("https://wabbajackpush.b-cdn.net/" +(string)file.Path.FileNameWithoutExtension, result);
}
}
}
}

2
Wabbajack.BuildServer/AppSettings.cs
View File

@ -15,6 +15,8 @@ namespace Wabbajack.BuildServer
public string TempFolder { get; set; }
public AbsolutePath TempPath => (AbsolutePath)TempFolder;
public bool JobScheduler { get; set; }
public bool JobRunner { get; set; }

19
Wabbajack.BuildServer/Controllers/UploadedFiles.cs
View File

@ -50,7 +50,7 @@ namespace Wabbajack.BuildServer.Controllers
_writeLocks.GetOrAdd(key, new AsyncLock());
System.IO.File.Create(Path.Combine("public", "tmp_files", key)).Close();
await using var fs = _settings.TempPath.Combine(key).Create();
Utils.Log($"Starting Ingest for {key}");
return Ok(key);
}
@ -62,20 +62,24 @@ namespace Wabbajack.BuildServer.Controllers
{
if (!Key.All(a => HexChars.Contains(a)))
return BadRequest("NOT A VALID FILENAME");
Utils.Log($"Writing at position {Offset} in ingest file {Key}");
var ms = new MemoryStream();
await Request.Body.CopyToAsync(ms);
ms.Position = 0;
Utils.Log($"Writing {ms.Length} at position {Offset} in ingest file {Key}");
long position;
using (var _ = await _writeLocks[Key].Wait())
await using (var file = System.IO.File.Open(Path.Combine("public", "tmp_files", Key), FileMode.Open, FileAccess.Write, FileShare.ReadWrite))
{
await using var file = _settings.TempPath.Combine(Key).WriteShared();
file.Position = Offset;
await ms.CopyToAsync(file);
position = file.Position;
position = Offset + ms.Length;
}
Utils.Log($"Wrote {ms.Length} as position {Offset} result {position}");
return Ok(position);
}
@ -132,7 +136,8 @@ namespace Wabbajack.BuildServer.Controllers
var originalName = $"{parts[0]}{parts[2]}";
var finalPath = "public".RelativeTo(AbsolutePath.EntryPoint).Combine("files", finalName);
"public".RelativeTo(AbsolutePath.EntryPoint).MoveTo(finalPath);
_settings.TempPath.Combine(Key).MoveTo(finalPath);
var hash = await finalPath.FileHashAsync();
if (expectedHash != hash)
@ -151,8 +156,8 @@ namespace Wabbajack.BuildServer.Controllers
Size = finalPath.Size,
CDNName = "wabbajackpush"
};
await Db.UploadedFiles.InsertOneAsync(record);
await Db.Jobs.InsertOneAsync(new Job
await SQL.AddUploadedFile(record);
await SQL.EnqueueJob(new Job
{
Priority = Job.JobPriority.High, Payload = new UploadToCDN {FileId = record.Id}
});

5
Wabbajack.BuildServer/Models/Jobs/UploadToCDN.cs
View File

@ -28,6 +28,11 @@ namespace Wabbajack.BuildServer.Models.Jobs
TOP:
var file = await db.UploadedFiles.AsQueryable().Where(f => f.Id == FileId).FirstOrDefaultAsync();
if (settings.BunnyCDN_User == "TEST" && settings.BunnyCDN_Password == "TEST")
{
return JobResult.Success();
}
using (var client = new FtpClient("storage.bunnycdn.com"))
{
client.Credentials = new NetworkCredential(settings.BunnyCDN_User, settings.BunnyCDN_Password);

1
Wabbajack.Common/Consts.cs
View File

@ -107,6 +107,7 @@ namespace Wabbajack.Common
public static string WabbajackCacheLocation = "http://build.wabbajack.org/nexus_api_cache/";
public static string WabbajackCacheHostname = "build.wabbajack.org";
public static Uri WabbajackBuildServerUri = new Uri("https://build.wabbajack.org");
public static int WabbajackCachePort = 80;
public static int MaxHTTPRetries = 4;
public static RelativePath MO2ModFolderName = (RelativePath)"mods";

5
Wabbajack.Common/Paths.cs
View File

@ -384,6 +384,11 @@ namespace Wabbajack.Common
{
return File.Open(_path, FileMode.Open, FileAccess.Read);
}
public FileStream WriteShared()
{
return File.Open(_path, FileMode.Open, FileAccess.Write, FileShare.ReadWrite);
}
}
public struct RelativePath : IPath, IEquatable<RelativePath>, IComparable<RelativePath>

8
Wabbajack.Common/Utils.cs
View File

@ -1138,6 +1138,14 @@ namespace Wabbajack.Common
return bytes.ToHex();
}
public static byte[] RandomData(int size)
{
var random = new Random();
byte[] bytes = new byte[size];
random.NextBytes(bytes);
return bytes;
}
public static async Task CopyFileAsync(string src, string dest)
{
await using var s = File.OpenRead(src);

52
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -21,24 +21,25 @@ namespace Wabbajack.Lib.FileUploader
{
public static IObservable<bool> HaveAuthorAPIKey => Utils.HaveEncryptedJsonObservable("author-api-key.txt");
public static async Task<string> GetAPIKey()
public static async Task<string> GetAPIKey(string apiKey = null)
{
return (await Consts.LocalAppDataPath.Combine("author-api-key.txt").ReadAllTextAsync()).Trim();
return apiKey ?? (await Consts.LocalAppDataPath.Combine("author-api-key.txt").ReadAllTextAsync()).Trim();
}
public static readonly Uri UploadURL = new Uri("https://build.wabbajack.org/upload_file");
public static Uri UploadURL => new Uri($"{Consts.WabbajackBuildServerUri}upload_file");
public static long BLOCK_SIZE = (long)1024 * 1024 * 2;
public static int MAX_CONNECTIONS = 8;
public static Task<string> UploadFile(WorkQueue queue, AbsolutePath filename, Action<double> progressFn)
public static Task<string> UploadFile(AbsolutePath filename, Action<double> progressFn, string apikey=null)
{
var tcs = new TaskCompletionSource<string>();
Task.Run(async () =>
{
var client = await GetAuthorizedClient();
var client = await GetAuthorizedClient(apikey);
var fsize = filename.Size;
var hash_task = filename.FileHashAsync();
var hashTask = filename.FileHashAsync();
Utils.Log($"{UploadURL}/{filename.FileName.ToString()}/start");
var response = await client.PutAsync($"{UploadURL}/{filename.FileName.ToString()}/start", new StringContent(""));
if (!response.IsSuccessStatusCode)
{
@ -58,36 +59,39 @@ namespace Wabbajack.Lib.FileUploader
{
iqueue.Report("Starting Upload", Percent.One);
await Blocks(fsize)
.PMap(iqueue, async block_idx =>
.PMap(iqueue, async blockIdx =>
{
if (tcs.Task.IsFaulted) return;
var block_offset = block_idx * BLOCK_SIZE;
var block_size = block_offset + BLOCK_SIZE > fsize
? fsize - block_offset
var blockOffset = blockIdx * BLOCK_SIZE;
var blockSize = blockOffset + BLOCK_SIZE > fsize
? fsize - blockOffset
: BLOCK_SIZE;
Interlocked.Add(ref sent, block_size);
Interlocked.Add(ref sent, blockSize);
progressFn((double)sent / fsize);
await using var fs = filename.OpenRead();
fs.Position = block_offset;
var data = new byte[block_size];
var data = new byte[blockSize];
await using (var fs = filename.OpenRead())
{
fs.Position = blockOffset;
await fs.ReadAsync(data, 0, data.Length);
}
response = await client.PutAsync(UploadURL + $"/{key}/data/{block_offset}",
var offsetResponse = await client.PutAsync(UploadURL + $"/{key}/data/{blockOffset}",
new ByteArrayContent(data));
if (!response.IsSuccessStatusCode)
if (!offsetResponse.IsSuccessStatusCode)
{
tcs.SetException(new Exception($"Put Error: {response.StatusCode} {response.ReasonPhrase}"));
Utils.Log(await offsetResponse.Content.ReadAsStringAsync());
tcs.SetException(new Exception($"Put Error: {offsetResponse.StatusCode} {offsetResponse.ReasonPhrase}"));
return;
}
var val = long.Parse(await response.Content.ReadAsStringAsync());
if (val != block_offset + data.Length)
var val = long.Parse(await offsetResponse.Content.ReadAsStringAsync());
if (val != blockOffset + data.Length)
{
tcs.SetResult($"Sync Error {val} vs {block_offset + data.Length}");
tcs.SetException(new Exception($"Sync Error {val} vs {block_offset + data.Length}"));
tcs.SetResult($"Sync Error {val} vs {blockOffset + data.Length} Offset {blockOffset} Size {data.Length}");
tcs.SetException(new Exception($"Sync Error {val} vs {blockOffset + data.Length}"));
}
});
}
@ -95,7 +99,7 @@ namespace Wabbajack.Lib.FileUploader
if (!tcs.Task.IsFaulted)
{
progressFn(1.0);
var hash = (await hash_task).ToHex();
var hash = (await hashTask).ToHex();
response = await client.PutAsync(UploadURL + $"/{key}/finish/{hash}", new StringContent(""));
if (response.IsSuccessStatusCode)
tcs.SetResult(await response.Content.ReadAsStringAsync());
@ -109,10 +113,10 @@ namespace Wabbajack.Lib.FileUploader
return tcs.Task;
}
public static async Task<Common.Http.Client> GetAuthorizedClient()
public static async Task<Common.Http.Client> GetAuthorizedClient(string apiKey=null)
{
var client = new Common.Http.Client();
client.Headers.Add(("X-API-KEY", await GetAPIKey()));
client.Headers.Add(("X-API-KEY", await GetAPIKey(apiKey)));
return client;
}

2
Wabbajack/View Models/Settings/AuthorFilesVM.cs
View File

@ -59,7 +59,7 @@ namespace Wabbajack
_isUploading.OnNext(true);
try
{
FinalUrl = await AuthorAPI.UploadFile(Queue, Picker.TargetPath,
FinalUrl = await AuthorAPI.UploadFile(Picker.TargetPath,
progress => UploadProgress = progress);
}
catch (Exception ex)