Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #460 from wabbajack-tools/fix-uploading-again

Browse Source 
Fix file uploading and implement server-side hash verification.
This commit is contained in:
Timothy Baldridge 2020-01-31 22:42:37 -07:00 committed by GitHub
commit d0d99095d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 39 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
Wabbajack.BuildServer/Controllers/UploadedFiles.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@ -26,6 +27,7 @@ namespace Wabbajack.BuildServer.Controllers
{
public class UploadedFiles : AControllerBase<UploadedFiles>
{
private static ConcurrentDictionary<string, AsyncLock> _writeLocks = new ConcurrentDictionary<string, AsyncLock>();
private AppSettings _settings;
public UploadedFiles(ILogger<UploadedFiles> logger, DBContext db, AppSettings settings) : base(logger, db)
@ -39,6 +41,9 @@ namespace Wabbajack.BuildServer.Controllers
{
var guid = Guid.NewGuid();
var key = Encoding.UTF8.GetBytes($"{Path.GetFileNameWithoutExtension(Name)}|{guid.ToString()}|{Path.GetExtension(Name)}").ToHex();
_writeLocks.GetOrAdd(key, new AsyncLock());
System.IO.File.Create(Path.Combine("public", "files", key)).Close();
Utils.Log($"Starting Ingest for {key}");
return Ok(key);
@ -52,18 +57,27 @@ namespace Wabbajack.BuildServer.Controllers
if (!Key.All(a => HexChars.Contains(a)))
return BadRequest("NOT A VALID FILENAME");
Utils.Log($"Writing at position {Offset} in ingest file {Key}");
var ms = new MemoryStream();
await Request.Body.CopyToAsync(ms);
ms.Position = 0;
long position;
using (var _ = await _writeLocks[Key].Wait())
await using (var file = System.IO.File.Open(Path.Combine("public", "files", Key), FileMode.Open, FileAccess.Write, FileShare.ReadWrite))
{
file.Position = Offset;
await Request.Body.CopyToAsync(file);
return Ok(file.Position.ToString());
await ms.CopyToAsync(file);
position = file.Position;
}
return Ok(position);
}
[HttpPut]
[Route("upload_file/{Key}/finish")]
public async Task<IActionResult> UploadFileFinish(string Key)
[Route("upload_file/{Key}/finish/{xxHashAsHex}")]
public async Task<IActionResult> UploadFileFinish(string Key, string xxHashAsHex)
{
var expectedHash = xxHashAsHex.FromHex().ToBase64();
var user = User.FindFirstValue(ClaimTypes.Name);
if (!Key.All(a => HexChars.Contains(a)))
return BadRequest("NOT A VALID FILENAME");
@ -74,8 +88,14 @@ namespace Wabbajack.BuildServer.Controllers
var final_path = Path.Combine("public", "files", final_name);
System.IO.File.Move(Path.Combine("public", "files", Key), final_path);
var hash = await final_path.FileHashAsync();
if (expectedHash != hash)
{
System.IO.File.Delete(final_path);
return BadRequest($"Bad Hash, Expected: {expectedHash} Got: {hash}");
}
_writeLocks.TryRemove(Key, out var _);
var record = new UploadedFile
{
Id = parts[1],

16
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -1,4 +1,6 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
@ -38,6 +40,9 @@ namespace Wabbajack.Lib.FileUploader
var handler = new HttpClientHandler {MaxConnectionsPerServer = MAX_CONNECTIONS};
var client = new HttpClient(handler);
var fsize = new FileInfo(filename).Length;
var hash_task = filename.FileHashAsync();
client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey());
var response = await client.PutAsync(UploadURL+$"/{Path.GetFileName(filename)}/start", new StringContent(""));
if (!response.IsSuccessStatusCode)
@ -46,12 +51,18 @@ namespace Wabbajack.Lib.FileUploader
return;
}
IEnumerable<long> Blocks(long fsize)
{
for (long block = 0; block * BLOCK_SIZE < fsize; block ++)
yield return block;
}
var key = await response.Content.ReadAsStringAsync();
long sent = 0;
using (var iqueue = new WorkQueue(MAX_CONNECTIONS))
{
iqueue.Report("Starting Upload", 1);
await Enumerable.Range(0, (int)(fsize / BLOCK_SIZE))
await Blocks(fsize)
.PMap(iqueue, async block_idx =>
{
if (tcs.Task.IsFaulted) return;
@ -93,7 +104,8 @@ namespace Wabbajack.Lib.FileUploader
if (!tcs.Task.IsFaulted)
{
progressFn(1.0);
response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent(""));
var hash = (await hash_task).FromBase64().ToHex();
response = await client.PutAsync(UploadURL + $"/{key}/finish/{hash}", new StringContent(""));
if (response.IsSuccessStatusCode)
tcs.SetResult(await response.Content.ReadAsStringAsync());
else