Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rewrote the uploader to use a block based approach

Browse Source
This commit is contained in:
Timothy Baldridge 2020-01-18 22:51:12 -07:00
parent 72977ec0b3
commit f2359ab273
7 changed files with 136 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
Wabbajack.BuildServer/Controllers/UploadedFiles.cs
View File

@ -1,8 +1,10 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Security.Claims;
using System.Text;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
@ -14,6 +16,7 @@ using MongoDB.Driver.Linq;
using Nettle;
using Wabbajack.BuildServer.Models;
using Wabbajack.Common;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.BuildServer.Controllers
{
@ -24,16 +27,60 @@ namespace Wabbajack.BuildServer.Controllers
}
[HttpPost]
[Authorize]
[Route("upload_file")]
public async Task<IActionResult> UploadFile(IList<IFormFile> files)
[HttpPut]
[Route("upload_file/{Name}/start")]
public async Task<IActionResult> UploadFileStreaming(string Name)
{
var guid = Guid.NewGuid();
var key = Encoding.UTF8.GetBytes($"{Path.GetFileNameWithoutExtension(Name)}|{guid.ToString()}|{Path.GetExtension(Name)}").ToHex();
System.IO.File.Create(Path.Combine("public", "files", key)).Close();
Utils.Log($"Starting Ingest for {key}");
return Ok(key);
}
static private HashSet<char> HexChars = new HashSet<char>("abcdef1234567890");
[HttpPut]
[Route("upload_file/{Key}/data/{Offset}")]
public async Task<IActionResult> UploadFilePart(string Key, long Offset)
{
if (!Key.All(a => HexChars.Contains(a)))
return BadRequest("NOT A VALID FILENAME");
Utils.Log($"Writing at position {Offset} in ingest file {Key}");
await using (var file = System.IO.File.Open(Path.Combine("public", "files", Key), FileMode.Open, FileAccess.Write))
{
file.Position = Offset;
await Request.Body.CopyToAsync(file);
return Ok(file.Position.ToString());
}
}
[HttpPut]
[Route("upload_file/{Key}/finish")]
public async Task<IActionResult> UploadFileFinish(string Key)
{
var user = User.FindFirstValue(ClaimTypes.Name);
UploadedFile result = null;
result = await UploadedFile.Ingest(Db, files.First(), user);
return Ok(result.Uri.ToString());
if (!Key.All(a => HexChars.Contains(a)))
return BadRequest("NOT A VALID FILENAME");
var parts = Encoding.UTF8.GetString(Key.FromHex()).Split('|');
var final_name = $"{parts[0]}-{parts[1]}{parts[2]}";
var original_name = $"{parts[0]}{parts[2]}";
var final_path = Path.Combine("public", "files", final_name);
System.IO.File.Move(Path.Combine("public", "files", Key), final_path);
var hash = await final_path.FileHashAsync();
var record = new UploadedFile
{
Id = parts[1],
Hash = hash,
Name = original_name,
Uploader = user,
Size = new FileInfo(final_path).Length
};
await Db.UploadedFiles.InsertOneAsync(record);
return Ok(record.Uri);
}
private static readonly Func<object, string> HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@"
<html><body>
@ -69,5 +116,7 @@ namespace Wabbajack.BuildServer.Controllers
Content = response
};
}
}
}

5
Wabbajack.BuildServer/JobManager.cs
View File

@ -4,6 +4,7 @@ using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nettle;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
@ -106,8 +107,8 @@ namespace Wabbajack.BuildServer
private async Task ScheduledJob<T>(TimeSpan span, Job.JobPriority priority) where T : AJobPayload, new()
{
if (!Settings.RunBackEndJobs && typeof(T).IsSubclassOf(typeof(IBackEndJob))) return;
if (!Settings.RunFrontEndJobs && typeof(T).IsSubclassOf(typeof(IFrontEndJob))) return;
if (!Settings.RunBackEndJobs && typeof(T).ImplementsInterface(typeof(IBackEndJob))) return;
if (!Settings.RunFrontEndJobs && typeof(T).ImplementsInterface(typeof(IFrontEndJob))) return;
try
{
var jobs = await Db.Jobs.AsQueryable()

13
Wabbajack.BuildServer/Models/UploadedFile.cs
View File

@ -21,18 +21,5 @@ namespace Wabbajack.BuildServer.Models
public string MungedName => $"{Path.GetFileNameWithoutExtension(Name)}-{Id}{Path.GetExtension(Name)}";
[BsonIgnore] public object Uri => $"https://build.wabbajack.org/files/{MungedName}";
public static async Task<UploadedFile> Ingest(DBContext db, IFormFile src, string uploader)
{
var record = new UploadedFile {Uploader = uploader, Name = src.FileName, Id = Guid.NewGuid().ToString()};
var dest_path = $@"public\\files\\{record.MungedName}";
using (var stream = File.OpenWrite(dest_path))
await src.CopyToAsync(stream);
record.Size = new FileInfo(dest_path).Length;
record.Hash = await dest_path.FileHashAsync();
await db.UploadedFiles.InsertOneAsync(record);
return record;
}
}
}

12
Wabbajack.BuildServer/Startup.cs
View File

@ -26,6 +26,7 @@ using Swashbuckle.AspNetCore.Swagger;
using Wabbajack.BuildServer.Controllers;
using Wabbajack.BuildServer.Models;
using Microsoft.AspNetCore.Mvc.NewtonsoftJson;
using Microsoft.AspNetCore.StaticFiles;
using Wabbajack.BuildServer.Controllers;
using Microsoft.Extensions.FileProviders;
using Directory = System.IO.Directory;
@ -89,8 +90,13 @@ namespace Wabbajack.BuildServer
app.UseHttpsRedirection();
app.UseGraphiQl();
app.UseDeveloperExceptionPage();
var provider = new FileExtensionContentTypeProvider();
provider.Mappings[".rar"] = "application/x-rar-compressed";
provider.Mappings[".7z"] = "application/x-7z-compressed";
provider.Mappings[".zip"] = "application/zip";
provider.Mappings[".wabbajack"] = "application/zip";
app.UseStaticFiles();
//app.UseHttpsRedirection();
app.UseSwagger();
app.UseSwaggerUI(c =>
@ -106,7 +112,9 @@ namespace Wabbajack.BuildServer
app.UseFileServer(new FileServerOptions
{
FileProvider = new PhysicalFileProvider(
Path.Combine(Directory.GetCurrentDirectory(), "public"))
Path.Combine(Directory.GetCurrentDirectory(), "public")),
StaticFileOptions = {ServeUnknownFileTypes = true},
EnableDirectoryBrowsing = true
});
app.UseEndpoints(endpoints =>

66
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -1,5 +1,7 @@
using System;
using System.Net.Http;
using System.Reactive.Linq;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using Wabbajack.Common;
@ -18,5 +20,69 @@ namespace Wabbajack.Lib.FileUploader
return File.ReadAllText(Path.Combine(Consts.LocalAppDataPath, "author-api-key.txt")).Trim();
}
public static bool HasAPIKey => File.Exists(Path.Combine(Consts.LocalAppDataPath, "author-api-key.txt"));
public static readonly Uri UploadURL = new Uri("https://build.wabbajack.org/upload_file");
public static long BLOCK_SIZE = (long)1024 * 1024 * 8;
public static Task<string> UploadFile(WorkQueue queue, string filename)
{
var tcs = new TaskCompletionSource<string>();
queue.QueueTask(async () =>
{
using (var stream =
new StatusFileStream(File.OpenRead(filename), $"Uploading {Path.GetFileName(filename)}", queue))
{
var client = new HttpClient();
client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey());
var response = await client.PutAsync(UploadURL+$"/{Path.GetFileName(filename)}/start", new StringContent(""));
if (!response.IsSuccessStatusCode)
{
tcs.SetResult("FAILED");
return;
}
var key = await response.Content.ReadAsStringAsync();
var data = new byte[BLOCK_SIZE];
while (stream.Position < stream.Length)
{
var old_offset = stream.Position;
var new_size = Math.Min(stream.Length - stream.Position, BLOCK_SIZE);
if (new_size != data.Length)
data = new byte[new_size];
stream.ReadAsync(data, 0, data.Length);
response = await client.PutAsync(UploadURL + $"/{key}/data/{old_offset}",
new ByteArrayContent(data));
if (!response.IsSuccessStatusCode)
{
tcs.SetResult("FAILED");
return;
}
var val = long.Parse(await response.Content.ReadAsStringAsync());
if (val != old_offset + data.Length)
{
tcs.SetResult("Sync Error");
return;
}
}
response = await client.PutAsync(UploadURL + $"/{key}/finish", new StringContent(""));
if (response.IsSuccessStatusCode)
tcs.SetResult(await response.Content.ReadAsStringAsync());
else
tcs.SetResult("FAILED");
}
});
return tcs.Task;
}
}
}

27
Wabbajack.Lib/GraphQL/GraphQLService.cs
View File

@ -18,7 +18,6 @@ namespace Wabbajack.Lib.GraphQL
public class GraphQLService
{
public static readonly Uri BaseURL = new Uri("https://build.wabbajack.org/graphql");
public static readonly Uri UploadURL = new Uri("https://build.wabbajack.org/upload_file");
public static async Task<List<UploadedFile>> GetUploadedFiles()
{
@ -41,31 +40,5 @@ namespace Wabbajack.Lib.GraphQL
return result.GetDataFieldAs<List<UploadedFile>>("uploadedFiles");
}
public static Task<string> UploadFile(WorkQueue queue, string filename)
{
var tcs = new TaskCompletionSource<string>();
queue.QueueTask(async () =>
{
using (var stream =
new StatusFileStream(File.OpenRead(filename), $"Uploading {Path.GetFileName(filename)}", queue))
{
var client = new HttpClient();
client.DefaultRequestHeaders.Add("X-API-KEY", AuthorAPI.GetAPIKey());
var content = new StreamContent(stream);
var form = new MultipartFormDataContent
{
{content, "files", Path.GetFileName(filename)}
};
content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
var response = await client.PostAsync(UploadURL, form);
if (response.IsSuccessStatusCode)
tcs.SetResult(await response.Content.ReadAsStringAsync());
else
tcs.SetResult("FAILED");
}
});
return tcs.Task;
}
}
}

2
Wabbajack/View Models/Settings/AuthorFilesVM.cs
View File

@ -51,7 +51,7 @@ namespace Wabbajack
Upload = ReactiveCommand.Create(async () =>
{
SelectedFile = await GraphQLService.UploadFile(Queue, SelectedFile);
SelectedFile = await AuthorAPI.UploadFile(Queue, SelectedFile);
});
}
}