using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Security.Claims; using System.Text; using System.Threading.Tasks; using FluentFTP; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using MongoDB.Driver; using MongoDB.Driver.Linq; using Nettle; using Org.BouncyCastle.Crypto.Engines; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Path = Alphaleonis.Win32.Filesystem.Path; using AlphaFile = Alphaleonis.Win32.Filesystem.File; namespace Wabbajack.BuildServer.Controllers { public class UploadedFiles : AControllerBase { private static ConcurrentDictionary _writeLocks = new ConcurrentDictionary(); private AppSettings _settings; public UploadedFiles(ILogger logger, DBContext db, AppSettings settings, SqlService sql) : base(logger, db, sql) { _settings = settings; } [HttpPut] [Route("upload_file/{Name}/start")] public async Task UploadFileStreaming(string Name) { var guid = Guid.NewGuid(); var key = Encoding.UTF8.GetBytes($"{Path.GetFileNameWithoutExtension(Name)}|{guid.ToString()}|{Path.GetExtension(Name)}").ToHex(); _writeLocks.GetOrAdd(key, new AsyncLock()); System.IO.File.Create(Path.Combine("public", "tmp_files", key)).Close(); Utils.Log($"Starting Ingest for {key}"); return Ok(key); } static private HashSet HexChars = new HashSet("abcdef1234567890"); [HttpPut] [Route("upload_file/{Key}/data/{Offset}")] public async Task UploadFilePart(string Key, long Offset) { if (!Key.All(a => HexChars.Contains(a))) return BadRequest("NOT A VALID FILENAME"); Utils.Log($"Writing at position {Offset} in ingest file {Key}"); var ms = new MemoryStream(); await Request.Body.CopyToAsync(ms); ms.Position = 0; long position; using (var _ = await _writeLocks[Key].Wait()) await using (var file = System.IO.File.Open(Path.Combine("public", "tmp_files", Key), FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) { file.Position = Offset; await ms.CopyToAsync(file); position = file.Position; } return Ok(position); } [Authorize] [HttpGet] [Route("clean_http_uploads")] public async Task CleanUploads() { var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync(); var seen = new HashSet(); var duplicate = new List(); foreach (var file in files) { if (seen.Contains(file.Name)) duplicate.Add(file); else seen.Add(file.Name); } using (var client = new FtpClient("storage.bunnycdn.com")) { client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password); await client.ConnectAsync(); foreach (var dup in duplicate) { var final_path = Path.Combine("public", "files", dup.MungedName); Utils.Log($"Cleaning upload {final_path}"); if (AlphaFile.Exists(final_path)) AlphaFile.Delete(final_path); if (await client.FileExistsAsync(dup.MungedName)) await client.DeleteFileAsync(dup.MungedName); await Db.UploadedFiles.DeleteOneAsync(f => f.Id == dup.Id); } } return Ok(new {Remain = seen.ToArray(), Deleted = duplicate.ToArray()}.ToJSON(prettyPrint:true)); } [HttpPut] [Route("upload_file/{Key}/finish/{xxHashAsHex}")] public async Task UploadFileFinish(string Key, string xxHashAsHex) { var expectedHash = xxHashAsHex.FromHex().ToBase64(); var user = User.FindFirstValue(ClaimTypes.Name); if (!Key.All(a => HexChars.Contains(a))) return BadRequest("NOT A VALID FILENAME"); var parts = Encoding.UTF8.GetString(Key.FromHex()).Split('|'); var final_name = $"{parts[0]}-{parts[1]}{parts[2]}"; var original_name = $"{parts[0]}{parts[2]}"; var final_path = Path.Combine("public", "files", final_name); System.IO.File.Move(Path.Combine("public", "tmp_files", Key), final_path); var hash = await final_path.FileHashAsync(); if (expectedHash != hash) { System.IO.File.Delete(final_path); return BadRequest($"Bad Hash, Expected: {expectedHash} Got: {hash}"); } _writeLocks.TryRemove(Key, out var _); var record = new UploadedFile { Id = parts[1], Hash = hash, Name = original_name, Uploader = user, Size = new FileInfo(final_path).Length, CDNName = "wabbajackpush" }; await Db.UploadedFiles.InsertOneAsync(record); await Db.Jobs.InsertOneAsync(new Job { Priority = Job.JobPriority.High, Payload = new UploadToCDN {FileId = record.Id} }); return Ok(record.Uri); } private static readonly Func HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@" {{each $.files }} {{/each}}
{{$.Name}}{{$.Size}}{{$.Date}}{{$.Uploader}}
"); [HttpGet] [Route("uploaded_files")] public async Task UploadedFilesGet() { var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync(); var response = HandleGetListTemplate(new { files = files.Select(file => new { Link = file.Uri, Size = file.Size.ToFileSizeString(), file.Name, Date = file.UploadDate, file.Uploader }) }); return new ContentResult { ContentType = "text/html", StatusCode = (int) HttpStatusCode.OK, Content = response }; } [HttpGet] [Route("uploaded_files/list")] [Authorize] public async Task ListMyFiles() { var user = User.FindFirstValue(ClaimTypes.Name); Utils.Log($"List Uploaded Files {user}"); var files = await Db.UploadedFiles.AsQueryable().Where(f => f.Uploader == user).ToListAsync(); return Ok(files.OrderBy(f => f.UploadDate).Select(f => f.MungedName).ToArray().ToJSON(prettyPrint:true)); } [HttpDelete] [Route("uploaded_files/{name}")] [Authorize] public async Task DeleteMyFile(string name) { var user = User.FindFirstValue(ClaimTypes.Name); Utils.Log($"Delete Uploaded File {user} {name}"); var files = await Db.UploadedFiles.AsQueryable().Where(f => f.Uploader == user).ToListAsync(); var to_delete = files.First(f => f.MungedName == name); if (AlphaFile.Exists(Path.Combine("public", "files", to_delete.MungedName))) AlphaFile.Delete(Path.Combine("public", "files", to_delete.MungedName)); using (var client = new FtpClient("storage.bunnycdn.com")) { client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password); await client.ConnectAsync(); if (await client.FileExistsAsync(to_delete.MungedName)) await client.DeleteFileAsync(to_delete.MungedName); } var result = await Db.UploadedFiles.DeleteOneAsync(f => f.Id == to_delete.Id); if (result.DeletedCount == 1) return Ok($"Deleted {name}"); return NotFound(name); } } }