using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Security.Claims; using System.Text; using System.Threading.Tasks; using FluentFTP; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using MongoDB.Driver; using MongoDB.Driver.Linq; using Nettle; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Org.BouncyCastle.Crypto.Engines; using Wabbajack.BuildServer.Model.Models; using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.Jobs; using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Path = Alphaleonis.Win32.Filesystem.Path; using AlphaFile = Alphaleonis.Win32.Filesystem.File; namespace Wabbajack.BuildServer.Controllers { public class UploadedFiles : AControllerBase { private static ConcurrentDictionary _writeLocks = new ConcurrentDictionary(); private AppSettings _settings; public UploadedFiles(ILogger logger, DBContext db, AppSettings settings, SqlService sql) : base(logger, db, sql) { _settings = settings; } [HttpPut] [Route("upload_file/{Name}/start")] public async Task UploadFileStreaming(string Name) { var guid = Guid.NewGuid(); var key = Encoding.UTF8.GetBytes($"{Path.GetFileNameWithoutExtension(Name)}|{guid.ToString()}|{Path.GetExtension(Name)}").ToHex(); _writeLocks.GetOrAdd(key, new AsyncLock()); await using var fs = _settings.TempPath.Combine(key).Create(); Utils.Log($"Starting Ingest for {key}"); return Ok(key); } static private HashSet HexChars = new HashSet("abcdef1234567890"); [HttpPut] [Route("upload_file/{Key}/data/{Offset}")] public async Task UploadFilePart(string Key, long Offset) { if (!Key.All(a => HexChars.Contains(a))) return BadRequest("NOT A VALID FILENAME"); var ms = new MemoryStream(); await Request.Body.CopyToAsync(ms); ms.Position = 0; Utils.Log($"Writing {ms.Length} at position {Offset} in ingest file {Key}"); long position; using (var _ = await _writeLocks[Key].Wait()) { await using var file = _settings.TempPath.Combine(Key).WriteShared(); file.Position = Offset; await ms.CopyToAsync(file); position = Offset + ms.Length; } Utils.Log($"Wrote {ms.Length} as position {Offset} result {position}"); return Ok(position); } [Authorize] [HttpGet] [Route("clean_http_uploads")] public async Task CleanUploads() { var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync(); var seen = new HashSet(); var duplicate = new List(); foreach (var file in files) { if (seen.Contains(file.Name)) duplicate.Add(file); else seen.Add(file.Name); } using (var client = new FtpClient("storage.bunnycdn.com")) { client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password); await client.ConnectAsync(); foreach (var dup in duplicate) { var final_path = Path.Combine("public", "files", dup.MungedName); Utils.Log($"Cleaning upload {final_path}"); if (AlphaFile.Exists(final_path)) AlphaFile.Delete(final_path); if (await client.FileExistsAsync(dup.MungedName)) await client.DeleteFileAsync(dup.MungedName); await Db.UploadedFiles.DeleteOneAsync(f => f.Id == dup.Id); } } return Ok(new {Remain = seen.ToArray(), Deleted = duplicate.ToArray()}.ToJSON(prettyPrint:true)); } [HttpPut] [Route("upload_file/{Key}/finish/{xxHashAsHex}")] public async Task UploadFileFinish(string Key, string xxHashAsHex) { var expectedHash = Hash.FromHex(xxHashAsHex); var user = User.FindFirstValue(ClaimTypes.Name); if (!Key.All(a => HexChars.Contains(a))) return BadRequest("NOT A VALID FILENAME"); var parts = Encoding.UTF8.GetString(Key.FromHex()).Split('|'); var finalName = $"{parts[0]}-{parts[1]}{parts[2]}"; var originalName = $"{parts[0]}{parts[2]}"; var finalPath = "public".RelativeTo(AbsolutePath.EntryPoint).Combine("files", finalName); _settings.TempPath.Combine(Key).MoveTo(finalPath); var hash = await finalPath.FileHashAsync(); if (expectedHash != hash) { finalPath.Delete(); return BadRequest($"Bad Hash, Expected: {expectedHash} Got: {hash}"); } _writeLocks.TryRemove(Key, out var _); var record = new UploadedFile { Id = Guid.Parse(parts[1]), Hash = hash, Name = originalName, Uploader = user, Size = finalPath.Size, CDNName = "wabbajackpush" }; await SQL.AddUploadedFile(record); await SQL.EnqueueJob(new Job { Priority = Job.JobPriority.High, Payload = new UploadToCDN {FileId = record.Id} }); return Ok(record.Uri); } private static readonly Func HandleGetListTemplate = NettleEngine.GetCompiler().Compile(@" {{each $.files }} {{/each}}
{{$.Name}}{{$.Size}}{{$.Date}}{{$.Uploader}}
"); [HttpGet] [Route("uploaded_files")] public async Task UploadedFilesGet() { var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync(); var response = HandleGetListTemplate(new { files = files.Select(file => new { Link = file.Uri, Size = file.Size.ToFileSizeString(), file.Name, Date = file.UploadDate, file.Uploader }) }); return new ContentResult { ContentType = "text/html", StatusCode = (int) HttpStatusCode.OK, Content = response }; } [HttpGet] [Route("uploaded_files/list")] [Authorize] public async Task ListMyFiles() { var user = User.FindFirstValue(ClaimTypes.Name); Utils.Log($"List Uploaded Files {user}"); var files = await SQL.AllUploadedFilesForUser(user); return Ok(files.OrderBy(f => f.UploadDate).Select(f => f.MungedName ).ToArray().ToJSON(prettyPrint:true)); } [HttpDelete] [Route("uploaded_files/{name}")] [Authorize] public async Task DeleteMyFile(string name) { var user = User.FindFirstValue(ClaimTypes.Name); Utils.Log($"Delete Uploaded File {user} {name}"); var files = await Db.UploadedFiles.AsQueryable().Where(f => f.Uploader == user).ToListAsync(); var to_delete = files.First(f => f.MungedName == name); if (AlphaFile.Exists(Path.Combine("public", "files", to_delete.MungedName))) AlphaFile.Delete(Path.Combine("public", "files", to_delete.MungedName)); using (var client = new FtpClient("storage.bunnycdn.com")) { client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password); await client.ConnectAsync(); if (await client.FileExistsAsync(to_delete.MungedName)) await client.DeleteFileAsync(to_delete.MungedName); } var result = await Db.UploadedFiles.DeleteOneAsync(f => f.Id == to_delete.Id); if (result.DeletedCount == 1) return Ok($"Deleted {name}"); return NotFound(name); } [HttpGet] [Route("ingest/uploaded_files/{name}")] [Authorize] public async Task IngestMongoDB(string name) { var fullPath = name.RelativeTo((AbsolutePath)_settings.TempFolder); await using var fs = fullPath.OpenRead(); var files = new List(); using var rdr = new JsonTextReader(new StreamReader(fs)) {SupportMultipleContent = true}; while (await rdr.ReadAsync()) { dynamic obj = await JObject.LoadAsync(rdr); var uf = new UploadedFile { Id = Guid.Parse((string)obj._id), Name = obj.Name, Size = long.Parse((string)(obj.Size["$numberLong"] ?? obj.Size["$numberInt"])), Hash = Hash.FromBase64((string)obj.Hash), Uploader = obj.Uploader, UploadDate = long.Parse(((string)obj.UploadDate["$date"]["$numberLong"]).Substring(0, 10)).AsUnixTime(), CDNName = obj.CDNName }; files.Add(uf); await SQL.AddUploadedFile(uf); } return Ok(files.Count); } } }