From 44b78111a3d60585ae9293b1e068097d4baa69f5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 2 Mar 2020 16:16:15 -0700 Subject: [PATCH] Server side cleanup, give authors the ability to delete their own files. Some sanity and logging checks for --- .../Controllers/ModlistUpdater.cs | 59 ++++++++++++- .../Controllers/UploadedFiles.cs | 84 +++++++++++++++++++ Wabbajack.BuildServer/JobManager.cs | 2 +- .../Models/Jobs/UpdateModLists.cs | 22 ++++- Wabbajack.BuildServer/Models/PatchArchive.cs | 3 + Wabbajack.CLI/OptionsDefinition.cs | 4 +- Wabbajack.CLI/Program.cs | 2 + Wabbajack.CLI/Verbs/DeleteFile.cs | 19 +++++ Wabbajack.CLI/Verbs/MyFiles.cs | 19 +++++ Wabbajack.Common/Http/Client.cs | 6 ++ Wabbajack.Common/Utils.cs | 7 +- Wabbajack.Lib/ClientAPI.cs | 3 +- Wabbajack.Lib/FileUploader/AuthorAPI.cs | 12 +++ 13 files changed, 233 insertions(+), 9 deletions(-) create mode 100644 Wabbajack.CLI/Verbs/DeleteFile.cs create mode 100644 Wabbajack.CLI/Verbs/MyFiles.cs diff --git a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs index ad20947e..c4ced4d6 100644 --- a/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs +++ b/Wabbajack.BuildServer/Controllers/ModlistUpdater.cs @@ -1,7 +1,11 @@ using System; +using System.Collections.Generic; using System.Linq; +using System.Net; using System.Threading.Tasks; using Alphaleonis.Win32.Filesystem; +using FluentFTP; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using MongoDB.Driver; @@ -15,6 +19,7 @@ using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Wabbajack.Lib.NexusApi; using AlphaFile = Alphaleonis.Win32.Filesystem.File; +using Directory = System.IO.Directory; namespace Wabbajack.BuildServer.Controllers { @@ -31,6 +36,55 @@ namespace Wabbajack.BuildServer.Controllers _sql = sql; } + [HttpGet] + [Authorize] + [Route("/delete_updates")] + public async Task DeleteUpdates() + { + var lists = await Db.ModListStatus.AsQueryable().ToListAsync(); + var archives = lists.SelectMany(list => list.DetailedStatus.Archives) + .Select(a => a.Archive.Hash.FromBase64().ToHex()) + .ToHashSet(); + + var toDelete = new List(); + var toSave = new List(); + using (var client = new FtpClient("storage.bunnycdn.com")) + { + client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password); + await client.ConnectAsync(); + + foreach (var file in Directory.GetFiles("updates")) + { + var relativeName = Path.GetFileName(file); + var parts = Path.GetFileName(file).Split('_', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 2) continue; + + if (parts[0] == parts[1]) + { + toDelete.Add(relativeName); + continue; + } + + if (!archives.Contains(parts[0])) + toDelete.Add(relativeName); + else + toSave.Add(relativeName); + } + + foreach (var delete in toDelete) + { + Utils.Log($"Deleting update {delete}"); + if (await client.FileExistsAsync($"updates/{delete}")) + await client.DeleteFileAsync($"updates/{delete}"); + if (AlphaFile.Exists($"updates\\{delete}")) + AlphaFile.Delete($"updates\\{delete}"); + + } + } + + return Ok(new {Save = toSave.ToArray(), Delete = toDelete.ToArray()}.ToJSON(prettyPrint:true)); + } + [HttpGet] [Route("/alternative/{xxHash}")] public async Task GetAlternative(string xxHash) @@ -55,7 +109,7 @@ namespace Wabbajack.BuildServer.Controllers return NotFound("No alternative available"); } - Utils.Log($"Found {newArchive.State.PrimaryKeyString} as an alternative to {startingHash}"); + Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}"); if (newArchive.Hash == null) { Db.Jobs.InsertOne(new Job @@ -76,6 +130,9 @@ namespace Wabbajack.BuildServer.Controllers return Accepted("Enqueued for Processing"); } + if (startingHash == newArchive.Hash) + return NotFound("End hash same as old hash"); + if (!AlphaFile.Exists(PatchArchive.CdnPath(startingHash, newArchive.Hash))) { Db.Jobs.InsertOne(new Job diff --git a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs index 874cb3a8..a7bfb5ab 100644 --- a/Wabbajack.BuildServer/Controllers/UploadedFiles.cs +++ b/Wabbajack.BuildServer/Controllers/UploadedFiles.cs @@ -7,6 +7,7 @@ using System.Net; using System.Security.Claims; using System.Text; using System.Threading.Tasks; +using FluentFTP; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Identity; @@ -15,6 +16,7 @@ using Microsoft.Extensions.Logging; using MongoDB.Driver; using MongoDB.Driver.Linq; using Nettle; +using Org.BouncyCastle.Crypto.Engines; using Wabbajack.BuildServer.Models; using Wabbajack.BuildServer.Models.JobQueue; using Wabbajack.BuildServer.Models.Jobs; @@ -22,6 +24,7 @@ using Wabbajack.Common; using Wabbajack.Lib; using Wabbajack.Lib.Downloaders; using Path = Alphaleonis.Win32.Filesystem.Path; +using AlphaFile = Alphaleonis.Win32.Filesystem.File; namespace Wabbajack.BuildServer.Controllers { @@ -73,6 +76,46 @@ namespace Wabbajack.BuildServer.Controllers return Ok(position); } + [Authorize] + [HttpGet] + [Route("clean_http_uploads")] + public async Task CleanUploads() + { + var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync(); + var seen = new HashSet(); + var duplicate = new List(); + + foreach (var file in files) + { + if (seen.Contains(file.Name)) + duplicate.Add(file); + else + seen.Add(file.Name); + } + + using (var client = new FtpClient("storage.bunnycdn.com")) + { + client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password); + await client.ConnectAsync(); + + foreach (var dup in duplicate) + { + var final_path = Path.Combine("public", "files", dup.MungedName); + Utils.Log($"Cleaning upload {final_path}"); + + if (AlphaFile.Exists(final_path)) + AlphaFile.Delete(final_path); + + if (await client.FileExistsAsync(dup.MungedName)) + await client.DeleteFileAsync(dup.MungedName); + await Db.UploadedFiles.DeleteOneAsync(f => f.Id == dup.Id); + } + } + + return Ok(new {Remain = seen.ToArray(), Deleted = duplicate.ToArray()}.ToJSON(prettyPrint:true)); + } + + [HttpPut] [Route("upload_file/{Key}/finish/{xxHashAsHex}")] public async Task UploadFileFinish(string Key, string xxHashAsHex) @@ -151,6 +194,47 @@ namespace Wabbajack.BuildServer.Controllers Content = response }; } + + [HttpGet] + [Route("uploaded_files/list")] + [Authorize] + public async Task ListMyFiles() + { + var user = User.FindFirstValue(ClaimTypes.Name); + Utils.Log($"List Uploaded Files {user}"); + var files = await Db.UploadedFiles.AsQueryable().Where(f => f.Uploader == user).ToListAsync(); + return Ok(files.OrderBy(f => f.UploadDate).Select(f => f.MungedName).ToArray().ToJSON(prettyPrint:true)); + } + + [HttpDelete] + [Route("uploaded_files/{name}")] + [Authorize] + public async Task DeleteMyFile(string name) + { + var user = User.FindFirstValue(ClaimTypes.Name); + Utils.Log($"Delete Uploaded File {user} {name}"); + var files = await Db.UploadedFiles.AsQueryable().Where(f => f.Uploader == user).ToListAsync(); + + var to_delete = files.First(f => f.MungedName == name); + + if (AlphaFile.Exists(Path.Combine("public", "files", to_delete.MungedName))) + AlphaFile.Delete(Path.Combine("public", "files", to_delete.MungedName)); + + using (var client = new FtpClient("storage.bunnycdn.com")) + { + client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password); + await client.ConnectAsync(); + if (await client.FileExistsAsync(to_delete.MungedName)) + await client.DeleteFileAsync(to_delete.MungedName); + + } + + var result = await Db.UploadedFiles.DeleteOneAsync(f => f.Id == to_delete.Id); + if (result.DeletedCount == 1) + return Ok($"Deleted {name}"); + return NotFound(name); + } + } diff --git a/Wabbajack.BuildServer/JobManager.cs b/Wabbajack.BuildServer/JobManager.cs index a6e73c58..9f5bc9a9 100644 --- a/Wabbajack.BuildServer/JobManager.cs +++ b/Wabbajack.BuildServer/JobManager.cs @@ -82,7 +82,7 @@ namespace Wabbajack.BuildServer while (true) { await KillOrphanedJobs(); - await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.High); + await ScheduledJob(TimeSpan.FromHours(1), Job.JobPriority.High); await ScheduledJob(TimeSpan.FromMinutes(30), Job.JobPriority.High); await ScheduledJob(TimeSpan.FromHours(2), Job.JobPriority.Low); await ScheduledJob(TimeSpan.FromHours(24), Job.JobPriority.High); diff --git a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs index ad9e9931..136d3456 100644 --- a/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs +++ b/Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs @@ -127,6 +127,11 @@ namespace Wabbajack.BuildServer.Models.Jobs if (await ValidateNexusFast(db, state)) return true; } + else if (archive.State is GoogleDriveDownloader.State) + { + // Disabled for now + return true; + } else if (archive.State is HTTPDownloader.State hstate && hstate.Url.StartsWith("https://wabbajack")) { @@ -142,16 +147,27 @@ namespace Wabbajack.BuildServer.Models.Jobs // ignored } + Utils.Log($"{archive.State.PrimaryKeyString} is broken, looking for upgrade: {archive.Name}"); var result = await ClientAPI.GetModUpgrade(archive.Hash); - if (result != null) return true; + + if (result != null) + { + Utils.Log($"{archive.State.PrimaryKeyString} is broken, upgraded to {result.State.PrimaryKeyString} {result.Name}"); + return true; + } + + Utils.Log($"{archive.State.PrimaryKeyString} is broken, no alternative found"); + return false; } - catch (Exception) + catch (Exception ex) { - return true; + Utils.Log(ex.ToString()); + return false; } return false; + } private async Task ValidateNexusFast(DBContext db, NexusDownloader.State state) diff --git a/Wabbajack.BuildServer/Models/PatchArchive.cs b/Wabbajack.BuildServer/Models/PatchArchive.cs index b554af30..46ae68ea 100644 --- a/Wabbajack.BuildServer/Models/PatchArchive.cs +++ b/Wabbajack.BuildServer/Models/PatchArchive.cs @@ -25,6 +25,9 @@ namespace Wabbajack.BuildServer.Models var srcPath = settings.PathForArchive(Src); var destHash = (await db.DownloadStates.AsQueryable().Where(s => s.Key == DestPK).FirstOrDefaultAsync()).Hash; var destPath = settings.PathForArchive(destHash); + + if (Src == destHash) + return JobResult.Success(); Utils.Log($"Creating Patch ({Src} -> {DestPK})"); var cdnPath = CdnPath(Src, destHash); diff --git a/Wabbajack.CLI/OptionsDefinition.cs b/Wabbajack.CLI/OptionsDefinition.cs index a2ab8ad7..31143da8 100644 --- a/Wabbajack.CLI/OptionsDefinition.cs +++ b/Wabbajack.CLI/OptionsDefinition.cs @@ -14,7 +14,9 @@ namespace Wabbajack.CLI typeof(UpdateModlists), typeof(UpdateNexusCache), typeof(ChangeDownload), - typeof(ServerLog) + typeof(ServerLog), + typeof(MyFiles), + typeof(DeleteFile) }; } } diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index 9d9ba5e6..17e61ede 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -17,6 +17,8 @@ namespace Wabbajack.CLI (UpdateNexusCache opts) => opts.Execute(), (ChangeDownload opts) => opts.Execute(), (ServerLog opts) => opts.Execute(), + (MyFiles opts) => opts.Execute(), + (DeleteFile opts) => opts.Execute(), errs => 1); } } diff --git a/Wabbajack.CLI/Verbs/DeleteFile.cs b/Wabbajack.CLI/Verbs/DeleteFile.cs new file mode 100644 index 00000000..c89b698a --- /dev/null +++ b/Wabbajack.CLI/Verbs/DeleteFile.cs @@ -0,0 +1,19 @@ +using System; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Lib.FileUploader; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("delete-uploaded-file", HelpText = "Delete a file you uploaded to the CDN. Cannot delete other user's files")] + public class DeleteFile : AVerb + { + [Option('n', "name", Required = true, HelpText = @"Full name (as returned by my-files) of the file")] + public string Name { get; set; } + protected override async Task Run() + { + Console.WriteLine(await AuthorAPI.DeleteFile(Name)); + return 0; + } + } +} diff --git a/Wabbajack.CLI/Verbs/MyFiles.cs b/Wabbajack.CLI/Verbs/MyFiles.cs new file mode 100644 index 00000000..6dfcb63b --- /dev/null +++ b/Wabbajack.CLI/Verbs/MyFiles.cs @@ -0,0 +1,19 @@ +using System; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Lib.FileUploader; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("my-files", HelpText = "List files I have uploaded to the CDN (requires Author API key)")] + public class MyFiles : AVerb + { + protected override async Task Run() + { + var files = await AuthorAPI.GetMyFiles(); + foreach (var file in files) + Console.WriteLine(file); + return 0; + } + } +} diff --git a/Wabbajack.Common/Http/Client.cs b/Wabbajack.Common/Http/Client.cs index f2e6764c..c54073e4 100644 --- a/Wabbajack.Common/Http/Client.cs +++ b/Wabbajack.Common/Http/Client.cs @@ -36,6 +36,12 @@ namespace Wabbajack.Common.Http var request = new HttpRequestMessage(HttpMethod.Get, url); return await SendStringAsync(request); } + + public async Task DeleteStringAsync(string url) + { + var request = new HttpRequestMessage(HttpMethod.Delete, url); + return await SendStringAsync(request); + } private async Task SendStringAsync(HttpRequestMessage request) { diff --git a/Wabbajack.Common/Utils.cs b/Wabbajack.Common/Utils.cs index 63fd6713..b7fe6a2c 100644 --- a/Wabbajack.Common/Utils.cs +++ b/Wabbajack.Common/Utils.cs @@ -536,10 +536,13 @@ namespace Wabbajack.Common public static string ToJSON(this T obj, TypeNameHandling handling = TypeNameHandling.All, - TypeNameAssemblyFormatHandling format = TypeNameAssemblyFormatHandling.Full) + TypeNameAssemblyFormatHandling format = TypeNameAssemblyFormatHandling.Full, + bool prettyPrint = false) { return JsonConvert.SerializeObject(obj, Formatting.Indented, - new JsonSerializerSettings {TypeNameHandling = handling, TypeNameAssemblyFormatHandling = format}); + new JsonSerializerSettings {TypeNameHandling = handling, + TypeNameAssemblyFormatHandling = format, + Formatting = prettyPrint ? Formatting.Indented : Formatting.None}); } public static T FromJSON(this string filename, diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index d059b570..1cd031c6 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -8,7 +8,8 @@ namespace Wabbajack.Lib public static Common.Http.Client GetClient() { var client = new Common.Http.Client(); - client.Headers.Add((Consts.MetricsKeyHeader, Utils.FromEncryptedJson(Consts.MetricsKeyHeader))); + if (Utils.HaveEncryptedJson(Consts.MetricsKeyHeader)) + client.Headers.Add((Consts.MetricsKeyHeader, Utils.FromEncryptedJson(Consts.MetricsKeyHeader))); return client; } diff --git a/Wabbajack.Lib/FileUploader/AuthorAPI.cs b/Wabbajack.Lib/FileUploader/AuthorAPI.cs index d843871c..552e5ade 100644 --- a/Wabbajack.Lib/FileUploader/AuthorAPI.cs +++ b/Wabbajack.Lib/FileUploader/AuthorAPI.cs @@ -175,5 +175,17 @@ namespace Wabbajack.Lib.FileUploader { return await GetAuthorizedClient().GetStringAsync($"https://{Consts.WabbajackCacheHostname}/heartbeat/logs"); } + + public static async Task> GetMyFiles() + { + return (await GetAuthorizedClient().GetStringAsync($"https://{Consts.WabbajackCacheHostname}/uploaded_files/list")).FromJSONString(); + } + + public static async Task DeleteFile(string name) + { + var result = await GetAuthorizedClient() + .DeleteStringAsync($"https://{Consts.WabbajackCacheHostname}/uploaded_files/{name}"); + return result; + } } }