Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Server side cleanup, give authors the ability to delete their own files. Some sanity and logging checks for

Browse Source
This commit is contained in:
Timothy Baldridge 2020-03-02 16:16:15 -07:00
parent 1994b4279c
commit 44b78111a3
13 changed files with 233 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
Wabbajack.BuildServer/Controllers/ModlistUpdater.cs
View File

@ -1,7 +1,11 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using Alphaleonis.Win32.Filesystem;
using FluentFTP;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
@ -15,6 +19,7 @@ using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.NexusApi;
using AlphaFile = Alphaleonis.Win32.Filesystem.File;
using Directory = System.IO.Directory;
namespace Wabbajack.BuildServer.Controllers
{
@ -31,6 +36,55 @@ namespace Wabbajack.BuildServer.Controllers
_sql = sql;
}
[HttpGet]
[Authorize]
[Route("/delete_updates")]
public async Task<IActionResult> DeleteUpdates()
{
var lists = await Db.ModListStatus.AsQueryable().ToListAsync();
var archives = lists.SelectMany(list => list.DetailedStatus.Archives)
.Select(a => a.Archive.Hash.FromBase64().ToHex())
.ToHashSet();
var toDelete = new List<string>();
var toSave = new List<string>();
using (var client = new FtpClient("storage.bunnycdn.com"))
{
client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password);
await client.ConnectAsync();
foreach (var file in Directory.GetFiles("updates"))
{
var relativeName = Path.GetFileName(file);
var parts = Path.GetFileName(file).Split('_', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length != 2) continue;
if (parts[0] == parts[1])
{
toDelete.Add(relativeName);
continue;
}
if (!archives.Contains(parts[0]))
toDelete.Add(relativeName);
else
toSave.Add(relativeName);
}
foreach (var delete in toDelete)
{
Utils.Log($"Deleting update {delete}");
if (await client.FileExistsAsync($"updates/{delete}"))
await client.DeleteFileAsync($"updates/{delete}");
if (AlphaFile.Exists($"updates\\{delete}"))
AlphaFile.Delete($"updates\\{delete}");
}
}
return Ok(new {Save = toSave.ToArray(), Delete = toDelete.ToArray()}.ToJSON(prettyPrint:true));
}
[HttpGet]
[Route("/alternative/{xxHash}")]
public async Task<IActionResult> GetAlternative(string xxHash)
@ -55,7 +109,7 @@ namespace Wabbajack.BuildServer.Controllers
return NotFound("No alternative available");
}
Utils.Log($"Found {newArchive.State.PrimaryKeyString} as an alternative to {startingHash}");
Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}");
if (newArchive.Hash == null)
{
Db.Jobs.InsertOne(new Job
@ -76,6 +130,9 @@ namespace Wabbajack.BuildServer.Controllers
return Accepted("Enqueued for Processing");
}
if (startingHash == newArchive.Hash)
return NotFound("End hash same as old hash");
if (!AlphaFile.Exists(PatchArchive.CdnPath(startingHash, newArchive.Hash)))
{
Db.Jobs.InsertOne(new Job

84
Wabbajack.BuildServer/Controllers/UploadedFiles.cs
View File

@ -7,6 +7,7 @@ using System.Net;
using System.Security.Claims;
using System.Text;
using System.Threading.Tasks;
using FluentFTP;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Identity;
@ -15,6 +16,7 @@ using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using Nettle;
using Org.BouncyCastle.Crypto.Engines;
using Wabbajack.BuildServer.Models;
using Wabbajack.BuildServer.Models.JobQueue;
using Wabbajack.BuildServer.Models.Jobs;
@ -22,6 +24,7 @@ using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Path = Alphaleonis.Win32.Filesystem.Path;
using AlphaFile = Alphaleonis.Win32.Filesystem.File;
namespace Wabbajack.BuildServer.Controllers
{
@ -73,6 +76,46 @@ namespace Wabbajack.BuildServer.Controllers
return Ok(position);
}
[Authorize]
[HttpGet]
[Route("clean_http_uploads")]
public async Task<IActionResult> CleanUploads()
{
var files = await Db.UploadedFiles.AsQueryable().OrderByDescending(f => f.UploadDate).ToListAsync();
var seen = new HashSet<string>();
var duplicate = new List<UploadedFile>();
foreach (var file in files)
{
if (seen.Contains(file.Name))
duplicate.Add(file);
else
seen.Add(file.Name);
}
using (var client = new FtpClient("storage.bunnycdn.com"))
{
client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password);
await client.ConnectAsync();
foreach (var dup in duplicate)
{
var final_path = Path.Combine("public", "files", dup.MungedName);
Utils.Log($"Cleaning upload {final_path}");
if (AlphaFile.Exists(final_path))
AlphaFile.Delete(final_path);
if (await client.FileExistsAsync(dup.MungedName))
await client.DeleteFileAsync(dup.MungedName);
await Db.UploadedFiles.DeleteOneAsync(f => f.Id == dup.Id);
}
}
return Ok(new {Remain = seen.ToArray(), Deleted = duplicate.ToArray()}.ToJSON(prettyPrint:true));
}
[HttpPut]
[Route("upload_file/{Key}/finish/{xxHashAsHex}")]
public async Task<IActionResult> UploadFileFinish(string Key, string xxHashAsHex)
@ -151,6 +194,47 @@ namespace Wabbajack.BuildServer.Controllers
Content = response
};
}
[HttpGet]
[Route("uploaded_files/list")]
[Authorize]
public async Task<IActionResult> ListMyFiles()
{
var user = User.FindFirstValue(ClaimTypes.Name);
Utils.Log($"List Uploaded Files {user}");
var files = await Db.UploadedFiles.AsQueryable().Where(f => f.Uploader == user).ToListAsync();
return Ok(files.OrderBy(f => f.UploadDate).Select(f => f.MungedName).ToArray().ToJSON(prettyPrint:true));
}
[HttpDelete]
[Route("uploaded_files/{name}")]
[Authorize]
public async Task<IActionResult> DeleteMyFile(string name)
{
var user = User.FindFirstValue(ClaimTypes.Name);
Utils.Log($"Delete Uploaded File {user} {name}");
var files = await Db.UploadedFiles.AsQueryable().Where(f => f.Uploader == user).ToListAsync();
var to_delete = files.First(f => f.MungedName == name);
if (AlphaFile.Exists(Path.Combine("public", "files", to_delete.MungedName)))
AlphaFile.Delete(Path.Combine("public", "files", to_delete.MungedName));
using (var client = new FtpClient("storage.bunnycdn.com"))
{
client.Credentials = new NetworkCredential(_settings.BunnyCDN_User, _settings.BunnyCDN_Password);
await client.ConnectAsync();
if (await client.FileExistsAsync(to_delete.MungedName))
await client.DeleteFileAsync(to_delete.MungedName);
}
var result = await Db.UploadedFiles.DeleteOneAsync(f => f.Id == to_delete.Id);
if (result.DeletedCount == 1)
return Ok($"Deleted {name}");
return NotFound(name);
}
}

2
Wabbajack.BuildServer/JobManager.cs
View File

@ -82,7 +82,7 @@ namespace Wabbajack.BuildServer
while (true)
{
await KillOrphanedJobs();
await ScheduledJob<GetNexusUpdatesJob>(TimeSpan.FromHours(2), Job.JobPriority.High);
await ScheduledJob<GetNexusUpdatesJob>(TimeSpan.FromHours(1), Job.JobPriority.High);
await ScheduledJob<UpdateModLists>(TimeSpan.FromMinutes(30), Job.JobPriority.High);
await ScheduledJob<EnqueueAllArchives>(TimeSpan.FromHours(2), Job.JobPriority.Low);
await ScheduledJob<EnqueueAllGameFiles>(TimeSpan.FromHours(24), Job.JobPriority.High);

22
Wabbajack.BuildServer/Models/Jobs/UpdateModLists.cs
View File

@ -127,6 +127,11 @@ namespace Wabbajack.BuildServer.Models.Jobs
if (await ValidateNexusFast(db, state)) return true;
}
else if (archive.State is GoogleDriveDownloader.State)
{
// Disabled for now
return true;
}
else if (archive.State is HTTPDownloader.State hstate &&
hstate.Url.StartsWith("https://wabbajack"))
{
@ -142,16 +147,27 @@ namespace Wabbajack.BuildServer.Models.Jobs
// ignored
}
Utils.Log($"{archive.State.PrimaryKeyString} is broken, looking for upgrade: {archive.Name}");
var result = await ClientAPI.GetModUpgrade(archive.Hash);
if (result != null) return true;
if (result != null)
{
Utils.Log($"{archive.State.PrimaryKeyString} is broken, upgraded to {result.State.PrimaryKeyString} {result.Name}");
return true;
}
Utils.Log($"{archive.State.PrimaryKeyString} is broken, no alternative found");
return false;
}
catch (Exception)
catch (Exception ex)
{
return true;
Utils.Log(ex.ToString());
return false;
}
return false;
}
private async Task<bool> ValidateNexusFast(DBContext db, NexusDownloader.State state)

3
Wabbajack.BuildServer/Models/PatchArchive.cs
View File

@ -25,6 +25,9 @@ namespace Wabbajack.BuildServer.Models
var srcPath = settings.PathForArchive(Src);
var destHash = (await db.DownloadStates.AsQueryable().Where(s => s.Key == DestPK).FirstOrDefaultAsync()).Hash;
var destPath = settings.PathForArchive(destHash);
if (Src == destHash)
return JobResult.Success();
Utils.Log($"Creating Patch ({Src} -> {DestPK})");
var cdnPath = CdnPath(Src, destHash);

4
Wabbajack.CLI/OptionsDefinition.cs
View File

@ -14,7 +14,9 @@ namespace Wabbajack.CLI
typeof(UpdateModlists),
typeof(UpdateNexusCache),
typeof(ChangeDownload),
typeof(ServerLog)
typeof(ServerLog),
typeof(MyFiles),
typeof(DeleteFile)
};
}
}

2
Wabbajack.CLI/Program.cs
View File

@ -17,6 +17,8 @@ namespace Wabbajack.CLI
(UpdateNexusCache opts) => opts.Execute(),
(ChangeDownload opts) => opts.Execute(),
(ServerLog opts) => opts.Execute(),
(MyFiles opts) => opts.Execute(),
(DeleteFile opts) => opts.Execute(),
errs => 1);
}
}

19
Wabbajack.CLI/Verbs/DeleteFile.cs Normal file
View File

@ -0,0 +1,19 @@
using System;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Lib.FileUploader;
namespace Wabbajack.CLI.Verbs
{
[Verb("delete-uploaded-file", HelpText = "Delete a file you uploaded to the CDN. Cannot delete other user's files")]
public class DeleteFile : AVerb
{
[Option('n', "name", Required = true, HelpText = @"Full name (as returned by my-files) of the file")]
public string Name { get; set; }
protected override async Task<int> Run()
{
Console.WriteLine(await AuthorAPI.DeleteFile(Name));
return 0;
}
}
}

19
Wabbajack.CLI/Verbs/MyFiles.cs Normal file
View File

@ -0,0 +1,19 @@
using System;
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Lib.FileUploader;
namespace Wabbajack.CLI.Verbs
{
[Verb("my-files", HelpText = "List files I have uploaded to the CDN (requires Author API key)")]
public class MyFiles : AVerb
{
protected override async Task<int> Run()
{
var files = await AuthorAPI.GetMyFiles();
foreach (var file in files)
Console.WriteLine(file);
return 0;
}
}
}

6
Wabbajack.Common/Http/Client.cs
View File

@ -36,6 +36,12 @@ namespace Wabbajack.Common.Http
var request = new HttpRequestMessage(HttpMethod.Get, url);
return await SendStringAsync(request);
}
public async Task<string> DeleteStringAsync(string url)
{
var request = new HttpRequestMessage(HttpMethod.Delete, url);
return await SendStringAsync(request);
}
private async Task<string> SendStringAsync(HttpRequestMessage request)
{

7
Wabbajack.Common/Utils.cs
View File

@ -536,10 +536,13 @@ namespace Wabbajack.Common
public static string ToJSON<T>(this T obj,
TypeNameHandling handling = TypeNameHandling.All,
TypeNameAssemblyFormatHandling format = TypeNameAssemblyFormatHandling.Full)
TypeNameAssemblyFormatHandling format = TypeNameAssemblyFormatHandling.Full,
bool prettyPrint = false)
{
return JsonConvert.SerializeObject(obj, Formatting.Indented,
new JsonSerializerSettings {TypeNameHandling = handling, TypeNameAssemblyFormatHandling = format});
new JsonSerializerSettings {TypeNameHandling = handling,
TypeNameAssemblyFormatHandling = format,
Formatting = prettyPrint ? Formatting.Indented : Formatting.None});
}
public static T FromJSON<T>(this string filename,

3
Wabbajack.Lib/ClientAPI.cs
View File

@ -8,7 +8,8 @@ namespace Wabbajack.Lib
public static Common.Http.Client GetClient()
{
var client = new Common.Http.Client();
client.Headers.Add((Consts.MetricsKeyHeader, Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader)));
if (Utils.HaveEncryptedJson(Consts.MetricsKeyHeader))
client.Headers.Add((Consts.MetricsKeyHeader, Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader)));
return client;
}

12
Wabbajack.Lib/FileUploader/AuthorAPI.cs
View File

@ -175,5 +175,17 @@ namespace Wabbajack.Lib.FileUploader
{
return await GetAuthorizedClient().GetStringAsync($"https://{Consts.WabbajackCacheHostname}/heartbeat/logs");
}
public static async Task<IEnumerable<string>> GetMyFiles()
{
return (await GetAuthorizedClient().GetStringAsync($"https://{Consts.WabbajackCacheHostname}/uploaded_files/list")).FromJSONString<string[]>();
}
public static async Task<string> DeleteFile(string name)
{
var result = await GetAuthorizedClient()
.DeleteStringAsync($"https://{Consts.WabbajackCacheHostname}/uploaded_files/{name}");
return result;
}
}
}