From 252ff675c6ae883e9a2d1b950c67f6d97df2daa8 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 17 Jan 2022 20:56:29 -0700 Subject: [PATCH] Add nexus upload code --- Wabbajack.CLI/Program.cs | 1 + Wabbajack.CLI/Verbs/UploadToNexus.cs | 47 ++++++ .../DTOs/UploadDefinition.cs | 120 +++++++++++++++ Wabbajack.Networking.NexusApi/NexusApi.cs | 145 ++++++++++++++++++ 4 files changed, 313 insertions(+) create mode 100644 Wabbajack.CLI/Verbs/UploadToNexus.cs create mode 100644 Wabbajack.Networking.NexusApi/DTOs/UploadDefinition.cs diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index 27e22063..d0e83053 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -67,6 +67,7 @@ internal class Program services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); }).Build(); diff --git a/Wabbajack.CLI/Verbs/UploadToNexus.cs b/Wabbajack.CLI/Verbs/UploadToNexus.cs new file mode 100644 index 00000000..21ff73b2 --- /dev/null +++ b/Wabbajack.CLI/Verbs/UploadToNexus.cs @@ -0,0 +1,47 @@ +using System.CommandLine; +using System.CommandLine.Invocation; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.Common; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Networking.NexusApi; +using Wabbajack.Networking.NexusApi.DTOs; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; + + +namespace Wabbajack.CLI.Verbs; + +public class UploadToNexus : IVerb +{ + private readonly ILogger _logger; + private readonly NexusApi _client; + private readonly DTOSerializer _dtos; + + public UploadToNexus(ILogger logger, NexusApi wjClient, DTOSerializer dtos) + { + _logger = logger; + _client = wjClient; + _dtos = dtos; + } + public Command MakeCommand() + { + var command = new Command("upload-to-nexus"); + command.Add(new Option(new[] {"-d", "-definition"}, "Definition JSON file")); + command.Description = "Uploads a file to the Nexus defined by the given .json definition file"; + command.Handler = CommandHandler.Create(Run); + return command; + } + + public async Task Run(AbsolutePath definition) + { + var d = await definition.FromJson(_dtos); + + await _client.UploadFile(d); + + + return 0; + } + +} \ No newline at end of file diff --git a/Wabbajack.Networking.NexusApi/DTOs/UploadDefinition.cs b/Wabbajack.Networking.NexusApi/DTOs/UploadDefinition.cs new file mode 100644 index 00000000..e402b52d --- /dev/null +++ b/Wabbajack.Networking.NexusApi/DTOs/UploadDefinition.cs @@ -0,0 +1,120 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Text.Json.Serialization; +using System.Web; +using Wabbajack.DTOs; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; + +namespace Wabbajack.Networking.NexusApi.DTOs; + +public enum Category : int +{ + Main = 1, + Updates = 2, + Optional = 3, + Old = 4, + Misc = 5, + Archives = 7 +} + +public enum ChunkStatus +{ + NoContent, + Waiting, + Done +} + +public class UploadDefinition +{ + public const long ChunkSize = 5242880; // 5MB chunks + public Game Game { get; set; } + + [JsonIgnore] + public long GameId => Game.MetaData().NexusGameId; + + public string Name { get; set; } + public AbsolutePath Path { get; set; } + + public string Version { get; set; } + + public string Category { get; set; } + + public bool NewExisting { get; set; } + + public long OldFileId { get; set; } + + public bool RemoveOldVersion { get; set; } + + public string BriefOverview { get; set; } + + public string FileUUID { get; set; } = ""; + + public long FileSize => Path.Size(); + public long ModId { get; set; } + + public long TotalChunks => (long) Math.Ceiling(FileSize / (double) ChunkSize); + public string ResumableIdentifier => FileSize + "-" + Path.FileName.ToString().Replace(".", "").Replace(" ", ""); + public string ResumableRelativePath => HttpUtility.UrlEncode(Path.FileName.ToString()); + public bool SetAsMain { get; set; } + + public IEnumerable Chunks() + { + + var size = FileSize; + + if (size <= ChunkSize) + { + + yield return new Chunk + { + Index = 0, + Offset = 0, + Size = size + }; + yield break; + } + + for (long block = 0; block * ChunkSize < size; block++) { + yield return new Chunk + { + Index = block, + Size = Math.Min(ChunkSize, size - block * ChunkSize), + Offset = block * ChunkSize + }; + } + } +} + +public class Chunk +{ + public long Index { get; set; } + public long Size { get; set; } + public long Offset { get; set; } +} + +public class ChunkStatusResult +{ + [JsonPropertyName("filename")] + public string Filename { get; set; } + + [JsonPropertyName("status")] + public bool Status { get; set; } + + [JsonPropertyName("uuid")] + public string UUID { get; set; } +} + +public class FileStatusResult +{ + [JsonPropertyName("file_chunks_reassembled")] + public bool FileChunksAssembled { get; set; } + + [JsonPropertyName("s3_upload_complete")] + public bool S3UploadComplete { get; set; } + + [JsonPropertyName("virus_total_result")] + public int VirusTotalStatus { get; set; } + +} \ No newline at end of file diff --git a/Wabbajack.Networking.NexusApi/NexusApi.cs b/Wabbajack.Networking.NexusApi/NexusApi.cs index 66bbbb3e..be2da24a 100644 --- a/Wabbajack.Networking.NexusApi/NexusApi.cs +++ b/Wabbajack.Networking.NexusApi/NexusApi.cs @@ -1,16 +1,22 @@ using System; +using System.IO; using System.Linq; +using System.Net; using System.Net.Http; using System.Net.Http.Headers; +using System.Net.Http.Json; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Wabbajack.Common; using Wabbajack.DTOs; using Wabbajack.DTOs.Logins; using Wabbajack.Networking.Http; using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Networking.NexusApi.DTOs; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; using Wabbajack.Server.DTOs; @@ -165,4 +171,143 @@ public class NexusApi var msg = await GenerateMessage(HttpMethod.Get, Endpoints.Updates, game.MetaData().NexusName, "1m"); return await Send(msg, token); } + + public async Task ChunkStatus(UploadDefinition definition, Chunk chunk) + { + var msg = new HttpRequestMessage(); + msg.Method = HttpMethod.Get; + + var query = + $"resumableChunkNumber={chunk.Index + 1}&resumableCurrentChunkSize={chunk.Size}&resumableTotalSize={definition.FileSize}" + + $"&resumableType=&resumableIdentifier={definition.ResumableIdentifier}&resumableFilename={definition.ResumableRelativePath}" + + $"&resumableRelativePath={definition.ResumableRelativePath}&resumableTotalChunks={definition.Chunks().Count()}"; + + msg.RequestUri = new Uri($"https://upload.nexusmods.com/uploads/chunk?{query}"); + + using var result = await _client.SendAsync(msg); + if (!result.IsSuccessStatusCode) + throw new HttpException(result); + if (result.StatusCode == HttpStatusCode.NoContent) + return DTOs.ChunkStatus.NoContent; + + var status = await result.Content.ReadFromJsonAsync(); + return status?.Status ?? false ? DTOs.ChunkStatus.Done : DTOs.ChunkStatus.Waiting; + } + + public async Task UploadChunk(UploadDefinition d, Chunk chunk) + { + var form = new MultipartFormDataContent(); + form.Add(new StringContent((chunk.Index+1).ToString()), "resumableChunkNumber"); + form.Add(new StringContent(UploadDefinition.ChunkSize.ToString()), "resumableChunkSize"); + form.Add(new StringContent(chunk.Size.ToString()), "resumableCurrentChunkSize"); + form.Add(new StringContent(d.FileSize.ToString()), "resumableTotalSize"); + form.Add(new StringContent(""), "resumableType"); + form.Add(new StringContent(d.ResumableIdentifier), "resumableIdentifier"); + form.Add(new StringContent(d.ResumableRelativePath), "resumableFilename"); + form.Add(new StringContent(d.ResumableRelativePath), "resumableRelativePath"); + form.Add(new StringContent(d.Chunks().Count().ToString()), "resumableTotalChunks"); + + await using var ms = new MemoryStream(); + await using var fs = d.Path.Open(FileMode.Open, FileAccess.Read, FileShare.Read); + fs.Position = chunk.Offset; + await fs.CopyToLimitAsync(ms, (int)chunk.Size, CancellationToken.None); + ms.Position = 0; + + form.Add(new StreamContent(ms), "file", "blob"); + + var msg = new HttpRequestMessage(HttpMethod.Post, "https://upload.nexusmods.com/uploads/chunk"); + msg.Content = form; + + var result = await _client.SendAsync(msg); + if (result.StatusCode != HttpStatusCode.OK) + throw new HttpException(result); + + var response = await result.Content.ReadFromJsonAsync(_jsonOptions); + return response; + } + public async Task UploadFile(UploadDefinition d) + { + _logger.LogInformation("Checking Access"); + await CheckAccess(); + + _logger.LogInformation("Checking chunk status"); + + var numberOfChunks = d.Chunks().Count(); + var chunkStatus = new ChunkStatusResult(); + foreach (var chunk in d.Chunks()) + { + var status = await ChunkStatus(d, chunk); + _logger.LogInformation("({Index}/{MaxChunks}) Chunk status: {Status}", chunk.Index, numberOfChunks, status); + if (status == DTOs.ChunkStatus.NoContent) + { + _logger.LogInformation("({Index}/{MaxChunks}) Uploading", chunk.Index, numberOfChunks); + chunkStatus = await UploadChunk(d, chunk); + } + } + + await WaitForFileStatus(chunkStatus); + + await AddFile(d, chunkStatus); + + } + + private async Task CheckAccess() + { + var msg = new HttpRequestMessage(HttpMethod.Get, "https://www.nexusmods.com/users/myaccount"); + msg.AddCookies((await ApiKey.Get())!.Cookies); + using var response = await _client.SendAsync(msg); + var body = await response.Content.ReadAsStringAsync(); + + if (body.Contains("You are not allowed to access this area!")) + throw new HttpException(403, "Nexus Cookies are incorrect"); + } + + private async Task AddFile(UploadDefinition d, ChunkStatusResult status) + { + _logger.LogInformation("Saving file update {Name} to {Game}:{ModId}", d.Path.FileName, d.Game, d.ModId); + + var msg = new HttpRequestMessage(HttpMethod.Post, + "https://www.nexusmods.com/Core/Libs/Common/Managers/Mods?AddFile"); + msg.Headers.Referrer = + new Uri( + $"https://www.nexusmods.com/{d.Game.MetaData().NexusName}/mods/edit/?id={d.ModId}&game_id={d.GameId}&step=files"); + + msg.AddCookies((await ApiKey.Get())!.Cookies); + var form = new MultipartFormDataContent(); + form.Add(new StringContent(d.GameId.ToString()), "game_id"); + form.Add(new StringContent(d.Name), "name"); + form.Add(new StringContent(d.Version), "file-version"); + form.Add(new StringContent((d.RemoveOldVersion ? 1 : 0).ToString()), "update-version"); + form.Add(new StringContent(((int)Enum.Parse(d.Category, true)).ToString()), "category"); + form.Add(new StringContent((d.NewExisting ? 1 : 0).ToString()), "new-existing"); + form.Add(new StringContent(d.OldFileId.ToString()), "old_file_id"); + form.Add(new StringContent((d.RemoveOldVersion ? 1 : 0).ToString()), "remove-old-version"); + form.Add(new StringContent(d.BriefOverview), "brief-overview"); + form.Add(new StringContent((d.SetAsMain ? 1 : 0).ToString()), "set_as_main_nmm"); + form.Add(new StringContent(status.UUID), "file_uuid"); + form.Add(new StringContent(d.FileSize.ToString()), "file_size"); + form.Add(new StringContent(d.ModId.ToString()), "mod_id"); + form.Add(new StringContent(d.ModId.ToString()), "id"); + form.Add(new StringContent("save"), "action"); + form.Add(new StringContent(status.Filename), "uploaded_file"); + form.Add(new StringContent(d.Path.FileName.ToString()), "original_file"); + msg.Content = form; + + using var result = await _client.SendAsync(msg); + if (!result.IsSuccessStatusCode) + throw new HttpException(result); + } + + private async Task WaitForFileStatus(ChunkStatusResult chunkStatus) + { + while (true) + { + _logger.LogInformation("Checking file status of {Uuid}", chunkStatus.UUID); + var data = await _client.GetFromJsonAsync( + $"https://upload.nexusmods.com/uploads/check_status?id={chunkStatus.UUID}"); + if (data.FileChunksAssembled) + return data; + await Task.Delay(TimeSpan.FromSeconds(5)); + } + } } \ No newline at end of file