diff --git a/Wabbajack.CLI/Verbs/ForceHeal.cs b/Wabbajack.CLI/Verbs/ForceHeal.cs new file mode 100644 index 00000000..61bbe0c5 --- /dev/null +++ b/Wabbajack.CLI/Verbs/ForceHeal.cs @@ -0,0 +1,97 @@ +using System; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FluentFTP.Helpers; +using Microsoft.Extensions.Logging; +using Wabbajack.Common; +using Wabbajack.Compiler.PatchCache; +using Wabbajack.Downloaders; +using Wabbajack.DTOs; +using Wabbajack.DTOs.ModListValidation; +using Wabbajack.DTOs.ServerResponses; +using Wabbajack.Installer; +using Wabbajack.Networking.WabbajackClientApi; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; +using Wabbajack.VFS; + +namespace Wabbajack.CLI.Verbs; + +public class ForceHeal +{ + private readonly ILogger _logger; + private readonly Client _client; + private readonly DownloadDispatcher _downloadDispatcher; + private readonly FileHashCache _fileHashCache; + + public ForceHeal(ILogger logger, Client client, DownloadDispatcher downloadDispatcher, FileHashCache hashCache) + { + _logger = logger; + _client = client; + _downloadDispatcher = downloadDispatcher; + _fileHashCache = hashCache; + } + + public Command MakeCommand() + { + var command = new Command("force-heal"); + command.Add(new Option(new[] {"-f", "-from"}, "Old File")); + command.Add(new Option(new[] {"-t", "-to"}, "New File")); + command.Description = "Creates a patch from New file to Old File and uploads it"; + command.Handler = CommandHandler.Create(Run); + return command; + } + + public async Task Run(AbsolutePath from, AbsolutePath to) + { + var fromResolved = await Resolve(from); + var toResolved = await Resolve(to); + + _logger.LogInformation("Creating patch"); + var outData = new MemoryStream(); + OctoDiff.Create( await @from.ReadAllBytesAsync(), await to.ReadAllBytesAsync(), outData); + + _logger.LogInformation("Created {Size} patch", outData.Length.FileSizeToString()); + + outData.Position = 0; + + var validated = new ValidatedArchive + { + Original = fromResolved, + PatchedFrom = toResolved, + Status = ArchiveStatus.Updated + }; + + validated = await _client.UploadPatch(validated, outData); + + _logger.LogInformation("Adding patch to forced_healing.json"); + await _client.AddForceHealedPatch(validated); + _logger.LogInformation("Done, validation should trigger soon"); + return 0; + } + + private async Task Resolve(AbsolutePath file) + { + var meta = file.WithExtension(Ext.Meta); + if (!meta.FileExists()) + throw new Exception($"Meta not found {meta}"); + + var ini = meta.LoadIniFile(); + var state = await _downloadDispatcher.ResolveArchive(ini["General"].ToDictionary(d => d.KeyName, d => d.Value)); + + _logger.LogInformation("Hashing {File}", file.FileName); + var hash = await _fileHashCache.FileHashCachedAsync(file, CancellationToken.None); + + return new Archive + { + Hash = hash, + Name = file.FileName.ToString(), + Size = file.Size(), + State = state! + }; + } +} \ No newline at end of file diff --git a/Wabbajack.Networking.GitHub/Client.cs b/Wabbajack.Networking.GitHub/Client.cs index a337c13d..082ed1bf 100644 --- a/Wabbajack.Networking.GitHub/Client.cs +++ b/Wabbajack.Networking.GitHub/Client.cs @@ -64,4 +64,15 @@ public class Client throw new Exception("List not found or user not authorized"); } + + public async Task<(string Sha, string Content)> GetData(string owner, string repo, string path) + { + var result = (await _client.Repository.Content.GetAllContents(owner, repo, path))[0]; + return (result.Sha, result.Content); + } + + public async Task PutData(string owner, string repo, string path, string message, string content, string oldSha) + { + await _client.Repository.Content.UpdateFile(owner, repo, path, new UpdateFileRequest(message, content, oldSha)); + } } \ No newline at end of file diff --git a/Wabbajack.Networking.WabbajackClientApi/Client.cs b/Wabbajack.Networking.WabbajackClientApi/Client.cs index 184befa5..44191923 100644 --- a/Wabbajack.Networking.WabbajackClientApi/Client.cs +++ b/Wabbajack.Networking.WabbajackClientApi/Client.cs @@ -5,6 +5,7 @@ using System.IO.Compression; using System.Linq; using System.Net.Http; using System.Net.Http.Json; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -16,6 +17,7 @@ using Wabbajack.DTOs.Logins; using Wabbajack.DTOs.ModListValidation; using Wabbajack.DTOs.Validation; using Wabbajack.Hashing.xxHash64; +using Wabbajack.Networking.Http; using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Paths; using Wabbajack.Paths.IO; @@ -59,6 +61,8 @@ public class Client var msg = new HttpRequestMessage(method, uri); var key = (await _token.Get())!; msg.Headers.Add(_configuration.MetricsKeyHeader, key.MetricsKey); + if (!string.IsNullOrWhiteSpace(key.AuthorKey)) + msg.Headers.Add(_configuration.AuthorKeyHeader, key.AuthorKey); return msg; } @@ -209,4 +213,40 @@ public class Client { return new Uri($"{_configuration.PatchBaseAddress}{upgradeHash.ToHex()}_{archiveHash.ToHex()}"); } + + public async Task UploadPatch(ValidatedArchive validated, MemoryStream outData) + { + throw new NotImplementedException(); + } + + public async Task AddForceHealedPatch(ValidatedArchive validated) + { + var oldData = await GetGithubFile("wabbajack-tools", "mod-lists", "configs/forced_healing.json"); + var content = oldData.Content.Append(validated).ToArray(); + await UpdateGitHubFile("wabbajack-tools", "mod-lists", "configs/forced_healing.json", content, oldData.Sha); + } + + private async Task UpdateGitHubFile(string owner, string repo, string path, T content, string oldSha) + { + var json = _dtos.Serialize(content); + var msg = await MakeMessage(HttpMethod.Post, + new Uri($"{_configuration.BuildServerUrl}/github/?owner={owner}&repo={repo}&path={path}&oldSha={oldSha}")); + + msg.Content = new StringContent(json, Encoding.UTF8, "application/json"); + using var result = await _client.SendAsync(msg); + if (!result.IsSuccessStatusCode) + throw new HttpException(result); + } + + private async Task<(string Sha, T Content)> GetGithubFile(string owner, string repo, string path) + { + var msg = await MakeMessage(HttpMethod.Get, + new Uri($"{_configuration.BuildServerUrl}/github/?owner={owner}&repo={repo}&path={path}")); + using var oldData = await _client.SendAsync(msg); + if (!oldData.IsSuccessStatusCode) + throw new HttpException(oldData); + + var sha = oldData.Headers.GetValues(_configuration.ResponseShaHeader).First(); + return (sha, (await oldData.Content.ReadFromJsonAsync())!); + } } \ No newline at end of file diff --git a/Wabbajack.Networking.WabbajackClientApi/Configuration.cs b/Wabbajack.Networking.WabbajackClientApi/Configuration.cs index 33bc1e0e..4bbc211b 100644 --- a/Wabbajack.Networking.WabbajackClientApi/Configuration.cs +++ b/Wabbajack.Networking.WabbajackClientApi/Configuration.cs @@ -7,6 +7,10 @@ public class Configuration public Uri ServerUri { get; set; } = new("https://build.wabbajack.org"); public string MetricsKey { get; set; } public string MetricsKeyHeader { get; set; } = "x-metrics-key"; + public string AuthorKeyHeader { get; set; } = "x-api-key"; + + public string ResponseShaHeader { get; set; } = "x-response-sha"; + public Uri ServerAllowList { get; set; } = new("https://raw.githubusercontent.com/wabbajack-tools/opt-out-lists/master/ServerWhitelist.yml"); diff --git a/Wabbajack.Server/AppSettings.cs b/Wabbajack.Server/AppSettings.cs index 9cd068cb..c5306103 100644 --- a/Wabbajack.Server/AppSettings.cs +++ b/Wabbajack.Server/AppSettings.cs @@ -19,6 +19,8 @@ public class AppSettings public string HamWebHook { get; set; } = null; public string AuthoredFilesFolder { get; set; } + + public string PatchesFilesFolder { get; set; } public string MetricsFolder { get; set; } = ""; public string TarLogPath { get; set; } public string GitHubKey { get; set; } = ""; diff --git a/Wabbajack.Server/Controllers/Github.cs b/Wabbajack.Server/Controllers/Github.cs new file mode 100644 index 00000000..087f7dcc --- /dev/null +++ b/Wabbajack.Server/Controllers/Github.cs @@ -0,0 +1,52 @@ +using System.Security.Claims; +using System.Text; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Wabbajack.Common; +using Wabbajack.Networking.GitHub; +using Wabbajack.Server.DTOs; +using Wabbajack.Server.Services; + +namespace Wabbajack.Server.Controllers; + +[Authorize(Roles = "Author")] +[Route("/github")] +public class Github : ControllerBase +{ + private readonly Client _client; + private readonly ILogger _logger; + private readonly DiscordWebHook _discord; + + public Github(ILogger logger, Client client, DiscordWebHook discord) + { + _client = client; + _logger = logger; + _discord = discord; + } + + [HttpGet] + public async Task GetContent([FromQuery] string owner, [FromQuery] string repo, [FromQuery] string path) + { + var (sha, content) = await _client.GetData(owner, repo, path); + Response.StatusCode = 200; + Response.Headers.Add("x-content-sha", sha); + await Response.WriteAsync(content); + } + + [HttpPost] + public async Task SetContent([FromQuery] string owner, [FromQuery] string repo, [FromQuery] string path, [FromQuery] string oldSha) + { + var user = User.FindFirstValue(ClaimTypes.Name)!; + _logger.LogInformation("Updating {Owner}/{Repo}/{Path} on behalf of {User}", owner, repo, path, user); + + await _discord.Send(Channel.Ham, + new DiscordMessage {Content = $"Updating {owner}/{repo}/{path} on behalf of {user}"}); + + var content = Encoding.UTF8.GetString(await Request.Body.ReadAllAsync()); + await _client.PutData(owner, repo, path, $"Update on behalf of {user}", content, oldSha); + return Ok(); + } + +} \ No newline at end of file diff --git a/Wabbajack.Server/Controllers/Patches.cs b/Wabbajack.Server/Controllers/Patches.cs new file mode 100644 index 00000000..3359628e --- /dev/null +++ b/Wabbajack.Server/Controllers/Patches.cs @@ -0,0 +1,75 @@ +using System.Security.Claims; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Wabbajack.BuildServer; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; +using Wabbajack.Server.DTOs; +using Wabbajack.Server.Services; + +namespace Wabbajack.Server.Controllers; + +[ApiController] +[Authorize(Roles = "Author")] +[Route("/patches")] +public class Patches : ControllerBase +{ + private readonly AppSettings _settings; + private readonly DiscordWebHook _discord; + private readonly DTOSerializer _dtos; + + public Patches(AppSettings appSettings, DiscordWebHook discord, DTOSerializer dtos) + { + _settings = appSettings; + _discord = discord; + _dtos = dtos; + } + + [HttpPost] + public async Task WritePart(CancellationToken token, [FromQuery] string name, [FromQuery] long start) + { + var path = GetPath(name); + if (!path.FileExists()) + { + + var user = User.FindFirstValue(ClaimTypes.Name)!; + await _discord.Send(Channel.Ham, + new DiscordMessage {Content = $"{user} is uploading a new forced-healing patch file"}); + } + + await using var file = path.Open(FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read); + file.Position = start; + var hash = await Request.Body.HashingCopy(file, token); + await file.FlushAsync(token); + return Ok(hash.ToHex()); + } + + private AbsolutePath GetPath(string name) + { + return _settings.PatchesFilesFolder.ToAbsolutePath().Combine(name); + } + + [HttpGet] + [Route("list")] + public async Task ListPatches(CancellationToken token) + { + var root = _settings.PatchesFilesFolder.ToAbsolutePath(); + var files = root.EnumerateFiles() + .ToDictionary(f => f.RelativeTo(root).ToString(), f => f.Size()); + return Ok(_dtos.Serialize(files)); + } + + [HttpDelete] + public async Task DeletePart([FromQuery] string name) + { + var user = User.FindFirstValue(ClaimTypes.Name)!; + await _discord.Send(Channel.Ham, + new DiscordMessage {Content = $"{user} is deleting a new forced-healing patch file"}); + + GetPath(name).Delete(); + return Ok(name); + } + +} \ No newline at end of file diff --git a/Wabbajack.Server/appsettings.json b/Wabbajack.Server/appsettings.json index dbce5ffa..0f249129 100644 --- a/Wabbajack.Server/appsettings.json +++ b/Wabbajack.Server/appsettings.json @@ -11,6 +11,7 @@ "MetricsFolder": "c:\\tmp\\server_metrics", "AuthoredFilesFolder": "c:\\tmp\\server_authored_files", "AuthorAPIKeyFile": "c:\\tmp\\author_keys.txt", + "PatchesFilesFolder": "c:\\tmp\\patches", "GitHubKey": "" }, "AllowedHosts": "*"