Force healing reimplementation, phase 1

Wabbajack.CLI/Verbs/ForceHeal.cs

@@ -0,0 +1,97 @@
+using System;
+using System.CommandLine;
+using System.CommandLine.Invocation;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using FluentFTP.Helpers;
+using Microsoft.Extensions.Logging;
+using Wabbajack.Common;
+using Wabbajack.Compiler.PatchCache;
+using Wabbajack.Downloaders;
+using Wabbajack.DTOs;
+using Wabbajack.DTOs.ModListValidation;
+using Wabbajack.DTOs.ServerResponses;
+using Wabbajack.Installer;
+using Wabbajack.Networking.WabbajackClientApi;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+using Wabbajack.VFS;
+
+namespace Wabbajack.CLI.Verbs;
+
+public class ForceHeal
+{
+    private readonly ILogger<ForceHeal> _logger;
+    private readonly Client _client;
+    private readonly DownloadDispatcher _downloadDispatcher;
+    private readonly FileHashCache _fileHashCache;
+
+    public ForceHeal(ILogger<ForceHeal> logger, Client client, DownloadDispatcher downloadDispatcher, FileHashCache hashCache)
+    {
+        _logger = logger;
+        _client = client;
+        _downloadDispatcher = downloadDispatcher;
+        _fileHashCache = hashCache;
+    }
+
+    public Command MakeCommand()
+    {
+        var command = new Command("force-heal");
+        command.Add(new Option<AbsolutePath>(new[] {"-f", "-from"}, "Old File"));
+        command.Add(new Option<string>(new[] {"-t", "-to"}, "New File"));
+        command.Description = "Creates a patch from New file to Old File and uploads it";
+        command.Handler = CommandHandler.Create(Run);
+        return command;
+    }
+
+    public async Task<int> Run(AbsolutePath from, AbsolutePath to)
+    {
+        var fromResolved = await Resolve(from);
+        var toResolved = await Resolve(to);
+
+        _logger.LogInformation("Creating patch");
+        var outData = new MemoryStream();
+        OctoDiff.Create( await @from.ReadAllBytesAsync(), await to.ReadAllBytesAsync(), outData);
+        
+        _logger.LogInformation("Created {Size} patch", outData.Length.FileSizeToString());
+
+        outData.Position = 0;
+        
+        var validated = new ValidatedArchive
+        {
+            Original = fromResolved,
+            PatchedFrom = toResolved,
+            Status = ArchiveStatus.Updated
+        };
+        
+        validated = await _client.UploadPatch(validated, outData);
+
+        _logger.LogInformation("Adding patch to forced_healing.json");
+        await _client.AddForceHealedPatch(validated);
+        _logger.LogInformation("Done, validation should trigger soon");
+        return 0;
+    }
+
+    private async Task<Archive> Resolve(AbsolutePath file)
+    {
+        var meta = file.WithExtension(Ext.Meta);
+        if (!meta.FileExists())
+            throw new Exception($"Meta not found {meta}");
+        
+        var ini = meta.LoadIniFile();
+        var state = await _downloadDispatcher.ResolveArchive(ini["General"].ToDictionary(d => d.KeyName, d => d.Value));
+
+        _logger.LogInformation("Hashing {File}", file.FileName);
+        var hash = await _fileHashCache.FileHashCachedAsync(file, CancellationToken.None);
+        
+        return new Archive
+        {
+            Hash = hash,
+            Name = file.FileName.ToString(),
+            Size = file.Size(),
+            State = state!
+        };
+    }
+}

Wabbajack.Networking.GitHub/Client.cs

@@ -64,4 +64,15 @@ public class Client
 
         throw new Exception("List not found or user not authorized");
     }
+
+    public async Task<(string Sha, string Content)> GetData(string owner, string repo, string path)
+    {
+        var result = (await _client.Repository.Content.GetAllContents(owner, repo, path))[0];
+        return (result.Sha, result.Content);
+    }
+
+    public async Task PutData(string owner, string repo, string path, string message, string content, string oldSha)
+    {
+        await _client.Repository.Content.UpdateFile(owner, repo, path, new UpdateFileRequest(message, content, oldSha));
+    }
 }

Wabbajack.Networking.WabbajackClientApi/Client.cs

@@ -5,6 +5,7 @@ using System.IO.Compression;
 using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Json;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
@@ -16,6 +17,7 @@ using Wabbajack.DTOs.Logins;
 using Wabbajack.DTOs.ModListValidation;
 using Wabbajack.DTOs.Validation;
 using Wabbajack.Hashing.xxHash64;
+using Wabbajack.Networking.Http;
 using Wabbajack.Networking.Http.Interfaces;
 using Wabbajack.Paths;
 using Wabbajack.Paths.IO;
@@ -59,6 +61,8 @@ public class Client
         var msg = new HttpRequestMessage(method, uri);
         var key = (await _token.Get())!;
         msg.Headers.Add(_configuration.MetricsKeyHeader, key.MetricsKey);
+        if (!string.IsNullOrWhiteSpace(key.AuthorKey))
+            msg.Headers.Add(_configuration.AuthorKeyHeader, key.AuthorKey);
         return msg;
     }
 
@@ -209,4 +213,40 @@ public class Client
     {
         return new Uri($"{_configuration.PatchBaseAddress}{upgradeHash.ToHex()}_{archiveHash.ToHex()}");
     }
+
+    public async Task<ValidatedArchive> UploadPatch(ValidatedArchive validated, MemoryStream outData)
+    {
+        throw new NotImplementedException();
+    }
+
+    public async Task AddForceHealedPatch(ValidatedArchive validated)
+    {
+        var oldData = await GetGithubFile<ValidatedArchive[]>("wabbajack-tools", "mod-lists", "configs/forced_healing.json");
+        var content = oldData.Content.Append(validated).ToArray();
+        await UpdateGitHubFile("wabbajack-tools", "mod-lists", "configs/forced_healing.json", content, oldData.Sha);
+    }
+
+    private async Task UpdateGitHubFile<T>(string owner, string repo, string path, T content, string oldSha)
+    {
+        var json = _dtos.Serialize(content);
+        var msg = await MakeMessage(HttpMethod.Post,
+            new Uri($"{_configuration.BuildServerUrl}/github/?owner={owner}&repo={repo}&path={path}&oldSha={oldSha}"));
+
+        msg.Content = new StringContent(json, Encoding.UTF8, "application/json");
+        using var result = await _client.SendAsync(msg);
+        if (!result.IsSuccessStatusCode)
+            throw new HttpException(result);
+    }
+
+    private async Task<(string Sha, T Content)> GetGithubFile<T>(string owner, string repo, string path)
+    {
+        var msg = await MakeMessage(HttpMethod.Get,
+            new Uri($"{_configuration.BuildServerUrl}/github/?owner={owner}&repo={repo}&path={path}"));
+        using var oldData = await _client.SendAsync(msg);
+        if (!oldData.IsSuccessStatusCode)
+            throw new HttpException(oldData);
+
+        var sha = oldData.Headers.GetValues(_configuration.ResponseShaHeader).First();
+        return (sha, (await oldData.Content.ReadFromJsonAsync<T>())!);
+    }
 }

Wabbajack.Networking.WabbajackClientApi/Configuration.cs

@@ -7,6 +7,10 @@ public class Configuration
     public Uri ServerUri { get; set; } = new("https://build.wabbajack.org");
     public string MetricsKey { get; set; }
     public string MetricsKeyHeader { get; set; } = "x-metrics-key";
+    public string AuthorKeyHeader { get; set; } = "x-api-key";
+
+    public string ResponseShaHeader { get; set; } = "x-response-sha";
+
 
     public Uri ServerAllowList { get; set; } =
         new("https://raw.githubusercontent.com/wabbajack-tools/opt-out-lists/master/ServerWhitelist.yml");

Wabbajack.Server/AppSettings.cs

@@ -19,6 +19,8 @@ public class AppSettings
     public string HamWebHook { get; set; } = null;
     
     public string AuthoredFilesFolder { get; set; }
+    
+    public string PatchesFilesFolder { get; set; }
     public string MetricsFolder { get; set; } = "";
     public string TarLogPath { get; set; }
     public string GitHubKey { get; set; } = "";

Wabbajack.Server/Controllers/Github.cs

@@ -0,0 +1,52 @@
+using System.Security.Claims;
+using System.Text;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using Wabbajack.Common;
+using Wabbajack.Networking.GitHub;
+using Wabbajack.Server.DTOs;
+using Wabbajack.Server.Services;
+
+namespace Wabbajack.Server.Controllers;
+
+[Authorize(Roles = "Author")]
+[Route("/github")]
+public class Github : ControllerBase
+{
+    private readonly Client _client;
+    private readonly ILogger<Github> _logger;
+    private readonly DiscordWebHook _discord;
+
+    public Github(ILogger<Github> logger, Client client, DiscordWebHook discord)
+    {
+        _client = client;
+        _logger = logger;
+        _discord = discord;
+    }
+
+    [HttpGet]
+    public async Task GetContent([FromQuery] string owner, [FromQuery] string repo, [FromQuery] string path)
+    {
+        var (sha, content) = await _client.GetData(owner, repo, path);
+        Response.StatusCode = 200;
+        Response.Headers.Add("x-content-sha", sha);
+        await Response.WriteAsync(content);
+    }
+
+    [HttpPost]
+    public async Task<IActionResult> SetContent([FromQuery] string owner, [FromQuery] string repo, [FromQuery] string path, [FromQuery] string oldSha)
+    {
+        var user = User.FindFirstValue(ClaimTypes.Name)!;
+        _logger.LogInformation("Updating {Owner}/{Repo}/{Path} on behalf of {User}", owner, repo, path, user);
+        
+        await _discord.Send(Channel.Ham,
+            new DiscordMessage {Content = $"Updating {owner}/{repo}/{path} on behalf of {user}"});
+        
+        var content = Encoding.UTF8.GetString(await Request.Body.ReadAllAsync());
+        await _client.PutData(owner, repo, path, $"Update on behalf of {user}", content, oldSha);
+        return Ok();
+    }
+    
+}

Wabbajack.Server/Controllers/Patches.cs

@@ -0,0 +1,75 @@
+using System.Security.Claims;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+using Wabbajack.BuildServer;
+using Wabbajack.DTOs.JsonConverters;
+using Wabbajack.Hashing.xxHash64;
+using Wabbajack.Paths;
+using Wabbajack.Paths.IO;
+using Wabbajack.Server.DTOs;
+using Wabbajack.Server.Services;
+
+namespace Wabbajack.Server.Controllers;
+
+[ApiController]
+[Authorize(Roles = "Author")]
+[Route("/patches")]
+public class Patches : ControllerBase
+{
+    private readonly AppSettings _settings;
+    private readonly DiscordWebHook _discord;
+    private readonly DTOSerializer _dtos;
+
+    public Patches(AppSettings appSettings, DiscordWebHook discord, DTOSerializer dtos)
+    {
+        _settings = appSettings;
+        _discord = discord;
+        _dtos = dtos;
+    }
+
+    [HttpPost]
+    public async Task<IActionResult> WritePart(CancellationToken token, [FromQuery] string name, [FromQuery] long start)
+    {
+        var path = GetPath(name);
+        if (!path.FileExists())
+        {
+            
+            var user = User.FindFirstValue(ClaimTypes.Name)!;
+            await _discord.Send(Channel.Ham,
+                new DiscordMessage {Content = $"{user} is uploading a new forced-healing patch file"});
+        }
+
+        await using var file = path.Open(FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read);
+        file.Position = start;
+        var hash = await Request.Body.HashingCopy(file, token);
+        await file.FlushAsync(token);
+        return Ok(hash.ToHex());
+    }
+
+    private AbsolutePath GetPath(string name)
+    {
+        return _settings.PatchesFilesFolder.ToAbsolutePath().Combine(name);
+    }
+
+    [HttpGet]
+    [Route("list")]
+    public async Task<IActionResult> ListPatches(CancellationToken token)
+    {
+        var root = _settings.PatchesFilesFolder.ToAbsolutePath();
+        var files = root.EnumerateFiles()
+            .ToDictionary(f => f.RelativeTo(root).ToString(), f => f.Size());
+        return Ok(_dtos.Serialize(files));
+    }
+
+    [HttpDelete]
+    public async Task<IActionResult> DeletePart([FromQuery] string name)
+    {
+        var user = User.FindFirstValue(ClaimTypes.Name)!;
+        await _discord.Send(Channel.Ham,
+            new DiscordMessage {Content = $"{user} is deleting a new forced-healing patch file"});
+        
+        GetPath(name).Delete();
+        return Ok(name);
+    }
+    
+}

Wabbajack.Server/appsettings.json

@@ -11,6 +11,7 @@
     "MetricsFolder": "c:\\tmp\\server_metrics",
     "AuthoredFilesFolder": "c:\\tmp\\server_authored_files",
     "AuthorAPIKeyFile": "c:\\tmp\\author_keys.txt",
+    "PatchesFilesFolder": "c:\\tmp\\patches",
     "GitHubKey": ""
   },
   "AllowedHosts": "*"