From 766bf8e7190a180ab5701c3668fd56b7c03ee13f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 13 Jul 2022 15:51:42 -0600 Subject: [PATCH] Can publish modlists --- .../View Models/Compilers/CompilerVM.cs | 16 +++ .../Views/Compilers/CompilerView.xaml.cs | 2 +- .../Client.cs | 113 +++++++++++++----- ...ajack.Networking.WabbajackClientApi.csproj | 1 + 4 files changed, 104 insertions(+), 28 deletions(-) diff --git a/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs b/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs index 390690b9..80058126 100644 --- a/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs +++ b/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs @@ -239,6 +239,15 @@ namespace Wabbajack await compiler.Begin(token); + if (PublishUpdate) + { + _logger.LogInformation("Publishing List"); + var downloadMetadata = _dtos.Deserialize( + await mo2Settings.OutputFile.WithExtension(Ext.Meta).WithExtension(Ext.Json).ReadAllTextAsync())!; + await _wjClient.PublishModlist(MachineUrl, System.Version.Parse(Version), mo2Settings.OutputFile, downloadMetadata); + } + _logger.LogInformation("Compiler Finished"); + State = CompilerState.Completed; } catch (Exception ex) @@ -259,6 +268,13 @@ namespace Wabbajack _logger.LogError("Preflight Check failed, list {MachineUrl} not found in any repository", MachineUrl); return false; } + + if (!System.Version.TryParse(Version, out var v)) + { + _logger.LogError("Bad Version Number {Version}", Version); + return false; + } + return true; } diff --git a/Wabbajack.App.Wpf/Views/Compilers/CompilerView.xaml.cs b/Wabbajack.App.Wpf/Views/Compilers/CompilerView.xaml.cs index 035e8bbc..8d692b8a 100644 --- a/Wabbajack.App.Wpf/Views/Compilers/CompilerView.xaml.cs +++ b/Wabbajack.App.Wpf/Views/Compilers/CompilerView.xaml.cs @@ -34,7 +34,7 @@ namespace Wabbajack .DisposeWith(disposables); ViewModel.WhenAny(vm => vm.State) - .Select(x => x is CompilerState.Errored or CompilerState.Completed) + .Select(x => x == CompilerState.Errored) .Select(failed => $"Compilation {(failed ? "Failed" : "Complete")}") .BindToStrict(this, x => x.CompilationComplete.TitleText.Text) .DisposeWith(disposables); diff --git a/Wabbajack.Networking.WabbajackClientApi/Client.cs b/Wabbajack.Networking.WabbajackClientApi/Client.cs index 5b9f2227..34b63806 100644 --- a/Wabbajack.Networking.WabbajackClientApi/Client.cs +++ b/Wabbajack.Networking.WabbajackClientApi/Client.cs @@ -11,6 +11,7 @@ using System.Threading; using System.Threading.Tasks; using System.Web; using Microsoft.Extensions.Logging; +using Octokit; using Wabbajack.Common; using Wabbajack.DTOs; using Wabbajack.DTOs.CDN; @@ -29,7 +30,7 @@ using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; -using YamlDotNet.Serialization.TypeInspectors; +using FileMode = System.IO.FileMode; namespace Wabbajack.Networking.WabbajackClientApi; @@ -151,7 +152,7 @@ public class Client $"https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/reports/{machineURL}/status.json", _dtos.Options))!; } - + IEnumerable Blocks(long size) { for (long block = 0; block * UploadedFileBlockSize < size; block++) @@ -197,13 +198,14 @@ public class Client var featured = await LoadFeaturedLists(); return await (await repos).PMapAll(async url => - (await _client.GetFromJsonAsync(_limiter, new HttpRequestMessage(HttpMethod.Get, url.Value), + (await _client.GetFromJsonAsync(_limiter, + new HttpRequestMessage(HttpMethod.Get, url.Value), _dtos.Options))!.Select(meta => - { - meta.RepositoryName = url.Key; - meta.Official = (meta.RepositoryName == "wj-featured" || featured.Contains(meta.NamespacedName)); - return meta; - })) + { + meta.RepositoryName = url.Key; + meta.Official = (meta.RepositoryName == "wj-featured" || featured.Contains(meta.NamespacedName)); + return meta; + })) .SelectMany(x => x) .ToArray(); } @@ -212,7 +214,8 @@ public class Client { var data = await _client.GetFromJsonAsync(_limiter, new HttpRequestMessage(HttpMethod.Get, - "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/featured_lists.json"), _dtos.Options); + "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/featured_lists.json"), + _dtos.Options); return data!.ToHashSet(StringComparer.CurrentCultureIgnoreCase); } @@ -233,7 +236,7 @@ public class Client { _logger.LogInformation("Uploading Patch {From} {To}", validated.Original.Hash, validated.PatchedFrom!.Hash); var name = $"{validated.Original.Hash.ToHex()}_{validated.PatchedFrom.Hash.ToHex()}"; - + var blocks = Blocks(data.Length).ToArray(); foreach (var block in blocks) { @@ -261,7 +264,8 @@ public class Client public async Task AddForceHealedPatch(ValidatedArchive validated) { - var oldData = await GetGithubFile("wabbajack-tools", "mod-lists", "configs/forced_healing.json"); + var oldData = + await GetGithubFile("wabbajack-tools", "mod-lists", "configs/forced_healing.json"); var content = oldData.Content.Append(validated).ToArray(); await UpdateGitHubFile("wabbajack-tools", "mod-lists", "configs/forced_healing.json", content, oldData.Sha); } @@ -278,10 +282,11 @@ public class Client throw new HttpException(result); } - private async Task<(string Sha, T Content)> GetGithubFile(string owner, string repo, string path, CancellationToken? token = null) + private async Task<(string Sha, T Content)> GetGithubFile(string owner, string repo, string path, + CancellationToken? token = null) { token ??= CancellationToken.None; - + var msg = await MakeMessage(HttpMethod.Get, new Uri($"{_configuration.BuildServerUrl}github/?owner={owner}&repo={repo}&path={path}")); using var oldData = await _client.SendAsync(msg, token.Value); @@ -297,13 +302,13 @@ public class Client { var hashAsHex = definition.Hash.ToHex(); _logger.LogInformation("Starting upload of {Name} ({Hash})", file.FileName, hashAsHex); - + using var result = await _client.SendAsync(await MakeMessage(HttpMethod.Put, new Uri($"{_configuration.BuildServerUrl}mirrored_files/create/{hashAsHex}"), new StringContent(_dtos.Serialize(definition), Encoding.UTF8, "application/json"))); if (!result.IsSuccessStatusCode) throw new HttpException(result); - + _logger.LogInformation("Uploading Parts"); await using var dataIn = file.Open(FileMode.Open); @@ -319,14 +324,14 @@ public class Client using var partResult = await _client.SendAsync(await MakeMessage(HttpMethod.Put, new Uri($"{_configuration.BuildServerUrl}mirrored_files/{hashAsHex}/part/{idx}"), new ByteArrayContent(data))); - + if (!partResult.IsSuccessStatusCode) throw new HttpException(result); } using var finalResult = await _client.SendAsync(await MakeMessage(HttpMethod.Put, new Uri($"{_configuration.BuildServerUrl}mirrored_files/{hashAsHex}/finish"))); - + if (!finalResult.IsSuccessStatusCode) throw new HttpException(result); } @@ -339,13 +344,16 @@ public class Client public async Task GetAllPatches(CancellationToken token) { - return (await _client.GetFromJsonAsync("https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/configs/forced_healing.json", _dtos.Options, token))!; + return (await _client.GetFromJsonAsync( + "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/configs/forced_healing.json", + _dtos.Options, token))!; } public async Task DeleteMirror(Hash hash) { _logger.LogInformation("Deleting mirror of {Hash}", hash); - var msg = await MakeMessage(HttpMethod.Delete, new Uri($"{_configuration.BuildServerUrl}mirrored_files/{hash.ToHex()}")); + var msg = await MakeMessage(HttpMethod.Delete, + new Uri($"{_configuration.BuildServerUrl}mirrored_files/{hash.ToHex()}")); var result = await _client.SendAsync(msg); if (!result.IsSuccessStatusCode) throw new HttpException(result); @@ -366,7 +374,8 @@ public class Client report.OnNext((Percent.Zero, "Creating file upload")); await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () => { - var msg = await MakeMessage(HttpMethod.Put, new Uri($"{_configuration.BuildServerUrl}authored_files/create")); + var msg = await MakeMessage(HttpMethod.Put, + new Uri($"{_configuration.BuildServerUrl}authored_files/create")); msg.Content = new StringContent(_dtos.Serialize(definition)); using var result = await _client.SendAsync(msg); HttpException.ThrowOnFailure(result); @@ -405,7 +414,8 @@ public class Client return await CircuitBreaker.WithAutoRetryAllAsync(_logger, async () => { var msg = await MakeMessage(HttpMethod.Put, - new Uri($"{_configuration.BuildServerUrl}authored_files/{definition.ServerAssignedUniqueId}/finish")); + new Uri( + $"{_configuration.BuildServerUrl}authored_files/{definition.ServerAssignedUniqueId}/finish")); msg.Content = new StringContent(_dtos.Serialize(definition)); using var result = await _client.SendAsync(msg); HttpException.ThrowOnFailure(result); @@ -415,17 +425,21 @@ public class Client }); return (report, tsk); } + public async Task GetForcedRemovals(CancellationToken token) { - return (await _client.GetFromJsonAsync("https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/configs/forced_removal.json", _dtos.Options, token))!; + return (await _client.GetFromJsonAsync( + "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/configs/forced_removal.json", + _dtos.Options, token))!; } public async Task GetSteamManifests(Game game, string version) { - var url = $"https://raw.githubusercontent.com/wabbajack-tools/indexed-game-files/master/{game}/{version}_steam_manifests.json"; + var url = + $"https://raw.githubusercontent.com/wabbajack-tools/indexed-game-files/master/{game}/{version}_steam_manifests.json"; return await _client.GetFromJsonAsync(url, _dtos.Options) ?? Array.Empty(); } - + public async Task ProxyHas(Uri uri) { var newUri = new Uri($"{_configuration.BuildServerUrl}proxy?uri={HttpUtility.UrlEncode(uri.ToString())}"); @@ -445,8 +459,9 @@ public class Client { if (archive.State is Manual && !await ProxyHas(uri)) return null; - - return new Uri($"{_configuration.BuildServerUrl}proxy?name={archive.Name}&hash={archive.Hash.ToHex()}&uri={HttpUtility.UrlEncode(uri.ToString())}"); + + return new Uri( + $"{_configuration.BuildServerUrl}proxy?name={archive.Name}&hash={archive.Hash.ToHex()}&uri={HttpUtility.UrlEncode(uri.ToString())}"); } public async Task GetCesiVfsEntry(Hash hash, CancellationToken token) @@ -464,4 +479,48 @@ public class Client HttpException.ThrowOnFailure(response); return (await _dtos.DeserializeAsync(await response.Content.ReadAsStreamAsync(token), token))!; } -} \ No newline at end of file + + public async Task PublishModlist(string namespacedName, Version version, AbsolutePath modList, DownloadMetadata metadata) + { + var pair = namespacedName.Split("/"); + var wjRepoName = pair[0]; + var machineUrl = pair[1]; + + var repoUrl = (await LoadRepositories())[wjRepoName]; + + var decomposed = repoUrl.LocalPath.Split("/"); + var owner = decomposed[1]; + var repoName = decomposed[2]; + var path = string.Join("/", decomposed[4..]); + + _logger.LogInformation("Uploading modlist {MachineUrl}", namespacedName); + + var (progress, uploadTask) = await UploadAuthorFile(modList); + progress.Subscribe(x => _logger.LogInformation(x.Message)); + var downloadUrl = await uploadTask; + + _logger.LogInformation("Publishing modlist {MachineUrl}", namespacedName); + + var creds = new Credentials((await _token.Get())!.AuthorKey); + var ghClient = new GitHubClient(new ProductHeaderValue("wabbajack")) {Credentials = creds}; + + var oldData = + (await ghClient.Repository.Content.GetAllContents(owner, repoName, path)) + .First(); + var oldContent = _dtos.Deserialize(oldData.Content); + var list = oldContent.First(c => c.Links.MachineURL == machineUrl); + list.Version = version; + list.DownloadMetadata = metadata; + list.Links.Download = downloadUrl.ToString(); + list.DateUpdated = DateTime.UtcNow; + + + var newContent = _dtos.Serialize(oldContent, true); + // the website requires all names be in lowercase; + newContent = GameRegistry.Games.Keys.Aggregate(newContent, + (current, g) => current.Replace($"\"game\": \"{g}\",", $"\"game\": \"{g.ToString().ToLower()}\",")); + + var updateRequest = new UpdateFileRequest($"New release of {machineUrl}", newContent, oldData.Sha); + await ghClient.Repository.Content.UpdateFile(owner, repoName, path, updateRequest); + } +} diff --git a/Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj b/Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj index 1e625c0f..4e3da365 100644 --- a/Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj +++ b/Wabbajack.Networking.WabbajackClientApi/Wabbajack.Networking.WabbajackClientApi.csproj @@ -9,6 +9,7 @@ +