From bb33dea03acf8c7570d76d136b2d382decd60bc3 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 19 Jul 2021 23:08:12 -0600 Subject: [PATCH] Full path of files should now work --- Wabbajack.Lib/ACompiler.cs | 37 ++++++++++++++++++- Wabbajack.Lib/AuthorApi/Client.cs | 7 ++-- Wabbajack.Lib/GitHub/Client.cs | 9 +++-- Wabbajack.Lib/GitHub/UpdateRequest.cs | 15 ++++++++ Wabbajack.Lib/MO2Compiler.cs | 13 ++++++- .../ModListRegistry/ModListMetadata.cs | 2 - Wabbajack.Lib/NativeCompiler.cs | 12 +++++- .../Controllers/AuthorControls.cs | 15 +++++--- Wabbajack.Server/Controllers/AuthoredFiles.cs | 1 + Wabbajack.Test/GitHubTests.cs | 14 +++++-- Wabbajack/Settings.cs | 3 ++ Wabbajack/View Models/Compilers/CompilerVM.cs | 6 ++- .../View Models/Compilers/MO2CompilerVM.cs | 20 +++++++++- .../Compilers/ModlistSettingsEditorVM.cs | 7 ++++ Wabbajack/Views/Compilers/CompilerView.xaml | 9 +++++ .../Views/Compilers/CompilerView.xaml.cs | 12 +++++- 16 files changed, 153 insertions(+), 29 deletions(-) create mode 100644 Wabbajack.Lib/GitHub/UpdateRequest.cs diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index 3d144d74..54e06bb0 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -14,6 +14,8 @@ using Org.BouncyCastle.Asn1.Cms; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.FileUploader; +using Wabbajack.Lib.GitHub; using Wabbajack.Lib.ModListRegistry; using Wabbajack.VirtualFileSystem; @@ -34,9 +36,10 @@ namespace Wabbajack.Lib public string? ModListName, ModListAuthor, ModListDescription, ModListWebsite, ModlistReadme; public Version? ModlistVersion; protected Version? WabbajackVersion; + public UpdateRequest? PublishData; public ACompiler(int steps, string modlistName, AbsolutePath sourcePath, AbsolutePath downloadsPath, - AbsolutePath outputModListName) + AbsolutePath outputModListName, UpdateRequest? publishData) : base(steps) { SourcePath = sourcePath; @@ -48,8 +51,11 @@ namespace Wabbajack.Lib Settings = new CompilerSettings(); ModListOutputFolder = AbsolutePath.EntryPoint.Combine("output_folder", Guid.NewGuid().ToString()); CompilingGame = new GameMetaData(); + PublishData = publishData; } + + /// /// Set to true to include game files during compilation, only ever disabled /// in testing (to speed up tests) @@ -171,6 +177,35 @@ namespace Wabbajack.Lib return true; } + public async Task PreflightChecks() + { + Utils.Log($"Running preflight checks"); + if (PublishData == null) return; + + var ourLists = await (await AuthorApi.Client.Create()).GetMyModlists(); + if (ourLists.All(l => l.MachineURL != PublishData.MachineUrl)) + { + Utils.ErrorThrow(new CriticalFailureIntervention( + $"Cannot publish to {PublishData.MachineUrl}, you are not listed as a maintainer", + "Cannot publish")); + } + } + + public async Task PublishModlist() + { + if (PublishData == null) return; + var api = await AuthorApi.Client.Create(); + Utils.Log($"Uploading modlist {PublishData!.MachineUrl}"); + + var metadata = ((AbsolutePath)(ModListOutputFile + ".meta.json")).FromJson(); + var uri = await api.UploadFile(Queue, ModListOutputFile, (s, percent) => Utils.Status(s, percent)); + PublishData.DownloadUrl = uri; + PublishData.DownloadMetadata = metadata; + + Utils.Log($"Publishing modlist {PublishData!.MachineUrl}"); + await api.UpdateModListInformation(PublishData); + } + protected async Task IndexGameFileHashes() { if (UseGamePaths) diff --git a/Wabbajack.Lib/AuthorApi/Client.cs b/Wabbajack.Lib/AuthorApi/Client.cs index cf22696b..ed081c38 100644 --- a/Wabbajack.Lib/AuthorApi/Client.cs +++ b/Wabbajack.Lib/AuthorApi/Client.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Net.Http; using System.Threading.Tasks; using Wabbajack.Common; +using Wabbajack.Lib.GitHub; using Wabbajack.Lib.ModListRegistry; namespace Wabbajack.Lib.AuthorApi @@ -77,12 +78,12 @@ namespace Wabbajack.Lib.AuthorApi return definition; } - public async Task UpdateModListInformation(string machineUrl, DownloadMetadata metadata) + public async Task UpdateModListInformation(UpdateRequest metadata) { await CircuitBreaker.WithAutoRetryAllAsync(async () => { - Utils.Log($"Updating modlist information for {machineUrl} - {metadata.Version}"); - using var result = await _client.PostAsync($"{Consts.WabbajackBuildServerUri}author_controls/{machineUrl}/download_metadata", + Utils.Log($"Updating modlist information for {metadata.MachineUrl} - {metadata.Version}"); + using var result = await _client.PostAsync($"{Consts.WabbajackBuildServerUri}author_controls/lists/download_metadata", new StringContent(metadata.ToJson())); }); } diff --git a/Wabbajack.Lib/GitHub/Client.cs b/Wabbajack.Lib/GitHub/Client.cs index 6e2ab776..3a4c1798 100644 --- a/Wabbajack.Lib/GitHub/Client.cs +++ b/Wabbajack.Lib/GitHub/Client.cs @@ -56,18 +56,19 @@ namespace Wabbajack.Lib.GitHub await _client.Repository.Content.UpdateFile("wabbajack-tools", "mod-lists", PathNames[file], updateRequest); } - public async Task UpdateList(string maintainer, string machineUrl, DownloadMetadata newData) + public async Task UpdateList(string maintainer, UpdateRequest newData) { foreach (var file in Enum.GetValues()) { var data = await GetData(file); - var found = data.Lists.FirstOrDefault(l => l.Links.MachineURL == machineUrl && l.Maintainers.Contains(maintainer)); + var found = data.Lists.FirstOrDefault(l => l.Links.MachineURL == newData.MachineUrl && l.Maintainers.Contains(maintainer)); if (found == null) continue; - found.DownloadMetadata = newData; + found.DownloadMetadata = newData.DownloadMetadata; found.Version = newData.Version; + found.Links.Download = newData.DownloadUrl.ToString(); - await WriteData(file, machineUrl, data.Hash, data.Lists); + await WriteData(file, newData.MachineUrl, data.Hash, data.Lists); return; } diff --git a/Wabbajack.Lib/GitHub/UpdateRequest.cs b/Wabbajack.Lib/GitHub/UpdateRequest.cs new file mode 100644 index 00000000..214eb2a3 --- /dev/null +++ b/Wabbajack.Lib/GitHub/UpdateRequest.cs @@ -0,0 +1,15 @@ +using System; +using Wabbajack.Common.Serialization.Json; +using Wabbajack.Lib.ModListRegistry; + +namespace Wabbajack.Lib.GitHub +{ + [JsonName("UpdateRequest")] + public class UpdateRequest + { + public string MachineUrl { get; set; } = ""; + public Version? Version { get; set; } = new(); + public Uri DownloadUrl { get; set; } = new("https://www.wabbajack.org"); + public DownloadMetadata DownloadMetadata { get; set; } = new(); + } +} diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 2ec804fd..da6702df 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -8,6 +8,7 @@ using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.GitHub; using Wabbajack.Lib.Validation; using Wabbajack.VirtualFileSystem; @@ -15,8 +16,8 @@ namespace Wabbajack.Lib { public class MO2Compiler : ACompiler { - public MO2Compiler(AbsolutePath sourcePath, AbsolutePath downloadsPath, string mo2Profile, AbsolutePath outputFile) - : base(21, mo2Profile, sourcePath, downloadsPath, outputFile) + public MO2Compiler(AbsolutePath sourcePath, AbsolutePath downloadsPath, string mo2Profile, AbsolutePath outputFile, UpdateRequest? publishData = null) + : base(23, mo2Profile, sourcePath, downloadsPath, outputFile, publishData) { MO2Profile = mo2Profile; MO2Ini = SourcePath.Combine("ModOrganizer.ini").LoadIniFile(); @@ -25,6 +26,7 @@ namespace Wabbajack.Lib GamePath = CompilingGame.GameLocation(); } + public UpdateRequest? PublishData { get; set; } public AbsolutePath MO2ModsFolder => SourcePath.Combine(Consts.MO2ModFolderName); public string MO2Profile { get; } @@ -58,6 +60,10 @@ namespace Wabbajack.Lib FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam; UpdateTracker.Reset(); + + UpdateTracker.NextStep("Running Preflight Checks"); + await PreflightChecks(); + UpdateTracker.NextStep("Gathering information"); Utils.Log("Loading compiler Settings"); @@ -362,6 +368,9 @@ namespace Wabbajack.Lib UpdateTracker.NextStep("Exporting Modlist"); await ExportModList(); + + UpdateTracker.NextStep("Publishing Modlist"); + await PublishModlist(); ResetMembers(); diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index 9234b21d..a5f72a7a 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -138,8 +138,6 @@ namespace Wabbajack.Lib.ModListRegistry { public Hash Hash { get; set; } public long Size { get; set; } - - public Version Version { get; set; } = new(); public long NumberOfArchives { get; set; } public long SizeOfArchives { get; set; } public long NumberOfInstalledFiles { get; set; } diff --git a/Wabbajack.Lib/NativeCompiler.cs b/Wabbajack.Lib/NativeCompiler.cs index 6abc0bee..81553f09 100644 --- a/Wabbajack.Lib/NativeCompiler.cs +++ b/Wabbajack.Lib/NativeCompiler.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; +using Wabbajack.Lib.GitHub; using Wabbajack.Lib.Validation; using Wabbajack.VirtualFileSystem; @@ -12,8 +13,8 @@ namespace Wabbajack.Lib { public class NativeCompiler : ACompiler { - public NativeCompiler(NativeCompilerSettings settings, AbsolutePath sourcePath, AbsolutePath downloadsPath, AbsolutePath outputModListPath) - : base(3, settings.ModListName, sourcePath, downloadsPath, outputModListPath) + public NativeCompiler(NativeCompilerSettings settings, AbsolutePath sourcePath, AbsolutePath downloadsPath, AbsolutePath outputModListPath, UpdateRequest? publishData = null) + : base(5, settings.ModListName, sourcePath, downloadsPath, outputModListPath, publishData) { CompilingGame = settings.CompilingGame.MetaData(); GamePath = CompilingGame.GameLocation(); @@ -34,6 +35,10 @@ namespace Wabbajack.Lib FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam; UpdateTracker.Reset(); + + UpdateTracker.NextStep("Running Preflight Checks"); + await PreflightChecks(); + UpdateTracker.NextStep("Gathering information"); Utils.Log($"Compiling Game: {CompilingGame.Game}"); @@ -243,6 +248,9 @@ namespace Wabbajack.Lib UpdateTracker.NextStep("Exporting Modlist"); await ExportModList(); + + UpdateTracker.NextStep("Publishing Modlist"); + await PublishModlist(); ResetMembers(); diff --git a/Wabbajack.Server/Controllers/AuthorControls.cs b/Wabbajack.Server/Controllers/AuthorControls.cs index c722a3c5..e1822a08 100644 --- a/Wabbajack.Server/Controllers/AuthorControls.cs +++ b/Wabbajack.Server/Controllers/AuthorControls.cs @@ -16,6 +16,7 @@ using Wabbajack.Common; using Wabbajack.Lib.GitHub; using Wabbajack.Lib.ModListRegistry; using Wabbajack.Server.DataLayer; +using Wabbajack.Server.Services; namespace Wabbajack.BuildServer.Controllers { @@ -25,11 +26,13 @@ namespace Wabbajack.BuildServer.Controllers { private ILogger _logger; private SqlService _sql; + private readonly QuickSync _quickSync; - public AuthorControls(ILogger logger, SqlService sql) + public AuthorControls(ILogger logger, SqlService sql, QuickSync quickSync) { _logger = logger; _sql = sql; + _quickSync = quickSync; } [Route("login/{authorKey}")] @@ -56,19 +59,21 @@ namespace Wabbajack.BuildServer.Controllers return Ok(lists); } - [Route("lists/{machineUrl}/download_metadata")] + [Route("lists/download_metadata")] [HttpPost] - public async Task AuthorLists(string machineUrl) + public async Task PostDownloadMetadata() { var user = User.FindFirstValue(ClaimTypes.Name); - var data = (await Request.Body.ReadAllTextAsync()).FromJsonString(); + var data = (await Request.Body.ReadAllTextAsync()).FromJsonString(); var client = await Client.Get(); try { - await client.UpdateList(user, machineUrl, data); + await client.UpdateList(user, data); + await _quickSync.Notify(); } catch (Exception ex) { + _logger.LogError(ex, "During posting of download_metadata"); return BadRequest(ex); } return Ok(data); diff --git a/Wabbajack.Server/Controllers/AuthoredFiles.cs b/Wabbajack.Server/Controllers/AuthoredFiles.cs index e1faac95..8260a003 100644 --- a/Wabbajack.Server/Controllers/AuthoredFiles.cs +++ b/Wabbajack.Server/Controllers/AuthoredFiles.cs @@ -141,6 +141,7 @@ namespace Wabbajack.BuildServer.Controllers var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId); if (definition.Author != user) return Forbid("File Id does not match authorized user"); + await _discord.Send(Channel.Ham, new DiscordMessage() {Content = $"{user} is deleting {definition.MungedName}, {definition.Size.ToFileSizeString()} to be freed"}); _logger.Log(LogLevel.Information, $"Finalizing file upload {definition.OriginalFileName}"); await DeleteFolderOrSilentlyFail($"{definition.MungedName}"); diff --git a/Wabbajack.Test/GitHubTests.cs b/Wabbajack.Test/GitHubTests.cs index 58f54aef..d487fecc 100644 --- a/Wabbajack.Test/GitHubTests.cs +++ b/Wabbajack.Test/GitHubTests.cs @@ -24,17 +24,23 @@ namespace Wabbajack.Test NumberOfInstalledFiles = rnd.Next(1000), SizeOfInstalledFiles = rnd.Next(1000000), Size = rnd.Next(10000), - Version = new Version(1, 0, rnd.Next(10), 0) + }; - await client.UpdateList("ci_tester", "ci_test", meta); + var update = new UpdateRequest + { + DownloadMetadata = meta, + DownloadUrl = new Uri($"https://www.google.com/{rnd.Next()}"), + MachineUrl = "ci_test", + Version = new Version(1, rnd.Next(10), rnd.Next(10), rnd.Next(10)) + }; + await client.UpdateList("ci_tester", update); var updated = await client.GetData(Client.List.CI); var lst = updated.Lists.FirstOrDefault(l => l.Links.MachineURL == "ci_test"); var newMeta = lst!.DownloadMetadata!; Assert.Equal(meta.Hash, newMeta.Hash); Assert.Equal(meta.Size, newMeta.Size); - Assert.Equal(meta.Version, newMeta.Version); - Assert.Equal(lst.Version, newMeta.Version); + Assert.Equal(update.Version, lst.Version); Assert.Equal(meta.NumberOfArchives, newMeta.NumberOfArchives); Assert.Equal(meta.NumberOfInstalledFiles, newMeta.NumberOfInstalledFiles); diff --git a/Wabbajack/Settings.cs b/Wabbajack/Settings.cs index 7ec19c8f..8b11a0e0 100644 --- a/Wabbajack/Settings.cs +++ b/Wabbajack/Settings.cs @@ -180,7 +180,10 @@ namespace Wabbajack public string Website { get; set; } public string Readme { get; set; } public bool IsNSFW { get; set; } + + public string MachineUrl { get; set; } public AbsolutePath SplashScreen { get; set; } + public bool Publish { get; set; } } [JsonName("MO2CompilationSettings")] diff --git a/Wabbajack/View Models/Compilers/CompilerVM.cs b/Wabbajack/View Models/Compilers/CompilerVM.cs index 9ab5ac1d..4d6a18cb 100644 --- a/Wabbajack/View Models/Compilers/CompilerVM.cs +++ b/Wabbajack/View Models/Compilers/CompilerVM.cs @@ -16,6 +16,8 @@ using System.Windows.Media.Imaging; using Wabbajack.Common; using Wabbajack.Common.StatusFeed; using Wabbajack.Lib; +using Wabbajack.Lib.AuthorApi; +using Wabbajack.Lib.FileUploader; namespace Wabbajack { @@ -65,11 +67,11 @@ namespace Wabbajack private readonly ObservableAsPropertyHelper<(int CurrentCPUs, int DesiredCPUs)> _CurrentCpuCount; public (int CurrentCPUs, int DesiredCPUs) CurrentCpuCount => _CurrentCpuCount.Value; - + public CompilerVM(MainWindowVM mainWindowVM) : base(mainWindowVM) { MWVM = mainWindowVM; - + OutputLocation = new FilePickerVM() { ExistCheckOption = FilePickerVM.CheckOptions.IfPathNotEmpty, diff --git a/Wabbajack/View Models/Compilers/MO2CompilerVM.cs b/Wabbajack/View Models/Compilers/MO2CompilerVM.cs index 12e92bf7..73fe851c 100644 --- a/Wabbajack/View Models/Compilers/MO2CompilerVM.cs +++ b/Wabbajack/View Models/Compilers/MO2CompilerVM.cs @@ -2,6 +2,7 @@ using ReactiveUI; using ReactiveUI.Fody.Helpers; using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Reactive.Disposables; @@ -10,6 +11,9 @@ using System.Threading.Tasks; using DynamicData; using Wabbajack.Common; using Wabbajack.Lib; +using Wabbajack.Lib.AuthorApi; +using Wabbajack.Lib.FileUploader; +using Wabbajack.Lib.GitHub; using WebSocketSharp; namespace Wabbajack @@ -34,6 +38,7 @@ namespace Wabbajack public ACompiler ActiveCompilation { get; private set; } private readonly ObservableAsPropertyHelper _modlistSettings; + private readonly IObservable> _authorKeys; public ModlistSettingsEditorVM ModlistSettings => _modlistSettings.Value; [Reactive] @@ -201,11 +206,20 @@ namespace Wabbajack try { ACompiler compiler; + UpdateRequest request = null; + if (ModlistSettings.Publish) + { + request = new UpdateRequest + { + MachineUrl = ModlistSettings.MachineUrl.Trim(), + Version = ModlistSettings.Version, + }; + } if (ModListLocation.TargetPath.FileName == Consts.NativeSettingsJson) { var settings = ModListLocation.TargetPath.FromJson(); - compiler = new NativeCompiler(settings, Mo2Folder, DownloadLocation.TargetPath, outputFile) + compiler = new NativeCompiler(settings, Mo2Folder, DownloadLocation.TargetPath, outputFile, request) { ModListName = ModlistSettings.ModListName, ModListAuthor = ModlistSettings.AuthorText, @@ -219,11 +233,13 @@ namespace Wabbajack } else { + compiler = new MO2Compiler( sourcePath: Mo2Folder, downloadsPath: DownloadLocation.TargetPath, mo2Profile: MOProfile, - outputFile: outputFile) + outputFile: outputFile, + publishData: request) { ModListName = ModlistSettings.ModListName, ModListAuthor = ModlistSettings.AuthorText, diff --git a/Wabbajack/View Models/Compilers/ModlistSettingsEditorVM.cs b/Wabbajack/View Models/Compilers/ModlistSettingsEditorVM.cs index 5d7256cf..517e28e9 100644 --- a/Wabbajack/View Models/Compilers/ModlistSettingsEditorVM.cs +++ b/Wabbajack/View Models/Compilers/ModlistSettingsEditorVM.cs @@ -34,6 +34,9 @@ namespace Wabbajack [Reactive] public string Readme { get; set; } + [Reactive] public string MachineUrl { get; set; } = ""; + [Reactive] public bool Publish { get; set; } = false; + [Reactive] public string Website { get; set; } @@ -85,6 +88,8 @@ namespace Wabbajack Website = _settings.Website; VersionText = _settings.Version; IsNSFW = _settings.IsNSFW; + MachineUrl = _settings.MachineUrl; + Publish = _settings.Publish; } public void Save() @@ -97,6 +102,8 @@ namespace Wabbajack _settings.SplashScreen = ImagePath.TargetPath; _settings.Website = Website; _settings.IsNSFW = IsNSFW; + _settings.MachineUrl = MachineUrl; + _settings.Publish = Publish; } } } diff --git a/Wabbajack/Views/Compilers/CompilerView.xaml b/Wabbajack/Views/Compilers/CompilerView.xaml index 4cdd8a1a..7be5cb90 100644 --- a/Wabbajack/Views/Compilers/CompilerView.xaml +++ b/Wabbajack/Views/Compilers/CompilerView.xaml @@ -133,6 +133,15 @@ x:Name="NSFWSetting" Content="NSFW" ToolTip="Select this if your Modlist has adult themed content such as SexLab or other mods involving sexual acts. Nude body replacer do not fall under this category neither do revealing outfits or gore." /> + + + x.CurrentModlistSettings.IsNSFW, x => x.NSFWSetting.IsChecked) .DisposeWith(dispose); - + + this.BindStrict(ViewModel, x => x.CurrentModlistSettings.MachineUrl, x => x.MachineUrl.Text) + .DisposeWith(dispose); + + this.BindStrict(ViewModel, x => x.CurrentModlistSettings.Publish, x => x.PublishUpdate.IsChecked) + .DisposeWith(dispose); + // Bottom Compiler Settings this.WhenAny(x => x.ViewModel.StartedCompilation) .Select(started => started ? Visibility.Hidden : Visibility.Visible)