diff --git a/Compression.BSA/BSA/Builder/BSABuilder.cs b/Compression.BSA/BSA/Builder/BSABuilder.cs index d241df3e..fd16f8fd 100644 --- a/Compression.BSA/BSA/Builder/BSABuilder.cs +++ b/Compression.BSA/BSA/Builder/BSABuilder.cs @@ -272,7 +272,7 @@ namespace Compression.BSA case VersionType.SSE: { var r = new MemoryStream(); - await using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L08_HC}, true)) + await using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L12_MAX}, true)) { await _srcData.CopyToWithStatusAsync(_srcData.Length, w, $"Compressing {_path}"); } diff --git a/Wabbajack.CLI/OptionsDefinition.cs b/Wabbajack.CLI/OptionsDefinition.cs index ec4695e9..c398b133 100644 --- a/Wabbajack.CLI/OptionsDefinition.cs +++ b/Wabbajack.CLI/OptionsDefinition.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using Markdig.Syntax.Inlines; using Wabbajack.CLI.Verbs; @@ -6,42 +7,8 @@ namespace Wabbajack.CLI { public class OptionsDefinition { - public static readonly Type[] AllOptions = { - typeof(OptionsDefinition), - typeof(Encrypt), - typeof(Decrypt), - typeof(Validate), - typeof(DownloadUrl), - typeof(UpdateModlists), - typeof(UpdateNexusCache), - typeof(ChangeDownload), - typeof(ServerLog), - typeof(MyFiles), - typeof(DeleteFile), - typeof(Changelog), - typeof(FindSimilar), - typeof(BSADump), - typeof(MigrateGameFolderFiles), - typeof(HashFile), - typeof(InlinedFileReport), - typeof(ExtractBSA), - typeof(PurgeNexusCache), - typeof(ForceHealing), - typeof(HashVariants), - typeof(ParseMeta), - typeof(NoPatch), - typeof(NexusPermissions), - typeof(ExportServerGameFiles), - typeof(HashGamefiles), - typeof(Backup), - typeof(Restore), - typeof(PurgeArchive), - typeof(AllKnownDownloadStates), - typeof(VerifyAllDownloads), - typeof(HashBenchmark), - typeof(StressTestURL), - typeof(MirrorFolder), - typeof(DownloadFromMeta) - }; + public static Type[] AllOptions => + typeof(OptionsDefinition).Assembly.GetTypes().Where(t => t.IsAssignableTo(typeof(AVerb))).ToArray(); + } } diff --git a/Wabbajack.CLI/Properties/launchSettings.json b/Wabbajack.CLI/Properties/launchSettings.json index fa6e610c..37532044 100644 --- a/Wabbajack.CLI/Properties/launchSettings.json +++ b/Wabbajack.CLI/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "Wabbajack.CLI": { "commandName": "Project", - "commandLineArgs": "download-from-meta -m \"c:\\tmp\\test (2).meta\" -o c:\\tmp\\out.7z" + "commandLineArgs": "upload-mod-to-cdn -m \"Generated Files\" -f \"Sanitarium Generated Files\" -r \"C:\\Modding\\Sanitarium\"" } } } \ No newline at end of file diff --git a/Wabbajack.CLI/Verbs/UploadModToCDN.cs b/Wabbajack.CLI/Verbs/UploadModToCDN.cs new file mode 100644 index 00000000..ab441096 --- /dev/null +++ b/Wabbajack.CLI/Verbs/UploadModToCDN.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using CommandLine; +using Wabbajack.Common; +using Wabbajack.Lib.AuthorApi; +using Wabbajack.VirtualFileSystem; + +namespace Wabbajack.CLI.Verbs +{ + [Verb("upload-mod-to-cdn", HelpText = "Compresses and uploads one or more folders in MO2 onto the CDN and updates the MO2's downloads folder to match.")] + public class UploadModToCDN : AVerb + { + [Option('m', "mods", Required = true, HelpText = @"ModOrganizer2 mod names to export")] + public IEnumerable Mods { get; set; } = Array.Empty(); + + [Option('f', "filename", Required = true, HelpText = @"Human friendly filename for the created download")] + public string Filename { get; set; } = ""; + + [Option('r', "root", Required = true, HelpText = @"The root MO2 folder")] + public string _MO2Path { get; set; } = ""; + + public AbsolutePath MO2Path => (AbsolutePath)_MO2Path; + protected override async Task Run() + { + var ini = MO2Path.Combine(Consts.ModOrganizer2Ini).LoadIniFile(); + var downloadsFolder = (AbsolutePath)(ini.Settings.download_directory ?? MO2Path.Combine("downloads")); + var fileFixed = downloadsFolder.Combine(Filename).ReplaceExtension(new Extension(".7z")); + + var folders = Mods.Select(m => MO2Path.Combine("mods", m)).ToArray(); + + Utils.Log("Compressing files"); + await FileExtractor2.CompressFiles(fileFixed, folders, Utils.Log); + Utils.Log($"Final Size: {fileFixed.Size.ToFileSizeString()}"); + + Utils.Log("Uploading to CDN"); + var queue = new WorkQueue(); + var url = await (await Client.Create()).UploadFile(queue, fileFixed,(s, p) => Utils.Log($"{p} - {s}")); + Utils.Log("Updating Meta"); + await fileFixed.WithExtension(new Extension(Consts.MetaFileExtension)).WriteAllLinesAsync( + "[General]", + "installed=true", + $"directURL={url}"); + + return ExitCode.Ok; + } + } +} diff --git a/Wabbajack.Common/Json.cs b/Wabbajack.Common/Json.cs index 57320008..0282cb3e 100644 --- a/Wabbajack.Common/Json.cs +++ b/Wabbajack.Common/Json.cs @@ -65,27 +65,32 @@ namespace Wabbajack.Common using var tw = new StreamWriter(stream, new UTF8Encoding(false), bufferSize: 1024, leaveOpen: true); using var writer = new JsonTextWriter(tw); - JsonSerializerSettings settings = (useGenericSettings, prettyPrint) switch + JsonSerializerSettings settings = SettingsForOptions(useGenericSettings, prettyPrint); + + var ser = JsonSerializer.Create(settings); + ser.Serialize(writer, obj); + } + + private static JsonSerializerSettings SettingsForOptions(bool useGenericSettings, bool prettyPrint) + { + return (useGenericSettings, prettyPrint) switch { (true, true) => GenericJsonSettings, (false, true) => JsonSettingsPretty, (false, false) => JsonSettings, (true, false) => GenericJsonSettings }; - - var ser = JsonSerializer.Create(settings); - ser.Serialize(writer, obj); } - + public static async ValueTask ToJsonAsync(this T obj, AbsolutePath path, bool useGenericSettings = false, bool prettyPrint = false) { await using var fs = await path.Create(); obj.ToJson(fs, useGenericSettings, prettyPrint: prettyPrint); } - public static string ToJson(this T obj, bool useGenericSettings = false) + public static string ToJson(this T obj, bool useGenericSettings = false, bool prettyPrint = false) { - return JsonConvert.SerializeObject(obj, useGenericSettings ? GenericJsonSettings : JsonSettings); + return JsonConvert.SerializeObject(obj, SettingsForOptions(useGenericSettings, prettyPrint)); } public static T FromJson(this AbsolutePath filename) diff --git a/Wabbajack.Launcher/MainWindowVM.cs b/Wabbajack.Launcher/MainWindowVM.cs index e8fd1eb8..e6ab126f 100644 --- a/Wabbajack.Launcher/MainWindowVM.cs +++ b/Wabbajack.Launcher/MainWindowVM.cs @@ -154,9 +154,12 @@ namespace Wabbajack.Launcher .OrderByDescending(v => Version.TryParse(Path.GetFileName(v), out var ver) ? ver : new Version(0, 0, 0, 0)) .FirstOrDefault(); + + var filename = Path.Combine(wjFolder, "Wabbajack.exe"); + await CreateBatchFile(filename); var info = new ProcessStartInfo { - FileName = Path.Combine(wjFolder, "Wabbajack.exe"), + FileName = filename, Arguments = string.Join(" ", Environment.GetCommandLineArgs().Skip(1).Select(s => s.Contains(' ') ? '\"' + s + '\"' : s)), WorkingDirectory = wjFolder, }; @@ -181,6 +184,15 @@ namespace Wabbajack.Launcher } } + private async Task CreateBatchFile(string filename) + { + filename = Path.Combine(Path.GetDirectoryName(filename), "wabbajack-cli.exe"); + var data = $"\"{filename}\" %*"; + var file = Path.Combine(Directory.GetCurrentDirectory(), "wabbajack-cli.bat"); + if (File.Exists(file) && await File.ReadAllTextAsync(file) == data) return; + await File.WriteAllTextAsync(file, data); + } + private void UpdateProgress(object sender, DownloadProgressChangedEventArgs e) { Status = $"Downloading {_version.Tag} ({e.ProgressPercentage}%)..."; diff --git a/Wabbajack.Lib/ACompiler.cs b/Wabbajack.Lib/ACompiler.cs index 92e64a7b..54e06bb0 100644 --- a/Wabbajack.Lib/ACompiler.cs +++ b/Wabbajack.Lib/ACompiler.cs @@ -14,6 +14,8 @@ using Org.BouncyCastle.Asn1.Cms; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.FileUploader; +using Wabbajack.Lib.GitHub; using Wabbajack.Lib.ModListRegistry; using Wabbajack.VirtualFileSystem; @@ -34,9 +36,10 @@ namespace Wabbajack.Lib public string? ModListName, ModListAuthor, ModListDescription, ModListWebsite, ModlistReadme; public Version? ModlistVersion; protected Version? WabbajackVersion; + public UpdateRequest? PublishData; public ACompiler(int steps, string modlistName, AbsolutePath sourcePath, AbsolutePath downloadsPath, - AbsolutePath outputModListName) + AbsolutePath outputModListName, UpdateRequest? publishData) : base(steps) { SourcePath = sourcePath; @@ -48,8 +51,11 @@ namespace Wabbajack.Lib Settings = new CompilerSettings(); ModListOutputFolder = AbsolutePath.EntryPoint.Combine("output_folder", Guid.NewGuid().ToString()); CompilingGame = new GameMetaData(); + PublishData = publishData; } + + /// /// Set to true to include game files during compilation, only ever disabled /// in testing (to speed up tests) @@ -171,6 +177,35 @@ namespace Wabbajack.Lib return true; } + public async Task PreflightChecks() + { + Utils.Log($"Running preflight checks"); + if (PublishData == null) return; + + var ourLists = await (await AuthorApi.Client.Create()).GetMyModlists(); + if (ourLists.All(l => l.MachineURL != PublishData.MachineUrl)) + { + Utils.ErrorThrow(new CriticalFailureIntervention( + $"Cannot publish to {PublishData.MachineUrl}, you are not listed as a maintainer", + "Cannot publish")); + } + } + + public async Task PublishModlist() + { + if (PublishData == null) return; + var api = await AuthorApi.Client.Create(); + Utils.Log($"Uploading modlist {PublishData!.MachineUrl}"); + + var metadata = ((AbsolutePath)(ModListOutputFile + ".meta.json")).FromJson(); + var uri = await api.UploadFile(Queue, ModListOutputFile, (s, percent) => Utils.Status(s, percent)); + PublishData.DownloadUrl = uri; + PublishData.DownloadMetadata = metadata; + + Utils.Log($"Publishing modlist {PublishData!.MachineUrl}"); + await api.UpdateModListInformation(PublishData); + } + protected async Task IndexGameFileHashes() { if (UseGamePaths) @@ -307,7 +342,7 @@ namespace Wabbajack.Lib await ModListOutputFolder.Combine("sig") .WriteAllBytesAsync(((await ModListOutputFolder.Combine("modlist").FileHashAsync()) ?? Hash.Empty).ToArray()); - await ClientAPI.SendModListDefinition(ModList); + //await ClientAPI.SendModListDefinition(ModList); await ModListOutputFile.DeleteAsync(); diff --git a/Wabbajack.Lib/AuthorApi/Client.cs b/Wabbajack.Lib/AuthorApi/Client.cs index 01692ecf..ed081c38 100644 --- a/Wabbajack.Lib/AuthorApi/Client.cs +++ b/Wabbajack.Lib/AuthorApi/Client.cs @@ -7,6 +7,8 @@ using System.Linq; using System.Net.Http; using System.Threading.Tasks; using Wabbajack.Common; +using Wabbajack.Lib.GitHub; +using Wabbajack.Lib.ModListRegistry; namespace Wabbajack.Lib.AuthorApi { @@ -76,6 +78,34 @@ namespace Wabbajack.Lib.AuthorApi return definition; } + public async Task UpdateModListInformation(UpdateRequest metadata) + { + await CircuitBreaker.WithAutoRetryAllAsync(async () => + { + Utils.Log($"Updating modlist information for {metadata.MachineUrl} - {metadata.Version}"); + using var result = await _client.PostAsync($"{Consts.WabbajackBuildServerUri}author_controls/lists/download_metadata", + new StringContent(metadata.ToJson())); + }); + } + + public async Task> GetMyModlists() + { + var myLists = await _client.GetJsonAsync($"{Consts.WabbajackBuildServerUri}author_controls/lists"); + List<(string MachineURL, Version Version)> lists = new(); + var client = await GitHub.Client.Get(); + foreach (var file in Enum.GetValues()) + { + foreach (var lst in (await client.GetData(file)).Lists) + { + if (myLists.Contains(lst.Links.MachineURL)) + { + lists.Add((lst.Links.MachineURL, lst.Version ?? new Version())); + } + } + } + return lists; + } + public async Task UploadFile(WorkQueue queue, AbsolutePath path, Action progressFn) { var definition = await GenerateFileDefinition(queue, path, progressFn); diff --git a/Wabbajack.Lib/GitHub/Client.cs b/Wabbajack.Lib/GitHub/Client.cs new file mode 100644 index 00000000..3a4c1798 --- /dev/null +++ b/Wabbajack.Lib/GitHub/Client.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Octokit; +using Wabbajack.Common; +using Wabbajack.Lib.ModListRegistry; + +namespace Wabbajack.Lib.GitHub +{ + public class Client + { + private readonly GitHubClient _client; + + private Client(GitHubClient client) + { + _client = client; + } + public static async Task Get() + { + if (!Utils.HaveEncryptedJson("github-key")) + return new Client(new GitHubClient(ProductHeaderValue.Parse("wabbajack"))); + + var key = Encoding.UTF8.GetString(await Utils.FromEncryptedData("github-key")); + var creds = new Credentials(key); + return new Client(new GitHubClient(ProductHeaderValue.Parse("wabbajack_build_server")){Credentials = creds}); + } + + public enum List + { + CI, + Unlisted, + Utility, + Published + } + + public Dictionary PathNames = new() + { + {List.CI, "ci_lists.json"}, + {List.Unlisted, "unlisted_modlists.json"}, + {List.Utility, "utility_modlists.json"}, + {List.Published, "modlists.json"} + }; + + public async Task<(string Hash, IReadOnlyList Lists)> GetData(List lst) + { + var content = + (await _client.Repository.Content.GetAllContents("wabbajack-tools", "mod-lists", PathNames[lst])).First(); + return (content.Sha, content.Content.FromJsonString()); + } + + private async Task WriteData(List file, string machinenUrl, string dataHash, IReadOnlyList dataLists) + { + var updateRequest = new UpdateFileRequest($"New release of {machinenUrl}", dataLists.ToJson(prettyPrint:true), dataHash); + await _client.Repository.Content.UpdateFile("wabbajack-tools", "mod-lists", PathNames[file], updateRequest); + } + + public async Task UpdateList(string maintainer, UpdateRequest newData) + { + foreach (var file in Enum.GetValues()) + { + var data = await GetData(file); + var found = data.Lists.FirstOrDefault(l => l.Links.MachineURL == newData.MachineUrl && l.Maintainers.Contains(maintainer)); + if (found == null) continue; + + found.DownloadMetadata = newData.DownloadMetadata; + found.Version = newData.Version; + found.Links.Download = newData.DownloadUrl.ToString(); + + await WriteData(file, newData.MachineUrl, data.Hash, data.Lists); + return; + } + + throw new Exception("List not found or user not authorized"); + } + + + } +} diff --git a/Wabbajack.Lib/GitHub/UpdateRequest.cs b/Wabbajack.Lib/GitHub/UpdateRequest.cs new file mode 100644 index 00000000..214eb2a3 --- /dev/null +++ b/Wabbajack.Lib/GitHub/UpdateRequest.cs @@ -0,0 +1,15 @@ +using System; +using Wabbajack.Common.Serialization.Json; +using Wabbajack.Lib.ModListRegistry; + +namespace Wabbajack.Lib.GitHub +{ + [JsonName("UpdateRequest")] + public class UpdateRequest + { + public string MachineUrl { get; set; } = ""; + public Version? Version { get; set; } = new(); + public Uri DownloadUrl { get; set; } = new("https://www.wabbajack.org"); + public DownloadMetadata DownloadMetadata { get; set; } = new(); + } +} diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index 2ec804fd..da6702df 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -8,6 +8,7 @@ using Alphaleonis.Win32.Filesystem; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.Downloaders; +using Wabbajack.Lib.GitHub; using Wabbajack.Lib.Validation; using Wabbajack.VirtualFileSystem; @@ -15,8 +16,8 @@ namespace Wabbajack.Lib { public class MO2Compiler : ACompiler { - public MO2Compiler(AbsolutePath sourcePath, AbsolutePath downloadsPath, string mo2Profile, AbsolutePath outputFile) - : base(21, mo2Profile, sourcePath, downloadsPath, outputFile) + public MO2Compiler(AbsolutePath sourcePath, AbsolutePath downloadsPath, string mo2Profile, AbsolutePath outputFile, UpdateRequest? publishData = null) + : base(23, mo2Profile, sourcePath, downloadsPath, outputFile, publishData) { MO2Profile = mo2Profile; MO2Ini = SourcePath.Combine("ModOrganizer.ini").LoadIniFile(); @@ -25,6 +26,7 @@ namespace Wabbajack.Lib GamePath = CompilingGame.GameLocation(); } + public UpdateRequest? PublishData { get; set; } public AbsolutePath MO2ModsFolder => SourcePath.Combine(Consts.MO2ModFolderName); public string MO2Profile { get; } @@ -58,6 +60,10 @@ namespace Wabbajack.Lib FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam; UpdateTracker.Reset(); + + UpdateTracker.NextStep("Running Preflight Checks"); + await PreflightChecks(); + UpdateTracker.NextStep("Gathering information"); Utils.Log("Loading compiler Settings"); @@ -362,6 +368,9 @@ namespace Wabbajack.Lib UpdateTracker.NextStep("Exporting Modlist"); await ExportModList(); + + UpdateTracker.NextStep("Publishing Modlist"); + await PublishModlist(); ResetMembers(); diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index 96259c05..a5f72a7a 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -21,6 +21,9 @@ namespace Wabbajack.Lib.ModListRegistry [JsonProperty("author")] public string Author { get; set; } = string.Empty; + [JsonProperty("maintainers")] public string[] + Maintainers { get; set; } = Array.Empty(); + [JsonProperty("game")] public Game Game { get; set; } @@ -135,7 +138,6 @@ namespace Wabbajack.Lib.ModListRegistry { public Hash Hash { get; set; } public long Size { get; set; } - public long NumberOfArchives { get; set; } public long SizeOfArchives { get; set; } public long NumberOfInstalledFiles { get; set; } diff --git a/Wabbajack.Lib/NativeCompiler.cs b/Wabbajack.Lib/NativeCompiler.cs index 6abc0bee..81553f09 100644 --- a/Wabbajack.Lib/NativeCompiler.cs +++ b/Wabbajack.Lib/NativeCompiler.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; +using Wabbajack.Lib.GitHub; using Wabbajack.Lib.Validation; using Wabbajack.VirtualFileSystem; @@ -12,8 +13,8 @@ namespace Wabbajack.Lib { public class NativeCompiler : ACompiler { - public NativeCompiler(NativeCompilerSettings settings, AbsolutePath sourcePath, AbsolutePath downloadsPath, AbsolutePath outputModListPath) - : base(3, settings.ModListName, sourcePath, downloadsPath, outputModListPath) + public NativeCompiler(NativeCompilerSettings settings, AbsolutePath sourcePath, AbsolutePath downloadsPath, AbsolutePath outputModListPath, UpdateRequest? publishData = null) + : base(5, settings.ModListName, sourcePath, downloadsPath, outputModListPath, publishData) { CompilingGame = settings.CompilingGame.MetaData(); GamePath = CompilingGame.GameLocation(); @@ -34,6 +35,10 @@ namespace Wabbajack.Lib FileExtractor2.FavorPerfOverRAM = FavorPerfOverRam; UpdateTracker.Reset(); + + UpdateTracker.NextStep("Running Preflight Checks"); + await PreflightChecks(); + UpdateTracker.NextStep("Gathering information"); Utils.Log($"Compiling Game: {CompilingGame.Game}"); @@ -243,6 +248,9 @@ namespace Wabbajack.Lib UpdateTracker.NextStep("Exporting Modlist"); await ExportModList(); + + UpdateTracker.NextStep("Publishing Modlist"); + await PublishModlist(); ResetMembers(); diff --git a/Wabbajack.Lib/Wabbajack.Lib.csproj b/Wabbajack.Lib/Wabbajack.Lib.csproj index 53ae8ee3..ea8d2e71 100644 --- a/Wabbajack.Lib/Wabbajack.Lib.csproj +++ b/Wabbajack.Lib/Wabbajack.Lib.csproj @@ -37,6 +37,9 @@ 2.1.1 + + 0.50.0 + 14.1.1 diff --git a/Wabbajack.Server/Controllers/AuthorControls.cs b/Wabbajack.Server/Controllers/AuthorControls.cs index efdaa536..e1822a08 100644 --- a/Wabbajack.Server/Controllers/AuthorControls.cs +++ b/Wabbajack.Server/Controllers/AuthorControls.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Net; using System.Reflection; @@ -12,7 +13,10 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Nettle; using Wabbajack.Common; +using Wabbajack.Lib.GitHub; +using Wabbajack.Lib.ModListRegistry; using Wabbajack.Server.DataLayer; +using Wabbajack.Server.Services; namespace Wabbajack.BuildServer.Controllers { @@ -22,11 +26,13 @@ namespace Wabbajack.BuildServer.Controllers { private ILogger _logger; private SqlService _sql; + private readonly QuickSync _quickSync; - public AuthorControls(ILogger logger, SqlService sql) + public AuthorControls(ILogger logger, SqlService sql, QuickSync quickSync) { _logger = logger; _sql = sql; + _quickSync = quickSync; } [Route("login/{authorKey}")] @@ -36,6 +42,42 @@ namespace Wabbajack.BuildServer.Controllers Response.Cookies.Append(ApiKeyAuthenticationHandler.ApiKeyHeaderName, authorKey); return Redirect($"{Consts.WabbajackBuildServerUri}author_controls/home"); } + + [Route("lists")] + [HttpGet] + public async Task AuthorLists() + { + var user = User.FindFirstValue(ClaimTypes.Name); + List lists = new(); + var client = await Client.Get(); + foreach (var file in Enum.GetValues()) + { + lists.AddRange((await client.GetData(file)).Lists.Where(l => l.Maintainers.Contains(user)) + .Select(lst => lst.Links.MachineURL)); + } + + return Ok(lists); + } + + [Route("lists/download_metadata")] + [HttpPost] + public async Task PostDownloadMetadata() + { + var user = User.FindFirstValue(ClaimTypes.Name); + var data = (await Request.Body.ReadAllTextAsync()).FromJsonString(); + var client = await Client.Get(); + try + { + await client.UpdateList(user, data); + await _quickSync.Notify(); + } + catch (Exception ex) + { + _logger.LogError(ex, "During posting of download_metadata"); + return BadRequest(ex); + } + return Ok(data); + } private static async Task HomePageTemplate(object o) { diff --git a/Wabbajack.Server/Controllers/AuthoredFiles.cs b/Wabbajack.Server/Controllers/AuthoredFiles.cs index e1faac95..8260a003 100644 --- a/Wabbajack.Server/Controllers/AuthoredFiles.cs +++ b/Wabbajack.Server/Controllers/AuthoredFiles.cs @@ -141,6 +141,7 @@ namespace Wabbajack.BuildServer.Controllers var definition = await _sql.GetCDNFileDefinition(serverAssignedUniqueId); if (definition.Author != user) return Forbid("File Id does not match authorized user"); + await _discord.Send(Channel.Ham, new DiscordMessage() {Content = $"{user} is deleting {definition.MungedName}, {definition.Size.ToFileSizeString()} to be freed"}); _logger.Log(LogLevel.Information, $"Finalizing file upload {definition.OriginalFileName}"); await DeleteFolderOrSilentlyFail($"{definition.MungedName}"); diff --git a/Wabbajack.Test/GitHubTests.cs b/Wabbajack.Test/GitHubTests.cs new file mode 100644 index 00000000..d487fecc --- /dev/null +++ b/Wabbajack.Test/GitHubTests.cs @@ -0,0 +1,50 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using System.Windows.Documents; +using Octokit; +using Wabbajack.Common; +using Wabbajack.Lib.GitHub; +using Wabbajack.Lib.ModListRegistry; +using Xunit; + +namespace Wabbajack.Test +{ + public class GitHubTests + { + [Fact] + public async Task CanLogIntoGithub() + { + var client = await Wabbajack.Lib.GitHub.Client.Get(); + var rnd = new Random(); + var meta = new DownloadMetadata + { + Hash = Hash.FromLong(rnd.Next()), + NumberOfArchives = rnd.Next(100), + NumberOfInstalledFiles = rnd.Next(1000), + SizeOfInstalledFiles = rnd.Next(1000000), + Size = rnd.Next(10000), + + }; + var update = new UpdateRequest + { + DownloadMetadata = meta, + DownloadUrl = new Uri($"https://www.google.com/{rnd.Next()}"), + MachineUrl = "ci_test", + Version = new Version(1, rnd.Next(10), rnd.Next(10), rnd.Next(10)) + }; + await client.UpdateList("ci_tester", update); + + var updated = await client.GetData(Client.List.CI); + var lst = updated.Lists.FirstOrDefault(l => l.Links.MachineURL == "ci_test"); + var newMeta = lst!.DownloadMetadata!; + Assert.Equal(meta.Hash, newMeta.Hash); + Assert.Equal(meta.Size, newMeta.Size); + Assert.Equal(update.Version, lst.Version); + + Assert.Equal(meta.NumberOfArchives, newMeta.NumberOfArchives); + Assert.Equal(meta.NumberOfInstalledFiles, newMeta.NumberOfInstalledFiles); + Assert.Equal(meta.SizeOfInstalledFiles, newMeta.SizeOfInstalledFiles); + } + } +} diff --git a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs index 9762660b..7bdcff92 100644 --- a/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs +++ b/Wabbajack.VirtualFileSystem/FileExtractor2/FileExtractor.cs @@ -51,6 +51,7 @@ namespace Wabbajack.VirtualFileSystem public static bool FavorPerfOverRAM { get; set; } + public static async Task> GatheringExtract(WorkQueue queue, IStreamFactory sFn, Predicate shouldExtract, Func> mapfn, AbsolutePath? tempFolder = null, @@ -280,5 +281,30 @@ namespace Wabbajack.VirtualFileSystem return 0; }); } + + + /// + /// Compresses all the files with 7zip + /// + /// + /// + public static async Task CompressFiles(AbsolutePath to, AbsolutePath[] filesAndFolders, Action status) + { + var args = new List() {"a", to, "-mx9"}; + args.AddRange(filesAndFolders.Cast()); + var process = new ProcessHelper + { + Path = @"Extractors\7z.exe".RelativeTo(AbsolutePath.EntryPoint), + Arguments = args, + ThrowOnNonZeroExitCode = true + }; + + using var _ = process.Output.Where(o => o.Type == ProcessHelper.StreamType.Output) + .Subscribe(l => status(l.Line)); + + await process.Start(); + return; + + } } } diff --git a/Wabbajack/Settings.cs b/Wabbajack/Settings.cs index 7ec19c8f..8b11a0e0 100644 --- a/Wabbajack/Settings.cs +++ b/Wabbajack/Settings.cs @@ -180,7 +180,10 @@ namespace Wabbajack public string Website { get; set; } public string Readme { get; set; } public bool IsNSFW { get; set; } + + public string MachineUrl { get; set; } public AbsolutePath SplashScreen { get; set; } + public bool Publish { get; set; } } [JsonName("MO2CompilationSettings")] diff --git a/Wabbajack/View Models/Compilers/CompilerVM.cs b/Wabbajack/View Models/Compilers/CompilerVM.cs index 9ab5ac1d..4d6a18cb 100644 --- a/Wabbajack/View Models/Compilers/CompilerVM.cs +++ b/Wabbajack/View Models/Compilers/CompilerVM.cs @@ -16,6 +16,8 @@ using System.Windows.Media.Imaging; using Wabbajack.Common; using Wabbajack.Common.StatusFeed; using Wabbajack.Lib; +using Wabbajack.Lib.AuthorApi; +using Wabbajack.Lib.FileUploader; namespace Wabbajack { @@ -65,11 +67,11 @@ namespace Wabbajack private readonly ObservableAsPropertyHelper<(int CurrentCPUs, int DesiredCPUs)> _CurrentCpuCount; public (int CurrentCPUs, int DesiredCPUs) CurrentCpuCount => _CurrentCpuCount.Value; - + public CompilerVM(MainWindowVM mainWindowVM) : base(mainWindowVM) { MWVM = mainWindowVM; - + OutputLocation = new FilePickerVM() { ExistCheckOption = FilePickerVM.CheckOptions.IfPathNotEmpty, diff --git a/Wabbajack/View Models/Compilers/MO2CompilerVM.cs b/Wabbajack/View Models/Compilers/MO2CompilerVM.cs index 12e92bf7..73fe851c 100644 --- a/Wabbajack/View Models/Compilers/MO2CompilerVM.cs +++ b/Wabbajack/View Models/Compilers/MO2CompilerVM.cs @@ -2,6 +2,7 @@ using ReactiveUI; using ReactiveUI.Fody.Helpers; using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Reactive.Disposables; @@ -10,6 +11,9 @@ using System.Threading.Tasks; using DynamicData; using Wabbajack.Common; using Wabbajack.Lib; +using Wabbajack.Lib.AuthorApi; +using Wabbajack.Lib.FileUploader; +using Wabbajack.Lib.GitHub; using WebSocketSharp; namespace Wabbajack @@ -34,6 +38,7 @@ namespace Wabbajack public ACompiler ActiveCompilation { get; private set; } private readonly ObservableAsPropertyHelper _modlistSettings; + private readonly IObservable> _authorKeys; public ModlistSettingsEditorVM ModlistSettings => _modlistSettings.Value; [Reactive] @@ -201,11 +206,20 @@ namespace Wabbajack try { ACompiler compiler; + UpdateRequest request = null; + if (ModlistSettings.Publish) + { + request = new UpdateRequest + { + MachineUrl = ModlistSettings.MachineUrl.Trim(), + Version = ModlistSettings.Version, + }; + } if (ModListLocation.TargetPath.FileName == Consts.NativeSettingsJson) { var settings = ModListLocation.TargetPath.FromJson(); - compiler = new NativeCompiler(settings, Mo2Folder, DownloadLocation.TargetPath, outputFile) + compiler = new NativeCompiler(settings, Mo2Folder, DownloadLocation.TargetPath, outputFile, request) { ModListName = ModlistSettings.ModListName, ModListAuthor = ModlistSettings.AuthorText, @@ -219,11 +233,13 @@ namespace Wabbajack } else { + compiler = new MO2Compiler( sourcePath: Mo2Folder, downloadsPath: DownloadLocation.TargetPath, mo2Profile: MOProfile, - outputFile: outputFile) + outputFile: outputFile, + publishData: request) { ModListName = ModlistSettings.ModListName, ModListAuthor = ModlistSettings.AuthorText, diff --git a/Wabbajack/View Models/Compilers/ModlistSettingsEditorVM.cs b/Wabbajack/View Models/Compilers/ModlistSettingsEditorVM.cs index 5d7256cf..517e28e9 100644 --- a/Wabbajack/View Models/Compilers/ModlistSettingsEditorVM.cs +++ b/Wabbajack/View Models/Compilers/ModlistSettingsEditorVM.cs @@ -34,6 +34,9 @@ namespace Wabbajack [Reactive] public string Readme { get; set; } + [Reactive] public string MachineUrl { get; set; } = ""; + [Reactive] public bool Publish { get; set; } = false; + [Reactive] public string Website { get; set; } @@ -85,6 +88,8 @@ namespace Wabbajack Website = _settings.Website; VersionText = _settings.Version; IsNSFW = _settings.IsNSFW; + MachineUrl = _settings.MachineUrl; + Publish = _settings.Publish; } public void Save() @@ -97,6 +102,8 @@ namespace Wabbajack _settings.SplashScreen = ImagePath.TargetPath; _settings.Website = Website; _settings.IsNSFW = IsNSFW; + _settings.MachineUrl = MachineUrl; + _settings.Publish = Publish; } } } diff --git a/Wabbajack/Views/Compilers/CompilerView.xaml b/Wabbajack/Views/Compilers/CompilerView.xaml index 4cdd8a1a..7be5cb90 100644 --- a/Wabbajack/Views/Compilers/CompilerView.xaml +++ b/Wabbajack/Views/Compilers/CompilerView.xaml @@ -133,6 +133,15 @@ x:Name="NSFWSetting" Content="NSFW" ToolTip="Select this if your Modlist has adult themed content such as SexLab or other mods involving sexual acts. Nude body replacer do not fall under this category neither do revealing outfits or gore." /> + + + x.CurrentModlistSettings.IsNSFW, x => x.NSFWSetting.IsChecked) .DisposeWith(dispose); - + + this.BindStrict(ViewModel, x => x.CurrentModlistSettings.MachineUrl, x => x.MachineUrl.Text) + .DisposeWith(dispose); + + this.BindStrict(ViewModel, x => x.CurrentModlistSettings.Publish, x => x.PublishUpdate.IsChecked) + .DisposeWith(dispose); + // Bottom Compiler Settings this.WhenAny(x => x.ViewModel.StartedCompilation) .Select(started => started ? Visibility.Hidden : Visibility.Visible)