diff --git a/CHANGELOG.md b/CHANGELOG.md index b64b9422..419c95bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ ### Changelog -#### Version - 2.1.3.0 - 7/15/2020 +#### Version - 2.1.4.0 - ** +* List ingestion now supports compression and processes on a background threaded +* Support for validation of unlisted modlists + +#### Version - 2.1.3.0 - 7/16/2020 * Filters from the FilePicker are now being used * Wabbajack will continue working even if the build server is down * Fixed an issue where the main window does not appear after the splash screen diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index eae4b28e..497f1123 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -82,6 +82,7 @@ namespace Wabbajack.Common public static string ServerWhitelistURL = "https://raw.githubusercontent.com/wabbajack-tools/opt-out-lists/master/ServerWhitelist.yml"; public static string ModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/modlists.json"; + public static string UnlistedModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/unlisted_modlists.json"; public static string ModlistSummaryURL = "http://build.wabbajack.org/lists/status.json"; public static string UserAgent { @@ -126,6 +127,8 @@ namespace Wabbajack.Common public static AbsolutePath SettingsFile => LocalAppDataPath.Combine("settings.json"); public static RelativePath SettingsIni = (RelativePath)"settings.ini"; public static byte SettingsVersion => 2; + public static string CompressedBodyHeader = "x-compressed-body"; + public static AbsolutePath CefCacheLocation = @"CEF".RelativeTo(LocalAppDataPath); public static Extension SeqExtension = new Extension(".seq"); diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index faa39f37..8baeb55a 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -1,10 +1,13 @@ using System; using System.Collections.Generic; -using System.Linq; + using System.IO; + using System.IO.Compression; + using System.Linq; using System.Net; using System.Net.Http; using System.Text; using System.Threading.Tasks; + using K4os.Compression.LZ4.Internal; using Org.BouncyCastle.Crypto.Agreement.Srp; using Wabbajack.Common; using Wabbajack.Common.Exceptions; @@ -136,7 +139,15 @@ namespace Wabbajack.Lib var client = await GetClient(); if (BuildServerStatus.IsBuildServerDown) return; - await client.PostAsync($"{Consts.WabbajackBuildServerUri}list_definitions/ingest", new StringContent(modList.ToJson(), Encoding.UTF8, "application/json")); + var data = Encoding.UTF8.GetBytes(modList.ToJson()); + + await using var fs = new MemoryStream(); + await using var gzip = new GZipStream(fs, CompressionLevel.Optimal, true); + await gzip.WriteAsync(data); + await gzip.DisposeAsync(); + + client.Headers.Add((Consts.CompressedBodyHeader, "gzip")); + await client.PostAsync($"{Consts.WabbajackBuildServerUri}list_definitions/ingest", new ByteArrayContent(fs.ToArray())); } public static async Task GetExistingGameFiles(WorkQueue queue, Game game) diff --git a/Wabbajack.Lib/Http/Client.cs b/Wabbajack.Lib/Http/Client.cs index 0f8fcc55..7b5a3ab2 100644 --- a/Wabbajack.Lib/Http/Client.cs +++ b/Wabbajack.Lib/Http/Client.cs @@ -110,6 +110,7 @@ namespace Wabbajack.Lib.Http if (retries > Consts.MaxHTTPRetries) throw; retries++; + Utils.LogStraightToFile(ex.ToString()); Utils.Log($"Http Connect error to {msg.RequestUri} retry {retries}"); await Task.Delay(100 * retries); msg = CloneMessage(msg); diff --git a/Wabbajack.Lib/MO2Compiler.cs b/Wabbajack.Lib/MO2Compiler.cs index eeeea422..f8e8f8aa 100644 --- a/Wabbajack.Lib/MO2Compiler.cs +++ b/Wabbajack.Lib/MO2Compiler.cs @@ -4,15 +4,12 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Net; using System.Threading; using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Lib.CompilationSteps; using Wabbajack.Lib.Downloaders; -using Wabbajack.Lib.FileUploader; using Wabbajack.Lib.Validation; -using Wabbajack.VirtualFileSystem; using Path = Alphaleonis.Win32.Filesystem.Path; namespace Wabbajack.Lib diff --git a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs index f8516ec4..e0877ee8 100644 --- a/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs +++ b/Wabbajack.Lib/ModListRegistry/ModListMetadata.cs @@ -62,7 +62,7 @@ namespace Wabbajack.Lib.ModListRegistry public static async Task> LoadFromGithub() { - var client = new Wabbajack.Lib.Http.Client(); + var client = new Http.Client(); Utils.Log("Loading ModLists from GitHub"); var metadataResult = client.GetStringAsync(Consts.ModlistMetadataURL); var summaryResult = client.GetStringAsync(Consts.ModlistSummaryURL); @@ -83,6 +83,21 @@ namespace Wabbajack.Lib.ModListRegistry return metadata.OrderBy(m => (m.ValidationSummary?.HasFailures ?? false ? 1 : 0, m.Title)).ToList(); } + + public static async Task> LoadUnlistedFromGithub() + { + try + { + var client = new Http.Client(); + return (await client.GetStringAsync(Consts.ModlistMetadataURL)).FromJsonString>(); + } + catch (Exception ex) + { + Utils.LogStatus("Error loading unlisted modlists"); + return new List(); + } + + } public async ValueTask NeedsDownload(AbsolutePath modlistPath) { diff --git a/Wabbajack.Server.Test/ListIngestionTests.cs b/Wabbajack.Server.Test/ListIngestionTests.cs new file mode 100644 index 00000000..cf8396c9 --- /dev/null +++ b/Wabbajack.Server.Test/ListIngestionTests.cs @@ -0,0 +1,34 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Wabbajack.BuildServer.Test; +using Wabbajack.Common; +using Wabbajack.Lib; +using Xunit; +using Xunit.Abstractions; + +namespace Wabbajack.Server.Test +{ + public class ListIngestionTests : ABuildServerSystemTest + { + public ListIngestionTests(ITestOutputHelper output, SingletonAdaptor fixture) : base(output, fixture) + { + } + + [Fact] + public async Task CanIngestModLists() + { + await ClientAPI.SendModListDefinition(new ModList {Name = "sup"}); + await Task.Delay(500); + + Assert.Contains(AbsolutePath.EntryPoint.Combine("mod_list_definitions") + .EnumerateFiles(false), + f => DateTime.UtcNow - f.LastModifiedUtc < TimeSpan.FromSeconds(15)); + + var data = AbsolutePath.EntryPoint.Combine("mod_list_definitions").EnumerateFiles(false) + .OrderByDescending(f => f.LastModifiedUtc).First().FromJson(); + + Assert.Equal("sup", data.Name); + } + } +} diff --git a/Wabbajack.Server/Controllers/ListDefinitions.cs b/Wabbajack.Server/Controllers/ListDefinitions.cs index 5e1368c8..4cbb72fe 100644 --- a/Wabbajack.Server/Controllers/ListDefinitions.cs +++ b/Wabbajack.Server/Controllers/ListDefinitions.cs @@ -1,7 +1,9 @@ using System; using System.IO; +using System.IO.Compression; using System.Linq; using System.Reflection; +using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; @@ -35,16 +37,42 @@ namespace Wabbajack.BuildServer.Controllers public async Task PostIngest() { var user = Request.Headers[Consts.MetricsKeyHeader].First(); + var use_gzip = Request.Headers[Consts.CompressedBodyHeader].Any(); _logger.Log(LogLevel.Information, $"Ingesting Modlist Definition for {user}"); - var modlistBytes = await Request.Body.ReadAllAsync(); - var modlist = new MemoryStream(modlistBytes).FromJson(); - var file = AbsolutePath.EntryPoint.Combine("mod_list_definitions") - .Combine($"{user}_{DateTime.UtcNow.ToFileTimeUtc()}.json"); - file.Parent.CreateDirectory(); - await using var stream = await file.OpenWrite(); - modlist.ToJson(stream); - _logger.Log(LogLevel.Information, $"Done Ingesting Modlist Definition for {user}"); + var modlistBytes = await Request.Body.ReadAllAsync(); + + + + _logger.LogInformation("Spawning ingestion task"); + var tsk = Task.Run(async () => + { + try + { + if (use_gzip) + { + await using var os = new MemoryStream(); + await using var gZipStream = + new GZipStream(new MemoryStream(modlistBytes), CompressionMode.Decompress); + await gZipStream.CopyToAsync(os); + modlistBytes = os.ToArray(); + } + + var modlist = new MemoryStream(modlistBytes).FromJson(); + + var file = AbsolutePath.EntryPoint.Combine("mod_list_definitions") + .Combine($"{user}_{DateTime.UtcNow.ToFileTimeUtc()}.json"); + file.Parent.CreateDirectory(); + await using var stream = await file.Create(); + modlist.ToJson(stream); + _logger.Log(LogLevel.Information, $"Done Ingesting Modlist Definition for {user}"); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error ingesting uploaded modlist"); + } + }); + return Accepted(0); } diff --git a/Wabbajack.Server/Controllers/ListsStatus.cs b/Wabbajack.Server/Controllers/ListsStatus.cs index 54a1efb8..5a4e57f9 100644 --- a/Wabbajack.Server/Controllers/ListsStatus.cs +++ b/Wabbajack.Server/Controllers/ListsStatus.cs @@ -82,7 +82,7 @@ namespace Wabbajack.BuildServer.Controllers
    {{each $.failed }} {{if $.HasUrl}} -
  • {{$.Name}}
  • +
  • {{$.Name}}
  • {{else}}
  • {{$.Name}}
  • {{/if}} @@ -94,7 +94,7 @@ namespace Wabbajack.BuildServer.Controllers
      {{each $.updated }} {{if $.HasUrl}} -
    • {{$.Name}}
    • +
    • {{$.Name}}
    • {{else}}
    • {{$.Name}}
    • {{/if}} @@ -106,7 +106,7 @@ namespace Wabbajack.BuildServer.Controllers
        {{each $.updating }} {{if $.HasUrl}} -
      • {{$.Name}}
      • +
      • {{$.Name}}
      • {{else}}
      • {{$.Name}}
      • {{/if}} @@ -117,7 +117,7 @@ namespace Wabbajack.BuildServer.Controllers
          {{each $.passed }} {{if $.HasUrl}} -
        • {{$.Name}}
        • +
        • {{$.Name}}
        • {{else}}
        • {{$.Name}}
        • {{/if}} diff --git a/Wabbajack.Server/Services/ModListDownloader.cs b/Wabbajack.Server/Services/ModListDownloader.cs index d73da914..f18dc957 100644 --- a/Wabbajack.Server/Services/ModListDownloader.cs +++ b/Wabbajack.Server/Services/ModListDownloader.cs @@ -1,5 +1,6 @@ using System; using System.IO.Compression; +using System.Linq; using System.Runtime.CompilerServices; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; @@ -60,7 +61,9 @@ namespace Wabbajack.Server.Services public async Task CheckForNewLists() { int downloaded = 0; - var lists = await ModlistMetadata.LoadFromGithub(); + var lists = (await ModlistMetadata.LoadFromGithub()) + .Concat(await ModlistMetadata.LoadUnlistedFromGithub()).ToList(); + foreach (var list in lists) { try