Merge pull request #964 from wabbajack-tools/ingestion-compression

Fixes for broken archives names on server links

CHANGELOG.md

@@ -1,6 +1,10 @@
 ### Changelog
 
-#### Version - 2.1.3.0 - 7/15/2020
+#### Version - 2.1.4.0 - **
+* List ingestion now supports compression and processes on a background threaded
+* Support for validation of unlisted modlists
+
+#### Version - 2.1.3.0 - 7/16/2020
 * Filters from the FilePicker are now being used
 * Wabbajack will continue working even if the build server is down
 * Fixed an issue where the main window does not appear after the splash screen

Wabbajack.Common/Consts.cs

@@ -82,6 +82,7 @@ namespace Wabbajack.Common
 
         public static string ServerWhitelistURL = "https://raw.githubusercontent.com/wabbajack-tools/opt-out-lists/master/ServerWhitelist.yml";
         public static string ModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/modlists.json";
+        public static string UnlistedModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/unlisted_modlists.json";
         public static string ModlistSummaryURL = "http://build.wabbajack.org/lists/status.json";
         public static string UserAgent
         {
@@ -126,6 +127,8 @@ namespace Wabbajack.Common
         public static AbsolutePath SettingsFile => LocalAppDataPath.Combine("settings.json");
         public static RelativePath SettingsIni = (RelativePath)"settings.ini";
         public static byte SettingsVersion => 2;
+        public static string CompressedBodyHeader = "x-compressed-body";
+
         public static AbsolutePath CefCacheLocation = @"CEF".RelativeTo(LocalAppDataPath);
 
         public static Extension SeqExtension = new Extension(".seq");

Wabbajack.Lib/ClientAPI.cs

@@ -1,10 +1,13 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
+ using System.IO;
+ using System.IO.Compression;
+ using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Text;
 using System.Threading.Tasks;
+ using K4os.Compression.LZ4.Internal;
  using Org.BouncyCastle.Crypto.Agreement.Srp;
  using Wabbajack.Common;
 using Wabbajack.Common.Exceptions;
@@ -136,7 +139,15 @@ namespace Wabbajack.Lib
             var client = await GetClient();
             if (BuildServerStatus.IsBuildServerDown)
                 return;
-            await client.PostAsync($"{Consts.WabbajackBuildServerUri}list_definitions/ingest", new StringContent(modList.ToJson(), Encoding.UTF8, "application/json"));
+            var data = Encoding.UTF8.GetBytes(modList.ToJson());
+            
+            await using var fs = new MemoryStream();
+            await using var gzip = new GZipStream(fs, CompressionLevel.Optimal, true);
+            await gzip.WriteAsync(data);
+            await gzip.DisposeAsync();
+            
+            client.Headers.Add((Consts.CompressedBodyHeader, "gzip"));
+            await client.PostAsync($"{Consts.WabbajackBuildServerUri}list_definitions/ingest", new ByteArrayContent(fs.ToArray()));
         }
 
         public static async Task<Archive[]> GetExistingGameFiles(WorkQueue queue, Game game)

Wabbajack.Lib/Http/Client.cs

@@ -110,6 +110,7 @@ namespace Wabbajack.Lib.Http
                 if (retries > Consts.MaxHTTPRetries) throw;
 
                 retries++;
+                Utils.LogStraightToFile(ex.ToString());
                 Utils.Log($"Http Connect error to {msg.RequestUri} retry {retries}");
                 await Task.Delay(100 * retries);
                 msg = CloneMessage(msg);

Wabbajack.Lib/MO2Compiler.cs

@@ -4,15 +4,12 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Net;
 using System.Threading;
 using System.Threading.Tasks;
 using Wabbajack.Common;
 using Wabbajack.Lib.CompilationSteps;
 using Wabbajack.Lib.Downloaders;
-using Wabbajack.Lib.FileUploader;
 using Wabbajack.Lib.Validation;
-using Wabbajack.VirtualFileSystem;
 using Path = Alphaleonis.Win32.Filesystem.Path;
 
 namespace Wabbajack.Lib

Wabbajack.Lib/ModListRegistry/ModListMetadata.cs

@@ -62,7 +62,7 @@ namespace Wabbajack.Lib.ModListRegistry
 
         public static async Task<List<ModlistMetadata>> LoadFromGithub()
         {
-            var client = new Wabbajack.Lib.Http.Client();
+            var client = new Http.Client();
             Utils.Log("Loading ModLists from GitHub");
             var metadataResult = client.GetStringAsync(Consts.ModlistMetadataURL);
             var summaryResult = client.GetStringAsync(Consts.ModlistSummaryURL);
@@ -84,6 +84,21 @@ namespace Wabbajack.Lib.ModListRegistry
             return metadata.OrderBy(m => (m.ValidationSummary?.HasFailures ?? false ? 1 : 0, m.Title)).ToList();
         }
 
+        public static async Task<List<ModlistMetadata>> LoadUnlistedFromGithub()
+        {
+            try
+            {
+                var client = new Http.Client();
+                return (await client.GetStringAsync(Consts.ModlistMetadataURL)).FromJsonString<List<ModlistMetadata>>();
+            }
+            catch (Exception ex)
+            {
+                Utils.LogStatus("Error loading unlisted modlists");
+                return new List<ModlistMetadata>();
+            }
+
+        }
+        
         public async ValueTask<bool> NeedsDownload(AbsolutePath modlistPath)
         {
             if (!modlistPath.Exists) return true;

Wabbajack.Server.Test/ListIngestionTests.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Wabbajack.BuildServer.Test;
+using Wabbajack.Common;
+using Wabbajack.Lib;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Wabbajack.Server.Test
+{
+    public class ListIngestionTests : ABuildServerSystemTest
+    {
+        public ListIngestionTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
+        {
+        }
+
+        [Fact]
+        public async Task CanIngestModLists()
+        {
+            await ClientAPI.SendModListDefinition(new ModList {Name = "sup"});
+            await Task.Delay(500);
+            
+            Assert.Contains(AbsolutePath.EntryPoint.Combine("mod_list_definitions")
+                .EnumerateFiles(false),
+                f => DateTime.UtcNow - f.LastModifiedUtc < TimeSpan.FromSeconds(15));
+
+            var data = AbsolutePath.EntryPoint.Combine("mod_list_definitions").EnumerateFiles(false)
+                .OrderByDescending(f => f.LastModifiedUtc).First().FromJson<ModList>();
+
+            Assert.Equal("sup", data.Name);
+        }
+    }
+}

Wabbajack.Server/Controllers/ListDefinitions.cs

@@ -1,7 +1,9 @@
 using System;
 using System.IO;
+using System.IO.Compression;
 using System.Linq;
 using System.Reflection;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
@@ -35,16 +37,42 @@ namespace Wabbajack.BuildServer.Controllers
         public async Task<IActionResult> PostIngest()
         {
             var user = Request.Headers[Consts.MetricsKeyHeader].First();
+            var use_gzip = Request.Headers[Consts.CompressedBodyHeader].Any();
             _logger.Log(LogLevel.Information, $"Ingesting Modlist Definition for {user}");
+
             var modlistBytes = await Request.Body.ReadAllAsync();
+            
+
+
+            _logger.LogInformation("Spawning ingestion task");
+            var tsk = Task.Run(async () =>
+            {
+                try
+                {
+                    if (use_gzip)
+                    {
+                        await using var os = new MemoryStream();
+                        await using var gZipStream =
+                            new GZipStream(new MemoryStream(modlistBytes), CompressionMode.Decompress);
+                        await gZipStream.CopyToAsync(os);
+                        modlistBytes = os.ToArray();
+                    }
+
                     var modlist = new MemoryStream(modlistBytes).FromJson<ModList>();
 
                     var file = AbsolutePath.EntryPoint.Combine("mod_list_definitions")
                         .Combine($"{user}_{DateTime.UtcNow.ToFileTimeUtc()}.json");
                     file.Parent.CreateDirectory();
-            await using var stream = await file.OpenWrite();
+                    await using var stream = await file.Create();
                     modlist.ToJson(stream);
                     _logger.Log(LogLevel.Information, $"Done Ingesting Modlist Definition for {user}");
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError(ex, "Error ingesting uploaded modlist");
+                }
+            });
+            
             return Accepted(0);
         }
         

Wabbajack.Server/Controllers/ListsStatus.cs

@@ -82,7 +82,7 @@ namespace Wabbajack.BuildServer.Controllers
                 <ul>
                 {{each $.failed }}
                 {{if $.HasUrl}}
-                <a href='{{$.Url}}'><li>{{$.Name}}</li></a>
+                <li><a href='{{$.Url}}'>{{$.Name}}</a></li>
                 {{else}}
                 <li>{{$.Name}}</li>
                 {{/if}}
@@ -94,7 +94,7 @@ namespace Wabbajack.BuildServer.Controllers
                 <ul>
                 {{each $.updated }}
                 {{if $.HasUrl}}
-                <a href='{{$.Url}}'><li>{{$.Name}}</li></a>
+                <li><a href='{{$.Url}}'>{{$.Name}}</a></li>
                 {{else}}
                 <li>{{$.Name}}</li>
                 {{/if}}
@@ -106,7 +106,7 @@ namespace Wabbajack.BuildServer.Controllers
                 <ul>
                 {{each $.updating }}
                 {{if $.HasUrl}}
-                <a href='{{$.Url}}'><li>{{$.Name}}</li></a>
+                <li><a href='{{$.Url}}'>{{$.Name}}</a></li>
                 {{else}}
                 <li>{{$.Name}}</li>
                 {{/if}}
@@ -117,7 +117,7 @@ namespace Wabbajack.BuildServer.Controllers
                 <ul>
                 {{each $.passed }}
                 {{if $.HasUrl}}
-                <a href='{{$.Url}}'><li>{{$.Name}}</li></a>
+                <li><a href='{{$.Url}}'>{{$.Name}}</a></li>
                 {{else}}
                 <li>{{$.Name}}</li>
                 {{/if}}

Wabbajack.Server/Services/ModListDownloader.cs

@@ -1,5 +1,6 @@
 using System;
 using System.IO.Compression;
+using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
@@ -60,7 +61,9 @@ namespace Wabbajack.Server.Services
         public async Task<int> CheckForNewLists()
         {
             int downloaded = 0;
-            var lists = await ModlistMetadata.LoadFromGithub();
+            var lists = (await ModlistMetadata.LoadFromGithub())
+                .Concat(await ModlistMetadata.LoadUnlistedFromGithub()).ToList();
+            
             foreach (var list in lists)
             {
                 try