Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #964 from wabbajack-tools/ingestion-compression

Browse Source 
Fixes for broken archives names on server links
This commit is contained in:
Timothy Baldridge 2020-07-16 15:23:12 -07:00 committed by GitHub
commit 372fd2d7f7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 116 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -1,6 +1,10 @@
### Changelog
#### Version - 2.1.3.0 - 7/15/2020
#### Version - 2.1.4.0 - **
* List ingestion now supports compression and processes on a background threaded
* Support for validation of unlisted modlists
#### Version - 2.1.3.0 - 7/16/2020
* Filters from the FilePicker are now being used
* Wabbajack will continue working even if the build server is down
* Fixed an issue where the main window does not appear after the splash screen

3
Wabbajack.Common/Consts.cs
View File

@ -82,6 +82,7 @@ namespace Wabbajack.Common
public static string ServerWhitelistURL = "https://raw.githubusercontent.com/wabbajack-tools/opt-out-lists/master/ServerWhitelist.yml";
public static string ModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/modlists.json";
public static string UnlistedModlistMetadataURL = "https://raw.githubusercontent.com/wabbajack-tools/mod-lists/master/unlisted_modlists.json";
public static string ModlistSummaryURL = "http://build.wabbajack.org/lists/status.json";
public static string UserAgent
{
@ -126,6 +127,8 @@ namespace Wabbajack.Common
public static AbsolutePath SettingsFile => LocalAppDataPath.Combine("settings.json");
public static RelativePath SettingsIni = (RelativePath)"settings.ini";
public static byte SettingsVersion => 2;
public static string CompressedBodyHeader = "x-compressed-body";
public static AbsolutePath CefCacheLocation = @"CEF".RelativeTo(LocalAppDataPath);
public static Extension SeqExtension = new Extension(".seq");

15
Wabbajack.Lib/ClientAPI.cs
View File

@ -1,10 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using K4os.Compression.LZ4.Internal;
using Org.BouncyCastle.Crypto.Agreement.Srp;
using Wabbajack.Common;
using Wabbajack.Common.Exceptions;
@ -136,7 +139,15 @@ namespace Wabbajack.Lib
var client = await GetClient();
if (BuildServerStatus.IsBuildServerDown)
return;
await client.PostAsync($"{Consts.WabbajackBuildServerUri}list_definitions/ingest", new StringContent(modList.ToJson(), Encoding.UTF8, "application/json"));
var data = Encoding.UTF8.GetBytes(modList.ToJson());
await using var fs = new MemoryStream();
await using var gzip = new GZipStream(fs, CompressionLevel.Optimal, true);
await gzip.WriteAsync(data);
await gzip.DisposeAsync();
client.Headers.Add((Consts.CompressedBodyHeader, "gzip"));
await client.PostAsync($"{Consts.WabbajackBuildServerUri}list_definitions/ingest", new ByteArrayContent(fs.ToArray()));
}
public static async Task<Archive[]> GetExistingGameFiles(WorkQueue queue, Game game)

1
Wabbajack.Lib/Http/Client.cs
View File

@ -110,6 +110,7 @@ namespace Wabbajack.Lib.Http
if (retries > Consts.MaxHTTPRetries) throw;
retries++;
Utils.LogStraightToFile(ex.ToString());
Utils.Log($"Http Connect error to {msg.RequestUri} retry {retries}");
await Task.Delay(100 * retries);
msg = CloneMessage(msg);

3
Wabbajack.Lib/MO2Compiler.cs
View File

@ -4,15 +4,12 @@ using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Lib.CompilationSteps;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.FileUploader;
using Wabbajack.Lib.Validation;
using Wabbajack.VirtualFileSystem;
using Path = Alphaleonis.Win32.Filesystem.Path;
namespace Wabbajack.Lib

17
Wabbajack.Lib/ModListRegistry/ModListMetadata.cs
View File

@ -62,7 +62,7 @@ namespace Wabbajack.Lib.ModListRegistry
public static async Task<List<ModlistMetadata>> LoadFromGithub()
{
var client = new Wabbajack.Lib.Http.Client();
var client = new Http.Client();
Utils.Log("Loading ModLists from GitHub");
var metadataResult = client.GetStringAsync(Consts.ModlistMetadataURL);
var summaryResult = client.GetStringAsync(Consts.ModlistSummaryURL);
@ -83,6 +83,21 @@ namespace Wabbajack.Lib.ModListRegistry
return metadata.OrderBy(m => (m.ValidationSummary?.HasFailures ?? false ? 1 : 0, m.Title)).ToList();
}
public static async Task<List<ModlistMetadata>> LoadUnlistedFromGithub()
{
try
{
var client = new Http.Client();
return (await client.GetStringAsync(Consts.ModlistMetadataURL)).FromJsonString<List<ModlistMetadata>>();
}
catch (Exception ex)
{
Utils.LogStatus("Error loading unlisted modlists");
return new List<ModlistMetadata>();
}
}
public async ValueTask<bool> NeedsDownload(AbsolutePath modlistPath)
{

34
Wabbajack.Server.Test/ListIngestionTests.cs Normal file
View File

@ -0,0 +1,34 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.BuildServer.Test;
using Wabbajack.Common;
using Wabbajack.Lib;
using Xunit;
using Xunit.Abstractions;
namespace Wabbajack.Server.Test
{
public class ListIngestionTests : ABuildServerSystemTest
{
public ListIngestionTests(ITestOutputHelper output, SingletonAdaptor<BuildServerFixture> fixture) : base(output, fixture)
{
}
[Fact]
public async Task CanIngestModLists()
{
await ClientAPI.SendModListDefinition(new ModList {Name = "sup"});
await Task.Delay(500);
Assert.Contains(AbsolutePath.EntryPoint.Combine("mod_list_definitions")
.EnumerateFiles(false),
f => DateTime.UtcNow - f.LastModifiedUtc < TimeSpan.FromSeconds(15));
var data = AbsolutePath.EntryPoint.Combine("mod_list_definitions").EnumerateFiles(false)
.OrderByDescending(f => f.LastModifiedUtc).First().FromJson<ModList>();
Assert.Equal("sup", data.Name);
}
}
}

44
Wabbajack.Server/Controllers/ListDefinitions.cs
View File

@ -1,7 +1,9 @@
using System;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
@ -35,16 +37,42 @@ namespace Wabbajack.BuildServer.Controllers
public async Task<IActionResult> PostIngest()
{
var user = Request.Headers[Consts.MetricsKeyHeader].First();
var use_gzip = Request.Headers[Consts.CompressedBodyHeader].Any();
_logger.Log(LogLevel.Information, $"Ingesting Modlist Definition for {user}");
var modlistBytes = await Request.Body.ReadAllAsync();
var modlist = new MemoryStream(modlistBytes).FromJson<ModList>();
var file = AbsolutePath.EntryPoint.Combine("mod_list_definitions")
.Combine($"{user}_{DateTime.UtcNow.ToFileTimeUtc()}.json");
file.Parent.CreateDirectory();
await using var stream = await file.OpenWrite();
modlist.ToJson(stream);
_logger.Log(LogLevel.Information, $"Done Ingesting Modlist Definition for {user}");
var modlistBytes = await Request.Body.ReadAllAsync();
_logger.LogInformation("Spawning ingestion task");
var tsk = Task.Run(async () =>
{
try
{
if (use_gzip)
{
await using var os = new MemoryStream();
await using var gZipStream =
new GZipStream(new MemoryStream(modlistBytes), CompressionMode.Decompress);
await gZipStream.CopyToAsync(os);
modlistBytes = os.ToArray();
}
var modlist = new MemoryStream(modlistBytes).FromJson<ModList>();
var file = AbsolutePath.EntryPoint.Combine("mod_list_definitions")
.Combine($"{user}_{DateTime.UtcNow.ToFileTimeUtc()}.json");
file.Parent.CreateDirectory();
await using var stream = await file.Create();
modlist.ToJson(stream);
_logger.Log(LogLevel.Information, $"Done Ingesting Modlist Definition for {user}");
}
catch (Exception ex)
{
_logger.LogError(ex, "Error ingesting uploaded modlist");
}
});
return Accepted(0);
}

8
Wabbajack.Server/Controllers/ListsStatus.cs
View File

@ -82,7 +82,7 @@ namespace Wabbajack.BuildServer.Controllers
<ul>
{{each $.failed }}
{{if $.HasUrl}}
<a href='{{$.Url}}'><li>{{$.Name}}</li></a>
<li><a href='{{$.Url}}'>{{$.Name}}</a></li>
{{else}}
<li>{{$.Name}}</li>
{{/if}}
@ -94,7 +94,7 @@ namespace Wabbajack.BuildServer.Controllers
<ul>
{{each $.updated }}
{{if $.HasUrl}}
<a href='{{$.Url}}'><li>{{$.Name}}</li></a>
<li><a href='{{$.Url}}'>{{$.Name}}</a></li>
{{else}}
<li>{{$.Name}}</li>
{{/if}}
@ -106,7 +106,7 @@ namespace Wabbajack.BuildServer.Controllers
<ul>
{{each $.updating }}
{{if $.HasUrl}}
<a href='{{$.Url}}'><li>{{$.Name}}</li></a>
<li><a href='{{$.Url}}'>{{$.Name}}</a></li>
{{else}}
<li>{{$.Name}}</li>
{{/if}}
@ -117,7 +117,7 @@ namespace Wabbajack.BuildServer.Controllers
<ul>
{{each $.passed }}
{{if $.HasUrl}}
<a href='{{$.Url}}'><li>{{$.Name}}</li></a>
<li><a href='{{$.Url}}'>{{$.Name}}</a></li>
{{else}}
<li>{{$.Name}}</li>
{{/if}}

5
Wabbajack.Server/Services/ModListDownloader.cs
View File

@ -1,5 +1,6 @@
using System;
using System.IO.Compression;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
@ -60,7 +61,9 @@ namespace Wabbajack.Server.Services
public async Task<int> CheckForNewLists()
{
int downloaded = 0;
var lists = await ModlistMetadata.LoadFromGithub();
var lists = (await ModlistMetadata.LoadFromGithub())
.Concat(await ModlistMetadata.LoadUnlistedFromGithub()).ToList();
foreach (var list in lists)
{
try