Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Version 2.4.2.0

Browse Source
This commit is contained in:
Timothy Baldridge 2021-02-03 20:48:30 -07:00
parent cfa69948c7
commit c0d50a8216
20 changed files with 229 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CHANGELOG.md
View File

@ -1,6 +1,13 @@
### Changelog
#### Version - 2.4.1.2 - 1/29/2020
#### Version - 2.4.2.0 - 2/3/2020
* Rework the Nexus Manual Downloading process now with less jank
* Manual Nexus Downloading now explains why it's so painful, and how to fix it (get Premium)
* Manual Nexus Downloading no longer opens a ton of CEF processes
* Manual Nexus Downloading no longer prompts users to download files that don't exist
* Disabled CloudFlare DDOS mitigation for LoversLab downloading, the site is back to normal now
#### Version - 2.4.1.2 - 1/30/2020
* Don't install .meta files for files sourced from the game folder
* Fix bug MO2 archive name detection in .meta files (rare bug with FO4VR and other like games)
* Catch exceptions when ECS downloads manifest data

53
Wabbajack.Benchmark/WorkQueueBenchmarks.cs Normal file
View File

@ -0,0 +1,53 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using BenchmarkDotNet.Attributes;
using Wabbajack.Common;
namespace Wabbajack.Benchmark
{
public class WorkQueueBenchmarks
{
private int[] _itms;
private WorkQueue _queue;
private TempFile _file;
private Random _rdm;
[Params(2, 4, 8, 16, 32, 64, 128, 256)]
public int Threads { get; set; }
[GlobalSetup]
public async Task Setup()
{
_rdm = new Random((int)DateTime.Now.ToFileTimeUtc());
_itms = Enumerable.Range(0, Threads * 10).ToArray();
_queue = new WorkQueue(Threads);
_file = new TempFile();
await using var f = await _file.Path.Create();
var data = new byte[1024 * 1024 * 10]; // 1GB
_rdm.NextBytes(data);
await f.WriteAsync(data);
}
[GlobalCleanup]
public async Task Cleanup()
{
_queue.Dispose();
await _file.DisposeAsync();
}
/* [Benchmark]
public async Task SleepTask()
{
await _itms.PMap(_queue, async f => await Task.Delay(1));
}*/
[Benchmark]
public async Task FileHashTask()
{
await _itms.PMap(_queue, async f => await _file.Path.FileHashAsync());
}
}
}

5
Wabbajack.Common/Logging.cs
View File

@ -167,5 +167,10 @@ namespace Wabbajack.Common
Utils.Error(ex);
}
}
public static void ErrorMetric(Exception exception)
{
throw new NotImplementedException();
}
}
}

7
Wabbajack.Lib/ABatchProcessor.cs
View File

@ -168,13 +168,18 @@ namespace Wabbajack.Lib
Utils.Log("Starting Installer Task");
return Task.Run(async () =>
{
{
try
{
Utils.Log("Installation has Started");
_isRunning.OnNext(true);
return await _Begin(_cancel.Token);
}
catch (Exception ex)
{
var _ = Metrics.Error(this.GetType(), ex);
throw;
}
finally
{
Utils.Log("Vacuuming databases");

8
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -32,7 +32,7 @@ namespace Wabbajack.Lib.Downloaders
{
private DateTime LastRequestTime = default;
protected long RequestsPerMinute = 20;
protected long RequestsPerMinute = 60;
private TimeSpan RequestDelay => TimeSpan.FromMinutes(1) / RequestsPerMinute;
protected AbstractIPS4Downloader(Uri loginUri, string encryptedKeyName, string cookieDomain, string loginCookie = "ips4_member_id")
@ -182,7 +182,8 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Download(Archive a, AbsolutePath destination)
{
await ((IWaitForWindowDownloader)Downloader).WaitForNextRequestWindow();
if (Downloader.IsCloudFlareProtected)
await ((IWaitForWindowDownloader)Downloader).WaitForNextRequestWindow();
return await ResolveDownloadStream(a, destination, false);
}
@ -312,7 +313,8 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Verify(Archive a, CancellationToken? token)
{
await ((IWaitForWindowDownloader)Downloader).WaitForNextRequestWindow();
if (Downloader.IsCloudFlareProtected)
await ((IWaitForWindowDownloader)Downloader).WaitForNextRequestWindow();
await using var tp = new TempFile();
var isValid = await ResolveDownloadStream(a, tp.Path, true, token: token);
return isValid;

2
Wabbajack.Lib/Downloaders/LoversLabDownloader.cs
View File

@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders
public LoversLabDownloader() : base(new Uri("https://www.loverslab.com/login"),
"loverslabcookies", "loverslab.com")
{
IsCloudFlareProtected = true;
IsCloudFlareProtected = false;
}
protected override async Task WhileWaiting(IWebDriver browser)
{

7
Wabbajack.Lib/Http/ClientFactory.cs
View File

@ -1,5 +1,8 @@
using System;
using System.Net;
using System.Net.Security;
using System.Security.Authentication;
using Wabbajack.Common;
using SysHttp = System.Net.Http;
namespace Wabbajack.Lib.Http
@ -18,8 +21,10 @@ namespace Wabbajack.Lib.Http
CookieContainer = Cookies,
MaxConnectionsPerServer = 20,
PooledConnectionLifetime = TimeSpan.FromMilliseconds(100),
PooledConnectionIdleTimeout = TimeSpan.FromMilliseconds(100)
PooledConnectionIdleTimeout = TimeSpan.FromMilliseconds(100),
};
Utils.Log($"Configuring with SSL {_socketsHandler.SslOptions.EnabledSslProtocols}");
Client = new SysHttp.HttpClient(_socketsHandler);
}
}

8
Wabbajack.Lib/Metrics.cs
View File

@ -1,5 +1,7 @@
using System;
using System.Text.Encodings.Web;
using System.Threading.Tasks;
using System.Web;
using Microsoft.Win32;
using Wabbajack.Common;
@ -78,5 +80,11 @@ namespace Wabbajack.Lib
client.Headers.Add((Consts.MetricsKeyHeader, key));
await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{value}");
}
public static async Task Error(Type type, Exception exception)
{
await Send("Exception", type.Name + "|" + exception.Message);
await Send("ExceptionData" + type.Name + "|" + Consts.CurrentMinimumWabbajackVersion, HttpUtility.UrlEncode($"{exception.Message}\n{exception.StackTrace}"));
}
}
}

15
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -19,6 +19,10 @@ namespace Wabbajack.Lib.NexusApi
public class NexusApiClient : ViewModel, INexusApi
{
private static readonly string API_KEY_CACHE_FILE = "nexus.key_cache";
/// <summary>
/// Forces the client to do manual downloading via CEF (for testing)
/// </summary>
private static bool ManualTestingMode = true;
public Http.Client HttpClient { get; } = new();
@ -301,15 +305,18 @@ namespace Wabbajack.Lib.NexusApi
return await Get<T>(builder.ToString(), HttpClient.WithHeader((Consts.MetricsKeyHeader, await Metrics.GetMetricsKey())));
}
private static AsyncLock ManualDownloadLock = new();
public async Task<string> GetNexusDownloadLink(NexusDownloader.State archive)
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
var info = await GetModInfo(archive.Game, archive.ModID);
if (!info.available)
var fileInfo = await GetModFiles(archive.Game, archive.ModID);
if (!info.available || !fileInfo.files.Any(f => f.file_id == archive.FileID && f.category_name != null))
throw new Exception("Mod unavailable");
if (await IsPremium())
if (await IsPremium() && !ManualTestingMode)
{
if (HourlyRemaining <= 0 && DailyRemaining <= 0)
{
@ -323,6 +330,8 @@ namespace Wabbajack.Lib.NexusApi
try
{
using var _ = await ManualDownloadLock.WaitAsync();
await Task.Delay(1000);
Utils.Log($"Requesting manual download for {archive.Name} {archive.PrimaryKeyString}");
return (await Utils.Log(await ManuallyDownloadNexusFile.Create(archive)).Task).ToString();
}

6
Wabbajack.Server.Test/ModListValidationTests.cs
View File

@ -40,7 +40,7 @@ namespace Wabbajack.BuildServer.Test
Consts.ModlistMetadataURL = modlist.ToString();
var sql = Fixture.GetService<SqlService>();
var downloader = Fixture.GetService<ModListDownloader>();
await downloader.CheckForNewLists();
await downloader.Execute();
foreach (var list in ModListMetaData)
{
@ -48,7 +48,7 @@ namespace Wabbajack.BuildServer.Test
}
// Nothing has changed so we shouldn't be downloading anything this time
Assert.Equal(0, await downloader.CheckForNewLists());
Assert.Equal(0, await downloader.Execute());
}
@ -161,7 +161,7 @@ namespace Wabbajack.BuildServer.Test
{
var downloader = Fixture.GetService<ModListDownloader>();
await downloader.CheckForNewLists();
await downloader.Execute();
if (runNonNexus)
{

11
Wabbajack.Server/ApiKeyAuthorizationHandler.cs
View File

@ -29,6 +29,7 @@ namespace Wabbajack.BuildServer
{
private const string ProblemDetailsContentType = "application/problem+json";
private readonly SqlService _sql;
private static ConcurrentHashSet<string> _knownKeys = new();
private const string ApiKeyHeaderName = "X-Api-Key";
public ApiKeyAuthenticationHandler(
@ -44,7 +45,8 @@ namespace Wabbajack.BuildServer
protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
{
var metricsKey = Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault();
await LogRequest(metricsKey);
// Never needed this, disabled for now
//await LogRequest(metricsKey);
if (metricsKey != default)
{
if (await _sql.IsTarKey(metricsKey))
@ -82,15 +84,16 @@ namespace Wabbajack.BuildServer
return AuthenticateResult.Success(ticket);
}
if (!await _sql.ValidMetricsKey(metricsKey))
if (!_knownKeys.Contains(metricsKey) && !await _sql.ValidMetricsKey(metricsKey))
{
return AuthenticateResult.Fail("Invalid Metrics Key");
}
else
{
var claims = new List<Claim>();
_knownKeys.Add(metricsKey);
var claims = new List<Claim> {new(ClaimTypes.Role, "User")};
claims.Add(new Claim(ClaimTypes.Role, "User"));
var identity = new ClaimsIdentity(claims, Options.AuthenticationType);
var identities = new List<ClaimsIdentity> {identity};

10
Wabbajack.Server/Controllers/AuthoredFiles.cs
View File

@ -27,14 +27,16 @@ namespace Wabbajack.BuildServer.Controllers
private ILogger<AuthoredFiles> _logger;
private AppSettings _settings;
private CDNMirrorList _mirrorList;
private DiscordWebHook _discord;
public AuthoredFiles(ILogger<AuthoredFiles> logger, SqlService sql, AppSettings settings, CDNMirrorList mirrorList)
public AuthoredFiles(ILogger<AuthoredFiles> logger, SqlService sql, AppSettings settings, CDNMirrorList mirrorList, DiscordWebHook discord)
{
_sql = sql;
_logger = logger;
_settings = settings;
_mirrorList = mirrorList;
_discord = discord;
}
[HttpPut]
@ -77,6 +79,9 @@ namespace Wabbajack.BuildServer.Controllers
_logger.Log(LogLevel.Information, $"Creating File upload {definition.OriginalFileName}");
definition = await _sql.CreateAuthoredFile(definition, user);
await _discord.Send(Channel.Ham,
new DiscordMessage() {Content = $"{user} has started uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})"});
return Ok(definition.ServerAssignedUniqueId);
}
@ -101,6 +106,9 @@ namespace Wabbajack.BuildServer.Controllers
ms.Position = 0;
await UploadAsync(ms, $"{definition.MungedName}/definition.json.gz");
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"{user} has finished uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})"});
return Ok($"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{definition.MungedName}");
}

10
Wabbajack.Server/DataLayer/ModLists.cs
View File

@ -100,5 +100,15 @@ namespace Wabbajack.Server.DataLayer
Hash = t.Item2
}).ToList();
}
public async Task<int> PurgeList(string machineURL)
{
await using var conn = await Open();
var ret1 = await conn.ExecuteAsync(@" delete from dbo.ModListArchives where MachineURL = @machineURL",
new {machineURL});
var ret2 = await conn.ExecuteAsync(@" delete from dbo.ModLists where MachineURL = @machineURL",
new {machineURL});
return ret1 + ret2;
}
}
}

35
Wabbajack.Server/Services/DiscordFrontend.cs
View File

@ -58,23 +58,34 @@ namespace Wabbajack.Server.Services
}
await PurgeNexusCache(arg, parts[2]);
}
else if (parts[1] == "cyberpunk")
else if (parts[1] == "quick-sync")
{
var random = new Random();
var releaseDate = new DateTime(2020, 12, 10, 0, 0, 0, DateTimeKind.Utc);
var r = releaseDate - DateTime.UtcNow;
if (r < TimeSpan.Zero)
var options = await _quickSync.Report();
if (parts.Length != 3)
{
await ReplyTo(arg, "It's out, what are you doing here?");
var optionsStr = string.Join(", ", options.Select(o => o.Key.Name));
await ReplyTo(arg, $"Can't expect me to quicksync the whole damn world! Try: {optionsStr}");
}
else
{
var msgs = (await "cyberpunk_message.txt".RelativeTo(AbsolutePath.EntryPoint)
.ReadAllLinesAsync()).ToArray();
var msg = msgs[random.Next(0, msgs.Length)];
var fullmsg = String.Format(msg,
$"{r.Days} days, {r.Hours} hours, {r.Minutes} minutes, {r.Seconds} seconds");
await ReplyTo(arg, fullmsg);
foreach (var pair in options.Where(o => o.Key.Name == parts[2]))
{
await _quickSync.Notify(pair.Key);
await ReplyTo(arg, $"Notified {pair.Key}");
}
}
}
else if (parts[1] == "purge-list")
{
if (parts.Length != 3)
{
await ReplyTo(arg, $"Yeah, I'm not gonna purge the whole server...");
}
else
{
var deleted = await _sql.PurgeList(parts[2]);
await _quickSync.Notify<ModListDownloader>();
await ReplyTo(arg, $"Purged all traces of #{parts[2]} from the server, triggered list downloading. {deleted} records removed");
}
}
}

59
Wabbajack.Server/Services/ModListDownloader.cs
View File

@ -16,15 +16,14 @@ using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.Services
{
public class ModListDownloader
public class ModListDownloader : AbstractService<ModListDownloader, int>
{
private ILogger<ModListDownloader> _logger;
private AppSettings _settings;
private ArchiveMaintainer _maintainer;
private SqlService _sql;
private DiscordWebHook _discord;
public ModListDownloader(ILogger<ModListDownloader> logger, AppSettings settings, ArchiveMaintainer maintainer, SqlService sql, DiscordWebHook discord)
public ModListDownloader(ILogger<ModListDownloader> logger, AppSettings settings, ArchiveMaintainer maintainer, SqlService sql, DiscordWebHook discord, QuickSync quickSync)
: base(logger, settings, quickSync, TimeSpan.FromMinutes(1))
{
_logger = logger;
_settings = settings;
@ -33,32 +32,8 @@ namespace Wabbajack.Server.Services
_discord = discord;
}
public void Start()
{
if (_settings.RunBackEndJobs)
{
Task.Run(async () =>
{
while (true)
{
try
{
_logger.Log(LogLevel.Information, "Checking for updated mod lists");
await CheckForNewLists();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error checking list");
}
await Task.Delay(TimeSpan.FromMinutes(5));
}
});
}
}
public async Task<int> CheckForNewLists()
public override async Task<int> Execute()
{
int downloaded = 0;
var lists = (await ModlistMetadata.LoadFromGithub())
@ -68,6 +43,7 @@ namespace Wabbajack.Server.Services
{
try
{
ReportStarting(list.Links.MachineURL);
if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
continue;
@ -76,7 +52,10 @@ namespace Wabbajack.Server.Services
{
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"});
new DiscordMessage
{
Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
var tf = new TempFile();
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
if (state == null)
@ -110,7 +89,10 @@ namespace Wabbajack.Server.Services
{
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"});
new DiscordMessage
{
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
continue;
}
@ -122,7 +104,10 @@ namespace Wabbajack.Server.Services
{
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"});
new DiscordMessage
{
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
continue;
}
}
@ -133,7 +118,15 @@ namespace Wabbajack.Server.Services
{
_logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
await _discord.Send(Channel.Ham,
new DiscordMessage {Content = $"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"});
new DiscordMessage
{
Content =
$"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
});
}
finally
{
ReportEnding(list.Links.MachineURL);
}
}
_logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");

11
Wabbajack.Server/Services/QuickSync.cs
View File

@ -67,5 +67,16 @@ namespace Wabbajack.Server.Services
ct.Cancel();
}
}
public async Task Notify(Type t)
{
_logger.LogInformation($"Quicksync {t.Name}");
// Needs debugging
using var _ = await _lock.WaitAsync();
if (_syncs.TryGetValue(t, out var ct))
{
ct.Cancel();
}
}
}
}

24
Wabbajack.Test/MetricsTests.cs Normal file
View File

@ -0,0 +1,24 @@
using System;
using System.Threading.Tasks;
using Wabbajack.Lib;
using Xunit;
namespace Wabbajack.Test
{
public class MetricsTests
{
[Fact]
public async Task CanSendExceptions()
{
try
{
throw new Exception("Test Exception");
}
catch (Exception ex)
{
await Metrics.Error(this.GetType(), ex);
}
}
}
}

10
Wabbajack.sln
View File

@ -44,6 +44,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Server", "Wabbaja
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Server.Test", "Wabbajack.Server.Test\Wabbajack.Server.Test.csproj", "{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Benchmark", "Wabbajack.Benchmark\Wabbajack.Benchmark.csproj", "{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -165,6 +167,14 @@ Global
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|Any CPU.Build.0 = Release|Any CPU
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|x64.ActiveCfg = Release|Any CPU
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|x64.Build.0 = Release|Any CPU
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Debug|x64.ActiveCfg = Debug|Any CPU
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Debug|x64.Build.0 = Debug|Any CPU
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Release|Any CPU.Build.0 = Release|Any CPU
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Release|x64.ActiveCfg = Release|Any CPU
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

3
Wabbajack/View Models/Compilers/MO2CompilerVM.cs
View File

@ -230,8 +230,7 @@ namespace Wabbajack
ModlistIsNSFW = ModlistSettings.IsNSFW
};
}
using (ActiveCompilation = compiler
)
using (ActiveCompilation = compiler)
{
Parent.MWVM.Settings.Performance.SetProcessorSettings(ActiveCompilation);
var success = await ActiveCompilation.Begin();

2
Wabbajack/View Models/UserInterventionHandlers.cs
View File

@ -139,7 +139,7 @@ namespace Wabbajack
};
await vm.Driver.WaitForInitialized();
IWebDriver browser = new CefSharpWrapper(vm.Browser);
vm.Instructions = $"Please Download {state.Name} - {state.ModID} - {state.FileID}";
vm.Instructions = $"Click the highlighted file (get a NexusMods.com Premium account to automate this)";
browser.DownloadHandler = uri =>
{
manuallyDownloadNexusFile.Resume(uri);