mirror of
https://github.com/wabbajack-tools/wabbajack.git
synced 2024-08-30 18:42:17 +00:00
Version 2.4.2.0
This commit is contained in:
parent
cfa69948c7
commit
c0d50a8216
@ -1,6 +1,13 @@
|
|||||||
### Changelog
|
### Changelog
|
||||||
|
|
||||||
#### Version - 2.4.1.2 - 1/29/2020
|
#### Version - 2.4.2.0 - 2/3/2020
|
||||||
|
* Rework the Nexus Manual Downloading process now with less jank
|
||||||
|
* Manual Nexus Downloading now explains why it's so painful, and how to fix it (get Premium)
|
||||||
|
* Manual Nexus Downloading no longer opens a ton of CEF processes
|
||||||
|
* Manual Nexus Downloading no longer prompts users to download files that don't exist
|
||||||
|
* Disabled CloudFlare DDOS mitigation for LoversLab downloading, the site is back to normal now
|
||||||
|
|
||||||
|
#### Version - 2.4.1.2 - 1/30/2020
|
||||||
* Don't install .meta files for files sourced from the game folder
|
* Don't install .meta files for files sourced from the game folder
|
||||||
* Fix bug MO2 archive name detection in .meta files (rare bug with FO4VR and other like games)
|
* Fix bug MO2 archive name detection in .meta files (rare bug with FO4VR and other like games)
|
||||||
* Catch exceptions when ECS downloads manifest data
|
* Catch exceptions when ECS downloads manifest data
|
||||||
|
53
Wabbajack.Benchmark/WorkQueueBenchmarks.cs
Normal file
53
Wabbajack.Benchmark/WorkQueueBenchmarks.cs
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
using System;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using BenchmarkDotNet.Attributes;
|
||||||
|
using Wabbajack.Common;
|
||||||
|
|
||||||
|
namespace Wabbajack.Benchmark
|
||||||
|
{
|
||||||
|
public class WorkQueueBenchmarks
|
||||||
|
{
|
||||||
|
private int[] _itms;
|
||||||
|
private WorkQueue _queue;
|
||||||
|
private TempFile _file;
|
||||||
|
private Random _rdm;
|
||||||
|
|
||||||
|
[Params(2, 4, 8, 16, 32, 64, 128, 256)]
|
||||||
|
public int Threads { get; set; }
|
||||||
|
|
||||||
|
[GlobalSetup]
|
||||||
|
public async Task Setup()
|
||||||
|
{
|
||||||
|
_rdm = new Random((int)DateTime.Now.ToFileTimeUtc());
|
||||||
|
_itms = Enumerable.Range(0, Threads * 10).ToArray();
|
||||||
|
_queue = new WorkQueue(Threads);
|
||||||
|
|
||||||
|
_file = new TempFile();
|
||||||
|
await using var f = await _file.Path.Create();
|
||||||
|
var data = new byte[1024 * 1024 * 10]; // 1GB
|
||||||
|
|
||||||
|
_rdm.NextBytes(data);
|
||||||
|
await f.WriteAsync(data);
|
||||||
|
}
|
||||||
|
|
||||||
|
[GlobalCleanup]
|
||||||
|
public async Task Cleanup()
|
||||||
|
{
|
||||||
|
_queue.Dispose();
|
||||||
|
await _file.DisposeAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
/* [Benchmark]
|
||||||
|
public async Task SleepTask()
|
||||||
|
{
|
||||||
|
await _itms.PMap(_queue, async f => await Task.Delay(1));
|
||||||
|
}*/
|
||||||
|
|
||||||
|
[Benchmark]
|
||||||
|
public async Task FileHashTask()
|
||||||
|
{
|
||||||
|
await _itms.PMap(_queue, async f => await _file.Path.FileHashAsync());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -167,5 +167,10 @@ namespace Wabbajack.Common
|
|||||||
Utils.Error(ex);
|
Utils.Error(ex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static void ErrorMetric(Exception exception)
|
||||||
|
{
|
||||||
|
throw new NotImplementedException();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -168,13 +168,18 @@ namespace Wabbajack.Lib
|
|||||||
|
|
||||||
Utils.Log("Starting Installer Task");
|
Utils.Log("Starting Installer Task");
|
||||||
return Task.Run(async () =>
|
return Task.Run(async () =>
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
Utils.Log("Installation has Started");
|
Utils.Log("Installation has Started");
|
||||||
_isRunning.OnNext(true);
|
_isRunning.OnNext(true);
|
||||||
return await _Begin(_cancel.Token);
|
return await _Begin(_cancel.Token);
|
||||||
}
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
var _ = Metrics.Error(this.GetType(), ex);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
Utils.Log("Vacuuming databases");
|
Utils.Log("Vacuuming databases");
|
||||||
|
@ -32,7 +32,7 @@ namespace Wabbajack.Lib.Downloaders
|
|||||||
{
|
{
|
||||||
|
|
||||||
private DateTime LastRequestTime = default;
|
private DateTime LastRequestTime = default;
|
||||||
protected long RequestsPerMinute = 20;
|
protected long RequestsPerMinute = 60;
|
||||||
private TimeSpan RequestDelay => TimeSpan.FromMinutes(1) / RequestsPerMinute;
|
private TimeSpan RequestDelay => TimeSpan.FromMinutes(1) / RequestsPerMinute;
|
||||||
|
|
||||||
protected AbstractIPS4Downloader(Uri loginUri, string encryptedKeyName, string cookieDomain, string loginCookie = "ips4_member_id")
|
protected AbstractIPS4Downloader(Uri loginUri, string encryptedKeyName, string cookieDomain, string loginCookie = "ips4_member_id")
|
||||||
@ -182,7 +182,8 @@ namespace Wabbajack.Lib.Downloaders
|
|||||||
|
|
||||||
public override async Task<bool> Download(Archive a, AbsolutePath destination)
|
public override async Task<bool> Download(Archive a, AbsolutePath destination)
|
||||||
{
|
{
|
||||||
await ((IWaitForWindowDownloader)Downloader).WaitForNextRequestWindow();
|
if (Downloader.IsCloudFlareProtected)
|
||||||
|
await ((IWaitForWindowDownloader)Downloader).WaitForNextRequestWindow();
|
||||||
return await ResolveDownloadStream(a, destination, false);
|
return await ResolveDownloadStream(a, destination, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -312,7 +313,8 @@ namespace Wabbajack.Lib.Downloaders
|
|||||||
|
|
||||||
public override async Task<bool> Verify(Archive a, CancellationToken? token)
|
public override async Task<bool> Verify(Archive a, CancellationToken? token)
|
||||||
{
|
{
|
||||||
await ((IWaitForWindowDownloader)Downloader).WaitForNextRequestWindow();
|
if (Downloader.IsCloudFlareProtected)
|
||||||
|
await ((IWaitForWindowDownloader)Downloader).WaitForNextRequestWindow();
|
||||||
await using var tp = new TempFile();
|
await using var tp = new TempFile();
|
||||||
var isValid = await ResolveDownloadStream(a, tp.Path, true, token: token);
|
var isValid = await ResolveDownloadStream(a, tp.Path, true, token: token);
|
||||||
return isValid;
|
return isValid;
|
||||||
|
@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders
|
|||||||
public LoversLabDownloader() : base(new Uri("https://www.loverslab.com/login"),
|
public LoversLabDownloader() : base(new Uri("https://www.loverslab.com/login"),
|
||||||
"loverslabcookies", "loverslab.com")
|
"loverslabcookies", "loverslab.com")
|
||||||
{
|
{
|
||||||
IsCloudFlareProtected = true;
|
IsCloudFlareProtected = false;
|
||||||
}
|
}
|
||||||
protected override async Task WhileWaiting(IWebDriver browser)
|
protected override async Task WhileWaiting(IWebDriver browser)
|
||||||
{
|
{
|
||||||
|
@ -1,5 +1,8 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Net;
|
using System.Net;
|
||||||
|
using System.Net.Security;
|
||||||
|
using System.Security.Authentication;
|
||||||
|
using Wabbajack.Common;
|
||||||
using SysHttp = System.Net.Http;
|
using SysHttp = System.Net.Http;
|
||||||
|
|
||||||
namespace Wabbajack.Lib.Http
|
namespace Wabbajack.Lib.Http
|
||||||
@ -18,8 +21,10 @@ namespace Wabbajack.Lib.Http
|
|||||||
CookieContainer = Cookies,
|
CookieContainer = Cookies,
|
||||||
MaxConnectionsPerServer = 20,
|
MaxConnectionsPerServer = 20,
|
||||||
PooledConnectionLifetime = TimeSpan.FromMilliseconds(100),
|
PooledConnectionLifetime = TimeSpan.FromMilliseconds(100),
|
||||||
PooledConnectionIdleTimeout = TimeSpan.FromMilliseconds(100)
|
PooledConnectionIdleTimeout = TimeSpan.FromMilliseconds(100),
|
||||||
|
|
||||||
};
|
};
|
||||||
|
Utils.Log($"Configuring with SSL {_socketsHandler.SslOptions.EnabledSslProtocols}");
|
||||||
Client = new SysHttp.HttpClient(_socketsHandler);
|
Client = new SysHttp.HttpClient(_socketsHandler);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,5 +1,7 @@
|
|||||||
using System;
|
using System;
|
||||||
|
using System.Text.Encodings.Web;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
|
using System.Web;
|
||||||
using Microsoft.Win32;
|
using Microsoft.Win32;
|
||||||
using Wabbajack.Common;
|
using Wabbajack.Common;
|
||||||
|
|
||||||
@ -78,5 +80,11 @@ namespace Wabbajack.Lib
|
|||||||
client.Headers.Add((Consts.MetricsKeyHeader, key));
|
client.Headers.Add((Consts.MetricsKeyHeader, key));
|
||||||
await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{value}");
|
await client.GetAsync($"{Consts.WabbajackBuildServerUri}metrics/{action}/{value}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static async Task Error(Type type, Exception exception)
|
||||||
|
{
|
||||||
|
await Send("Exception", type.Name + "|" + exception.Message);
|
||||||
|
await Send("ExceptionData" + type.Name + "|" + Consts.CurrentMinimumWabbajackVersion, HttpUtility.UrlEncode($"{exception.Message}\n{exception.StackTrace}"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -19,6 +19,10 @@ namespace Wabbajack.Lib.NexusApi
|
|||||||
public class NexusApiClient : ViewModel, INexusApi
|
public class NexusApiClient : ViewModel, INexusApi
|
||||||
{
|
{
|
||||||
private static readonly string API_KEY_CACHE_FILE = "nexus.key_cache";
|
private static readonly string API_KEY_CACHE_FILE = "nexus.key_cache";
|
||||||
|
/// <summary>
|
||||||
|
/// Forces the client to do manual downloading via CEF (for testing)
|
||||||
|
/// </summary>
|
||||||
|
private static bool ManualTestingMode = true;
|
||||||
|
|
||||||
public Http.Client HttpClient { get; } = new();
|
public Http.Client HttpClient { get; } = new();
|
||||||
|
|
||||||
@ -301,15 +305,18 @@ namespace Wabbajack.Lib.NexusApi
|
|||||||
return await Get<T>(builder.ToString(), HttpClient.WithHeader((Consts.MetricsKeyHeader, await Metrics.GetMetricsKey())));
|
return await Get<T>(builder.ToString(), HttpClient.WithHeader((Consts.MetricsKeyHeader, await Metrics.GetMetricsKey())));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static AsyncLock ManualDownloadLock = new();
|
||||||
public async Task<string> GetNexusDownloadLink(NexusDownloader.State archive)
|
public async Task<string> GetNexusDownloadLink(NexusDownloader.State archive)
|
||||||
{
|
{
|
||||||
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
|
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
|
||||||
|
|
||||||
var info = await GetModInfo(archive.Game, archive.ModID);
|
var info = await GetModInfo(archive.Game, archive.ModID);
|
||||||
if (!info.available)
|
var fileInfo = await GetModFiles(archive.Game, archive.ModID);
|
||||||
|
if (!info.available || !fileInfo.files.Any(f => f.file_id == archive.FileID && f.category_name != null))
|
||||||
throw new Exception("Mod unavailable");
|
throw new Exception("Mod unavailable");
|
||||||
|
|
||||||
if (await IsPremium())
|
if (await IsPremium() && !ManualTestingMode)
|
||||||
{
|
{
|
||||||
if (HourlyRemaining <= 0 && DailyRemaining <= 0)
|
if (HourlyRemaining <= 0 && DailyRemaining <= 0)
|
||||||
{
|
{
|
||||||
@ -323,6 +330,8 @@ namespace Wabbajack.Lib.NexusApi
|
|||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
using var _ = await ManualDownloadLock.WaitAsync();
|
||||||
|
await Task.Delay(1000);
|
||||||
Utils.Log($"Requesting manual download for {archive.Name} {archive.PrimaryKeyString}");
|
Utils.Log($"Requesting manual download for {archive.Name} {archive.PrimaryKeyString}");
|
||||||
return (await Utils.Log(await ManuallyDownloadNexusFile.Create(archive)).Task).ToString();
|
return (await Utils.Log(await ManuallyDownloadNexusFile.Create(archive)).Task).ToString();
|
||||||
}
|
}
|
||||||
|
@ -40,7 +40,7 @@ namespace Wabbajack.BuildServer.Test
|
|||||||
Consts.ModlistMetadataURL = modlist.ToString();
|
Consts.ModlistMetadataURL = modlist.ToString();
|
||||||
var sql = Fixture.GetService<SqlService>();
|
var sql = Fixture.GetService<SqlService>();
|
||||||
var downloader = Fixture.GetService<ModListDownloader>();
|
var downloader = Fixture.GetService<ModListDownloader>();
|
||||||
await downloader.CheckForNewLists();
|
await downloader.Execute();
|
||||||
|
|
||||||
foreach (var list in ModListMetaData)
|
foreach (var list in ModListMetaData)
|
||||||
{
|
{
|
||||||
@ -48,7 +48,7 @@ namespace Wabbajack.BuildServer.Test
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Nothing has changed so we shouldn't be downloading anything this time
|
// Nothing has changed so we shouldn't be downloading anything this time
|
||||||
Assert.Equal(0, await downloader.CheckForNewLists());
|
Assert.Equal(0, await downloader.Execute());
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -161,7 +161,7 @@ namespace Wabbajack.BuildServer.Test
|
|||||||
{
|
{
|
||||||
|
|
||||||
var downloader = Fixture.GetService<ModListDownloader>();
|
var downloader = Fixture.GetService<ModListDownloader>();
|
||||||
await downloader.CheckForNewLists();
|
await downloader.Execute();
|
||||||
|
|
||||||
if (runNonNexus)
|
if (runNonNexus)
|
||||||
{
|
{
|
||||||
|
@ -29,6 +29,7 @@ namespace Wabbajack.BuildServer
|
|||||||
{
|
{
|
||||||
private const string ProblemDetailsContentType = "application/problem+json";
|
private const string ProblemDetailsContentType = "application/problem+json";
|
||||||
private readonly SqlService _sql;
|
private readonly SqlService _sql;
|
||||||
|
private static ConcurrentHashSet<string> _knownKeys = new();
|
||||||
private const string ApiKeyHeaderName = "X-Api-Key";
|
private const string ApiKeyHeaderName = "X-Api-Key";
|
||||||
|
|
||||||
public ApiKeyAuthenticationHandler(
|
public ApiKeyAuthenticationHandler(
|
||||||
@ -44,7 +45,8 @@ namespace Wabbajack.BuildServer
|
|||||||
protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
|
protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
|
||||||
{
|
{
|
||||||
var metricsKey = Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault();
|
var metricsKey = Request.Headers[Consts.MetricsKeyHeader].FirstOrDefault();
|
||||||
await LogRequest(metricsKey);
|
// Never needed this, disabled for now
|
||||||
|
//await LogRequest(metricsKey);
|
||||||
if (metricsKey != default)
|
if (metricsKey != default)
|
||||||
{
|
{
|
||||||
if (await _sql.IsTarKey(metricsKey))
|
if (await _sql.IsTarKey(metricsKey))
|
||||||
@ -82,15 +84,16 @@ namespace Wabbajack.BuildServer
|
|||||||
return AuthenticateResult.Success(ticket);
|
return AuthenticateResult.Success(ticket);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!await _sql.ValidMetricsKey(metricsKey))
|
if (!_knownKeys.Contains(metricsKey) && !await _sql.ValidMetricsKey(metricsKey))
|
||||||
{
|
{
|
||||||
return AuthenticateResult.Fail("Invalid Metrics Key");
|
return AuthenticateResult.Fail("Invalid Metrics Key");
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
var claims = new List<Claim>();
|
_knownKeys.Add(metricsKey);
|
||||||
|
|
||||||
|
var claims = new List<Claim> {new(ClaimTypes.Role, "User")};
|
||||||
|
|
||||||
claims.Add(new Claim(ClaimTypes.Role, "User"));
|
|
||||||
|
|
||||||
var identity = new ClaimsIdentity(claims, Options.AuthenticationType);
|
var identity = new ClaimsIdentity(claims, Options.AuthenticationType);
|
||||||
var identities = new List<ClaimsIdentity> {identity};
|
var identities = new List<ClaimsIdentity> {identity};
|
||||||
|
@ -27,14 +27,16 @@ namespace Wabbajack.BuildServer.Controllers
|
|||||||
private ILogger<AuthoredFiles> _logger;
|
private ILogger<AuthoredFiles> _logger;
|
||||||
private AppSettings _settings;
|
private AppSettings _settings;
|
||||||
private CDNMirrorList _mirrorList;
|
private CDNMirrorList _mirrorList;
|
||||||
|
private DiscordWebHook _discord;
|
||||||
|
|
||||||
|
|
||||||
public AuthoredFiles(ILogger<AuthoredFiles> logger, SqlService sql, AppSettings settings, CDNMirrorList mirrorList)
|
public AuthoredFiles(ILogger<AuthoredFiles> logger, SqlService sql, AppSettings settings, CDNMirrorList mirrorList, DiscordWebHook discord)
|
||||||
{
|
{
|
||||||
_sql = sql;
|
_sql = sql;
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
_settings = settings;
|
_settings = settings;
|
||||||
_mirrorList = mirrorList;
|
_mirrorList = mirrorList;
|
||||||
|
_discord = discord;
|
||||||
}
|
}
|
||||||
|
|
||||||
[HttpPut]
|
[HttpPut]
|
||||||
@ -77,6 +79,9 @@ namespace Wabbajack.BuildServer.Controllers
|
|||||||
_logger.Log(LogLevel.Information, $"Creating File upload {definition.OriginalFileName}");
|
_logger.Log(LogLevel.Information, $"Creating File upload {definition.OriginalFileName}");
|
||||||
|
|
||||||
definition = await _sql.CreateAuthoredFile(definition, user);
|
definition = await _sql.CreateAuthoredFile(definition, user);
|
||||||
|
|
||||||
|
await _discord.Send(Channel.Ham,
|
||||||
|
new DiscordMessage() {Content = $"{user} has started uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})"});
|
||||||
|
|
||||||
return Ok(definition.ServerAssignedUniqueId);
|
return Ok(definition.ServerAssignedUniqueId);
|
||||||
}
|
}
|
||||||
@ -101,6 +106,9 @@ namespace Wabbajack.BuildServer.Controllers
|
|||||||
ms.Position = 0;
|
ms.Position = 0;
|
||||||
await UploadAsync(ms, $"{definition.MungedName}/definition.json.gz");
|
await UploadAsync(ms, $"{definition.MungedName}/definition.json.gz");
|
||||||
|
|
||||||
|
await _discord.Send(Channel.Ham,
|
||||||
|
new DiscordMessage {Content = $"{user} has finished uploading {definition.OriginalFileName} ({definition.Size.ToFileSizeString()})"});
|
||||||
|
|
||||||
return Ok($"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{definition.MungedName}");
|
return Ok($"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{definition.MungedName}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -100,5 +100,15 @@ namespace Wabbajack.Server.DataLayer
|
|||||||
Hash = t.Item2
|
Hash = t.Item2
|
||||||
}).ToList();
|
}).ToList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async Task<int> PurgeList(string machineURL)
|
||||||
|
{
|
||||||
|
await using var conn = await Open();
|
||||||
|
var ret1 = await conn.ExecuteAsync(@" delete from dbo.ModListArchives where MachineURL = @machineURL",
|
||||||
|
new {machineURL});
|
||||||
|
var ret2 = await conn.ExecuteAsync(@" delete from dbo.ModLists where MachineURL = @machineURL",
|
||||||
|
new {machineURL});
|
||||||
|
return ret1 + ret2;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -58,23 +58,34 @@ namespace Wabbajack.Server.Services
|
|||||||
}
|
}
|
||||||
await PurgeNexusCache(arg, parts[2]);
|
await PurgeNexusCache(arg, parts[2]);
|
||||||
}
|
}
|
||||||
else if (parts[1] == "cyberpunk")
|
else if (parts[1] == "quick-sync")
|
||||||
{
|
{
|
||||||
var random = new Random();
|
var options = await _quickSync.Report();
|
||||||
var releaseDate = new DateTime(2020, 12, 10, 0, 0, 0, DateTimeKind.Utc);
|
if (parts.Length != 3)
|
||||||
var r = releaseDate - DateTime.UtcNow;
|
|
||||||
if (r < TimeSpan.Zero)
|
|
||||||
{
|
{
|
||||||
await ReplyTo(arg, "It's out, what are you doing here?");
|
var optionsStr = string.Join(", ", options.Select(o => o.Key.Name));
|
||||||
|
await ReplyTo(arg, $"Can't expect me to quicksync the whole damn world! Try: {optionsStr}");
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
var msgs = (await "cyberpunk_message.txt".RelativeTo(AbsolutePath.EntryPoint)
|
foreach (var pair in options.Where(o => o.Key.Name == parts[2]))
|
||||||
.ReadAllLinesAsync()).ToArray();
|
{
|
||||||
var msg = msgs[random.Next(0, msgs.Length)];
|
await _quickSync.Notify(pair.Key);
|
||||||
var fullmsg = String.Format(msg,
|
await ReplyTo(arg, $"Notified {pair.Key}");
|
||||||
$"{r.Days} days, {r.Hours} hours, {r.Minutes} minutes, {r.Seconds} seconds");
|
}
|
||||||
await ReplyTo(arg, fullmsg);
|
}
|
||||||
|
}
|
||||||
|
else if (parts[1] == "purge-list")
|
||||||
|
{
|
||||||
|
if (parts.Length != 3)
|
||||||
|
{
|
||||||
|
await ReplyTo(arg, $"Yeah, I'm not gonna purge the whole server...");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
var deleted = await _sql.PurgeList(parts[2]);
|
||||||
|
await _quickSync.Notify<ModListDownloader>();
|
||||||
|
await ReplyTo(arg, $"Purged all traces of #{parts[2]} from the server, triggered list downloading. {deleted} records removed");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -16,15 +16,14 @@ using Wabbajack.Server.DTOs;
|
|||||||
|
|
||||||
namespace Wabbajack.Server.Services
|
namespace Wabbajack.Server.Services
|
||||||
{
|
{
|
||||||
public class ModListDownloader
|
public class ModListDownloader : AbstractService<ModListDownloader, int>
|
||||||
{
|
{
|
||||||
private ILogger<ModListDownloader> _logger;
|
|
||||||
private AppSettings _settings;
|
|
||||||
private ArchiveMaintainer _maintainer;
|
private ArchiveMaintainer _maintainer;
|
||||||
private SqlService _sql;
|
private SqlService _sql;
|
||||||
private DiscordWebHook _discord;
|
private DiscordWebHook _discord;
|
||||||
|
|
||||||
public ModListDownloader(ILogger<ModListDownloader> logger, AppSettings settings, ArchiveMaintainer maintainer, SqlService sql, DiscordWebHook discord)
|
public ModListDownloader(ILogger<ModListDownloader> logger, AppSettings settings, ArchiveMaintainer maintainer, SqlService sql, DiscordWebHook discord, QuickSync quickSync)
|
||||||
|
: base(logger, settings, quickSync, TimeSpan.FromMinutes(1))
|
||||||
{
|
{
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
_settings = settings;
|
_settings = settings;
|
||||||
@ -33,32 +32,8 @@ namespace Wabbajack.Server.Services
|
|||||||
_discord = discord;
|
_discord = discord;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void Start()
|
|
||||||
{
|
|
||||||
if (_settings.RunBackEndJobs)
|
|
||||||
{
|
|
||||||
Task.Run(async () =>
|
|
||||||
{
|
|
||||||
while (true)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
_logger.Log(LogLevel.Information, "Checking for updated mod lists");
|
|
||||||
await CheckForNewLists();
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
_logger.LogError(ex, "Error checking list");
|
|
||||||
}
|
|
||||||
|
|
||||||
await Task.Delay(TimeSpan.FromMinutes(5));
|
public override async Task<int> Execute()
|
||||||
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<int> CheckForNewLists()
|
|
||||||
{
|
{
|
||||||
int downloaded = 0;
|
int downloaded = 0;
|
||||||
var lists = (await ModlistMetadata.LoadFromGithub())
|
var lists = (await ModlistMetadata.LoadFromGithub())
|
||||||
@ -68,6 +43,7 @@ namespace Wabbajack.Server.Services
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
ReportStarting(list.Links.MachineURL);
|
||||||
if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
|
if (await _sql.HaveIndexedModlist(list.Links.MachineURL, list.DownloadMetadata.Hash))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
@ -76,7 +52,10 @@ namespace Wabbajack.Server.Services
|
|||||||
{
|
{
|
||||||
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
|
_logger.Log(LogLevel.Information, $"Downloading {list.Links.MachineURL}");
|
||||||
await _discord.Send(Channel.Ham,
|
await _discord.Send(Channel.Ham,
|
||||||
new DiscordMessage {Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"});
|
new DiscordMessage
|
||||||
|
{
|
||||||
|
Content = $"Downloading {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
|
||||||
|
});
|
||||||
var tf = new TempFile();
|
var tf = new TempFile();
|
||||||
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
|
var state = DownloadDispatcher.ResolveArchive(list.Links.Download);
|
||||||
if (state == null)
|
if (state == null)
|
||||||
@ -110,7 +89,10 @@ namespace Wabbajack.Server.Services
|
|||||||
{
|
{
|
||||||
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
|
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
|
||||||
await _discord.Send(Channel.Ham,
|
await _discord.Send(Channel.Ham,
|
||||||
new DiscordMessage {Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"});
|
new DiscordMessage
|
||||||
|
{
|
||||||
|
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
|
||||||
|
});
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -122,7 +104,10 @@ namespace Wabbajack.Server.Services
|
|||||||
{
|
{
|
||||||
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
|
_logger.LogWarning($"Bad Modlist {list.Links.MachineURL}");
|
||||||
await _discord.Send(Channel.Ham,
|
await _discord.Send(Channel.Ham,
|
||||||
new DiscordMessage {Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"});
|
new DiscordMessage
|
||||||
|
{
|
||||||
|
Content = $"Bad Modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
|
||||||
|
});
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -133,7 +118,15 @@ namespace Wabbajack.Server.Services
|
|||||||
{
|
{
|
||||||
_logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
|
_logger.LogError(ex, $"Error downloading modlist {list.Links.MachineURL}");
|
||||||
await _discord.Send(Channel.Ham,
|
await _discord.Send(Channel.Ham,
|
||||||
new DiscordMessage {Content = $"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"});
|
new DiscordMessage
|
||||||
|
{
|
||||||
|
Content =
|
||||||
|
$"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
ReportEnding(list.Links.MachineURL);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");
|
_logger.Log(LogLevel.Information, $"Done checking modlists. Downloaded {downloaded} new lists");
|
||||||
|
@ -67,5 +67,16 @@ namespace Wabbajack.Server.Services
|
|||||||
ct.Cancel();
|
ct.Cancel();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async Task Notify(Type t)
|
||||||
|
{
|
||||||
|
_logger.LogInformation($"Quicksync {t.Name}");
|
||||||
|
// Needs debugging
|
||||||
|
using var _ = await _lock.WaitAsync();
|
||||||
|
if (_syncs.TryGetValue(t, out var ct))
|
||||||
|
{
|
||||||
|
ct.Cancel();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
24
Wabbajack.Test/MetricsTests.cs
Normal file
24
Wabbajack.Test/MetricsTests.cs
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
using System;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Wabbajack.Lib;
|
||||||
|
using Xunit;
|
||||||
|
|
||||||
|
namespace Wabbajack.Test
|
||||||
|
{
|
||||||
|
public class MetricsTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task CanSendExceptions()
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
throw new Exception("Test Exception");
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
await Metrics.Error(this.GetType(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
@ -44,6 +44,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Server", "Wabbaja
|
|||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Server.Test", "Wabbajack.Server.Test\Wabbajack.Server.Test.csproj", "{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Server.Test", "Wabbajack.Server.Test\Wabbajack.Server.Test.csproj", "{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}"
|
||||||
EndProject
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Benchmark", "Wabbajack.Benchmark\Wabbajack.Benchmark.csproj", "{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}"
|
||||||
|
EndProject
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|Any CPU = Debug|Any CPU
|
Debug|Any CPU = Debug|Any CPU
|
||||||
@ -165,6 +167,14 @@ Global
|
|||||||
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|Any CPU.Build.0 = Release|Any CPU
|
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|x64.ActiveCfg = Release|Any CPU
|
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|x64.Build.0 = Release|Any CPU
|
{9DEC8DC8-B6E0-469B-9571-C4BAC0776D07}.Release|x64.Build.0 = Release|Any CPU
|
||||||
|
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||||
|
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Debug|x64.Build.0 = Debug|Any CPU
|
||||||
|
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Release|x64.ActiveCfg = Release|Any CPU
|
||||||
|
{DDB89C1F-FFA8-4CC2-B202-2DF13FCBC6C7}.Release|x64.Build.0 = Release|Any CPU
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
|
@ -230,8 +230,7 @@ namespace Wabbajack
|
|||||||
ModlistIsNSFW = ModlistSettings.IsNSFW
|
ModlistIsNSFW = ModlistSettings.IsNSFW
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
using (ActiveCompilation = compiler
|
using (ActiveCompilation = compiler)
|
||||||
)
|
|
||||||
{
|
{
|
||||||
Parent.MWVM.Settings.Performance.SetProcessorSettings(ActiveCompilation);
|
Parent.MWVM.Settings.Performance.SetProcessorSettings(ActiveCompilation);
|
||||||
var success = await ActiveCompilation.Begin();
|
var success = await ActiveCompilation.Begin();
|
||||||
|
@ -139,7 +139,7 @@ namespace Wabbajack
|
|||||||
};
|
};
|
||||||
await vm.Driver.WaitForInitialized();
|
await vm.Driver.WaitForInitialized();
|
||||||
IWebDriver browser = new CefSharpWrapper(vm.Browser);
|
IWebDriver browser = new CefSharpWrapper(vm.Browser);
|
||||||
vm.Instructions = $"Please Download {state.Name} - {state.ModID} - {state.FileID}";
|
vm.Instructions = $"Click the highlighted file (get a NexusMods.com Premium account to automate this)";
|
||||||
browser.DownloadHandler = uri =>
|
browser.DownloadHandler = uri =>
|
||||||
{
|
{
|
||||||
manuallyDownloadNexusFile.Resume(uri);
|
manuallyDownloadNexusFile.Resume(uri);
|
||||||
|
Loading…
Reference in New Issue
Block a user