Merge branch 'master' into mega-json-fix

This commit is contained in:
Timothy Baldridge 2020-04-29 10:40:06 -06:00 committed by GitHub
commit d0f5e7acb4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 218 additions and 114 deletions

View File

@ -280,7 +280,7 @@ namespace Compression.BSA
case VersionType.SSE:
{
var r = new MemoryStream();
await using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L12_MAX}, true))
await using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L08_HC}, true))
{
await _srcData.CopyToWithStatusAsync(_srcData.Length, w, $"Compressing {_path}");
}

View File

@ -7,7 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.console" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />

View File

@ -7,7 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="Xunit.Priority" Version="1.1.6" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />

View File

@ -1,6 +1,8 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Identity;
using Wabbajack.BuildServer.Controllers;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
@ -24,7 +26,18 @@ namespace Wabbajack.BuildServer.BackendServices
{
try
{
var isValid = await archive.State.Verify(archive);
bool isValid;
switch (archive.State)
{
case GoogleDriveDownloader.State _:
case ManualDownloader.State _:
case HTTPDownloader.State s when new Uri(s.Url).Host.StartsWith("wabbajackpush"):
isValid = true;
break;
default:
isValid = await archive.State.Verify(archive);
break;
}
return (Archive: archive, IsValid: isValid);
}
catch (Exception ex)

View File

@ -44,7 +44,8 @@ namespace Wabbajack.BuildServer.Controllers
return Ok(new HeartbeatResult
{
Uptime = DateTime.Now - _startTime,
LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync
LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync,
LastListValidation = DateTime.UtcNow - ListValidation.SummariesLastChecked
});
}
@ -53,6 +54,8 @@ namespace Wabbajack.BuildServer.Controllers
{
public TimeSpan Uptime { get; set; }
public TimeSpan LastNexusUpdate { get; set; }
public TimeSpan LastListValidation { get; set; }
}
[HttpGet("only-authenticated")]

View File

@ -17,6 +17,7 @@ using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.BuildServer.Controllers
{
@ -24,7 +25,7 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/lists")]
public class ListValidation : AControllerBase<ListValidation>
{
enum ArchiveStatus
public enum ArchiveStatus
{
Valid,
InValid,
@ -37,6 +38,8 @@ namespace Wabbajack.BuildServer.Controllers
_updater = new ModlistUpdater(null, sql, settings);
_settings = settings;
Cache = cache;
_nexusClient = NexusApiClient.Get();
}
public static IMemoryCache Cache { get; set; }
@ -44,76 +47,97 @@ namespace Wabbajack.BuildServer.Controllers
public static void ResetCache()
{
Cache?.Remove(ModListSummariesKey);
SummariesLastChecked = DateTime.UnixEpoch;
ModListSummaries = null;
}
private static IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> ModListSummaries = null;
public static DateTime SummariesLastChecked = DateTime.UnixEpoch;
private static AsyncLock UpdateLock = new AsyncLock();
public async Task<IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)>> GetSummaries()
{
if (Cache.TryGetValue(ModListSummariesKey, out object result))
static bool TimesUp()
{
return (IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)>)result;
return DateTime.UtcNow - SummariesLastChecked > TimeSpan.FromMinutes(5);
}
if (ModListSummaries != null && !TimesUp())
{
return ModListSummaries;
}
var data = await SQL.GetValidationData();
using var queue = new WorkQueue();
var results = await data.ModLists.PMap(queue, async list =>
var task = Task.Run(async () =>
{
var (metadata, modList) = list;
var archives = await modList.Archives.PMap(queue, async archive =>
using var _ = await UpdateLock.WaitAsync();
if (ModListSummaries != null && !TimesUp())
{
var (_, result) = ValidateArchive(data, archive);
if (result == ArchiveStatus.InValid)
return ModListSummaries;
}
SummariesLastChecked = DateTime.UtcNow;
var data = await SQL.GetValidationData();
using var queue = new WorkQueue();
var results = await data.ModLists.PMap(queue, async list =>
{
var (metadata, modList) = list;
var archives = await modList.Archives.PMap(queue, async archive =>
{
var fixResult = await TryToFix(data, archive);
return fixResult;
var (_, result) = await ValidateArchive(data, archive);
if (result != ArchiveStatus.InValid) return (archive, result);
}
return await TryToFix(data, archive);
return (archive, result);
});
var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid);
var passCount = archives.Count(f =>
f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated);
var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating);
var summary = new ModListSummary
{
Checked = DateTime.UtcNow,
Failed = failedCount,
Passed = passCount,
Updating = updatingCount,
MachineURL = metadata.Links.MachineURL,
Name = metadata.Title,
};
var detailed = new DetailedStatus
{
Name = metadata.Title,
Checked = DateTime.UtcNow,
DownloadMetaData = metadata.DownloadMetadata,
HasFailures = failedCount > 0,
MachineName = metadata.Links.MachineURL,
Archives = archives.Select(a => new DetailedStatusItem
{
Archive = a.Item1,
IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating
}).ToList()
};
return (summary, detailed);
});
var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid);
var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated);
var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating);
var summary = new ModListSummary
{
Checked = DateTime.UtcNow,
Failed = failedCount,
Passed = passCount,
Updating = updatingCount,
MachineURL = metadata.Links.MachineURL,
Name = metadata.Title,
};
var detailed = new DetailedStatus
{
Name = metadata.Title,
Checked = DateTime.UtcNow,
DownloadMetaData = metadata.DownloadMetadata,
HasFailures = failedCount > 0,
MachineName = metadata.Links.MachineURL,
Archives = archives.Select(a => new DetailedStatusItem
{
Archive = a.Item1, IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating
}).ToList()
};
return (summary, detailed);
var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1));
Cache.Set(ModListSummariesKey, results, cacheOptions);
ModListSummaries = results;
return results;
});
var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1));
Cache.Set(ModListSummariesKey, results, cacheOptions);
return results;
var data = ModListSummaries;
if (data == null)
return await task;
return data;
}
private static (Archive archive, ArchiveStatus) ValidateArchive(SqlService.ValidationData data, Archive archive)
private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(SqlService.ValidationData data, Archive archive)
{
switch (archive.State)
{
@ -123,8 +147,10 @@ namespace Wabbajack.BuildServer.Controllers
case NexusDownloader.State nexusState when data.NexusFiles.Contains((
nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)):
return (archive, ArchiveStatus.Valid);
case NexusDownloader.State _:
return (archive, ArchiveStatus.InValid);
case NexusDownloader.State ns:
return (archive, await FastNexusModStats(ns));
case HTTPDownloader.State s when new Uri(s.Url).Host.StartsWith("wabbajackpush"):
return (archive, ArchiveStatus.Valid);
case ManualDownloader.State _:
return (archive, ArchiveStatus.Valid);
default:
@ -140,6 +166,47 @@ namespace Wabbajack.BuildServer.Controllers
}
}
private async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns)
{
var mod = await SQL.GetNexusModInfoString(ns.Game, ns.ModID);
var files = await SQL.GetModFiles(ns.Game, ns.ModID);
if (mod == null)
{
Utils.Log($"Found missing Nexus mod info {ns.Game} {ns.ModID}");
mod = await (await _nexusClient).GetModInfo(ns.Game, ns.ModID, false);
try
{
await SQL.AddNexusModInfo(ns.Game, ns.ModID, mod.updated_time, mod);
}
catch (Exception _)
{
// Could be a PK constraint failure
}
}
if (files == null)
{
Utils.Log($"Found missing Nexus mod file infos {ns.Game} {ns.ModID}");
files = await (await _nexusClient).GetModFiles(ns.Game, ns.ModID, false);
try
{
await SQL.AddNexusModFiles(ns.Game, ns.ModID, mod.updated_time, files);
}
catch (Exception _)
{
// Could be a PK constraint failure
}
}
if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID))
return ArchiveStatus.Valid;
return ArchiveStatus.InValid;
}
private static AsyncLock _findPatchLock = new AsyncLock();
private async Task<(Archive, ArchiveStatus)> TryToFix(SqlService.ValidationData data, Archive archive)
{
@ -219,6 +286,7 @@ namespace Wabbajack.BuildServer.Controllers
private AppSettings _settings;
private ModlistUpdater _updater;
private Task<NexusApiClient> _nexusClient;
[HttpGet]
[Route("status/{Name}.html")]

View File

@ -88,14 +88,12 @@ namespace Wabbajack.BuildServer.Controllers
public async Task<IActionResult> GetAlternative(string xxHash)
{
var startingHash = Hash.FromHex(xxHash);
Utils.Log($"Alternative requested for {startingHash}");
await Metric("requested_upgrade", startingHash.ToString());
var archive = await SQL.GetStateByHash(startingHash);
if (archive == null)
{
Utils.Log($"No original state for {startingHash}");
return NotFound("Original state not found");
}
@ -120,7 +118,6 @@ namespace Wabbajack.BuildServer.Controllers
}
Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}");
if (newArchive.Hash == Hash.Empty)
{
await SQL.EnqueueJob(new Job
@ -138,6 +135,7 @@ namespace Wabbajack.BuildServer.Controllers
}
}
});
Utils.Log($"Enqueued Index and Upgrade for {startingHash} -> {newArchive.State.PrimaryKeyString}");
return Accepted("Enqueued for Processing");
}
@ -155,6 +153,7 @@ namespace Wabbajack.BuildServer.Controllers
DestPK = newArchive.State.PrimaryKeyString
}
});
Utils.Log($"Enqueued Upgrade for {startingHash} -> {newArchive.State.PrimaryKeyString}");
}
return Ok(newArchive.ToJson());
}

View File

@ -18,7 +18,7 @@
<ItemGroup>
<PackageReference Include="CsvHelper" Version="15.0.5" />
<PackageReference Include="Dapper" Version="2.0.35" />
<PackageReference Include="FluentFTP" Version="32.3.3" />
<PackageReference Include="FluentFTP" Version="32.4.0" />
<PackageReference Include="graphiql" Version="2.0.0" />
<PackageReference Include="GraphQL" Version="3.0.0-preview-1352" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
@ -28,7 +28,7 @@
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Microsoft.OpenApi" Version="1.1.4" />
<PackageReference Include="Nettle" Version="1.3.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.3.2" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.4.0" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.1" />
</ItemGroup>

View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
namespace Wabbajack.Common.StoreHandlers
{
@ -13,7 +14,7 @@ namespace Wabbajack.Common.StoreHandlers
public class StoreHandler
{
private static readonly Lazy<StoreHandler> _instance = new Lazy<StoreHandler>(() => new StoreHandler(), true);
private static readonly Lazy<StoreHandler> _instance = new Lazy<StoreHandler>(() => new StoreHandler(), isThreadSafe: true);
public static StoreHandler Instance => _instance.Value;
private static readonly Lazy<SteamHandler> _steamHandler = new Lazy<SteamHandler>(() => new SteamHandler());
@ -72,6 +73,11 @@ namespace Wabbajack.Common.StoreHandlers
{
return StoreGames.FirstOrDefault(g => g.Game == game)?.Path;
}
public static void Warmup()
{
Task.Run(() => _instance.Value).FireAndForget();
}
}
public abstract class AStoreGame

View File

@ -18,7 +18,10 @@ namespace Wabbajack.Common
_cleanTask = Task.Run(() => "tmp_files".RelativeTo(AbsolutePath.EntryPoint).DeleteDirectory());
}
public static void Init()
/// <summary>
/// Starts the initialization in a background task
/// </summary>
public static void Warmup()
{
// Nothing to do, as work is done in static ctor
}

View File

@ -32,7 +32,7 @@
<PackageReference Include="Microsoft.Win32.Registry" Version="4.7.0" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="Octodiff" Version="1.2.1" />
<PackageReference Include="ReactiveUI" Version="11.3.1" />
<PackageReference Include="ReactiveUI" Version="11.3.8" />
<PackageReference Include="SharpZipLib" Version="1.2.0" />
<PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
<PackageReference Include="System.Net.Http" Version="4.3.4" />

View File

@ -133,28 +133,31 @@ namespace Wabbajack.Lib.Downloaders
private static MegaApiClient MegaApiClient => DownloadDispatcher.GetInstance<MegaDownloader>().MegaApiClient;
private static void MegaLogin()
private static AsyncLock _loginLock = new AsyncLock();
private static async Task MegaLogin()
{
using var _ = await _loginLock.WaitAsync();
if (MegaApiClient.IsLoggedIn)
return;
if (!Utils.HaveEncryptedJson(DataName))
{
Utils.Status("Logging into MEGA (as anonymous)");
MegaApiClient.LoginAnonymous();
await MegaApiClient.LoginAnonymousAsync();
}
else
{
Utils.Status("Logging into MEGA with saved credentials.");
var infos = Utils.FromEncryptedJson<MEGAAuthInfos>(DataName);
var authInfo = infos.ToAuthInfos();
MegaApiClient.Login(authInfo);
await MegaApiClient.LoginAsync(authInfo);
}
}
public override async Task<bool> Download(Archive a, AbsolutePath destination)
{
MegaLogin();
await MegaLogin();
var fileLink = new Uri(Url);
Utils.Status($"Downloading MEGA file: {a.Name}");
@ -164,7 +167,7 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Verify(Archive a)
{
MegaLogin();
await MegaLogin();
var fileLink = new Uri(Url);
try

View File

@ -35,10 +35,10 @@
<Version>2.1.0</Version>
</PackageReference>
<PackageReference Include="ReactiveUI">
<Version>11.3.1</Version>
<Version>11.3.8</Version>
</PackageReference>
<PackageReference Include="ReactiveUI.Fody">
<Version>11.3.1</Version>
<Version>11.3.8</Version>
</PackageReference>
<PackageReference Include="SharpCompress">
<Version>0.25.0</Version>
@ -65,7 +65,7 @@
<Version>1.0.0</Version>
</PackageReference>
<PackageReference Include="YoutubeExplode">
<Version>5.0.1</Version>
<Version>5.0.2</Version>
</PackageReference>
</ItemGroup>
<ItemGroup>

View File

@ -18,6 +18,9 @@ namespace Wabbajack.Lib.WebAutomation
public CefSharpWrapper(IWebBrowser browser)
{
_browser = browser;
_browser.DownloadHandler = new DownloadHandler(this);
_browser.LifeSpanHandler = new PopupBlocker(this);
}
public Task NavigateTo(Uri uri)
@ -33,11 +36,9 @@ namespace Wabbajack.Lib.WebAutomation
tcs.SetResult(true);
}
};
_browser.LoadingStateChanged += handler;
_browser.Load(uri.ToString());
_browser.DownloadHandler = new DownloadHandler(this);
_browser.LifeSpanHandler = new PopupBlocker(this);
return tcs.Task;
}
@ -80,7 +81,7 @@ namespace Wabbajack.Lib.WebAutomation
IWindowInfo windowInfo, IBrowserSettings browserSettings, ref bool noJavascriptAccess, out IWebBrowser? newBrowser)
{
// Block popups
newBrowser = null;
newBrowser = chromiumWebBrowser;
return true;
}
@ -90,7 +91,7 @@ namespace Wabbajack.Lib.WebAutomation
public bool DoClose(IWebBrowser chromiumWebBrowser, IBrowser browser)
{
return true;
return false;
}
public void OnBeforeClose(IWebBrowser chromiumWebBrowser, IBrowser browser)

View File

@ -27,7 +27,7 @@
<ItemGroup>
<PackageReference Include="CefSharp.Common" Version="79.1.360" />
<PackageReference Include="CefSharp.OffScreen" Version="79.1.360" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
<PackageReference Include="coverlet.collector" Version="1.2.1" />

View File

@ -1,15 +1,6 @@
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Data;
using System.Linq;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Interop;
using System.Windows.Media;
using Wabbajack.Common;
using Wabbajack.Common.StoreHandlers;
using Wabbajack.Util;
namespace Wabbajack
{
@ -23,7 +14,6 @@ namespace Wabbajack
CLIOld.ParseOptions(Environment.GetCommandLineArgs());
if (CLIArguments.Help)
CLIOld.DisplayHelpText();
var storeHandler = new StoreHandler();
}
}
}

View File

@ -6,6 +6,7 @@ using System.Windows;
using MahApps.Metro.Controls;
using Newtonsoft.Json;
using Wabbajack.Common;
using Wabbajack.Common.StoreHandlers;
using Wabbajack.Lib.LibCefHelpers;
using Wabbajack.Util;
using Application = System.Windows.Application;
@ -23,8 +24,6 @@ namespace Wabbajack
public MainWindow()
{
TempFolder.Init();
Helpers.Init();
// Wire any unhandled crashing exceptions to log before exiting
AppDomain.CurrentDomain.UnhandledException += (sender, e) =>
{
@ -44,22 +43,7 @@ namespace Wabbajack
Utils.Log(
$"System settings - ({p.SystemMemorySize.ToFileSizeString()} RAM), Display: {p.ScreenWidth} x {p.ScreenHeight} ({p.VideoMemorySize.ToFileSizeString()} VRAM - VideoMemorySizeMb={p.EnbLEVRAMSize})");
// Run logic to associate wabbajack lists with this app in the background
Task.Run(async () =>
{
var appPath = System.Reflection.Assembly.GetExecutingAssembly().Location;
try
{
if (!ModListAssociationManager.IsAssociated() || ModListAssociationManager.NeedsUpdating(appPath))
{
ModListAssociationManager.Associate(appPath);
}
}
catch (Exception e)
{
Utils.Log($"ExtensionManager had an exception:\n{e}");
}
}).FireAndForget();
Warmup();
// Load settings
if (CLIArguments.NoSettings || !MainSettings.TryLoadTypicalSettings(out var settings))
@ -100,6 +84,40 @@ namespace Wabbajack
_settings = settings;
}
/// <summary>
/// Starts some background initialization tasks spinning so they're already prepped when actually needed
/// </summary>
private void Warmup()
{
TempFolder.Warmup();
// ToDo
// Currently this is a blocking call. Perhaps upgrade to be run in a background task.
// Would first need to ensure users of CEF properly await the background initialization before use
Helpers.Init();
StoreHandler.Warmup();
Task.Run(AssociateListsWithWabbajack).FireAndForget();
}
/// <summary>
/// Run logic to associate wabbajack lists with this app in the background
/// </summary>
private void AssociateListsWithWabbajack()
{
var appPath = System.Reflection.Assembly.GetExecutingAssembly().Location;
try
{
if (!ModListAssociationManager.IsAssociated() || ModListAssociationManager.NeedsUpdating(appPath))
{
ModListAssociationManager.Associate(appPath);
}
}
catch (Exception e)
{
Utils.Log($"ExtensionManager had an exception:\n{e}");
}
}
private void RunWhenLoaded(Action a)
{
if (IsLoaded)

View File

@ -57,7 +57,7 @@
<ItemGroup>
<PackageReference Include="CefSharp.Wpf" Version="79.1.360" />
<PackageReference Include="DynamicData" Version="6.14.10" />
<PackageReference Include="DynamicData" Version="6.14.14" />
<PackageReference Include="Extended.Wpf.Toolkit" Version="3.8.1" />
<PackageReference Include="Fody" Version="6.1.1">
<PrivateAssets>all</PrivateAssets>
@ -72,9 +72,9 @@
<PackageReference Include="MahApps.Metro.IconPacks" Version="4.0.0" />
<PackageReference Include="PInvoke.Gdi32" Version="0.6.6" />
<PackageReference Include="PInvoke.User32" Version="0.6.6" />
<PackageReference Include="ReactiveUI" Version="11.3.1" />
<PackageReference Include="ReactiveUI.Fody" Version="11.3.1" />
<PackageReference Include="ReactiveUI.WPF" Version="11.3.1" />
<PackageReference Include="ReactiveUI" Version="11.3.8" />
<PackageReference Include="ReactiveUI.Fody" Version="11.3.8" />
<PackageReference Include="ReactiveUI.WPF" Version="11.3.8" />
<PackageReference Include="SharpDX.DXGI" Version="4.2.0" />
<PackageReference Include="WindowsAPICodePack-Shell" Version="1.1.1" />
<PackageReference Include="WPFThemes.DarkBlend" Version="1.0.8" />