Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Bunch of small server-side fixes, LZ4 compression speed, and better list validation

Browse Source
This commit is contained in:
Timothy Baldridge 2020-04-29 06:26:44 -06:00
parent e24cbbeb0f
commit 64540e2cab
13 changed files with 165 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Compression.BSA/BSABuilder.cs
View File

@ -280,7 +280,7 @@ namespace Compression.BSA
case VersionType.SSE:
{
var r = new MemoryStream();
await using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L12_MAX}, true))
await using (var w = LZ4Stream.Encode(r, new LZ4EncoderSettings {CompressionLevel = LZ4Level.L08_HC}, true))
{
await _srcData.CopyToWithStatusAsync(_srcData.Length, w, $"Compressing {_path}");
}

2
Wabbajack.App.Test/Wabbajack.App.Test.csproj
View File

@ -7,7 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.console" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />

2
Wabbajack.BuildServer.Test/Wabbajack.BuildServer.Test.csproj
View File

@ -7,7 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="Xunit.Priority" Version="1.1.6" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />

15
Wabbajack.BuildServer/BackendServices/ValidateNonNexusArchives.cs
View File

@ -1,6 +1,8 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Identity;
using Wabbajack.BuildServer.Controllers;
using Wabbajack.BuildServer.Model.Models;
using Wabbajack.Common;
using Wabbajack.Lib.Downloaders;
@ -24,7 +26,18 @@ namespace Wabbajack.BuildServer.BackendServices
{
try
{
var isValid = await archive.State.Verify(archive);
bool isValid;
switch (archive.State)
{
case GoogleDriveDownloader.State _:
case ManualDownloader.State _:
case HTTPDownloader.State s when new Uri(s.Url).Host.StartsWith("wabbajackpush"):
isValid = true;
break;
default:
isValid = await archive.State.Verify(archive);
break;
}
return (Archive: archive, IsValid: isValid);
}
catch (Exception ex)

5
Wabbajack.BuildServer/Controllers/Heartbeat.cs
View File

@ -44,7 +44,8 @@ namespace Wabbajack.BuildServer.Controllers
return Ok(new HeartbeatResult
{
Uptime = DateTime.Now - _startTime,
LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync
LastNexusUpdate = DateTime.Now - GetNexusUpdatesJob.LastNexusSync,
LastListValidation = DateTime.UtcNow - ListValidation.SummariesLastChecked
});
}
@ -53,6 +54,8 @@ namespace Wabbajack.BuildServer.Controllers
{
public TimeSpan Uptime { get; set; }
public TimeSpan LastNexusUpdate { get; set; }
public TimeSpan LastListValidation { get; set; }
}
[HttpGet("only-authenticated")]

178
Wabbajack.BuildServer/Controllers/ListValidation.cs
View File

@ -17,6 +17,7 @@ using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Lib.ModListRegistry;
using Wabbajack.Lib.NexusApi;
namespace Wabbajack.BuildServer.Controllers
{
@ -24,7 +25,7 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/lists")]
public class ListValidation : AControllerBase<ListValidation>
{
enum ArchiveStatus
public enum ArchiveStatus
{
Valid,
InValid,
@ -37,6 +38,8 @@ namespace Wabbajack.BuildServer.Controllers
_updater = new ModlistUpdater(null, sql, settings);
_settings = settings;
Cache = cache;
_nexusClient = NexusApiClient.Get();
}
public static IMemoryCache Cache { get; set; }
@ -44,76 +47,97 @@ namespace Wabbajack.BuildServer.Controllers
public static void ResetCache()
{
Cache?.Remove(ModListSummariesKey);
SummariesLastChecked = DateTime.UnixEpoch;
ModListSummaries = null;
}
private static IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> ModListSummaries = null;
public static DateTime SummariesLastChecked = DateTime.UnixEpoch;
private static AsyncLock UpdateLock = new AsyncLock();
public async Task<IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)>> GetSummaries()
{
if (Cache.TryGetValue(ModListSummariesKey, out object result))
static bool TimesUp()
{
return (IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)>)result;
return DateTime.UtcNow - SummariesLastChecked > TimeSpan.FromMinutes(5);
}
if (ModListSummaries != null && !TimesUp())
{
return ModListSummaries;
}
var data = await SQL.GetValidationData();
using var queue = new WorkQueue();
var results = await data.ModLists.PMap(queue, async list =>
var task = Task.Run(async () =>
{
var (metadata, modList) = list;
var archives = await modList.Archives.PMap(queue, async archive =>
using var _ = await UpdateLock.WaitAsync();
if (ModListSummaries != null && !TimesUp())
{
var (_, result) = ValidateArchive(data, archive);
if (result == ArchiveStatus.InValid)
return ModListSummaries;
}
SummariesLastChecked = DateTime.UtcNow;
var data = await SQL.GetValidationData();
using var queue = new WorkQueue();
var results = await data.ModLists.PMap(queue, async list =>
{
var (metadata, modList) = list;
var archives = await modList.Archives.PMap(queue, async archive =>
{
var fixResult = await TryToFix(data, archive);
return fixResult;
var (_, result) = await ValidateArchive(data, archive);
if (result != ArchiveStatus.InValid) return (archive, result);
}
return await TryToFix(data, archive);
return (archive, result);
});
var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid);
var passCount = archives.Count(f =>
f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated);
var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating);
var summary = new ModListSummary
{
Checked = DateTime.UtcNow,
Failed = failedCount,
Passed = passCount,
Updating = updatingCount,
MachineURL = metadata.Links.MachineURL,
Name = metadata.Title,
};
var detailed = new DetailedStatus
{
Name = metadata.Title,
Checked = DateTime.UtcNow,
DownloadMetaData = metadata.DownloadMetadata,
HasFailures = failedCount > 0,
MachineName = metadata.Links.MachineURL,
Archives = archives.Select(a => new DetailedStatusItem
{
Archive = a.Item1,
IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating
}).ToList()
};
return (summary, detailed);
});
var failedCount = archives.Count(f => f.Item2 == ArchiveStatus.InValid);
var passCount = archives.Count(f => f.Item2 == ArchiveStatus.Valid || f.Item2 == ArchiveStatus.Updated);
var updatingCount = archives.Count(f => f.Item2 == ArchiveStatus.Updating);
var summary = new ModListSummary
{
Checked = DateTime.UtcNow,
Failed = failedCount,
Passed = passCount,
Updating = updatingCount,
MachineURL = metadata.Links.MachineURL,
Name = metadata.Title,
};
var detailed = new DetailedStatus
{
Name = metadata.Title,
Checked = DateTime.UtcNow,
DownloadMetaData = metadata.DownloadMetadata,
HasFailures = failedCount > 0,
MachineName = metadata.Links.MachineURL,
Archives = archives.Select(a => new DetailedStatusItem
{
Archive = a.Item1, IsFailing = a.Item2 == ArchiveStatus.InValid || a.Item2 == ArchiveStatus.Updating
}).ToList()
};
return (summary, detailed);
var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1));
Cache.Set(ModListSummariesKey, results, cacheOptions);
ModListSummaries = results;
return results;
});
var cacheOptions = new MemoryCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromMinutes(1));
Cache.Set(ModListSummariesKey, results, cacheOptions);
return results;
var data = ModListSummaries;
if (data == null)
return await task;
return data;
}
private static (Archive archive, ArchiveStatus) ValidateArchive(SqlService.ValidationData data, Archive archive)
private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(SqlService.ValidationData data, Archive archive)
{
switch (archive.State)
{
@ -123,8 +147,10 @@ namespace Wabbajack.BuildServer.Controllers
case NexusDownloader.State nexusState when data.NexusFiles.Contains((
nexusState.Game.MetaData().NexusGameId, nexusState.ModID, nexusState.FileID)):
return (archive, ArchiveStatus.Valid);
case NexusDownloader.State _:
return (archive, ArchiveStatus.InValid);
case NexusDownloader.State ns:
return (archive, await FastNexusModStats(ns));
case HTTPDownloader.State s when new Uri(s.Url).Host.StartsWith("wabbajackpush"):
return (archive, ArchiveStatus.Valid);
case ManualDownloader.State _:
return (archive, ArchiveStatus.Valid);
default:
@ -140,6 +166,47 @@ namespace Wabbajack.BuildServer.Controllers
}
}
private async Task<ArchiveStatus> FastNexusModStats(NexusDownloader.State ns)
{
var mod = await SQL.GetNexusModInfoString(ns.Game, ns.ModID);
var files = await SQL.GetModFiles(ns.Game, ns.ModID);
if (mod == null)
{
Utils.Log($"Found missing Nexus mod info {ns.Game} {ns.ModID}");
mod = await (await _nexusClient).GetModInfo(ns.Game, ns.ModID, false);
try
{
await SQL.AddNexusModInfo(ns.Game, ns.ModID, mod.updated_time, mod);
}
catch (Exception _)
{
// Could be a PK constraint failure
}
}
if (files == null)
{
Utils.Log($"Found missing Nexus mod file infos {ns.Game} {ns.ModID}");
files = await (await _nexusClient).GetModFiles(ns.Game, ns.ModID, false);
try
{
await SQL.AddNexusModFiles(ns.Game, ns.ModID, mod.updated_time, files);
}
catch (Exception _)
{
// Could be a PK constraint failure
}
}
if (mod.available && files.files.Any(f => !string.IsNullOrEmpty(f.category_name) && f.file_id == ns.FileID))
return ArchiveStatus.Valid;
return ArchiveStatus.InValid;
}
private static AsyncLock _findPatchLock = new AsyncLock();
private async Task<(Archive, ArchiveStatus)> TryToFix(SqlService.ValidationData data, Archive archive)
{
@ -219,6 +286,7 @@ namespace Wabbajack.BuildServer.Controllers
private AppSettings _settings;
private ModlistUpdater _updater;
private Task<NexusApiClient> _nexusClient;
[HttpGet]
[Route("status/{Name}.html")]

5
Wabbajack.BuildServer/Controllers/ModlistUpdater.cs
View File

@ -88,14 +88,12 @@ namespace Wabbajack.BuildServer.Controllers
public async Task<IActionResult> GetAlternative(string xxHash)
{
var startingHash = Hash.FromHex(xxHash);
Utils.Log($"Alternative requested for {startingHash}");
await Metric("requested_upgrade", startingHash.ToString());
var archive = await SQL.GetStateByHash(startingHash);
if (archive == null)
{
Utils.Log($"No original state for {startingHash}");
return NotFound("Original state not found");
}
@ -120,7 +118,6 @@ namespace Wabbajack.BuildServer.Controllers
}
Utils.Log($"Found {newArchive.State.PrimaryKeyString} {newArchive.Name} as an alternative to {startingHash}");
if (newArchive.Hash == Hash.Empty)
{
await SQL.EnqueueJob(new Job
@ -138,6 +135,7 @@ namespace Wabbajack.BuildServer.Controllers
}
}
});
Utils.Log($"Enqueued Index and Upgrade for {startingHash} -> {newArchive.State.PrimaryKeyString}");
return Accepted("Enqueued for Processing");
}
@ -155,6 +153,7 @@ namespace Wabbajack.BuildServer.Controllers
DestPK = newArchive.State.PrimaryKeyString
}
});
Utils.Log($"Enqueued Upgrade for {startingHash} -> {newArchive.State.PrimaryKeyString}");
}
return Ok(newArchive.ToJson());
}

4
Wabbajack.BuildServer/Wabbajack.BuildServer.csproj
View File

@ -18,7 +18,7 @@
<ItemGroup>
<PackageReference Include="CsvHelper" Version="15.0.5" />
<PackageReference Include="Dapper" Version="2.0.35" />
<PackageReference Include="FluentFTP" Version="32.3.3" />
<PackageReference Include="FluentFTP" Version="32.4.0" />
<PackageReference Include="graphiql" Version="2.0.0" />
<PackageReference Include="GraphQL" Version="3.0.0-preview-1352" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.Core" Version="2.2.0" />
@ -28,7 +28,7 @@
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
<PackageReference Include="Microsoft.OpenApi" Version="1.1.4" />
<PackageReference Include="Nettle" Version="1.3.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.3.2" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.4.0" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.1" />
</ItemGroup>

2
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -32,7 +32,7 @@
<PackageReference Include="Microsoft.Win32.Registry" Version="4.7.0" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="Octodiff" Version="1.2.1" />
<PackageReference Include="ReactiveUI" Version="11.3.1" />
<PackageReference Include="ReactiveUI" Version="11.3.8" />
<PackageReference Include="SharpZipLib" Version="1.2.0" />
<PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
<PackageReference Include="System.Net.Http" Version="4.3.4" />

13
Wabbajack.Lib/Downloaders/MEGADownloader.cs
View File

@ -96,27 +96,30 @@ namespace Wabbajack.Lib.Downloaders
private static MegaApiClient MegaApiClient => DownloadDispatcher.GetInstance<MegaDownloader>().MegaApiClient;
private static void MegaLogin()
private static AsyncLock _loginLock = new AsyncLock();
private static async Task MegaLogin()
{
using var _ = await _loginLock.WaitAsync();
if (MegaApiClient.IsLoggedIn)
return;
if (!Utils.HaveEncryptedJson(DataName))
{
Utils.Status("Logging into MEGA (as anonymous)");
MegaApiClient.LoginAnonymous();
await MegaApiClient.LoginAnonymousAsync();
}
else
{
Utils.Status("Logging into MEGA with saved credentials.");
var authInfo = Utils.FromEncryptedJson<MegaApiClient.AuthInfos>(DataName);
MegaApiClient.Login(authInfo);
await MegaApiClient.LoginAsync(authInfo);
}
}
public override async Task<bool> Download(Archive a, AbsolutePath destination)
{
MegaLogin();
await MegaLogin();
var fileLink = new Uri(Url);
Utils.Status($"Downloading MEGA file: {a.Name}");
@ -126,7 +129,7 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Verify(Archive a)
{
MegaLogin();
await MegaLogin();
var fileLink = new Uri(Url);
try

6
Wabbajack.Lib/Wabbajack.Lib.csproj
View File

@ -35,10 +35,10 @@
<Version>2.1.0</Version>
</PackageReference>
<PackageReference Include="ReactiveUI">
<Version>11.3.1</Version>
<Version>11.3.8</Version>
</PackageReference>
<PackageReference Include="ReactiveUI.Fody">
<Version>11.3.1</Version>
<Version>11.3.8</Version>
</PackageReference>
<PackageReference Include="SharpCompress">
<Version>0.25.0</Version>
@ -65,7 +65,7 @@
<Version>1.0.0</Version>
</PackageReference>
<PackageReference Include="YoutubeExplode">
<Version>5.0.1</Version>
<Version>5.0.2</Version>
</PackageReference>
</ItemGroup>
<ItemGroup>

2
Wabbajack.Test/Wabbajack.Test.csproj
View File

@ -27,7 +27,7 @@
<ItemGroup>
<PackageReference Include="CefSharp.Common" Version="79.1.360" />
<PackageReference Include="CefSharp.OffScreen" Version="79.1.360" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
<PackageReference Include="coverlet.collector" Version="1.2.1" />

8
Wabbajack/Wabbajack.csproj
View File

@ -57,7 +57,7 @@
<ItemGroup>
<PackageReference Include="CefSharp.Wpf" Version="79.1.360" />
<PackageReference Include="DynamicData" Version="6.14.10" />
<PackageReference Include="DynamicData" Version="6.14.14" />
<PackageReference Include="Extended.Wpf.Toolkit" Version="3.8.1" />
<PackageReference Include="Fody" Version="6.1.1">
<PrivateAssets>all</PrivateAssets>
@ -72,9 +72,9 @@
<PackageReference Include="MahApps.Metro.IconPacks" Version="4.0.0" />
<PackageReference Include="PInvoke.Gdi32" Version="0.6.6" />
<PackageReference Include="PInvoke.User32" Version="0.6.6" />
<PackageReference Include="ReactiveUI" Version="11.3.1" />
<PackageReference Include="ReactiveUI.Fody" Version="11.3.1" />
<PackageReference Include="ReactiveUI.WPF" Version="11.3.1" />
<PackageReference Include="ReactiveUI" Version="11.3.8" />
<PackageReference Include="ReactiveUI.Fody" Version="11.3.8" />
<PackageReference Include="ReactiveUI.WPF" Version="11.3.8" />
<PackageReference Include="SharpDX.DXGI" Version="4.2.0" />
<PackageReference Include="WindowsAPICodePack-Shell" Version="1.1.1" />
<PackageReference Include="WPFThemes.DarkBlend" Version="1.0.8" />