Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Code for healing MEGA links

Browse Source
This commit is contained in:
Timothy Baldridge 2020-07-13 16:10:05 -06:00
parent a6d5b05013
commit 7bbcbfdbb3
17 changed files with 237 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -1,5 +1,11 @@
### Changelog
#### Version - 2.1.2.0 - 7/13/2020
* Can heal hand selected MEGA files
* Several backend fixes
* Reworked the ChangeDownload CLI command
* Fix for a VFS cache error when compiling lists that extract BSAs.
#### Version - 2.1.1.0 - 7/10/2020
* New CLI option for clearing nexus cache entries (authors only)
* Retry failed Move commands

3
Wabbajack.CLI/OptionsDefinition.cs
View File

@ -25,7 +25,8 @@ namespace Wabbajack.CLI
typeof(HashFile),
typeof(InlinedFileReport),
typeof(ExtractBSA),
typeof(PurgeNexusCache)
typeof(PurgeNexusCache),
typeof(ForceHealing)
};
}
}

40
Wabbajack.CLI/Verbs/ForceHealing.cs Normal file
View File

@ -0,0 +1,40 @@
using System.Threading.Tasks;
using CommandLine;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Lib.Downloaders;
namespace Wabbajack.CLI.Verbs
{
[Verb("force-healing", HelpText = "Forces a given source download to be healed by a given new-er download. The new download must be valid.")]
public class ForceHealing : AVerb
{
[Option('o', "old", Required = true, HelpText = "Old Archive (must have an attached .meta)")]
public string _old { get; set; } = "";
public AbsolutePath Old => (AbsolutePath)_old;
[Option('n', "new", Required = true, HelpText = "New Archive (must have an attached .meta)")]
public string _new { get; set; } = "";
public AbsolutePath New => (AbsolutePath)_new;
protected override async Task<ExitCode> Run()
{
Utils.Log("Loading Meta files");
var oldState = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(Old.WithExtension(Consts.MetaFileExtension).LoadIniFile());
var newState = (AbstractDownloadState)await DownloadDispatcher.ResolveArchive(New.WithExtension(Consts.MetaFileExtension).LoadIniFile());
Utils.Log("Hashing archives");
var oldHash = await Old.FileHashCachedAsync();
var newHash = await New.FileHashCachedAsync();
var oldArchive = new Archive(oldState) {Hash = oldHash, Size = Old.Size};
var newArchive = new Archive(newState) {Hash = newHash, Size = New.Size};
Utils.Log($"Contacting Server to request patch ({oldHash} -> {newHash}");
Utils.Log($"Response: {await ClientAPI.GetModUpgrade(oldArchive, newArchive, useAuthor: true)}");
return ExitCode.Ok;
}
}
}

4
Wabbajack.CLI/Wabbajack.CLI.csproj
View File

@ -6,8 +6,8 @@
<AssemblyName>wabbajack-cli</AssemblyName>
<Company>Wabbajack</Company>
<Platforms>x64</Platforms>
<AssemblyVersion>2.1.1.0</AssemblyVersion>
<FileVersion>2.1.1.0</FileVersion>
<AssemblyVersion>2.1.2.0</AssemblyVersion>
<FileVersion>2.1.2.0</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>An automated ModList installer</Description>
<PublishReadyToRun>true</PublishReadyToRun>

4
Wabbajack.Launcher/Wabbajack.Launcher.csproj
View File

@ -4,8 +4,8 @@
<OutputType>WinExe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
<UseWPF>true</UseWPF>
<AssemblyVersion>2.1.1.0</AssemblyVersion>
<FileVersion>2.1.1.0</FileVersion>
<AssemblyVersion>2.1.2.0</AssemblyVersion>
<FileVersion>2.1.2.0</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>Wabbajack Application Launcher</Description>
<PublishReadyToRun>true</PublishReadyToRun>

33
Wabbajack.Lib/ClientAPI.cs
View File

@ -25,21 +25,18 @@ using Wabbajack.Lib.Downloaders;
NewArchive = newArchive;
}
public bool IsValid
public async Task<bool> IsValid()
{
get
{
if (OldArchive.Size > 2_500_000_000 || NewArchive.Size > 2_500_000_000) return false;
if (OldArchive.Hash == NewArchive.Hash && OldArchive.State.PrimaryKeyString == NewArchive.State.PrimaryKeyString) return false;
if (OldArchive.State.GetType() != NewArchive.State.GetType())
return false;
if (OldArchive.State is IUpgradingState u)
{
return u.ValidateUpgrade(NewArchive.State);
}
if (OldArchive.Size > 2_500_000_000 || NewArchive.Size > 2_500_000_000) return false;
if (OldArchive.Hash == NewArchive.Hash && OldArchive.State.PrimaryKeyString == NewArchive.State.PrimaryKeyString) return false;
if (OldArchive.State.GetType() != NewArchive.State.GetType())
return false;
if (OldArchive.State is IUpgradingState u)
{
return await u.ValidateUpgrade(OldArchive.Hash, NewArchive.State);
}
return false;
}
}
@ -52,7 +49,7 @@ using Wabbajack.Lib.Downloaders;
return client;
}
public static async Task<Uri> GetModUpgrade(Archive oldArchive, Archive newArchive, TimeSpan? maxWait = null, TimeSpan? waitBetweenTries = null)
public static async Task<Uri> GetModUpgrade(Archive oldArchive, Archive newArchive, TimeSpan? maxWait = null, TimeSpan? waitBetweenTries = null, bool useAuthor = false)
{
maxWait ??= TimeSpan.FromMinutes(10);
waitBetweenTries ??= TimeSpan.FromSeconds(15);
@ -62,7 +59,7 @@ using Wabbajack.Lib.Downloaders;
RETRY:
var response = await (await GetClient())
var response = await (useAuthor ? await AuthorApi.Client.GetAuthorizedClient() : await GetClient())
.PostAsync($"{Consts.WabbajackBuildServerUri}mod_upgrade", new StringContent(request.ToJson(), Encoding.UTF8, "application/json"));
if (response.IsSuccessStatusCode)
@ -141,5 +138,13 @@ using Wabbajack.Lib.Downloaders;
}
return null;
}
public static async Task<Archive[]> GetModUpgrades(Hash src)
{
var client = await GetClient();
Utils.Log($"Looking for generic upgrade for {src} ({(long)src})");
var results = await client.GetJsonAsync<Archive[]>($"{Consts.WabbajackBuildServerUri}mod_upgrade/find/{src.ToHex()}");
return results;
}
}
}

58
Wabbajack.Lib/Downloaders/AbstractDownloadState.cs
View File

@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders
Task<bool> LoadMetaData();
}
public abstract class AbstractDownloadState
public abstract class AbstractDownloadState : IUpgradingState
{
public static List<Type> KnownSubTypes = new List<Type>
{
@ -102,5 +102,61 @@ namespace Wabbajack.Lib.Downloaders
{
return string.Join("\n", GetMetaIni());
}
public async Task<(Archive? Archive, TempFile NewFile)> ServerFindUpgrade(Archive a)
{
var alternatives = await ClientAPI.GetModUpgrades(a.Hash);
if (alternatives == default)
return default;
await DownloadDispatcher.PrepareAll(alternatives.Select(r => r.State));
Archive? selected = null;
foreach (var result in alternatives)
{
try
{
if (!await result.State.Verify(result)) continue;
selected = result;
break;
}
catch (Exception ex)
{
Utils.Log($"Verification error for failed for possible upgrade {result.State.PrimaryKeyString}");
Utils.Log(ex.ToString());
}
}
if (selected == null) return default;
var tmpFile = new TempFile();
if (await selected.State.Download(selected, tmpFile.Path))
{
return (selected, tmpFile);
}
await tmpFile.DisposeAsync();
return default;
}
public virtual async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
{
return await ServerFindUpgrade(a);
}
public virtual async Task<bool> ServerValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState)
{
var alternatives = await ClientAPI.GetModUpgrades(srcHash);
return alternatives?.Any(a => a.State.PrimaryKeyString == newArchiveState.PrimaryKeyString) ?? default;
}
public virtual async Task<bool> ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState)
{
return await ServerValidateUpgrade(srcHash, newArchiveState);
}
}
}

2
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -240,7 +240,7 @@ TOP:
}
public bool ValidateUpgrade(AbstractDownloadState newArchiveState)
public override async Task<bool> ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState)
{
var httpState = (State)newArchiveState;

2
Wabbajack.Lib/Downloaders/IUpgradingState.cs
View File

@ -13,6 +13,6 @@ namespace Wabbajack.Lib.Downloaders
/// <returns></returns>
public Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a);
bool ValidateUpgrade(AbstractDownloadState newArchiveState);
Task<bool> ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState);
}
}

9
Wabbajack.Lib/Downloaders/MEGADownloader.cs
View File

@ -194,10 +194,15 @@ namespace Wabbajack.Lib.Downloaders
}
}
public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
public override Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a)
{
return default;
return ServerFindUpgrade(a);
}
public override async Task<bool> ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState)
{
return await ServerValidateUpgrade(srcHash, newArchiveState);
}
}
public ReactiveCommand<Unit, Unit> TriggerLogin { get; }

2
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -277,7 +277,7 @@ namespace Wabbajack.Lib.Downloaders
return (newArchive, tempFile);
}
public bool ValidateUpgrade(AbstractDownloadState newArchiveState)
public async Task<bool> ValidateUpgrade(Hash srcHash, AbstractDownloadState newArchiveState)
{
var state = (State)newArchiveState;
return Game == state.Game && ModID == state.ModID;

17
Wabbajack.Lib/Metrics.cs
View File

@ -21,6 +21,7 @@ namespace Wabbajack.Lib
public static async ValueTask<string> GetMetricsKey()
{
TOP:
using var _ = await _creationLock.WaitAsync();
if (!Utils.HaveEncryptedJson(Consts.MetricsKeyHeader))
{
@ -61,9 +62,19 @@ namespace Wabbajack.Lib
// If there's a regkey and a file, return regkey
using (RegistryKey regKey = Registry.CurrentUser.CreateSubKey(@"Software\Wabbajack", RegistryKeyPermissionCheck.Default)!)
{
string key = await Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader)!;
regKey.SetValue("x-metrics-key", key);
return key;
try
{
string key = await Utils.FromEncryptedJson<string>(Consts.MetricsKeyHeader)!;
regKey.SetValue("x-metrics-key", key);
return key;
}
catch (Exception)
{
// Probably an encryption error
await Utils.DeleteEncryptedJson(Consts.MetricsKeyHeader);
goto TOP;
}
}
}
}

61
Wabbajack.Server/Controllers/ModUpgrade.cs
View File

@ -1,4 +1,7 @@
using System.Threading.Tasks;
using System;
using System.Linq;
using System.Security.Claims;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
@ -32,18 +35,50 @@ namespace Wabbajack.BuildServer.Controllers
[Route("/mod_upgrade")]
public async Task<IActionResult> PostModUpgrade()
{
var isAuthor = User.Claims.Any(c => c.Type == ClaimTypes.Role && c.Value == "Author");
var request = (await Request.Body.ReadAllTextAsync()).FromJsonString<ModUpgradeRequest>();
if (!request.IsValid)
if (!isAuthor)
{
_logger.Log(LogLevel.Information, $"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as upgrade is invalid");
return BadRequest("Invalid mod upgrade");
var srcDownload = await _sql.GetArchiveDownload(request.OldArchive.State.PrimaryKeyString,
request.OldArchive.Hash, request.OldArchive.Size);
var destDownload = await _sql.GetArchiveDownload(request.NewArchive.State.PrimaryKeyString,
request.NewArchive.Hash, request.NewArchive.Size);
if (srcDownload == default || destDownload == default ||
await _sql.FindPatch(srcDownload.Id, destDownload.Id) == default)
{
if (!await request.IsValid())
{
_logger.Log(LogLevel.Information,
$"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as upgrade is invalid");
return BadRequest("Invalid mod upgrade");
}
if (_settings.ValidateModUpgrades && !await _sql.HashIsInAModlist(request.OldArchive.Hash))
{
_logger.Log(LogLevel.Information,
$"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as src hash is not in a curated modlist");
return BadRequest("Hash is not in a recent modlist");
}
}
}
if (_settings.ValidateModUpgrades && !await _sql.HashIsInAModlist(request.OldArchive.Hash))
try
{
_logger.Log(LogLevel.Information, $"Upgrade requested from {request.OldArchive.Hash} to {request.NewArchive.Hash} rejected as src hash is not in a curated modlist");
return BadRequest("Hash is not in a recent modlist");
if (await request.OldArchive.State.Verify(request.OldArchive))
{
_logger.LogInformation(
$"Refusing to upgrade ({request.OldArchive.State.PrimaryKeyString}), old archive is valid");
return NotFound("File is Valid");
}
}
catch (Exception ex)
{
// ignore
}
var oldDownload = await _sql.GetOrEnqueueArchive(request.OldArchive);
var newDownload = await _sql.GetOrEnqueueArchive(request.NewArchive);
@ -77,5 +112,17 @@ namespace Wabbajack.BuildServer.Controllers
return Accepted();
}
[HttpGet]
[Authorize(Roles = "User")]
[Route("/mod_upgrade/find/{hashAsHex}")]
public async Task<IActionResult> FindUpgrade(string hashAsHex)
{
var hash = Hash.FromHex(hashAsHex);
var patches = await _sql.PatchesForSource(hash);
return Ok(patches.Select(p => p.Dest).ToList());
}
}
}

43
Wabbajack.Server/DataLayer/Patches.cs
View File

@ -47,7 +47,7 @@ namespace Wabbajack.Server.DataLayer
{
await using var conn = await Open();
var patch = await conn.QueryFirstOrDefaultAsync<(long, DateTime?, bool?, string)>(
@"SELECT p.PatchHash, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
@"SELECT p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads src ON p.SrcId = src.Id
LEFT JOIN dbo.ArchiveDownloads dest ON p.SrcId = dest.Id
@ -133,19 +133,19 @@ namespace Wabbajack.Server.DataLayer
var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
"SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId", new {SrcId = sourceDownload});
List<Patch> results = new List<Patch>();
foreach (var (srcId, destId, patchSize, finished, isFailed, failMessage) in patches)
{
results.Add( new Patch {
Src = await GetArchiveDownload(srcId),
Dest = await GetArchiveDownload(destId),
PatchSize = patchSize,
Finished = finished,
IsFailed = isFailed,
FailMessage = failMessage
});
}
return results;
return await AsPatches(patches);
}
public async Task<List<Patch>> PatchesForSource(Hash sourceHash)
{
await using var conn = await Open();
var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
@"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a ON p.SrcId = a.Id
WHERE a.Hash = @Hash AND p.Finished IS NOT NULL AND p.IsFailed = 0", new {Hash = sourceHash});
return await AsPatches(patches);
}
public async Task MarkPatchUsage(Guid srcId, Guid destId)
@ -174,21 +174,29 @@ namespace Wabbajack.Server.DataLayer
WHERE m.PrimaryKeyString is not null
AND (p.LastUsed < DATEADD(d, -7, getutcdate()) OR p.LastUsed is null and p.Finished < DATEADD(d, -7, getutcdate()))");
return await AsPatches(patches);
}
private async Task<List<Patch>> AsPatches(IEnumerable<(Guid, Guid, long, DateTime?, bool?, string)> patches)
{
List<Patch> results = new List<Patch>();
foreach (var (srcId, destId, patchSize, finished, isFailed, failMessage) in patches)
{
results.Add( new Patch {
Src = await GetArchiveDownload(srcId),
results.Add(new Patch
{
Src = await GetArchiveDownload(srcId),
Dest = await GetArchiveDownload(destId),
PatchSize = patchSize,
Finished = finished,
IsFailed = isFailed,
FailMessage = failMessage
});
}
}
return results;
}
public async Task DeletePatch(Patch patch)
{
await using var conn = await Open();
@ -225,5 +233,6 @@ namespace Wabbajack.Server.DataLayer
});
}
}
}

1
Wabbajack.Server/Startup.cs
View File

@ -136,6 +136,7 @@ namespace Wabbajack.Server
headers.Add("Access-Control-Allow-Methods", "POST, GET");
headers.Add("Access-Control-Allow-Headers", "Accept, Origin, Content-type");
headers.Add("X-ResponseTime-Ms", stopWatch.ElapsedMilliseconds.ToString());
headers.Add("Cache-Control", "no-cache");
return Task.CompletedTask;
});
await next(context);

4
Wabbajack.Server/Wabbajack.Server.csproj
View File

@ -3,8 +3,8 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
<AssemblyVersion>2.1.1.0</AssemblyVersion>
<FileVersion>2.1.1.0</FileVersion>
<AssemblyVersion>2.1.2.0</AssemblyVersion>
<FileVersion>2.1.2.0</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>Wabbajack Server</Description>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>

4
Wabbajack/Wabbajack.csproj
View File

@ -6,8 +6,8 @@
<UseWPF>true</UseWPF>
<Platforms>x64</Platforms>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
<AssemblyVersion>2.1.1.0</AssemblyVersion>
<FileVersion>2.1.1.0</FileVersion>
<AssemblyVersion>2.1.2.0</AssemblyVersion>
<FileVersion>2.1.2.0</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>An automated ModList installer</Description>
<PublishReadyToRun>true</PublishReadyToRun>