Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Server side fixes for LL upgrades and list validation

Browse Source
This commit is contained in:
Timothy Baldridge 2021-07-09 22:13:31 -06:00
parent d3725c5b74
commit 5cbed8dade
3 changed files with 125 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Wabbajack.Lib/Downloaders/AbstractIPS4OAuthDownloader.cs
View File

@ -260,6 +260,8 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a, Func<Archive, Task<AbsolutePath>> downloadResolver)
{
if (IsAttachment) return default;
var files = await GetFilesInGroup();
var nl = new Levenshtein();
@ -273,6 +275,8 @@ namespace Wabbajack.Lib.Downloaders
var tmpHash = await tmp.Path.FileHashAsync();
if (tmpHash == null) return default;
newFile.Hash = tmpHash.Value;
var state = ((TState)newFile.State);
await state.LoadMetaData();
return (newFile, tmp);
}

225
Wabbajack.Server/Services/ListValidator.cs
View File

@ -233,116 +233,131 @@ namespace Wabbajack.Server.Services
private AsyncLock _healLock = new AsyncLock();
private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive, ModlistMetadata modList)
{
using var _ = await _healLock.WaitAsync();
var srcDownload = await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size);
if (srcDownload == null || srcDownload.IsFailed == true)
try
{
_logger.Log(LogLevel.Information, $"Cannot heal {archive.State.PrimaryKeyString} Size: {archive.Size} Hash: {(long)archive.Hash} because it hasn't been previously successfully downloaded");
return (archive, ArchiveStatus.InValid);
}
var patches = await _sql.PatchesForSource(archive.Hash);
foreach (var patch in patches)
{
if (patch.Finished is null)
return (archive, ArchiveStatus.Updating);
if (patch.IsFailed == true)
return (archive, ArchiveStatus.InValid);
var (_, status) = await ValidateArchive(data, patch.Dest.Archive);
if (status == ArchiveStatus.Valid)
return (archive, ArchiveStatus.Updated);
}
var upgradeTime = DateTime.UtcNow;
_logger.LogInformation($"Validator Finding Upgrade for {archive.Hash} {archive.State.PrimaryKeyString}");
Func<Archive, Task<AbsolutePath>> resolver = async findIt =>
{
_logger.LogInformation($"Quick find for {findIt.State.PrimaryKeyString}");
var foundArchive = await _sql.GetArchiveDownload(findIt.State.PrimaryKeyString);
if (foundArchive == null)
using var _ = await _healLock.WaitAsync();
var srcDownload =
await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size);
if (srcDownload == null || srcDownload.IsFailed == true)
{
_logger.LogInformation($"No Quick find for {findIt.State.PrimaryKeyString}");
return default;
}
return _archives.TryGetPath(foundArchive.Archive.Hash, out var path) ? path : default;
};
if (archive.State is NexusDownloader.State)
{
DownloadDispatcher.GetInstance<NexusDownloader>().Client = await _nexus.GetClient();
}
var upgrade = await DownloadDispatcher.FindUpgrade(archive, resolver);
if (upgrade == default)
{
_logger.Log(LogLevel.Information, $"Cannot heal {archive.State.PrimaryKeyString} because an alternative wasn't found");
return (archive, ArchiveStatus.InValid);
}
_logger.LogInformation($"Upgrade {upgrade.Archive.State.PrimaryKeyString} found for {archive.State.PrimaryKeyString}");
{
}
var found = await _sql.GetArchiveDownload(upgrade.Archive.State.PrimaryKeyString, upgrade.Archive.Hash,
upgrade.Archive.Size);
Guid id;
if (found == null)
{
if (upgrade.NewFile.Path.Exists)
await _archives.Ingest(upgrade.NewFile.Path);
id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime);
}
else
{
id = found.Id;
}
var destDownload = await _sql.GetArchiveDownload(id);
if (destDownload.Archive.Hash == srcDownload.Archive.Hash && destDownload.Archive.State.PrimaryKeyString == srcDownload.Archive.State.PrimaryKeyString)
{
_logger.Log(LogLevel.Information, $"Can't heal because src and dest match");
return (archive, ArchiveStatus.InValid);
}
if (destDownload.Archive.Hash == default)
{
_logger.Log(LogLevel.Information, "Can't heal because we got back a default hash for the downloaded file");
return (archive, ArchiveStatus.InValid);
}
var existing = await _sql.FindPatch(srcDownload.Id, destDownload.Id);
if (existing == null)
{
if (await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload}))
{
_logger.Log(LogLevel.Information,
$"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`"
});
$"Cannot heal {archive.State.PrimaryKeyString} Size: {archive.Size} Hash: {(long)archive.Hash} because it hasn't been previously successfully downloaded");
return (archive, ArchiveStatus.InValid);
}
var patches = await _sql.PatchesForSource(archive.Hash);
foreach (var patch in patches)
{
if (patch.Finished is null)
return (archive, ArchiveStatus.Updating);
if (patch.IsFailed == true)
return (archive, ArchiveStatus.InValid);
var (_, status) = await ValidateArchive(data, patch.Dest.Archive);
if (status == ArchiveStatus.Valid)
return (archive, ArchiveStatus.Updated);
}
var upgradeTime = DateTime.UtcNow;
_logger.LogInformation(
$"Validator Finding Upgrade for {archive.Hash} {archive.State.PrimaryKeyString}");
Func<Archive, Task<AbsolutePath>> resolver = async findIt =>
{
_logger.LogInformation($"Quick find for {findIt.State.PrimaryKeyString}");
var foundArchive = await _sql.GetArchiveDownload(findIt.State.PrimaryKeyString);
if (foundArchive == null)
{
_logger.LogInformation($"No Quick find for {findIt.State.PrimaryKeyString}");
return default;
}
return _archives.TryGetPath(foundArchive.Archive.Hash, out var path) ? path : default;
};
if (archive.State is NexusDownloader.State)
{
DownloadDispatcher.GetInstance<NexusDownloader>().Client = await _nexus.GetClient();
}
var upgrade = await DownloadDispatcher.FindUpgrade(archive, resolver);
if (upgrade == default)
{
_logger.Log(LogLevel.Information,
$"Cannot heal {archive.State.PrimaryKeyString} because an alternative wasn't found");
return (archive, ArchiveStatus.InValid);
}
_logger.LogInformation(
$"Upgrade {upgrade.Archive.State.PrimaryKeyString} found for {archive.State.PrimaryKeyString}");
{
}
var found = await _sql.GetArchiveDownload(upgrade.Archive.State.PrimaryKeyString, upgrade.Archive.Hash,
upgrade.Archive.Size);
Guid id;
if (found == null)
{
if (upgrade.NewFile.Path.Exists)
await _archives.Ingest(upgrade.NewFile.Path);
id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime);
}
else
{
id = found.Id;
}
var destDownload = await _sql.GetArchiveDownload(id);
if (destDownload.Archive.Hash == srcDownload.Archive.Hash &&
destDownload.Archive.State.PrimaryKeyString == srcDownload.Archive.State.PrimaryKeyString)
{
_logger.Log(LogLevel.Information, $"Can't heal because src and dest match");
return (archive, ArchiveStatus.InValid);
}
if (destDownload.Archive.Hash == default)
{
_logger.Log(LogLevel.Information,
"Can't heal because we got back a default hash for the downloaded file");
return (archive, ArchiveStatus.InValid);
}
var existing = await _sql.FindPatch(srcDownload.Id, destDownload.Id);
if (existing == null)
{
if (await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload}))
{
_logger.Log(LogLevel.Information,
$"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}");
await _discord.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash} to auto-heal `{modList.Links.MachineURL}`"
});
}
}
await upgrade.NewFile.DisposeAsync();
_logger.LogInformation($"Patch in progress {archive.Hash} {archive.State.PrimaryKeyString}");
return (archive, ArchiveStatus.Updating);
}
catch (Exception ex)
{
_logger.LogError(ex, "During healing");
return (archive, ArchiveStatus.InValid);
}
await upgrade.NewFile.DisposeAsync();
_logger.LogInformation($"Patch in progress {archive.Hash} {archive.State.PrimaryKeyString}");
return (archive, ArchiveStatus.Updating);
}
private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(ValidationData data, Archive archive)

2
Wabbajack.Server/Services/ModListDownloader.cs
View File

@ -126,7 +126,7 @@ namespace Wabbajack.Server.Services
new DiscordMessage
{
Content =
$"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash}"
$"Error downloading modlist {list.Links.MachineURL} - {list.DownloadMetadata.Hash} - {ex.Message}"
});
}
finally