Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

List validator now tries to heal lists

Browse Source
This commit is contained in:
Timothy Baldridge 2020-05-20 06:18:47 -06:00
parent da3d87093a
commit 7d58dbc161
6 changed files with 127 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -217,9 +217,16 @@ TOP:
var tmpFile = new TempFile();
var newArchive = new Archive(this) {Name = a.Name};
if (!await Download(newArchive, tmpFile.Path))
try
{
if (!await Download(newArchive, tmpFile.Path))
return default;
}
catch (HttpRequestException)
{
return default;
}
newArchive.Hash = await tmpFile.Path.FileHashAsync();
newArchive.Size = tmpFile.Path.Size;

2
Wabbajack.Server.Test/ModlistUpdater.cs
View File

@ -76,7 +76,7 @@ namespace Wabbajack.Server.Test
Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_updates/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero));
}
[Fact]
[Fact]
public async Task TestEndToEndArchiveUpdating()
{
var modLists = await MakeModList();

22
Wabbajack.Server/DataLayer/ArchiveDownloads.cs
View File

@ -13,6 +13,26 @@ namespace Wabbajack.Server.DataLayer
{
public partial class SqlService
{
public async Task<Guid> AddKnownDownload(Archive a, DateTime downloadFinished)
{
await using var conn = await Open();
var Id = Guid.NewGuid();
await conn.ExecuteAsync(
"INSERT INTO ArchiveDownloads (Id, PrimaryKeyString, Size, Hash, DownloadState, Downloader, DownloadFinished, IsFailed) VALUES (@Id, @PrimaryKeyString, @Size, @Hash, @DownloadState, @Downloader, @DownloadFinished, @IsFailed)",
new
{
Id = Id,
PrimaryKeyString = a.State.PrimaryKeyString,
Size = a.Size == 0 ? null : (long?)a.Size,
Hash = a.Hash == default ? null : (Hash?)a.Hash,
DownloadState = a.State,
Downloader = AbstractDownloadState.TypeToName[a.State.GetType()],
DownloadFinished = downloadFinished,
IsFailed = false
});
return Id;
}
public async Task<Guid> EnqueueDownload(Archive a)
{
await using var conn = await Open();
@ -26,7 +46,7 @@ namespace Wabbajack.Server.DataLayer
Size = a.Size == 0 ? null : (long?)a.Size,
Hash = a.Hash == default ? null : (Hash?)a.Hash,
DownloadState = a.State,
Downloader = AbstractDownloadState.TypeToName[a.State.GetType()]
Downloader = AbstractDownloadState.TypeToName[a.State.GetType()],
});
return Id;
}

24
Wabbajack.Server/DataLayer/Patches.cs
View File

@ -1,7 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Wabbajack.Common;
using Wabbajack.Lib;
using Wabbajack.Server.DTOs;
namespace Wabbajack.Server.DataLayer
@ -121,5 +124,26 @@ namespace Wabbajack.Server.DataLayer
FailMessage = patch.Item6
};
}
public async Task<List<Patch>> PatchesForSource(Guid sourceDownload)
{
await using var conn = await Open();
var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
"SELECT SrcId, DestId, PatchSize, Finished, IsFailed, FailMessage FROM dbo.Patches WHERE SrcId = @SrcId", new {SrcId = sourceDownload});
List<Patch> results = new List<Patch>();
foreach (var (srcId, destId, patchSize, finished, isFinished, failMessage) in patches)
{
results.Add( new Patch {
Src = await GetArchiveDownload(srcId),
Dest = await GetArchiveDownload(destId),
PatchSize = patchSize,
Finished = finished,
IsFailed = isFinished,
FailMessage = failMessage
});
}
return results;
}
}
}

63
Wabbajack.Server/Services/ListValidator.cs
View File

@ -24,17 +24,19 @@ namespace Wabbajack.Server.Services
private SqlService _sql;
private DiscordWebHook _discord;
private NexusKeyMaintainance _nexus;
private ArchiveMaintainer _archives;
public IEnumerable<(ModListSummary Summary, DetailedStatus Detailed)> Summaries { get; private set; } =
new (ModListSummary Summary, DetailedStatus Detailed)[0];
public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus)
: base(logger, settings, TimeSpan.FromMinutes(10))
public ListValidator(ILogger<ListValidator> logger, AppSettings settings, SqlService sql, DiscordWebHook discord, NexusKeyMaintainance nexus, ArchiveMaintainer archives)
: base(logger, settings, TimeSpan.FromMinutes(5))
{
_sql = sql;
_discord = discord;
_nexus = nexus;
_archives = archives;
}
public override async Task<int> Execute()
@ -53,7 +55,8 @@ namespace Wabbajack.Server.Services
var archives = await modList.Archives.PMap(queue, async archive =>
{
var (_, result) = await ValidateArchive(data, archive);
// TODO : auto-healing goes here
if (result == ArchiveStatus.InValid)
return await TryToHeal(data, archive);
return (archive, result);
});
@ -128,7 +131,59 @@ namespace Wabbajack.Server.Services
Summaries = results;
return Summaries.Count(s => s.Summary.HasFailures);
}
private AsyncLock _healLock = new AsyncLock();
private async Task<(Archive, ArchiveStatus)> TryToHeal(ValidationData data, Archive archive)
{
using var _ = await _healLock.WaitAsync();
if (!(archive.State is IUpgradingState))
return (archive, ArchiveStatus.InValid);
var srcDownload = await _sql.GetArchiveDownload(archive.State.PrimaryKeyString, archive.Hash, archive.Size);
if (srcDownload == null || srcDownload.IsFailed == true)
{
return (archive, ArchiveStatus.InValid);
}
var patches = await _sql.PatchesForSource(srcDownload.Id);
foreach (var patch in patches)
{
if (patch.Finished is null)
return (archive, ArchiveStatus.Updating);
if (patch.IsFailed == true)
continue;
var (_, status) = await ValidateArchive(data, patch.Dest.Archive);
if (status == ArchiveStatus.Valid)
return (archive, ArchiveStatus.Updated);
}
var upgradeTime = DateTime.UtcNow;
var upgrade = await (archive.State as IUpgradingState)?.FindUpgrade(archive);
if (upgrade == default)
{
return (archive, ArchiveStatus.InValid);
}
await _archives.Ingest(upgrade.NewFile.Path);
var id = await _sql.AddKnownDownload(upgrade.Archive, upgradeTime);
var destDownload = await _sql.GetArchiveDownload(id);
await _sql.AddPatch(new Patch {Src = srcDownload, Dest = destDownload});
_logger.Log(LogLevel.Information, $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}");
await _discord.Send(Channel.Spam, new DiscordMessage { Content = $"Enqueued Patch from {srcDownload.Archive.Hash} to {destDownload.Archive.Hash}" });
upgrade.NewFile.Dispose();
return (archive, ArchiveStatus.Updating);
}
private async Task<(Archive archive, ArchiveStatus)> ValidateArchive(ValidationData data, Archive archive)
{
switch (archive.State)

13
Wabbajack.Server/Services/PatchBuilder.cs
View File

@ -70,11 +70,24 @@ namespace Wabbajack.Server.Services
var size = await ftpClient.GetFileSizeAsync(patchName);
await patch.Finish(_sql, size);
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Built {size.ToFileSizeString()} patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
}
catch (Exception ex)
{
_logger.LogError(ex, "Error while building patch");
await patch.Fail(_sql, ex.ToString());
await _discordWebHook.Send(Channel.Spam,
new DiscordMessage
{
Content =
$"Failure building patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString}"
});
}
count++;