Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Delete old patches from the CDN

Browse Source
This commit is contained in:
Timothy Baldridge 2020-06-29 15:57:09 -06:00
parent 2855b1d1ca
commit 2591c785fe
3 changed files with 94 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Wabbajack.Server.Test/ModlistUpdater.cs
View File

@ -36,6 +36,7 @@ namespace Wabbajack.Server.Test
var downloader = Fixture.GetService<ArchiveDownloader>();
var archiver = Fixture.GetService<ArchiveMaintainer>();
var patcher = Fixture.GetService<PatchBuilder>();
patcher.NoCleaning = true;
var sql = Fixture.GetService<SqlService>();
var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!");
@ -85,6 +86,7 @@ namespace Wabbajack.Server.Test
var downloader = Fixture.GetService<ArchiveDownloader>();
var archiver = Fixture.GetService<ArchiveMaintainer>();
var patcher = Fixture.GetService<PatchBuilder>();
patcher.NoCleaning = true;
var sql = Fixture.GetService<SqlService>();
var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!");

44
Wabbajack.Server/DataLayer/Patches.cs
View File

@ -156,5 +156,49 @@ namespace Wabbajack.Server.DataLayer
new {SrcId = srcId, DestId = destId});
}
public async Task<List<Patch>> GetOldPatches()
{
await using var conn = await Open();
var patches = await conn.QueryAsync<(Guid, Guid, long, DateTime?, bool?, string)>(
@"SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a ON p.SrcId = a.Id
LEFT JOIN dbo.ModListArchives m ON m.PrimaryKeyString = a.PrimaryKeyString AND m.Hash = a.Hash
WHERE m.PrimaryKeyString is null
UNION
SELECT p.SrcId, p.DestId, p.PatchSize, p.Finished, p.IsFailed, p.FailMessage
FROM dbo.Patches p
LEFT JOIN dbo.ArchiveDownloads a ON p.SrcId = a.Id
LEFT JOIN dbo.ModListArchives m ON m.PrimaryKeyString = a.PrimaryKeyString AND m.Hash = a.Hash
WHERE m.PrimaryKeyString is not null
AND (p.LastUsed < DATEADD(d, -7, getutcdate()) OR p.LastUsed is null and p.Finished < DATEADD(d, -7, getutcdate()))");
List<Patch> results = new List<Patch>();
foreach (var (srcId, destId, patchSize, finished, isFailed, failMessage) in patches)
{
results.Add( new Patch {
Src = await GetArchiveDownload(srcId),
Dest = await GetArchiveDownload(destId),
PatchSize = patchSize,
Finished = finished,
IsFailed = isFailed,
FailMessage = failMessage
});
}
return results;
}
public async Task DeletePatch(Patch patch)
{
await using var conn = await Open();
await conn.ExecuteAsync(@"DELETE FROM dbo.Patches WHERE SrcId = @SrcId AND DestId = @DestID",
new
{
SrcId = patch.Src.Id,
DestId = patch.Dest.Id
});
}
}
}

51
Wabbajack.Server/Services/PatchBuilder.cs
View File

@ -28,6 +28,8 @@ namespace Wabbajack.Server.Services
_sql = sql;
_maintainer = maintainer;
}
public bool NoCleaning { get; set; }
public override async Task<int> Execute()
{
@ -66,8 +68,6 @@ namespace Wabbajack.Server.Services
_maintainer.TryGetPath(patch.Src.Archive.Hash, out var srcPath);
_maintainer.TryGetPath(patch.Dest.Archive.Hash, out var destPath);
var patchName = $"{Consts.ArchiveUpdatesCDNFolder}\\{patch.Src.Archive.Hash.ToHex()}_{patch.Dest.Archive.Hash.ToHex()}";
await using var sigFile = new TempFile();
await using var patchFile = new TempFile();
await using var srcStream = await srcPath.OpenShared();
@ -78,7 +78,7 @@ namespace Wabbajack.Server.Services
await patchOutput.DisposeAsync();
var size = patchFile.Path.Size;
await UploadToCDN(patchFile.Path, patchName);
await UploadToCDN(patchFile.Path, PatchName(patch));
await patch.Finish(_sql, size);
@ -109,9 +109,46 @@ namespace Wabbajack.Server.Services
await _quickSync.Notify<ListValidator>();
}
if (!NoCleaning)
await CleanupOldPatches();
return count;
}
private static string PatchName(Patch patch)
{
return $"{Consts.ArchiveUpdatesCDNFolder}\\{patch.Src.Archive.Hash.ToHex()}_{patch.Dest.Archive.Hash.ToHex()}";
}
private async Task CleanupOldPatches()
{
var patches = await _sql.GetOldPatches();
using var client = await GetBunnyCdnFtpClient();
foreach (var patch in patches)
{
_logger.LogInformation($"Cleaning patch {patch.Src.Archive.Hash} -> {patch.Dest.Archive.Hash}");
/*
await _discordWebHook.Send(Channel.Ham,
new DiscordMessage
{
Content =
$"Removing patch from {patch.Src.Archive.State.PrimaryKeyString} to {patch.Dest.Archive.State.PrimaryKeyString} due it no longer being required by curated lists"
});
*/
if (!await DeleteFromCDN(client, PatchName(patch)))
{
_logger.LogWarning($"Patch file didn't exist {PatchName(patch)}");
}
await _sql.DeletePatch(patch);
var pendingPatch = await _sql.GetPendingPatch();
if (pendingPatch != default) break;
}
}
private async Task UploadToCDN(AbsolutePath patchFile, string patchName)
{
for (var times = 0; times < 5; times ++)
@ -136,6 +173,14 @@ namespace Wabbajack.Server.Services
_logger.Log(LogLevel.Error, $"Couldn't upload {patchFile} to {patchName}");
}
private async Task<bool> DeleteFromCDN(FtpClient client, string patchName)
{
if (!await client.FileExistsAsync(patchName))
return false;
await client.DeleteFileAsync(patchName);
return true;
}
private async Task<FtpClient> GetBunnyCdnFtpClient()
{
var info = await Utils.FromEncryptedJson<BunnyCdnFtpInfo>("bunny-cdn-ftp-info");