diff --git a/Wabbajack.Common/Consts.cs b/Wabbajack.Common/Consts.cs index 13827c09..c4b227ec 100644 --- a/Wabbajack.Common/Consts.cs +++ b/Wabbajack.Common/Consts.cs @@ -144,6 +144,8 @@ namespace Wabbajack.Common public static Uri WabbajackOrg = new Uri("https://www.wabbajack.org/"); public static long UPLOADED_FILE_BLOCK_SIZE = (long)1024 * 1024 * 2; + + public static string ArchiveUpdatesCDNFolder = "archive_updates"; } } diff --git a/Wabbajack.Common/Paths.cs b/Wabbajack.Common/Paths.cs index 612e0ed1..eae25994 100644 --- a/Wabbajack.Common/Paths.cs +++ b/Wabbajack.Common/Paths.cs @@ -340,7 +340,7 @@ namespace Wabbajack.Common public async Task WriteAllAsync(Stream data, bool disposeAfter = true) { await using var fs = Create(); - await fs.CopyToAsync(data); + await data.CopyToAsync(fs); if (disposeAfter) await data.DisposeAsync(); } diff --git a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs index 912306a2..1a104d5b 100644 --- a/Wabbajack.Lib/Downloaders/HTTPDownloader.cs +++ b/Wabbajack.Lib/Downloaders/HTTPDownloader.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Net.Http; using System.Net.Http.Headers; using System.Threading.Tasks; +using AngleSharp.Css; using Newtonsoft.Json; using Wabbajack.Common; using Wabbajack.Common.Serialization.Json; @@ -46,7 +47,7 @@ namespace Wabbajack.Lib.Downloaders } [JsonName("HttpDownloader")] - public class State : AbstractDownloadState + public class State : AbstractDownloadState, IUpgradingState { public string Url { get; } @@ -210,6 +211,28 @@ TOP: return new [] {"[General]", $"directURL={Url}"}; } + + public async Task<(Archive? Archive, TempFile NewFile)> FindUpgrade(Archive a) + { + var tmpFile = new TempFile(); + + var newArchive = new Archive(this) {Name = a.Name}; + + if (!await Download(newArchive, tmpFile.Path)) + return default; + + newArchive.Hash = await tmpFile.Path.FileHashAsync(); + newArchive.Size = tmpFile.Path.Size; + + return (newArchive, tmpFile); + + } + + public bool ValidateUpgrade(AbstractDownloadState newArchiveState) + { + var httpState = (State)newArchiveState; + return httpState.Url == Url; + } } } } diff --git a/Wabbajack.Server.Test/ModlistUpdater.cs b/Wabbajack.Server.Test/ModlistUpdater.cs index 418af94e..40463b75 100644 --- a/Wabbajack.Server.Test/ModlistUpdater.cs +++ b/Wabbajack.Server.Test/ModlistUpdater.cs @@ -73,8 +73,60 @@ namespace Wabbajack.Server.Test await Assert.ThrowsAsync(async () => await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); Assert.Equal(1, await patcher.Execute()); - Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_upgrades/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); + Assert.Equal(new Uri("https://wabbajacktest.b-cdn.net/archive_updates/79223277e28e1b7b_3286c571d95f5666"),await ClientAPI.GetModUpgrade(oldArchive, newArchive, TimeSpan.Zero, TimeSpan.Zero)); + } + [Fact] + public async Task TestEndToEndArchiveUpdating() + { + var modLists = await MakeModList(); + Consts.ModlistMetadataURL = modLists.ToString(); + + + var downloader = Fixture.GetService(); + var archiver = Fixture.GetService(); + var patcher = Fixture.GetService(); + + var sql = Fixture.GetService(); + var oldFileData = Encoding.UTF8.GetBytes("Cheese for Everyone!"); + var newFileData = Encoding.UTF8.GetBytes("Forks for Everyone!"); + var oldDataHash = oldFileData.xxHash(); + var newDataHash = newFileData.xxHash(); + + await "upgrading_file.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(oldFileData); + + var oldArchive = new Archive(new HTTPDownloader.State(MakeURL("upgrading_file.txt"))) + { + Size = oldFileData.Length, + Hash = oldDataHash + }; + + await IngestData(archiver, oldFileData); + await sql.EnqueueDownload(oldArchive); + var oldDownload = await sql.GetNextPendingDownload(); + await oldDownload.Finish(sql); + + + // Now update the file + await "upgrading_file.txt".RelativeTo(Fixture.ServerPublicFolder).WriteAllBytesAsync(newFileData); + + + using var tempFile = new TempFile(); + var pendingRequest = DownloadDispatcher.DownloadWithPossibleUpgrade(oldArchive, tempFile.Path); + + for (var times = 0; await downloader.Execute() == 0 && times < 40; times ++) + { + await Task.Delay(TimeSpan.FromMilliseconds(200)); + } + + + for (var times = 0; await patcher.Execute() == 0 && times < 40; times ++) + { + await Task.Delay(TimeSpan.FromMilliseconds(200)); + } + + Assert.True(await pendingRequest); + Assert.Equal(oldDataHash, await tempFile.Path.FileHashAsync()); } private async Task IngestData(ArchiveMaintainer am, byte[] data) diff --git a/Wabbajack.Server/Controllers/ModUpgrade.cs b/Wabbajack.Server/Controllers/ModUpgrade.cs index ea6c919e..d7de7265 100644 --- a/Wabbajack.Server/Controllers/ModUpgrade.cs +++ b/Wabbajack.Server/Controllers/ModUpgrade.cs @@ -44,7 +44,7 @@ namespace Wabbajack.BuildServer.Controllers { return Ok( - $"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/archive_upgrades/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); + $"https://{_settings.BunnyCDN_StorageZone}.b-cdn.net/{Consts.ArchiveUpdatesCDNFolder}/{request.OldArchive.Hash.ToHex()}_{request.NewArchive.Hash.ToHex()}"); } return NotFound("Patch creation failed"); diff --git a/Wabbajack.Server/Services/PatchBuilder.cs b/Wabbajack.Server/Services/PatchBuilder.cs index b150d73c..aa9961e8 100644 --- a/Wabbajack.Server/Services/PatchBuilder.cs +++ b/Wabbajack.Server/Services/PatchBuilder.cs @@ -50,7 +50,7 @@ namespace Wabbajack.Server.Services _maintainer.TryGetPath(patch.Src.Archive.Hash, out var srcPath); _maintainer.TryGetPath(patch.Dest.Archive.Hash, out var destPath); - var patchName = $"archive_updates\\{patch.Src.Archive.Hash}_{patch.Dest.Archive.Hash}"; + var patchName = $"{Consts.ArchiveUpdatesCDNFolder}\\{patch.Src.Archive.Hash.ToHex()}_{patch.Dest.Archive.Hash.ToHex()}"; using var sigFile = new TempFile(); await using var srcStream = srcPath.OpenShared(); @@ -58,8 +58,8 @@ namespace Wabbajack.Server.Services await using var sigStream = sigFile.Path.Create(); using var ftpClient = await GetBunnyCdnFtpClient(); - if (!await ftpClient.DirectoryExistsAsync("archive_updates")) - await ftpClient.CreateDirectoryAsync("archive_updates"); + if (!await ftpClient.DirectoryExistsAsync(Consts.ArchiveUpdatesCDNFolder)) + await ftpClient.CreateDirectoryAsync(Consts.ArchiveUpdatesCDNFolder); await using var patchOutput = await ftpClient.OpenWriteAsync(patchName);