From 19dfb40eca997b098654e49bab24484f8dc6cb54 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 19 Jul 2020 21:45:55 -0600 Subject: [PATCH 1/2] Retry on different CDN servers when we can't find a file on the default CDN server --- Wabbajack.Lib/ClientAPI.cs | 9 +++ .../Downloaders/WabbajackCDNDownloader.cs | 48 ++++++++++++++- Wabbajack.Lib/Http/Client.cs | 7 ++- Wabbajack.Server.Test/AuthoredFilesTests.cs | 23 ++++++++ Wabbajack.Server/Controllers/AuthoredFiles.cs | 17 +++++- Wabbajack.Server/Services/CDNMirrorList.cs | 58 +++++++++++++++++++ Wabbajack.Server/Startup.cs | 2 + 7 files changed, 158 insertions(+), 6 deletions(-) create mode 100644 Wabbajack.Server/Services/CDNMirrorList.cs diff --git a/Wabbajack.Lib/ClientAPI.cs b/Wabbajack.Lib/ClientAPI.cs index 8baeb55a..312c39f8 100644 --- a/Wabbajack.Lib/ClientAPI.cs +++ b/Wabbajack.Lib/ClientAPI.cs @@ -203,6 +203,15 @@ namespace Wabbajack.Lib return results; } + + public static async Task GetCDNMirrorList() + { + var client = await GetClient(); + Utils.Log($"Looking for CDN mirrors"); + var results = await client.GetJsonAsync($"{Consts.WabbajackBuildServerUri}authored_files/mirrors"); + return results; + } + public static async Task GetVirusScanResult(AbsolutePath path) { var client = await GetClient(); diff --git a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs index 452381fe..88fd5ed8 100644 --- a/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs +++ b/Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs @@ -2,6 +2,8 @@ using System.IO; using System.IO.Compression; using System.IO.MemoryMappedFiles; +using System.Net.Http; +using System.Threading; using System.Threading.Tasks; using Wabbajack.Common; using Wabbajack.Common.Exceptions; @@ -13,6 +15,9 @@ namespace Wabbajack.Lib.Downloaders { public class WabbajackCDNDownloader : IDownloader, IUrlDownloader { + public string[]? Mirrors; + public long TotalRetries; + public async Task GetDownloaderState(dynamic archiveINI, bool quickMode = false) { var url = (Uri)DownloaderUtils.GetDirectURL(archiveINI); @@ -60,12 +65,13 @@ namespace Wabbajack.Lib.Downloaders await using var fs = await destination.Create(); using var mmfile = MemoryMappedFile.CreateFromFile(fs, null, definition.Size, MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, false); var client = new Wabbajack.Lib.Http.Client(); + client.Headers.Add(("Host", Url.Host)); using var queue = new WorkQueue(); await definition.Parts.PMap(queue, async part => { Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(definition.Parts.Length - part.Index, definition.Parts.Length)); await using var ostream = mmfile.CreateViewStream(part.Offset, part.Size); - using var response = await client.GetAsync($"{Url}/parts/{part.Index}"); + using var response = await GetWithMirroredRetry(client, $"{Url}/parts/{part.Index}"); if (!response.IsSuccessStatusCode) throw new HttpException((int)response.StatusCode, response.ReasonPhrase); await response.Content.CopyToAsync(ostream); @@ -79,10 +85,48 @@ namespace Wabbajack.Lib.Downloaders return true; } + private async Task GetWithMirroredRetry(Http.Client client, string url) + { + int retries = 0; + var downloader = DownloadDispatcher.GetInstance(); + if (downloader.Mirrors != null) + url = ReplaceHost(downloader.Mirrors, url); + + TOP: + + try + { + return await client.GetAsync(url, retry: false); + } + catch (Exception ex) + { + if (retries > 5) + { + Utils.Log($"Tried to read from {retries} CDN servers, giving up"); + throw; + } + Utils.Log($"Error reading {url} retying with a mirror"); + Utils.Log(ex.ToString()); + downloader.Mirrors ??= await ClientAPI.GetCDNMirrorList(); + url = ReplaceHost(downloader.Mirrors, url); + retries += 1; + Interlocked.Increment(ref downloader.TotalRetries); + goto TOP; + } + } + + private string ReplaceHost(string[] hosts, string url) + { + var rnd = new Random(); + var builder = new UriBuilder(url) {Host = hosts[rnd.Next(0, hosts.Length)]}; + return builder.ToString(); + } + private async Task GetDefinition() { var client = new Wabbajack.Lib.Http.Client(); - using var data = await client.GetAsync(Url + "/definition.json.gz"); + client.Headers.Add(("Host", Url.Host)); + using var data = await GetWithMirroredRetry(client, Url + "/definition.json.gz"); await using var gz = new GZipStream(await data.Content.ReadAsStreamAsync(), CompressionMode.Decompress); return gz.FromJson(); } diff --git a/Wabbajack.Lib/Http/Client.cs b/Wabbajack.Lib/Http/Client.cs index 7b5a3ab2..dd24d978 100644 --- a/Wabbajack.Lib/Http/Client.cs +++ b/Wabbajack.Lib/Http/Client.cs @@ -15,10 +15,10 @@ namespace Wabbajack.Lib.Http { public List<(string, string?)> Headers = new List<(string, string?)>(); public List Cookies = new List(); - public async Task GetAsync(string url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true) + public async Task GetAsync(string url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, bool retry = true) { var request = new HttpRequestMessage(HttpMethod.Get, url); - return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions); + return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions, retry: retry); } public async Task GetAsync(Uri url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true) @@ -71,7 +71,7 @@ namespace Wabbajack.Lib.Http return await result.Content.ReadAsStringAsync(); } - public async Task SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true) + public async Task SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, bool retry = false) { foreach (var (k, v) in Headers) msg.Headers.Add(k, v); @@ -95,6 +95,7 @@ namespace Wabbajack.Lib.Http } catch (Exception ex) { + if (!retry) throw; if (ex is HttpException http) { if (http.Code != 503 && http.Code != 521) throw; diff --git a/Wabbajack.Server.Test/AuthoredFilesTests.cs b/Wabbajack.Server.Test/AuthoredFilesTests.cs index 5bce8623..ec88d435 100644 --- a/Wabbajack.Server.Test/AuthoredFilesTests.cs +++ b/Wabbajack.Server.Test/AuthoredFilesTests.cs @@ -2,10 +2,12 @@ using System.Linq; using System.Threading.Tasks; using Wabbajack.Common; +using Wabbajack.Common.Exceptions; using Wabbajack.Lib; using Wabbajack.Lib.AuthorApi; using Wabbajack.Lib.Downloaders; using Wabbajack.Server.DataLayer; +using Wabbajack.Server.Services; using Xunit; using Xunit.Abstractions; @@ -42,5 +44,26 @@ namespace Wabbajack.BuildServer.Test } + [Fact] + public async Task ServerGetsEdgeServerInfo() + { + var service = Fixture.GetService(); + Assert.True(await service.Execute() > 0); + Assert.NotEmpty(service.Mirrors); + Assert.True(DateTime.UtcNow - service.LastUpdate < TimeSpan.FromMinutes(1)); + + var servers = await ClientAPI.GetCDNMirrorList(); + Assert.Equal(service.Mirrors, servers); + + var state = new WabbajackCDNDownloader.State(new Uri("https://wabbajack.b-cdn.net/this_file_doesn_t_exist")); + await DownloadDispatcher.PrepareAll(new[] {state}); + await using var tmp = new TempFile(); + + await Assert.ThrowsAsync(async () => await state.Download(new Archive(state) {Name = "test"}, tmp.Path)); + var downloader = DownloadDispatcher.GetInstance(); + Assert.Equal(servers, downloader.Mirrors); + Assert.Equal(6, downloader.TotalRetries); + } + } } diff --git a/Wabbajack.Server/Controllers/AuthoredFiles.cs b/Wabbajack.Server/Controllers/AuthoredFiles.cs index 489442a8..7c51107c 100644 --- a/Wabbajack.Server/Controllers/AuthoredFiles.cs +++ b/Wabbajack.Server/Controllers/AuthoredFiles.cs @@ -1,4 +1,5 @@ using System; +using System.Globalization; using System.IO; using System.IO.Compression; using System.Net; @@ -14,6 +15,7 @@ using Wabbajack.Common; using Wabbajack.Lib.AuthorApi; using Wabbajack.Server.DataLayer; using Wabbajack.Server.DTOs; +using Wabbajack.Server.Services; namespace Wabbajack.BuildServer.Controllers { @@ -24,12 +26,15 @@ namespace Wabbajack.BuildServer.Controllers private SqlService _sql; private ILogger _logger; private AppSettings _settings; + private CDNMirrorList _mirrorList; - public AuthoredFiles(ILogger logger, SqlService sql, AppSettings settings) + + public AuthoredFiles(ILogger logger, SqlService sql, AppSettings settings, CDNMirrorList mirrorList) { _sql = sql; _logger = logger; _settings = settings; + _mirrorList = mirrorList; } [HttpPut] @@ -153,6 +158,7 @@ namespace Wabbajack.BuildServer.Controllers "); + [HttpGet] [AllowAnonymous] [Route("")] @@ -167,6 +173,15 @@ namespace Wabbajack.BuildServer.Controllers Content = response }; } + + [HttpGet] + [AllowAnonymous] + [Route("mirrors")] + public async Task GetMirrorList() + { + Response.Headers.Add("x-last-updated", _mirrorList.LastUpdate.ToString(CultureInfo.InvariantCulture)); + return Ok(_mirrorList.Mirrors); + } } diff --git a/Wabbajack.Server/Services/CDNMirrorList.cs b/Wabbajack.Server/Services/CDNMirrorList.cs new file mode 100644 index 00000000..e0f438ee --- /dev/null +++ b/Wabbajack.Server/Services/CDNMirrorList.cs @@ -0,0 +1,58 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; +using System.Xml.Linq; +using HtmlAgilityPack; +using Microsoft.Extensions.Logging; +using Wabbajack.BuildServer; +using Wabbajack.Common; + +namespace Wabbajack.Server.Services +{ + public class CDNMirrorList : AbstractService + { + public CDNMirrorList(ILogger logger, AppSettings settings, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromHours(1)) + { + } + public string[] Mirrors { get; private set; } + public DateTime LastUpdate { get; private set; } + + public override async Task Execute() + { + var client = new Lib.Http.Client(); + var json = await client.GetStringAsync("https://bunnycdn.com/api/system/edgeserverlist"); + client.Headers.Add(("Host", "wabbajack.b-cdn.net")); + using var queue = new WorkQueue(); + var mirrors = json.FromJsonString(); + _logger.LogInformation($"Found {mirrors.Length} edge severs"); + + var servers = (await mirrors + .PMap(queue, async ip => + { + try + { + // We use a volume server, so this file will only exist on some (lower cost) servers + using var result = await client.GetAsync( + $"https://{ip}/WABBAJACK_TEST_FILE.zip_48f799f6-39b2-4229-a329-7459c9965c2d/definition.json.gz", + errorsAsExceptions: false, retry: false); + var data = await result.Content.ReadAsByteArrayAsync(); + return (ip, use: result.IsSuccessStatusCode, size : data.Length); + } + catch (Exception) + { + return (ip, use : false, size: 0); + } + })) + .Where(r => r.use && r.size == 267) + .Select(r => r.ip) + .ToArray(); + _logger.LogInformation($"Found {servers.Length} valid mirrors"); + Mirrors = servers; + LastUpdate = DateTime.UtcNow; + return Mirrors.Length; + + } + } +} diff --git a/Wabbajack.Server/Startup.cs b/Wabbajack.Server/Startup.cs index ed45eaf8..9fae9f72 100644 --- a/Wabbajack.Server/Startup.cs +++ b/Wabbajack.Server/Startup.cs @@ -67,6 +67,7 @@ namespace Wabbajack.Server services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddMvc(); services.AddControllers() @@ -121,6 +122,7 @@ namespace Wabbajack.Server app.UseService(); app.UseService(); app.UseService(); + app.UseService(); app.Use(next => { From 695448ced8c7c4873014bf76ad7a943354ca7fa2 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 19 Jul 2020 21:47:20 -0600 Subject: [PATCH 2/2] Retry on different CDN servers when we can't find a file on the default CDN server --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 419c95bd..259275ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ * List ingestion now supports compression and processes on a background threaded * Support for validation of unlisted modlists +#### Version - 2.1.3.1 - 7/20/2020 +* Fix for direct links on Mediafire +* Support for backup mirrors when a given CDN edge node isn't available +* Several help message improvements + #### Version - 2.1.3.0 - 7/16/2020 * Filters from the FilePicker are now being used * Wabbajack will continue working even if the build server is down