Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #970 from wabbajack-tools/wabbajack-cdn-backup

Browse Source 
Wabbajack cdn backup
This commit is contained in:
Timothy Baldridge 2020-07-19 21:36:48 -07:00 committed by GitHub
commit decf215fd7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 163 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -7,6 +7,11 @@
* List ingestion now supports compression and processes on a background threaded
* Support for validation of unlisted modlists
#### Version - 2.1.3.1 - 7/20/2020
* Fix for direct links on Mediafire
* Support for backup mirrors when a given CDN edge node isn't available
* Several help message improvements
#### Version - 2.1.3.0 - 7/16/2020
* Filters from the FilePicker are now being used
* Wabbajack will continue working even if the build server is down

9
Wabbajack.Lib/ClientAPI.cs
View File

@ -203,6 +203,15 @@ namespace Wabbajack.Lib
return results;
}
public static async Task<string[]> GetCDNMirrorList()
{
var client = await GetClient();
Utils.Log($"Looking for CDN mirrors");
var results = await client.GetJsonAsync<string[]>($"{Consts.WabbajackBuildServerUri}authored_files/mirrors");
return results;
}
public static async Task<VirusScanner.Result> GetVirusScanResult(AbsolutePath path)
{
var client = await GetClient();

48
Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs
View File

@ -2,6 +2,8 @@
using System.IO;
using System.IO.Compression;
using System.IO.MemoryMappedFiles;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.Exceptions;
@ -13,6 +15,9 @@ namespace Wabbajack.Lib.Downloaders
{
public class WabbajackCDNDownloader : IDownloader, IUrlDownloader
{
public string[]? Mirrors;
public long TotalRetries;
public async Task<AbstractDownloadState?> GetDownloaderState(dynamic archiveINI, bool quickMode = false)
{
var url = (Uri)DownloaderUtils.GetDirectURL(archiveINI);
@ -60,12 +65,13 @@ namespace Wabbajack.Lib.Downloaders
await using var fs = await destination.Create();
using var mmfile = MemoryMappedFile.CreateFromFile(fs, null, definition.Size, MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, false);
var client = new Wabbajack.Lib.Http.Client();
client.Headers.Add(("Host", Url.Host));
using var queue = new WorkQueue();
await definition.Parts.PMap(queue, async part =>
{
Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(definition.Parts.Length - part.Index, definition.Parts.Length));
await using var ostream = mmfile.CreateViewStream(part.Offset, part.Size);
using var response = await client.GetAsync($"{Url}/parts/{part.Index}");
using var response = await GetWithMirroredRetry(client, $"{Url}/parts/{part.Index}");
if (!response.IsSuccessStatusCode)
throw new HttpException((int)response.StatusCode, response.ReasonPhrase);
await response.Content.CopyToAsync(ostream);
@ -79,10 +85,48 @@ namespace Wabbajack.Lib.Downloaders
return true;
}
private async Task<HttpResponseMessage> GetWithMirroredRetry(Http.Client client, string url)
{
int retries = 0;
var downloader = DownloadDispatcher.GetInstance<WabbajackCDNDownloader>();
if (downloader.Mirrors != null)
url = ReplaceHost(downloader.Mirrors, url);
TOP:
try
{
return await client.GetAsync(url, retry: false);
}
catch (Exception ex)
{
if (retries > 5)
{
Utils.Log($"Tried to read from {retries} CDN servers, giving up");
throw;
}
Utils.Log($"Error reading {url} retying with a mirror");
Utils.Log(ex.ToString());
downloader.Mirrors ??= await ClientAPI.GetCDNMirrorList();
url = ReplaceHost(downloader.Mirrors, url);
retries += 1;
Interlocked.Increment(ref downloader.TotalRetries);
goto TOP;
}
}
private string ReplaceHost(string[] hosts, string url)
{
var rnd = new Random();
var builder = new UriBuilder(url) {Host = hosts[rnd.Next(0, hosts.Length)]};
return builder.ToString();
}
private async Task<CDNFileDefinition> GetDefinition()
{
var client = new Wabbajack.Lib.Http.Client();
using var data = await client.GetAsync(Url + "/definition.json.gz");
client.Headers.Add(("Host", Url.Host));
using var data = await GetWithMirroredRetry(client, Url + "/definition.json.gz");
await using var gz = new GZipStream(await data.Content.ReadAsStreamAsync(), CompressionMode.Decompress);
return gz.FromJson<CDNFileDefinition>();
}

7
Wabbajack.Lib/Http/Client.cs
View File

@ -15,10 +15,10 @@ namespace Wabbajack.Lib.Http
{
public List<(string, string?)> Headers = new List<(string, string?)>();
public List<Cookie> Cookies = new List<Cookie>();
public async Task<HttpResponseMessage> GetAsync(string url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true)
public async Task<HttpResponseMessage> GetAsync(string url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, bool retry = true)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions);
return await SendAsync(request, responseHeadersRead, errorsAsExceptions: errorsAsExceptions, retry: retry);
}
public async Task<HttpResponseMessage> GetAsync(Uri url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true)
@ -71,7 +71,7 @@ namespace Wabbajack.Lib.Http
return await result.Content.ReadAsStringAsync();
}
public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true)
public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead, bool errorsAsExceptions = true, bool retry = false)
{
foreach (var (k, v) in Headers)
msg.Headers.Add(k, v);
@ -95,6 +95,7 @@ namespace Wabbajack.Lib.Http
}
catch (Exception ex)
{
if (!retry) throw;
if (ex is HttpException http)
{
if (http.Code != 503 && http.Code != 521) throw;

23
Wabbajack.Server.Test/AuthoredFilesTests.cs
View File

@ -2,10 +2,12 @@
using System.Linq;
using System.Threading.Tasks;
using Wabbajack.Common;
using Wabbajack.Common.Exceptions;
using Wabbajack.Lib;
using Wabbajack.Lib.AuthorApi;
using Wabbajack.Lib.Downloaders;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.Services;
using Xunit;
using Xunit.Abstractions;
@ -42,5 +44,26 @@ namespace Wabbajack.BuildServer.Test
}
[Fact]
public async Task ServerGetsEdgeServerInfo()
{
var service = Fixture.GetService<CDNMirrorList>();
Assert.True(await service.Execute() > 0);
Assert.NotEmpty(service.Mirrors);
Assert.True(DateTime.UtcNow - service.LastUpdate < TimeSpan.FromMinutes(1));
var servers = await ClientAPI.GetCDNMirrorList();
Assert.Equal(service.Mirrors, servers);
var state = new WabbajackCDNDownloader.State(new Uri("https://wabbajack.b-cdn.net/this_file_doesn_t_exist"));
await DownloadDispatcher.PrepareAll(new[] {state});
await using var tmp = new TempFile();
await Assert.ThrowsAsync<HttpException>(async () => await state.Download(new Archive(state) {Name = "test"}, tmp.Path));
var downloader = DownloadDispatcher.GetInstance<WabbajackCDNDownloader>();
Assert.Equal(servers, downloader.Mirrors);
Assert.Equal(6, downloader.TotalRetries);
}
}
}

17
Wabbajack.Server/Controllers/AuthoredFiles.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Net;
@ -14,6 +15,7 @@ using Wabbajack.Common;
using Wabbajack.Lib.AuthorApi;
using Wabbajack.Server.DataLayer;
using Wabbajack.Server.DTOs;
using Wabbajack.Server.Services;
namespace Wabbajack.BuildServer.Controllers
{
@ -24,12 +26,15 @@ namespace Wabbajack.BuildServer.Controllers
private SqlService _sql;
private ILogger<AuthoredFiles> _logger;
private AppSettings _settings;
private CDNMirrorList _mirrorList;
public AuthoredFiles(ILogger<AuthoredFiles> logger, SqlService sql, AppSettings settings)
public AuthoredFiles(ILogger<AuthoredFiles> logger, SqlService sql, AppSettings settings, CDNMirrorList mirrorList)
{
_sql = sql;
_logger = logger;
_settings = settings;
_mirrorList = mirrorList;
}
[HttpPut]
@ -153,6 +158,7 @@ namespace Wabbajack.BuildServer.Controllers
");
[HttpGet]
[AllowAnonymous]
[Route("")]
@ -167,6 +173,15 @@ namespace Wabbajack.BuildServer.Controllers
Content = response
};
}
[HttpGet]
[AllowAnonymous]
[Route("mirrors")]
public async Task<IActionResult> GetMirrorList()
{
Response.Headers.Add("x-last-updated", _mirrorList.LastUpdate.ToString(CultureInfo.InvariantCulture));
return Ok(_mirrorList.Mirrors);
}
}

58
Wabbajack.Server/Services/CDNMirrorList.cs Normal file
View File

@ -0,0 +1,58 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using System.Xml.Linq;
using HtmlAgilityPack;
using Microsoft.Extensions.Logging;
using Wabbajack.BuildServer;
using Wabbajack.Common;
namespace Wabbajack.Server.Services
{
public class CDNMirrorList : AbstractService<ListValidator, int>
{
public CDNMirrorList(ILogger<ListValidator> logger, AppSettings settings, QuickSync quickSync) : base(logger, settings, quickSync, TimeSpan.FromHours(1))
{
}
public string[] Mirrors { get; private set; }
public DateTime LastUpdate { get; private set; }
public override async Task<int> Execute()
{
var client = new Lib.Http.Client();
var json = await client.GetStringAsync("https://bunnycdn.com/api/system/edgeserverlist");
client.Headers.Add(("Host", "wabbajack.b-cdn.net"));
using var queue = new WorkQueue();
var mirrors = json.FromJsonString<string[]>();
_logger.LogInformation($"Found {mirrors.Length} edge severs");
var servers = (await mirrors
.PMap(queue, async ip =>
{
try
{
// We use a volume server, so this file will only exist on some (lower cost) servers
using var result = await client.GetAsync(
$"https://{ip}/WABBAJACK_TEST_FILE.zip_48f799f6-39b2-4229-a329-7459c9965c2d/definition.json.gz",
errorsAsExceptions: false, retry: false);
var data = await result.Content.ReadAsByteArrayAsync();
return (ip, use: result.IsSuccessStatusCode, size : data.Length);
}
catch (Exception)
{
return (ip, use : false, size: 0);
}
}))
.Where(r => r.use && r.size == 267)
.Select(r => r.ip)
.ToArray();
_logger.LogInformation($"Found {servers.Length} valid mirrors");
Mirrors = servers;
LastUpdate = DateTime.UtcNow;
return Mirrors.Length;
}
}
}

2
Wabbajack.Server/Startup.cs
View File

@ -67,6 +67,7 @@ namespace Wabbajack.Server
services.AddSingleton<DiscordWebHook>();
services.AddSingleton<NexusKeyMaintainance>();
services.AddSingleton<PatchBuilder>();
services.AddSingleton<CDNMirrorList>();
services.AddMvc();
services.AddControllers()
@ -121,6 +122,7 @@ namespace Wabbajack.Server
app.UseService<DiscordWebHook>();
app.UseService<NexusKeyMaintainance>();
app.UseService<PatchBuilder>();
app.UseService<CDNMirrorList>();
app.Use(next =>
{