Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

2.2.2.0 - Route CDN through proxy

Browse Source
This commit is contained in:
halgari 2020-08-31 17:29:48 -06:00
parent 5d916da1ab
commit 865a592e09
10 changed files with 77 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -1,5 +1,8 @@
### Changelog
#### Version - 2.2.2.0 - 8/31/2020
* Route CDN requests through a reverse proxy to improve reliability
#### Version - 2.2.1.7 - 8/25/2020
* HOTFIX - Fix 404 errors with mirrors

4
Wabbajack.CLI/Wabbajack.CLI.csproj
View File

@ -6,8 +6,8 @@
<AssemblyName>wabbajack-cli</AssemblyName>
<Company>Wabbajack</Company>
<Platforms>x64</Platforms>
<AssemblyVersion>2.2.1.7</AssemblyVersion>
<FileVersion>2.2.1.7</FileVersion>
<AssemblyVersion>2.2.2.0</AssemblyVersion>
<FileVersion>2.2.2.0</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>An automated ModList installer</Description>
<PublishReadyToRun>true</PublishReadyToRun>

4
Wabbajack.Launcher/Wabbajack.Launcher.csproj
View File

@ -4,8 +4,8 @@
<OutputType>WinExe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
<UseWPF>true</UseWPF>
<AssemblyVersion>2.2.1.7</AssemblyVersion>
<FileVersion>2.2.1.7</FileVersion>
<AssemblyVersion>2.2.2.0</AssemblyVersion>
<FileVersion>2.2.2.0</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>Wabbajack Application Launcher</Description>
<PublishReadyToRun>true</PublishReadyToRun>

6
Wabbajack.Lib/Downloaders/DownloadDispatcher.cs
View File

@ -138,6 +138,12 @@ namespace Wabbajack.Lib.Downloaders
var tempFile = new TempFile();
if (WabbajackCDNDownloader.DomainRemaps.TryGetValue(patchResult.Host, out var remap))
{
var builder = new UriBuilder(patchResult) {Host = remap};
patchResult = builder.Uri;
}
using var response = await (await ClientAPI.GetClient()).GetAsync(patchResult);
await tempFile.Path.WriteAllAsync(await response.Content.ReadAsStreamAsync());

62
Wabbajack.Lib/Downloaders/WabbajackCDNDownloader.cs
View File

@ -1,10 +1,12 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.IO.MemoryMappedFiles;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using AngleSharp;
using Wabbajack.Common;
using Wabbajack.Common.Exceptions;
using Wabbajack.Common.Serialization.Json;
@ -13,8 +15,18 @@ using Wabbajack.Lib.Validation;
namespace Wabbajack.Lib.Downloaders
{
public class WabbajackCDNDownloader : IDownloader, IUrlDownloader
{
public static Dictionary<string, string> DomainRemaps = new Dictionary<string, string>
{
{"wabbajack.b-cdn.net", "authored-files.wabbajack.org"},
{"wabbajack-mirror.b-cdn.net", "mirror.wabbajack.org"},
{"wabbajack-patches.b-cdn-net", "patches.wabbajack.org"},
{"wabbajacktest.b-cdn.net", "test-files.wabbajack.org"}
};
public string[]? Mirrors;
public long TotalRetries;
@ -65,16 +77,34 @@ namespace Wabbajack.Lib.Downloaders
await using var fs = await destination.Create();
using var mmfile = MemoryMappedFile.CreateFromFile(fs, null, definition.Size, MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, false);
var client = new Wabbajack.Lib.Http.Client();
client.Headers.Add(("Host", Url.Host));
if (!DomainRemaps.ContainsKey(Url.Host))
client.Headers.Add(("Host", Url.Host));
using var queue = new WorkQueue();
await definition.Parts.PMap(queue, async part =>
{
Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(definition.Parts.Length - part.Index, definition.Parts.Length));
await using var ostream = mmfile.CreateViewStream(part.Offset, part.Size);
using var response = await GetWithMirroredRetry(client, $"{Url}/parts/{part.Index}");
if (!response.IsSuccessStatusCode)
throw new HttpException((int)response.StatusCode, response.ReasonPhrase);
await response.Content.CopyToAsync(ostream);
if (DomainRemaps.TryGetValue(Url.Host, out var remap))
{
var builder = new UriBuilder(Url) {Host = remap};
using var response = await client.GetAsync($"{builder}/parts/{part.Index}");
if (!response.IsSuccessStatusCode)
throw new HttpException((int)response.StatusCode, response.ReasonPhrase);
await response.Content.CopyToAsync(ostream);
}
else
{
using var response = await GetWithMirroredRetry(client, $"{Url}/parts/{part.Index}");
if (!response.IsSuccessStatusCode)
throw new HttpException((int)response.StatusCode, response.ReasonPhrase);
await response.Content.CopyToAsync(ostream);
}
});
return true;
}
@ -121,14 +151,26 @@ namespace Wabbajack.Lib.Downloaders
var builder = new UriBuilder(url) {Host = hosts[rnd.Next(0, hosts.Length)]};
return builder.ToString();
}
private async Task<CDNFileDefinition> GetDefinition()
{
var client = new Wabbajack.Lib.Http.Client();
client.Headers.Add(("Host", Url.Host));
using var data = await GetWithMirroredRetry(client, Url + "/definition.json.gz");
await using var gz = new GZipStream(await data.Content.ReadAsStreamAsync(), CompressionMode.Decompress);
return gz.FromJson<CDNFileDefinition>();
if (DomainRemaps.TryGetValue(Url.Host, out var remap))
{
var builder = new UriBuilder(Url) {Host = remap};
using var data = await client.GetAsync(builder + "/definition.json.gz");
await using var gz = new GZipStream(await data.Content.ReadAsStreamAsync(),
CompressionMode.Decompress);
return gz.FromJson<CDNFileDefinition>();
}
else
{
client.Headers.Add(("Host", Url.Host));
using var data = await GetWithMirroredRetry(client, Url + "/definition.json.gz");
await using var gz = new GZipStream(await data.Content.ReadAsStreamAsync(),
CompressionMode.Decompress);
return gz.FromJson<CDNFileDefinition>();
}
}
public override IDownloader GetDownloader()

3
Wabbajack.Server.Test/AuthoredFilesTests.cs
View File

@ -61,8 +61,7 @@ namespace Wabbajack.BuildServer.Test
await Assert.ThrowsAsync<HttpException>(async () => await state.Download(new Archive(state) {Name = "test"}, tmp.Path));
var downloader = DownloadDispatcher.GetInstance<WabbajackCDNDownloader>();
Assert.Equal(servers, downloader.Mirrors);
Assert.Equal(6, downloader.TotalRetries);
Assert.Null(downloader.Mirrors); // Now works through a host remap
}
}

4
Wabbajack.Server/Wabbajack.Server.csproj
View File

@ -3,8 +3,8 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
<AssemblyVersion>2.2.1.7</AssemblyVersion>
<FileVersion>2.2.1.7</FileVersion>
<AssemblyVersion>2.2.2.0</AssemblyVersion>
<FileVersion>2.2.2.0</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>Wabbajack Server</Description>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>

13
Wabbajack.Test/DownloaderTests.cs
View File

@ -43,12 +43,14 @@ namespace Wabbajack.Test
_unsubMsgs.Dispose();
}
/*
[Fact]
public async Task TestAllPrepares()
{
foreach (var downloader in DownloadDispatcher.Downloaders)
await downloader.Prepare();
}
}*/
[Fact]
public async Task MegaDownload()
@ -330,6 +332,7 @@ namespace Wabbajack.Test
}
/*
[Fact]
public async Task VectorPlexusDownload()
{
@ -341,10 +344,6 @@ namespace Wabbajack.Test
Assert.NotNull(state);
/*var url_state = DownloadDispatcher.ResolveArchive("https://www.loverslab.com/files/file/11116-test-file-for-wabbajack-integration/?do=download&r=737123&confirm=1&t=1");
Assert.Equal("http://build.wabbajack.org/WABBAJACK_TEST_FILE.txt",
((HTTPDownloader.State)url_state).Url);
*/
var converted = RoundTripState(state);
Assert.True(await converted.Verify(new Archive(state: null!) { Size = 20}));
await using var filename = new TempFile();
@ -357,7 +356,7 @@ namespace Wabbajack.Test
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
}
}*/
[Fact]
public async Task YandexDownloader()
@ -408,6 +407,7 @@ namespace Wabbajack.Test
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
}
/* Site is down
[Fact]
public async Task TESAllDownloader()
{
@ -431,6 +431,7 @@ namespace Wabbajack.Test
Assert.Equal("Cheese for Everyone!", await filename.Path.ReadAllTextAsync());
}
*/
/* WAITING FOR APPROVAL BY MODERATOR
[Fact]

2
Wabbajack.Test/NexusTests.cs
View File

@ -15,8 +15,6 @@ namespace Wabbajack.Test
{
var results = (await NexusUpdatesFeeds.GetUpdates()).ToArray();
Assert.NotEmpty(results);
Utils.Log($"Loaded {results.Length} updates from the Nexus");
foreach (var result in results)

4
Wabbajack/Wabbajack.csproj
View File

@ -6,8 +6,8 @@
<UseWPF>true</UseWPF>
<Platforms>x64</Platforms>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier>
<AssemblyVersion>2.2.1.7</AssemblyVersion>
<FileVersion>2.2.1.7</FileVersion>
<AssemblyVersion>2.2.2.0</AssemblyVersion>
<FileVersion>2.2.2.0</FileVersion>
<Copyright>Copyright © 2019-2020</Copyright>
<Description>An automated ModList installer</Description>
<PublishReadyToRun>true</PublishReadyToRun>