Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Swap HttpClient over to SocketsHttpHandler

Browse Source
This commit is contained in:
Timothy Baldridge 2020-02-14 15:23:27 -07:00
parent b77db18bfa
commit 8d13a4c455
14 changed files with 124 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Compression.BSA/Compression.BSA.csproj
View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework> <TargetFramework>netcoreapp3.1</TargetFramework>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks> <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<Platforms>x64</Platforms> <Platforms>x64</Platforms>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier> <RuntimeIdentifier>win10-x64</RuntimeIdentifier>

62
Wabbajack.Common/Http/Client.cs Normal file
View File

@ -0,0 +1,62 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
namespace Wabbajack.Common.Http
{
public class Client
{
public List<(string, string)> Headers => new List<(string, string)>();
public List<Cookie> Cookies = new List<Cookie>();
public async Task<HttpResponseMessage> GetAsync(string url, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseContentRead)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
foreach (var (k, v) in Headers)
request.Headers.Add(k, v);
return await SendAsync(request, responseHeadersRead);
}
public async Task<string> GetStringAsync(string url)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
foreach (var (k, v) in Headers)
request.Headers.Add(k, v);
if (Cookies.Count > 0)
Cookies.ForEach(c => ClientFactory.Cookies.Add(c));
return await SendStringAsync(request);
}
private async Task<string> SendStringAsync(HttpRequestMessage request)
{
var result = await SendAsync(request);
return await result.Content.ReadAsStringAsync();
}
public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseContentRead)
{
int retries = 0;
TOP:
try
{
var response = await ClientFactory.Client.SendAsync(msg, responseHeadersRead);
return response;
}
catch (Exception)
{
if (retries > Consts.MaxHTTPRetries) throw;
retries++;
Utils.Log($"Http Connect error to {msg.RequestUri} retry {retries}");
await Task.Delay(100 * retries);
goto TOP;
}
}
}
}

25
Wabbajack.Common/Http/ClientFactory.cs Normal file
View File

@ -0,0 +1,25 @@
using System;
using System.Net;
using SysHttp = System.Net.Http;
namespace Wabbajack.Common.Http
{
public static class ClientFactory
{
private static SysHttp.SocketsHttpHandler _socketsHandler { get; }
internal static SysHttp.HttpClient Client { get; }
internal static CookieContainer Cookies { get; }
static ClientFactory()
{
Cookies = new CookieContainer();
_socketsHandler = new SysHttp.SocketsHttpHandler
{
CookieContainer = Cookies,
MaxConnectionsPerServer = 8,
PooledConnectionLifetime = TimeSpan.FromSeconds(2),
PooledConnectionIdleTimeout = TimeSpan.FromSeconds(2)
};
Client = new SysHttp.HttpClient(_socketsHandler);
}
}
}

3
Wabbajack.Common/Wabbajack.Common.csproj
View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework> <TargetFramework>netcoreapp3.1</TargetFramework>
<Platforms>x64</Platforms> <Platforms>x64</Platforms>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier> <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
@ -39,6 +39,7 @@
<PackageReference Include="ReactiveUI" Version="11.1.23" /> <PackageReference Include="ReactiveUI" Version="11.1.23" />
<PackageReference Include="SharpZipLib" Version="1.2.0" /> <PackageReference Include="SharpZipLib" Version="1.2.0" />
<PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" /> <PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
<PackageReference Include="System.Net.Http" Version="4.3.4" />
<PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.7.0" /> <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.7.0" />
<PackageReference Include="System.Security.Principal.Windows" Version="4.7.0" /> <PackageReference Include="System.Security.Principal.Windows" Version="4.7.0" />
<PackageReference Include="YamlDotNet" Version="8.1.0" /> <PackageReference Include="YamlDotNet" Version="8.1.0" />

4
Wabbajack.Lib/Downloaders/AbstractNeedsLoginDownloader.cs
View File

@ -21,7 +21,7 @@ namespace Wabbajack.Lib.Downloaders
private readonly string _encryptedKeyName; private readonly string _encryptedKeyName;
private readonly string _cookieDomain; private readonly string _cookieDomain;
private readonly string _cookieName; private readonly string _cookieName;
internal HttpClient AuthedClient; internal Common.Http.Client AuthedClient;
/// <summary> /// <summary>
/// Sets up all the login facilites needed for a INeedsLogin downloader based on having the user log /// Sets up all the login facilites needed for a INeedsLogin downloader based on having the user log
@ -81,7 +81,7 @@ namespace Wabbajack.Lib.Downloaders
return cookies; return cookies;
} }
public async Task<HttpClient> GetAuthedClient() public async Task<Common.Http.Client> GetAuthedClient()
{ {
Helpers.Cookie[] cookies; Helpers.Cookie[] cookies;
try try

2
Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs
View File

@ -53,7 +53,7 @@ namespace Wabbajack.Lib.Downloaders
private async Task<HTTPDownloader.State> ToHttpState() private async Task<HTTPDownloader.State> ToHttpState()
{ {
var initialURL = $"https://drive.google.com/uc?id={Id}&export=download"; var initialURL = $"https://drive.google.com/uc?id={Id}&export=download";
var client = new HttpClient(); var client = new Common.Http.Client();
var response = await client.GetAsync(initialURL); var response = await client.GetAsync(initialURL);
if (!response.IsSuccessStatusCode) if (!response.IsSuccessStatusCode)
throw new HttpException((int)response.StatusCode, response.ReasonPhrase); throw new HttpException((int)response.StatusCode, response.ReasonPhrase);

8
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -61,7 +61,7 @@ namespace Wabbajack.Lib.Downloaders
public List<string> Headers { get; set; } public List<string> Headers { get; set; }
[Exclude] [Exclude]
public HttpClient Client { get; set; } public Common.Http.Client Client { get; set; }
public override object[] PrimaryKey { get => new object[] {Url};} public override object[] PrimaryKey { get => new object[] {Url};}
@ -86,8 +86,8 @@ namespace Wabbajack.Lib.Downloaders
using (var fs = download ? File.Open(destination, FileMode.Create) : null) using (var fs = download ? File.Open(destination, FileMode.Create) : null)
{ {
var client = Client ?? new HttpClient(); var client = Client ?? new Common.Http.Client();
client.DefaultRequestHeaders.Add("User-Agent", Consts.UserAgent); client.Headers.Add(("User-Agent", Consts.UserAgent));
if (Headers != null) if (Headers != null)
foreach (var header in Headers) foreach (var header in Headers)
@ -95,7 +95,7 @@ namespace Wabbajack.Lib.Downloaders
var idx = header.IndexOf(':'); var idx = header.IndexOf(':');
var k = header.Substring(0, idx); var k = header.Substring(0, idx);
var v = header.Substring(idx + 1); var v = header.Substring(idx + 1);
client.DefaultRequestHeaders.Add(k, v); client.Headers.Add((k, v));
} }
long totalRead = 0; long totalRead = 0;

2
Wabbajack.Lib/Downloaders/MediaFireDownloader.cs
View File

@ -53,7 +53,7 @@ namespace Wabbajack.Lib.Downloaders
if (newURL == null || !newURL.StartsWith("http")) return null; if (newURL == null || !newURL.StartsWith("http")) return null;
return new HTTPDownloader.State() return new HTTPDownloader.State()
{ {
Client = new HttpClient(), Client = new Common.Http.Client(),
Url = newURL Url = newURL
}; };
} }

6
Wabbajack.Lib/LibCefHelpers/Init.cs
View File

@ -16,13 +16,15 @@ namespace Wabbajack.Lib.LibCefHelpers
{ {
public static class Helpers public static class Helpers
{ {
public static HttpClient GetClient(IEnumerable<Cookie> cookies, string referer) public static Common.Http.Client GetClient(IEnumerable<Cookie> cookies, string referer)
{ {
throw new NotImplementedException("tt");
/*
var container = ToCookieContainer(cookies); var container = ToCookieContainer(cookies);
var handler = new HttpClientHandler { CookieContainer = container }; var handler = new HttpClientHandler { CookieContainer = container };
var client = new HttpClient(handler); var client = new HttpClient(handler);
client.DefaultRequestHeaders.Referrer = new Uri(referer); client.DefaultRequestHeaders.Referrer = new Uri(referer);
return client; return client;*/
} }
private static CookieContainer ToCookieContainer(IEnumerable<Cookie> cookies) private static CookieContainer ToCookieContainer(IEnumerable<Cookie> cookies)

30
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -24,9 +24,7 @@ namespace Wabbajack.Lib.NexusApi
private static readonly string API_KEY_CACHE_FILE = "nexus.key_cache"; private static readonly string API_KEY_CACHE_FILE = "nexus.key_cache";
private static string _additionalEntropy = "vtP2HF6ezg"; private static string _additionalEntropy = "vtP2HF6ezg";
private static object _diskLock = new object(); public Common.Http.Client HttpClient { get; } = new Common.Http.Client();
public HttpClient HttpClient { get; } = new HttpClient(new HttpClientHandler {MaxConnectionsPerServer = Consts.MaxConnectionsPerServer}) {Timeout = TimeSpan.FromMinutes(1), DefaultRequestHeaders = {ConnectionClose = true}};
#region Authentication #region Authentication
@ -202,14 +200,13 @@ namespace Wabbajack.Lib.NexusApi
{ {
ApiKey = apiKey; ApiKey = apiKey;
HttpClient.BaseAddress = new Uri("https://api.nexusmods.com");
// set default headers for all requests to the Nexus API // set default headers for all requests to the Nexus API
var headers = HttpClient.DefaultRequestHeaders; var headers = HttpClient.Headers;
headers.Add("User-Agent", Consts.UserAgent); headers.Add(("User-Agent", Consts.UserAgent));
headers.Add("apikey", ApiKey); headers.Add(("apikey", ApiKey));
headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); headers.Add(("Accept", "application/json"));
headers.Add("Application-Name", Consts.AppName); headers.Add(("Application-Name", Consts.AppName));
headers.Add("Application-Version", $"{Assembly.GetEntryAssembly()?.GetName()?.Version ?? new Version(0, 1)}"); headers.Add(("Application-Version", $"{Assembly.GetEntryAssembly()?.GetName()?.Version ?? new Version(0, 1)}"));
if (!Directory.Exists(Consts.NexusCacheDirectory)) if (!Directory.Exists(Consts.NexusCacheDirectory))
Directory.CreateDirectory(Consts.NexusCacheDirectory); Directory.CreateDirectory(Consts.NexusCacheDirectory);
@ -329,19 +326,6 @@ namespace Wabbajack.Lib.NexusApi
return await GetCached<ModInfo>(url); return await GetCached<ModInfo>(url);
} }
public async Task<EndorsementResponse> EndorseMod(NexusDownloader.State mod)
{
Utils.Status($"Endorsing ${mod.GameName} - ${mod.ModID}");
var url = $"https://api.nexusmods.com/v1/games/{ConvertGameName(mod.GameName)}/mods/{mod.ModID}/endorse.json";
var content = new FormUrlEncodedContent(new Dictionary<string, string> { { "version", mod.Version } });
using (var stream = await HttpClient.PostStream(url, content))
{
return stream.FromJSON<EndorsementResponse>();
}
}
private class DownloadLink private class DownloadLink
{ {
public string URI { get; set; } public string URI { get; set; }

6
Wabbajack.Lib/StatusMessages/ManuallyDownloadFile.cs
View File

@ -12,8 +12,8 @@ namespace Wabbajack.Lib
public override string ShortDescription { get; } public override string ShortDescription { get; }
public override string ExtendedDescription { get; } public override string ExtendedDescription { get; }
private TaskCompletionSource<(Uri, HttpClient)> _tcs = new TaskCompletionSource<(Uri, HttpClient)>(); private TaskCompletionSource<(Uri, Common.Http.Client)> _tcs = new TaskCompletionSource<(Uri, Common.Http.Client)>();
public Task<(Uri, HttpClient)> Task => _tcs.Task; public Task<(Uri, Common.Http.Client)> Task => _tcs.Task;
private ManuallyDownloadFile(ManualDownloader.State state) private ManuallyDownloadFile(ManualDownloader.State state)
{ {
@ -30,7 +30,7 @@ namespace Wabbajack.Lib
_tcs.SetCanceled(); _tcs.SetCanceled();
} }
public void Resume(Uri s, HttpClient client) public void Resume(Uri s, Common.Http.Client client)
{ {
_tcs.SetResult((s, client)); _tcs.SetResult((s, client));
} }

2
Wabbajack.Lib/Wabbajack.Lib.csproj
View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework> <TargetFramework>netcoreapp3.1</TargetFramework>
<Platforms>x64</Platforms> <Platforms>x64</Platforms>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier> <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
</PropertyGroup> </PropertyGroup>

2
Wabbajack.VirtualFileSystem/Wabbajack.VirtualFileSystem.csproj
View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework> <TargetFramework>netcoreapp3.1</TargetFramework>
<Platforms>x64</Platforms> <Platforms>x64</Platforms>
<RuntimeIdentifier>win10-x64</RuntimeIdentifier> <RuntimeIdentifier>win10-x64</RuntimeIdentifier>
</PropertyGroup> </PropertyGroup>

16
Wabbajack.sln
View File

@ -38,7 +38,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Test", "Wabbajack
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.CLI", "Wabbajack.CLI\Wabbajack.CLI.csproj", "{685D8BB1-D178-4D2C-85C7-C54A36FB7454}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.CLI", "Wabbajack.CLI\Wabbajack.CLI.csproj", "{685D8BB1-D178-4D2C-85C7-C54A36FB7454}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Launcher", "Wabbajack.Launcher\Wabbajack.Launcher.csproj", "{D6856DBF-C959-4867-A8A8-343DA2D2715E}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wabbajack.Launcher", "Wabbajack.Launcher\Wabbajack.Launcher.csproj", "{D6856DBF-C959-4867-A8A8-343DA2D2715E}"
EndProject EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -84,12 +84,14 @@ Global
{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|Any CPU.ActiveCfg = Release|x64 {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|Any CPU.ActiveCfg = Release|x64
{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.ActiveCfg = Release|x64 {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.ActiveCfg = Release|x64
{37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.Build.0 = Release|x64 {37E4D421-8FD3-4D57-8F3A-7A511D6ED5C5}.Release|x64.Build.0 = Release|x64
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|Any CPU.ActiveCfg = Debug|x64 {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.ActiveCfg = Debug|x64 {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|Any CPU.Build.0 = Debug|Any CPU
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.Build.0 = Debug|x64 {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.ActiveCfg = Debug|Any CPU
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|Any CPU.ActiveCfg = Release|x64 {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Debug|x64.Build.0 = Debug|Any CPU
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.ActiveCfg = Release|x64 {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|Any CPU.ActiveCfg = Release|Any CPU
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.Build.0 = Release|x64 {DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|Any CPU.Build.0 = Release|Any CPU
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.ActiveCfg = Release|Any CPU
{DE18D89E-39C5-48FD-8E42-16235E3C4593}.Release|x64.Build.0 = Release|Any CPU
{6ED08CFB-B879-4B55-8741-663A4A3491CE}.Debug|Any CPU.ActiveCfg = Debug|x64 {6ED08CFB-B879-4B55-8741-663A4A3491CE}.Debug|Any CPU.ActiveCfg = Debug|x64
{6ED08CFB-B879-4B55-8741-663A4A3491CE}.Debug|x64.ActiveCfg = Debug|x64 {6ED08CFB-B879-4B55-8741-663A4A3491CE}.Debug|x64.ActiveCfg = Debug|x64
{6ED08CFB-B879-4B55-8741-663A4A3491CE}.Debug|x64.Build.0 = Debug|x64 {6ED08CFB-B879-4B55-8741-663A4A3491CE}.Debug|x64.Build.0 = Debug|x64