Mirrored_Repos/wabbajack
1
0
Fork 0
mirror of https://github.com/wabbajack-tools/wabbajack.git synced 2024-08-30 18:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Purge uses of HttpClient, fix a bug that caused Nexus's verify endpoint to be spammed.

Browse Source
This commit is contained in:
Timothy Baldridge 2020-02-25 22:05:33 -07:00
parent 076b060afd
commit 6491a1c3f4
10 changed files with 25 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Wabbajack.Common/Http/Client.cs
View File

@ -47,30 +47,12 @@ namespace Wabbajack.Common.Http
return await SendStringAsync(request);
}
public async Task<Stream> GetStreamAsync(string url)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
foreach (var (k, v) in Headers)
request.Headers.Add(k, v);
if (Cookies.Count > 0)
Cookies.ForEach(c => ClientFactory.Cookies.Add(c));
return await SendStreamAsync(request);
}
private async Task<string> SendStringAsync(HttpRequestMessage request)
{
var result = await SendAsync(request);
using var result = await SendAsync(request);
return await result.Content.ReadAsStringAsync();
}
private async Task<Stream> SendStreamAsync(HttpRequestMessage request)
{
var result = await SendAsync(request);
return await result.Content.ReadAsStreamAsync();
}
public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage msg, HttpCompletionOption responseHeadersRead = HttpCompletionOption.ResponseHeadersRead)
{

6
Wabbajack.Common/Http/ClientFactory.cs
View File

@ -15,9 +15,9 @@ namespace Wabbajack.Common.Http
_socketsHandler = new SysHttp.SocketsHttpHandler
{
CookieContainer = Cookies,
MaxConnectionsPerServer = 8,
PooledConnectionLifetime = TimeSpan.FromSeconds(2),
PooledConnectionIdleTimeout = TimeSpan.FromSeconds(2)
MaxConnectionsPerServer = 20,
PooledConnectionLifetime = TimeSpan.FromMilliseconds(100),
PooledConnectionIdleTimeout = TimeSpan.FromMilliseconds(100)
};
Client = new SysHttp.HttpClient(_socketsHandler);
}

6
Wabbajack.Lib/Downloaders/AbstractIPS4Downloader.cs
View File

@ -90,8 +90,8 @@ namespace Wabbajack.Lib.Downloaders
public override async Task<bool> Download(Archive a, string destination)
{
var stream = await ResolveDownloadStream();
using (var file = File.Open(destination, FileMode.Create))
await using var stream = await ResolveDownloadStream();
await using (var file = File.Open(destination, FileMode.Create))
{
stream.CopyTo(file);
}
@ -125,6 +125,7 @@ namespace Wabbajack.Lib.Downloaders
var streamResult = await downloader.AuthedClient.GetAsync(url);
if (streamResult.StatusCode != HttpStatusCode.OK)
{
streamResult.Dispose();
Utils.Error(new InvalidOperationException(), $"{downloader.SiteName} servers reported an error for file: {FileID}");
}
@ -141,6 +142,7 @@ namespace Wabbajack.Lib.Downloaders
Utils.Status($"Waiting for {secs} at the request of {downloader.SiteName}", Percent.FactoryPutInRange(x, secs));
await Task.Delay(1000);
}
streamResult.Dispose();
Utils.Status("Retrying download");
goto TOP;
}

11
Wabbajack.Lib/Downloaders/BethesdaNetDownloader.cs
View File

@ -134,7 +134,7 @@ namespace Wabbajack.Lib.Downloaders
foreach (var chunk in info.depot_list[0].file_list[0].chunk_list.OrderBy(c => c.index))
{
Utils.Status($"Downloading {a.Name}", Percent.FactoryPutInRange(chunk.index, max_chunks));
var got = await client.GetAsync(
using var got = await client.GetAsync(
$"https://content.cdp.bethesda.net/{collected.CDPProductId}/{collected.CDPPropertiesId}/{chunk.sha}");
var data = await got.Content.ReadAsByteArrayAsync();
if (collected.AESKey != null)
@ -224,7 +224,7 @@ namespace Wabbajack.Lib.Downloaders
info.CDPToken = (await posted.Content.ReadAsStringAsync()).FromJSONString<CDPLoginResponse>().token;
client.Headers.Add(("X-Access-Token", info.AccessToken));
var got = await client.GetAsync($"mods/ugc-workshop/content/get?content_id={ContentId}");
var got = await client.GetAsync($"https://api.bethesda.net/mods/ugc-workshop/content/get?content_id={ContentId}");
JObject data = JObject.Parse(await got.Content.ReadAsStringAsync());
var content = data["platform"]["response"]["content"];
@ -235,15 +235,18 @@ namespace Wabbajack.Lib.Downloaders
client.Headers.Add(("Authorization", $"Token {info.CDPToken}"));
client.Headers.Add(("Accept", "application/json"));
got.Dispose();
got = await client.GetAsync(
$"/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/tree/.json");
$"https://api.bethesda.net/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/tree/.json");
var tree = (await got.Content.ReadAsStringAsync()).FromJSONString<CDPTree>();
got.Dispose();
got = await client.PostAsync($"https://api.bethesda.net/mods/ugc-content/add-subscription", new StringContent($"{{\"content_id\": \"{ContentId}\"}}", Encoding.UTF8, "application/json"));
got.Dispose();
got = await client.GetAsync(
$"/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/depots/.json");
$"https://api.bethesda.net/cdp-user/projects/{info.CDPProductId}/branches/{info.CDPBranchId}/depots/.json");
var props_obj = JObject.Parse(await got.Content.ReadAsStringAsync()).Properties().First();
info.CDPPropertiesId = (int)props_obj.Value["properties_id"];

2
Wabbajack.Lib/Downloaders/GoogleDriveDownloader.cs
View File

@ -54,7 +54,7 @@ namespace Wabbajack.Lib.Downloaders
{
var initialURL = $"https://drive.google.com/uc?id={Id}&export=download";
var client = new Common.Http.Client();
var response = await client.GetAsync(initialURL);
using var response = await client.GetAsync(initialURL);
if (!response.IsSuccessStatusCode)
throw new HttpException((int)response.StatusCode, response.ReasonPhrase);
var regex = new Regex("(?<=/uc\\?export=download&amp;confirm=).*(?=;id=)");

8
Wabbajack.Lib/Downloaders/HTTPDownloader.cs
View File

@ -102,7 +102,7 @@ namespace Wabbajack.Lib.Downloaders
var bufferSize = 1024 * 32;
Utils.Status($"Starting Download {a?.Name ?? Url}", Percent.Zero);
var response = await client.GetAsync(Url, HttpCompletionOption.ResponseHeadersRead);
var response = await client.GetAsync(Url);
TOP:
if (!response.IsSuccessStatusCode)
@ -166,8 +166,8 @@ TOP:
var msg = new HttpRequestMessage(HttpMethod.Get, Url);
msg.Headers.Range = new RangeHeaderValue(totalRead, null);
response = await client.SendAsync(msg,
HttpCompletionOption.ResponseHeadersRead);
response.Dispose();
response = await client.SendAsync(msg);
goto TOP;
}
throw ex;
@ -185,7 +185,7 @@ TOP:
totalRead += read;
}
}
response.Dispose();
return true;
}
}

2
Wabbajack.Lib/Downloaders/NexusDownloader.cs
View File

@ -117,8 +117,8 @@ namespace Wabbajack.Lib.Downloaders
{
Utils.ErrorThrow(new UnconvertedError($"Aborting at the request of the user"));
}
_prepared = true;
}
_prepared = true;
}
}
}

2
Wabbajack.Lib/MO2Compiler.cs
View File

@ -358,7 +358,7 @@ namespace Wabbajack.Lib
{
var vf = VFS.Index.ByFullPath[f];
var client = new Common.Http.Client();
var response =
using var response =
await client.GetAsync(
$"http://build.wabbajack.org/indexed_files/{vf.Hash.FromBase64().ToHex()}/meta.ini");

2
Wabbajack.Lib/NexusApi/NexusApi.cs
View File

@ -224,7 +224,7 @@ namespace Wabbajack.Lib.NexusApi
TOP:
try
{
using var response = await HttpClient.GetAsync(url, HttpCompletionOption.ResponseContentRead);
using var response = await HttpClient.GetAsync(url);
UpdateRemaining(response);
if (!response.IsSuccessStatusCode)
{

9
Wabbajack.Lib/Validation/ValidateModlist.cs
View File

@ -39,15 +39,10 @@ namespace Wabbajack.Lib.Validation
public async Task LoadListsFromGithub()
{
var client = new Common.Http.Client();
Utils.Log("Loading Nexus mod permissions");
using (var result = await client.GetStreamAsync(Consts.ModPermissionsURL))
{
AuthorPermissions = result.FromYaml<Dictionary<string, Author>>();
Utils.Log($"Loaded permissions for {AuthorPermissions.Count} authors");
}
Utils.Log("Loading server whitelist");
using (var result = await client.GetStreamAsync(Consts.ServerWhitelistURL))
using (var response = await client.GetAsync(Consts.ServerWhitelistURL))
using (var result = await response.Content.ReadAsStreamAsync())
{
ServerWhitelist = result.FromYaml<ServerWhitelist>();
Utils.Log($"Loaded permissions for {ServerWhitelist.AllowedPrefixes.Count} servers and {ServerWhitelist.GoogleIDs.Count} Google Drive files");