From 1124dd645d0db872b01a0c476c205da057a8fd04 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Wed, 22 May 2024 11:29:28 -0700
Subject: [PATCH 1/7] Use `make_client` instead of calling `HTTP::Client`
Using `make_client` to create `HTTP::Client`, allows for a simple way to
easily add logic to all `HTTP::Client` initialized within Invidious.
---
src/invidious/routes/api/v1/search.cr | 4 +---
src/invidious/yt_backend/connection_pool.cr | 13 ++++---------
2 files changed, 5 insertions(+), 12 deletions(-)
diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr
index 2922b060..6785ef73 100644
--- a/src/invidious/routes/api/v1/search.cr
+++ b/src/invidious/routes/api/v1/search.cr
@@ -31,9 +31,7 @@ module Invidious::Routes::API::V1::Search
query = env.params.query["q"]? || ""
begin
- client = HTTP::Client.new("suggestqueries-clients6.youtube.com")
- client.before_request { |r| add_yt_headers(r) }
-
+ client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers = true)
url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt"
response = client.get(url).body
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index ca612083..84d857ec 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -30,11 +30,8 @@ struct YoutubeConnectionPool
response = yield conn
rescue ex
conn.close
- conn = HTTP::Client.new(url)
-
- conn.family = CONFIG.force_resolve
+ conn = make_client(url)
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
- conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
response = yield conn
ensure
pool.release(conn)
@@ -45,16 +42,14 @@ struct YoutubeConnectionPool
private def build_pool
DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
- conn = HTTP::Client.new(url)
- conn.family = CONFIG.force_resolve
+ conn = make_client(url, force_solve = true)
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
- conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
conn
end
end
end
-def make_client(url : URI, region = nil, force_resolve : Bool = false)
+def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_header : Bool = false)
client = HTTP::Client.new(url)
# Force the usage of a specific configured IP Family
@@ -62,7 +57,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false)
client.family = CONFIG.force_resolve
end
- client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
+ client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_header
client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds
From 3af668186997d21295247ed6e31c6fd4634fa511 Mon Sep 17 00:00:00 2001
From: syeopite <70992037+syeopite@users.noreply.github.com>
Date: Fri, 24 May 2024 13:11:14 -0700
Subject: [PATCH 2/7] Fix typo in argument to `make_client`
Co-authored-by: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com>
---
src/invidious/yt_backend/connection_pool.cr | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index 84d857ec..8563cc3e 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -42,7 +42,7 @@ struct YoutubeConnectionPool
private def build_pool
DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
- conn = make_client(url, force_solve = true)
+ conn = make_client(url, force_resolve = true)
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
conn
end
From ee89db49ba6242771921c9204d57f47f3edb8975 Mon Sep 17 00:00:00 2001
From: syeopite <70992037+syeopite@users.noreply.github.com>
Date: Sun, 16 Jun 2024 15:18:21 +0000
Subject: [PATCH 3/7] Typo
Co-authored-by: Samantaz Fox <coding@samantaz.fr>
---
src/invidious/yt_backend/connection_pool.cr | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index 8563cc3e..f7227d67 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -42,7 +42,7 @@ struct YoutubeConnectionPool
private def build_pool
DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
- conn = make_client(url, force_resolve = true)
+ conn = make_client(url, force_resolve: true)
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
conn
end
From bd48af825c27f08987ee039381a702ca91e52cb8 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sun, 16 Jun 2024 14:15:05 -0700
Subject: [PATCH 4/7] Search API: Fix named arg syntax to make_client
---
src/invidious/routes/api/v1/search.cr | 2 +-
src/invidious/yt_backend/connection_pool.cr | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr
index 6785ef73..59a30745 100644
--- a/src/invidious/routes/api/v1/search.cr
+++ b/src/invidious/routes/api/v1/search.cr
@@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Search
query = env.params.query["q"]? || ""
begin
- client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers = true)
+ client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true)
url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt"
response = client.get(url).body
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index f7227d67..0dc42261 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -49,7 +49,7 @@ struct YoutubeConnectionPool
end
end
-def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_header : Bool = false)
+def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false)
client = HTTP::Client.new(url)
# Force the usage of a specific configured IP Family
@@ -57,7 +57,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, force_you
client.family = CONFIG.force_resolve
end
- client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_header
+ client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_headers
client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds
From 7521902e88a4654378b5a0428f9c538d52dcb9db Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sat, 24 Aug 2024 19:37:04 -0700
Subject: [PATCH 5/7] Ensure IP family is always used when force_resolve
---
src/invidious/yt_backend/connection_pool.cr | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index 0dc42261..eaa94158 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -31,7 +31,6 @@ struct YoutubeConnectionPool
rescue ex
conn.close
conn = make_client(url)
- conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
response = yield conn
ensure
pool.release(conn)
@@ -42,9 +41,7 @@ struct YoutubeConnectionPool
private def build_pool
DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
- conn = make_client(url, force_resolve: true)
- conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
- conn
+ next make_client(url, force_resolve: true)
end
end
end
@@ -55,6 +52,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, force_you
# Force the usage of a specific configured IP Family
if force_resolve
client.family = CONFIG.force_resolve
+ client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC
end
client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_headers
From 46c58bd84cf2a867b897338bb2105648aed0118c Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sat, 24 Aug 2024 19:38:02 -0700
Subject: [PATCH 6/7] Pool: Use force_resolve in fallback new client
---
src/invidious/yt_backend/connection_pool.cr | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index eaa94158..d474d760 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -30,7 +30,7 @@ struct YoutubeConnectionPool
response = yield conn
rescue ex
conn.close
- conn = make_client(url)
+ conn = make_client(url, force_resolve: true)
response = yield conn
ensure
pool.release(conn)
From 6e39b9b303930f931b5a5a60c75528c4d9db3587 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sat, 24 Aug 2024 19:41:39 -0700
Subject: [PATCH 7/7] make_client: add YouTube headers on *.youtube.com
---
src/invidious/yt_backend/connection_pool.cr | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr
index d474d760..bff4df72 100644
--- a/src/invidious/yt_backend/connection_pool.cr
+++ b/src/invidious/yt_backend/connection_pool.cr
@@ -55,7 +55,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, force_you
client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC
end
- client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_headers
+ client.before_request { |r| add_yt_headers(r) } if url.host.try &.ends_with?("youtube.com") || force_youtube_headers
client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds