Mirrored_Repos/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2024-08-30 18:23:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Extract API routes from invidious.cr (2/?)

Browse Source 
- Video playback endpoints
- Search feed api
- Video info api
This commit is contained in:
syeopite 2021-08-12 23:31:12 -07:00
parent 66becbf46f
commit 6aa65593ef
No known key found for this signature in database
GPG Key ID: 6FA616E5A5294A82
13 changed files with 734 additions and 701 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

575
src/invidious.cr
View File

@ -364,6 +364,8 @@ Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :up
Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme
define_v1_api_routes() define_v1_api_routes()
define_api_manifest_routes()
define_video_playback_routes()
# Users # Users
@ -1639,69 +1641,6 @@ end
# API Endpoints # API Endpoints
get "/api/v1/videos/:id" do |env|
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
env.response.content_type = "application/json"
id = env.params.url["id"]
region = env.params.query["region"]?
begin
video = get_video(id, PG_DB, region: region)
rescue ex : VideoRedirect
env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
next error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex
next error_json(500, ex)
end
video.to_json(locale)
end
get "/api/v1/search" do |env|
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
region = env.params.query["region"]?
env.response.content_type = "application/json"
query = env.params.query["q"]?
query ||= ""
page = env.params.query["page"]?.try &.to_i?
page ||= 1
sort_by = env.params.query["sort_by"]?.try &.downcase
sort_by ||= "relevance"
date = env.params.query["date"]?.try &.downcase
date ||= ""
duration = env.params.query["duration"]?.try &.downcase
duration ||= ""
features = env.params.query["features"]?.try &.split(",").map { |feature| feature.downcase }
features ||= [] of String
content_type = env.params.query["type"]?.try &.downcase
content_type ||= "video"
begin
search_params = produce_search_params(page, sort_by, date, content_type, duration, features)
rescue ex
next error_json(400, ex)
end
count, search_results = search(query, search_params, region).as(Tuple)
JSON.build do |json|
json.array do
search_results.each do |item|
item.to_json(locale, json)
end
end
end
end
{"/api/v1/playlists/:plid", "/api/v1/auth/playlists/:plid"}.each do |route| {"/api/v1/playlists/:plid", "/api/v1/auth/playlists/:plid"}.each do |route|
get route do |env| get route do |env|
locale = LOCALES[env.get("preferences").as(Preferences).locale]? locale = LOCALES[env.get("preferences").as(Preferences).locale]?
@ -2245,516 +2184,6 @@ post "/api/v1/auth/tokens/unregister" do |env|
env.response.status_code = 204 env.response.status_code = 204
end end
get "/api/manifest/dash/id/videoplayback" do |env|
env.response.headers.delete("Content-Type")
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.redirect "/videoplayback?#{env.params.query}"
end
get "/api/manifest/dash/id/videoplayback/*" do |env|
env.response.headers.delete("Content-Type")
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.redirect env.request.path.lchop("/api/manifest/dash/id")
end
get "/api/manifest/dash/id/:id" do |env|
env.response.headers.add("Access-Control-Allow-Origin", "*")
env.response.content_type = "application/dash+xml"
local = env.params.query["local"]?.try &.== "true"
id = env.params.url["id"]
region = env.params.query["region"]?
# Since some implementations create playlists based on resolution regardless of different codecs,
# we can opt to only add a source to a representation if it has a unique height within that representation
unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }
begin
video = get_video(id, PG_DB, region: region)
rescue ex : VideoRedirect
next env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex
env.response.status_code = 403
next
end
if dashmpd = video.dash_manifest_url
manifest = YT_POOL.client &.get(URI.parse(dashmpd).request_target).body
manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
url = baseurl.lchop("<BaseURL>")
url = url.rchop("</BaseURL>")
if local
uri = URI.parse(url)
url = "#{uri.request_target}host/#{uri.host}/"
end
"<BaseURL>#{url}</BaseURL>"
end
next manifest
end
adaptive_fmts = video.adaptive_fmts
if local
adaptive_fmts.each do |fmt|
fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target)
end
end
audio_streams = video.audio_streams
video_streams = video.video_streams.sort_by { |stream| {stream["width"].as_i, stream["fps"].as_i} }.reverse
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("MPD", "xmlns": "urn:mpeg:dash:schema:mpd:2011",
"profiles": "urn:mpeg:dash:profile:full:2011", minBufferTime: "PT1.5S", type: "static",
mediaPresentationDuration: "PT#{video.length_seconds}S") do
xml.element("Period") do
i = 0
{"audio/mp4", "audio/webm"}.each do |mime_type|
mime_streams = audio_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type }
next if mime_streams.empty?
xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true) do
mime_streams.each do |fmt|
codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
bandwidth = fmt["bitrate"].as_i
itag = fmt["itag"].as_i
url = fmt["url"].as_s
xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do
xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011",
value: "2")
xml.element("BaseURL") { xml.text url }
xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do
xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}")
end
end
end
end
i += 1
end
potential_heights = {4320, 2160, 1440, 1080, 720, 480, 360, 240, 144}
{"video/mp4", "video/webm"}.each do |mime_type|
mime_streams = video_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type }
next if mime_streams.empty?
heights = [] of Int32
xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, scanType: "progressive") do
mime_streams.each do |fmt|
codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
bandwidth = fmt["bitrate"].as_i
itag = fmt["itag"].as_i
url = fmt["url"].as_s
width = fmt["width"].as_i
height = fmt["height"].as_i
# Resolutions reported by YouTube player (may not accurately reflect source)
height = potential_heights.min_by { |i| (height - i).abs }
next if unique_res && heights.includes? height
heights << height
xml.element("Representation", id: itag, codecs: codecs, width: width, height: height,
startWithSAP: "1", maxPlayoutRate: "1",
bandwidth: bandwidth, frameRate: fmt["fps"]) do
xml.element("BaseURL") { xml.text url }
xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do
xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}")
end
end
end
end
i += 1
end
end
end
end
end
get "/api/manifest/hls_variant/*" do |env|
response = YT_POOL.client &.get(env.request.path)
if response.status_code != 200
env.response.status_code = response.status_code
next
end
local = env.params.query["local"]?.try &.== "true"
env.response.content_type = "application/x-mpegURL"
env.response.headers.add("Access-Control-Allow-Origin", "*")
manifest = response.body
if local
manifest = manifest.gsub("https://www.youtube.com", HOST_URL)
manifest = manifest.gsub("index.m3u8", "index.m3u8?local=true")
end
manifest
end
get "/api/manifest/hls_playlist/*" do |env|
response = YT_POOL.client &.get(env.request.path)
if response.status_code != 200
env.response.status_code = response.status_code
next
end
local = env.params.query["local"]?.try &.== "true"
env.response.content_type = "application/x-mpegURL"
env.response.headers.add("Access-Control-Allow-Origin", "*")
manifest = response.body
if local
manifest = manifest.gsub(/^https:\/\/r\d---.{11}\.c\.youtube\.com[^\n]*/m) do |match|
path = URI.parse(match).path
path = path.lchop("/videoplayback/")
path = path.rchop("/")
path = path.gsub(/mime\/\w+\/\w+/) do |mimetype|
mimetype = mimetype.split("/")
mimetype[0] + "/" + mimetype[1] + "%2F" + mimetype[2]
end
path = path.split("/")
raw_params = {} of String => Array(String)
path.each_slice(2) do |pair|
key, value = pair
value = URI.decode_www_form(value)
if raw_params[key]?
raw_params[key] << value
else
raw_params[key] = [value]
end
end
raw_params = HTTP::Params.new(raw_params)
if fvip = raw_params["hls_chunk_host"].match(/r(?<fvip>\d+)---/)
raw_params["fvip"] = fvip["fvip"]
end
raw_params["local"] = "true"
"#{HOST_URL}/videoplayback?#{raw_params}"
end
end
manifest
end
# YouTube /videoplayback links expire after 6 hours,
# so we have a mechanism here to redirect to the latest version
get "/latest_version" do |env|
if env.params.query["download_widget"]?
download_widget = JSON.parse(env.params.query["download_widget"])
id = download_widget["id"].as_s
title = download_widget["title"].as_s
if label = download_widget["label"]?
env.redirect "/api/v1/captions/#{id}?label=#{label}&title=#{title}"
next
else
itag = download_widget["itag"].as_s.to_i
local = "true"
end
end
id ||= env.params.query["id"]?
itag ||= env.params.query["itag"]?.try &.to_i
region = env.params.query["region"]?
local ||= env.params.query["local"]?
local ||= "false"
local = local == "true"
if !id || !itag
env.response.status_code = 400
next
end
video = get_video(id, PG_DB, region: region)
fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag }
url = fmt.try &.["url"]?.try &.as_s
if !url
env.response.status_code = 404
next
end
url = URI.parse(url).request_target.not_nil! if local
url = "#{url}&title=#{title}" if title
env.redirect url
end
options "/videoplayback" do |env|
env.response.headers.delete("Content-Type")
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS"
env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range"
end
options "/videoplayback/*" do |env|
env.response.headers.delete("Content-Type")
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS"
env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range"
end
options "/api/manifest/dash/id/videoplayback" do |env|
env.response.headers.delete("Content-Type")
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS"
env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range"
end
options "/api/manifest/dash/id/videoplayback/*" do |env|
env.response.headers.delete("Content-Type")
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS"
env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range"
end
get "/videoplayback/*" do |env|
path = env.request.path
path = path.lchop("/videoplayback/")
path = path.rchop("/")
path = path.gsub(/mime\/\w+\/\w+/) do |mimetype|
mimetype = mimetype.split("/")
mimetype[0] + "/" + mimetype[1] + "%2F" + mimetype[2]
end
path = path.split("/")
raw_params = {} of String => Array(String)
path.each_slice(2) do |pair|
key, value = pair
value = URI.decode_www_form(value)
if raw_params[key]?
raw_params[key] << value
else
raw_params[key] = [value]
end
end
query_params = HTTP::Params.new(raw_params)
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.redirect "/videoplayback?#{query_params}"
end
get "/videoplayback" do |env|
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
query_params = env.params.query
fvip = query_params["fvip"]? || "3"
mns = query_params["mn"]?.try &.split(",")
mns ||= [] of String
if query_params["region"]?
region = query_params["region"]
query_params.delete("region")
end
if query_params["host"]? && !query_params["host"].empty?
host = "https://#{query_params["host"]}"
query_params.delete("host")
else
host = "https://r#{fvip}---#{mns.pop}.googlevideo.com"
end
url = "/videoplayback?#{query_params.to_s}"
headers = HTTP::Headers.new
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
client = make_client(URI.parse(host), region)
response = HTTP::Client::Response.new(500)
error = ""
5.times do
begin
response = client.head(url, headers)
if response.headers["Location"]?
location = URI.parse(response.headers["Location"])
env.response.headers["Access-Control-Allow-Origin"] = "*"
new_host = "#{location.scheme}://#{location.host}"
if new_host != host
host = new_host
client.close
client = make_client(URI.parse(new_host), region)
end
url = "#{location.request_target}&host=#{location.host}#{region ? "&region=#{region}" : ""}"
else
break
end
rescue Socket::Addrinfo::Error
if !mns.empty?
mn = mns.pop
end
fvip = "3"
host = "https://r#{fvip}---#{mn}.googlevideo.com"
client = make_client(URI.parse(host), region)
rescue ex
error = ex.message
end
end
if response.status_code >= 400
env.response.status_code = response.status_code
env.response.content_type = "text/plain"
next error
end
if url.includes? "&file=seg.ts"
if CONFIG.disabled?("livestreams")
next error_template(403, "Administrator has disabled this endpoint.")
end
begin
client.get(url, headers) do |response|
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = response.headers["Location"]?
location = URI.parse(location)
location = "#{location.request_target}&host=#{location.host}"
if region
location += "&region=#{region}"
end
next env.redirect location
end
IO.copy(response.body_io, env.response)
end
rescue ex
end
else
if query_params["title"]? && CONFIG.disabled?("downloads") ||
CONFIG.disabled?("dash")
next error_template(403, "Administrator has disabled this endpoint.")
end
content_length = nil
first_chunk = true
range_start, range_end = parse_range(env.request.headers["Range"]?)
chunk_start = range_start
chunk_end = range_end
if !chunk_end || chunk_end - chunk_start > HTTP_CHUNK_SIZE
chunk_end = chunk_start + HTTP_CHUNK_SIZE - 1
end
# TODO: Record bytes written so we can restart after a chunk fails
while true
if !range_end && content_length
range_end = content_length
end
if range_end && chunk_start > range_end
break
end
if range_end && chunk_end > range_end
chunk_end = range_end
end
headers["Range"] = "bytes=#{chunk_start}-#{chunk_end}"
begin
client.get(url, headers) do |response|
if first_chunk
if !env.request.headers["Range"]? && response.status_code == 206
env.response.status_code = 200
else
env.response.status_code = response.status_code
end
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "content-range"
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = response.headers["Location"]?
location = URI.parse(location)
location = "#{location.request_target}&host=#{location.host}#{region ? "&region=#{region}" : ""}"
env.redirect location
break
end
if title = query_params["title"]?
# https://blog.fastmail.com/2011/06/24/download-non-english-filenames/
env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}"
end
if !response.headers.includes_word?("Transfer-Encoding", "chunked")
content_length = response.headers["Content-Range"].split("/")[-1].to_i64
if env.request.headers["Range"]?
env.response.headers["Content-Range"] = "bytes #{range_start}-#{range_end || (content_length - 1)}/#{content_length}"
env.response.content_length = ((range_end.try &.+ 1) || content_length) - range_start
else
env.response.content_length = content_length
end
end
end
proxy_file(response, env)
end
rescue ex
if ex.message != "Error reading socket: Connection reset by peer"
break
else
client.close
client = make_client(URI.parse(host), region)
end
end
chunk_start = chunk_end + 1
chunk_end += HTTP_CHUNK_SIZE
first_chunk = false
end
end
client.close
end
get "/ggpht/*" do |env| get "/ggpht/*" do |env|
url = env.request.path.lchop("/ggpht") url = env.request.path.lchop("/ggpht")

1
src/invidious/channels/channels.cr
View File

@ -185,6 +185,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
if !author if !author
raise InfoException.new("Deleted or invalid channel") raise InfoException.new("Deleted or invalid channel")
end end
author = author.content author = author.content
# Auto-generated channels # Auto-generated channels

9
src/invidious/helpers/macros.cr
View File

@ -56,3 +56,12 @@ end
macro rendered(filename) macro rendered(filename)
render "src/invidious/views/#{{{filename}}}.ecr" render "src/invidious/views/#{{{filename}}}.ecr"
end end
# Similar to Kemals halt method but works in a
# method.
macro haltf(env, status_code = 200, response = "")
{{env}}.response.status_code = {{status_code}}
{{env}}.response.print {{response}}
{{env}}.response.close
return
end

237
src/invidious/routes/api/manifest.cr Normal file
View File

@ -0,0 +1,237 @@
module Invidious::Routes::APIManifest
# /api/manifest/dash/id/:id
def self.get_dash_video_id(env)
env.response.headers.add("Access-Control-Allow-Origin", "*")
env.response.content_type = "application/dash+xml"
local = env.params.query["local"]?.try &.== "true"
id = env.params.url["id"]
region = env.params.query["region"]?
# Since some implementations create playlists based on resolution regardless of different codecs,
# we can opt to only add a source to a representation if it has a unique height within that representation
unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }
begin
video = get_video(id, PG_DB, region: region)
rescue ex : VideoRedirect
return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex
haltf env, status_code: 403
end
if dashmpd = video.dash_manifest_url
manifest = YT_POOL.client &.get(URI.parse(dashmpd).request_target).body
manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
url = baseurl.lchop("<BaseURL>")
url = url.rchop("</BaseURL>")
if local
uri = URI.parse(url)
url = "#{uri.request_target}host/#{uri.host}/"
end
"<BaseURL>#{url}</BaseURL>"
end
return manifest
end
adaptive_fmts = video.adaptive_fmts
if local
adaptive_fmts.each do |fmt|
fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target)
end
end
audio_streams = video.audio_streams
video_streams = video.video_streams.sort_by { |stream| {stream["width"].as_i, stream["fps"].as_i} }.reverse
manifest = XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("MPD", "xmlns": "urn:mpeg:dash:schema:mpd:2011",
"profiles": "urn:mpeg:dash:profile:full:2011", minBufferTime: "PT1.5S", type: "static",
mediaPresentationDuration: "PT#{video.length_seconds}S") do
xml.element("Period") do
i = 0
{"audio/mp4", "audio/webm"}.each do |mime_type|
mime_streams = audio_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type }
next if mime_streams.empty?
xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true) do
mime_streams.each do |fmt|
codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
bandwidth = fmt["bitrate"].as_i
itag = fmt["itag"].as_i
url = fmt["url"].as_s
xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do
xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011",
value: "2")
xml.element("BaseURL") { xml.text url }
xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do
xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}")
end
end
end
end
i += 1
end
potential_heights = {4320, 2160, 1440, 1080, 720, 480, 360, 240, 144}
{"video/mp4", "video/webm"}.each do |mime_type|
mime_streams = video_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type }
next if mime_streams.empty?
heights = [] of Int32
xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, scanType: "progressive") do
mime_streams.each do |fmt|
codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
bandwidth = fmt["bitrate"].as_i
itag = fmt["itag"].as_i
url = fmt["url"].as_s
width = fmt["width"].as_i
height = fmt["height"].as_i
# Resolutions reported by YouTube player (may not accurately reflect source)
height = potential_heights.min_by { |i| (height - i).abs }
next if unique_res && heights.includes? height
heights << height
xml.element("Representation", id: itag, codecs: codecs, width: width, height: height,
startWithSAP: "1", maxPlayoutRate: "1",
bandwidth: bandwidth, frameRate: fmt["fps"]) do
xml.element("BaseURL") { xml.text url }
xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do
xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}")
end
end
end
end
i += 1
end
end
end
end
return manifest
end
# /api/manifest/dash/id/videoplayback
def self.get_dash_video_playback(env)
env.response.headers.delete("Content-Type")
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.redirect "/videoplayback?#{env.params.query}"
end
# /api/manifest/dash/id/videoplayback/*
def self.get_dash_video_playback_greedy(env)
env.response.headers.delete("Content-Type")
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.redirect env.request.path.lchop("/api/manifest/dash/id")
end
# /api/manifest/dash/id/videoplayback && /api/manifest/dash/id/videoplayback/*
def self.options_dash_video_playback(env)
env.response.headers.delete("Content-Type")
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS"
env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range"
end
# /api/manifest/hls_playlist/*
def self.get_hls_playlist(env)
response = YT_POOL.client &.get(env.request.path)
if response.status_code != 200
haltf env, status_code: response.status_code
end
local = env.params.query["local"]?.try &.== "true"
env.response.content_type = "application/x-mpegURL"
env.response.headers.add("Access-Control-Allow-Origin", "*")
manifest = response.body
if local
manifest = manifest.gsub(/^https:\/\/r\d---.{11}\.c\.youtube\.com[^\n]*/m) do |match|
path = URI.parse(match).path
path = path.lchop("/videoplayback/")
path = path.rchop("/")
path = path.gsub(/mime\/\w+\/\w+/) do |mimetype|
mimetype = mimetype.split("/")
mimetype[0] + "/" + mimetype[1] + "%2F" + mimetype[2]
end
path = path.split("/")
raw_params = {} of String => Array(String)
path.each_slice(2) do |pair|
key, value = pair
value = URI.decode_www_form(value)
if raw_params[key]?
raw_params[key] << value
else
raw_params[key] = [value]
end
end
raw_params = HTTP::Params.new(raw_params)
if fvip = raw_params["hls_chunk_host"].match(/r(?<fvip>\d+)---/)
raw_params["fvip"] = fvip["fvip"]
end
raw_params["local"] = "true"
"#{HOST_URL}/videoplayback?#{raw_params}"
end
end
manifest
end
# /api/manifest/hls_variant/*
def self.get_hls_variant(env)
response = YT_POOL.client &.get(env.request.path)
if response.status_code != 200
haltf env, status_code: response.status_code
end
local = env.params.query["local"]?.try &.== "true"
env.response.content_type = "application/x-mpegURL"
env.response.headers.add("Access-Control-Allow-Origin", "*")
manifest = response.body
if local
manifest = manifest.gsub("https://www.youtube.com", HOST_URL)
manifest = manifest.gsub("index.m3u8", "index.m3u8?local=true")
end
manifest
end
end
macro define_api_manifest_routes
Invidious::Routing.get "/api/manifest/dash/id/:id", Invidious::Routes::APIManifest, :get_dash_video_id
Invidious::Routing.get "/api/manifest/dash/id/videoplayback", Invidious::Routes::APIManifest, :get_dash_video_playback
Invidious::Routing.get "/api/manifest/dash/id/videoplayback/*", Invidious::Routes::APIManifest, :get_dash_video_playback_greedy
Invidious::Routing.options "/api/manifest/dash/id/videoplayback", Invidious::Routes::APIManifest, :options_dash_video_playback
Invidious::Routing.options "/api/manifest/dash/id/videoplayback/*", Invidious::Routes::APIManifest, :options_dash_video_playback
Invidious::Routing.get "/api/manifest/hls_playlist/*", Invidious::Routes::APIManifest, :get_hls_playlist
Invidious::Routing.get "/api/manifest/hls_variant/*", Invidious::Routes::APIManifest, :get_hls_variant
end

1
src/invidious/routes/api_v1/channels.cr → src/invidious/routes/api/v1/channels.cr
View File

@ -78,7 +78,6 @@ module Invidious::Routes::APIv1
json.field "subCount", channel.sub_count json.field "subCount", channel.sub_count
json.field "totalViews", channel.total_views json.field "totalViews", channel.total_views
json.field "joined", channel.joined.to_unix json.field "joined", channel.joined.to_unix
json.field "paid", channel.paid
json.field "autoGenerated", channel.auto_generated json.field "autoGenerated", channel.auto_generated
json.field "isFamilyFriendly", channel.is_family_friendly json.field "isFamilyFriendly", channel.is_family_friendly

0
src/invidious/routes/api_v1/feeds.cr → src/invidious/routes/api/v1/feeds.cr
View File

0
src/invidious/routes/api_v1/misc.cr → src/invidious/routes/api/v1/misc.cr
View File

13
src/invidious/routes/api_v1/routes.cr → src/invidious/routes/api/v1/routes.cr
View File

@ -3,17 +3,19 @@
macro define_v1_api_routes(base_url = "/api/v1") macro define_v1_api_routes(base_url = "/api/v1")
Invidious::Routing.get "#{{{base_url}}}/stats", Invidious::Routes::APIv1, :stats Invidious::Routing.get "#{{{base_url}}}/stats", Invidious::Routes::APIv1, :stats
# Widgets
Invidious::Routing.get "#{{{base_url}}}/storyboards/:id", Invidious::Routes::APIv1, :storyboards Invidious::Routing.get "#{{{base_url}}}/storyboards/:id", Invidious::Routes::APIv1, :storyboards
Invidious::Routing.get "#{{{base_url}}}/captions/:id", Invidious::Routes::APIv1, :captions Invidious::Routing.get "#{{{base_url}}}/captions/:id", Invidious::Routes::APIv1, :captions
Invidious::Routing.get "#{{{base_url}}}/annotations/:id", Invidious::Routes::APIv1, :annotations Invidious::Routing.get "#{{{base_url}}}/annotations/:id", Invidious::Routes::APIv1, :annotations
Invidious::Routing.get "#{{{base_url}}}/search/suggestions/:id", Invidious::Routes::APIv1, :search_suggestions Invidious::Routing.get "#{{{base_url}}}/search/suggestions/:id", Invidious::Routes::APIv1, :search_suggestions
Invidious::Routing.get "#{{{base_url}}}/comments/:id", Invidious::Routes::APIv1, :comments Invidious::Routing.get "#{{{base_url}}}/comments/:id", Invidious::Routes::APIv1, :comments
# Feeds
Invidious::Routing.get "#{{{base_url}}}/trending", Invidious::Routes::APIv1, :trending Invidious::Routing.get "#{{{base_url}}}/trending", Invidious::Routes::APIv1, :trending
Invidious::Routing.get "#{{{base_url}}}/popular", Invidious::Routes::APIv1, :popular Invidious::Routing.get "#{{{base_url}}}/popular", Invidious::Routes::APIv1, :popular
# Channels
Invidious::Routing.get "#{{{base_url}}}/channels/:ucid", Invidious::Routes::APIv1, :home Invidious::Routing.get "#{{{base_url}}}/channels/:ucid", Invidious::Routes::APIv1, :home
{% for route in { {% for route in {
{"home", "home"}, {"home", "home"},
{"videos", "videos"}, {"videos", "videos"},
@ -25,6 +27,11 @@ macro define_v1_api_routes(base_url = "/api/v1")
Invidious::Routing.get "#{{{base_url}}}/channels/#{{{route[0]}}}/:ucid", Invidious::Routes::APIv1, :{{route[1]}} Invidious::Routing.get "#{{{base_url}}}/channels/#{{{route[0]}}}/:ucid", Invidious::Routes::APIv1, :{{route[1]}}
Invidious::Routing.get "#{{{base_url}}}/channels/:ucid/#{{{route[0]}}}", Invidious::Routes::APIv1, :{{route[1]}} Invidious::Routing.get "#{{{base_url}}}/channels/:ucid/#{{{route[0]}}}", Invidious::Routes::APIv1, :{{route[1]}}
{% end %} {% end %}
# Search
Invidious::Routing.get "#{{{base_url}}}/search", Invidious::Routes::APIv1, :search
Invidious::Routing.get "#{{{base_url}}}/videos/:id", Invidious::Routes::APIv1, :videos
Invidious::Routing.get "#{{{base_url}}}/search", Invidious::Routes::APIv1, :search
end end

101
src/invidious/routes/api/v1/search.cr Normal file
View File

@ -0,0 +1,101 @@
module Invidious::Routes::APIv1
def self.search(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
region = env.params.query["region"]?
env.response.content_type = "application/json"
query = env.params.query["q"]?
query ||= ""
page = env.params.query["page"]?.try &.to_i?
page ||= 1
sort_by = env.params.query["sort_by"]?.try &.downcase
sort_by ||= "relevance"
date = env.params.query["date"]?.try &.downcase
date ||= ""
duration = env.params.query["duration"]?.try &.downcase
duration ||= ""
features = env.params.query["features"]?.try &.split(",").map { |feature| feature.downcase }
features ||= [] of String
content_type = env.params.query["type"]?.try &.downcase
content_type ||= "video"
begin
search_params = produce_search_params(page, sort_by, date, content_type, duration, features)
rescue ex
return error_json(400, ex)
end
count, search_results = search(query, search_params, region).as(Tuple)
JSON.build do |json|
json.array do
search_results.each do |item|
item.to_json(locale, json)
end
end
end
end
def self.channel_search(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
query = env.params.query["q"]?
query ||= ""
page = env.params.query["page"]?.try &.to_i?
page ||= 1
count, search_results = channel_search(query, page, ucid)
JSON.build do |json|
json.array do
search_results.each do |item|
item.to_json(locale, json)
end
end
end
end
def self.search_suggestions(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
region = env.params.query["region"]?
env.response.content_type = "application/json"
query = env.params.query["q"]?
query ||= ""
begin
headers = HTTP::Headers{":authority" => "suggestqueries.google.com"}
response = YT_POOL.client &.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback", headers).body
body = response[35..-2]
body = JSON.parse(body).as_a
suggestions = body[1].as_a[0..-2]
JSON.build do |json|
json.object do
json.field "query", body[0].as_s
json.field "suggestions" do
json.array do
suggestions.each do |suggestion|
json.string suggestion[0].as_s
end
end
end
end
end
rescue ex
return error_json(500, ex)
end
end
end

182
src/invidious/routes/api_v1/widgets.cr → src/invidious/routes/api/v1/videos.cr
View File

@ -1,10 +1,5 @@
module Invidious::Routes::APIv1 module Invidious::Routes::APIv1
# Fetches YouTube storyboards def self.videos(env)
#
# Which are sprites containing x * y preview
# thumbnails for individual scenes in a video.
# See https://support.jwplayer.com/articles/how-to-add-preview-thumbnails
def self.storyboards(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]? locale = LOCALES[env.get("preferences").as(Preferences).locale]?
env.response.content_type = "application/json" env.response.content_type = "application/json"
@ -18,66 +13,10 @@ module Invidious::Routes::APIv1
env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex rescue ex
env.response.status_code = 500 return error_json(500, ex)
return
end end
storyboards = video.storyboards video.to_json(locale)
width = env.params.query["width"]?
height = env.params.query["height"]?
if !width && !height
response = JSON.build do |json|
json.object do
json.field "storyboards" do
generate_storyboards(json, id, storyboards)
end
end
end
return response
end
env.response.content_type = "text/vtt"
storyboard = storyboards.select { |storyboard| width == "#{storyboard[:width]}" || height == "#{storyboard[:height]}" }
if storyboard.empty?
env.response.status_code = 404
return
else
storyboard = storyboard[0]
end
String.build do |str|
str << <<-END_VTT
WEBVTT
END_VTT
start_time = 0.milliseconds
end_time = storyboard[:interval].milliseconds
storyboard[:storyboard_count].times do |i|
url = storyboard[:url]
authority = /(i\d?).ytimg.com/.match(url).not_nil![1]?
url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "")
url = "#{HOST_URL}/sb/#{authority}/#{url}"
storyboard[:storyboard_height].times do |j|
storyboard[:storyboard_width].times do |k|
str << <<-END_CUE
#{start_time}.000 --> #{end_time}.000
#{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]}
END_CUE
start_time += storyboard[:interval].milliseconds
end_time += storyboard[:interval].milliseconds
end
end
end
end
end end
def self.captions(env) def self.captions(env)
@ -206,6 +145,87 @@ module Invidious::Routes::APIv1
webvtt webvtt
end end
# Fetches YouTube storyboards
#
# Which are sprites containing x * y preview
# thumbnails for individual scenes in a video.
# See https://support.jwplayer.com/articles/how-to-add-preview-thumbnails
def self.storyboards(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
env.response.content_type = "application/json"
id = env.params.url["id"]
region = env.params.query["region"]?
begin
video = get_video(id, PG_DB, region: region)
rescue ex : VideoRedirect
env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex
env.response.status_code = 500
return
end
storyboards = video.storyboards
width = env.params.query["width"]?
height = env.params.query["height"]?
if !width && !height
response = JSON.build do |json|
json.object do
json.field "storyboards" do
generate_storyboards(json, id, storyboards)
end
end
end
return response
end
env.response.content_type = "text/vtt"
storyboard = storyboards.select { |storyboard| width == "#{storyboard[:width]}" || height == "#{storyboard[:height]}" }
if storyboard.empty?
env.response.status_code = 404
return
else
storyboard = storyboard[0]
end
String.build do |str|
str << <<-END_VTT
WEBVTT
END_VTT
start_time = 0.milliseconds
end_time = storyboard[:interval].milliseconds
storyboard[:storyboard_count].times do |i|
url = storyboard[:url]
authority = /(i\d?).ytimg.com/.match(url).not_nil![1]?
url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "")
url = "#{HOST_URL}/sb/#{authority}/#{url}"
storyboard[:storyboard_height].times do |j|
storyboard[:storyboard_width].times do |k|
str << <<-END_CUE
#{start_time}.000 --> #{end_time}.000
#{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]}
END_CUE
start_time += storyboard[:interval].milliseconds
end_time += storyboard[:interval].milliseconds
end
end
end
end
end
def self.annotations(env) def self.annotations(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]? locale = LOCALES[env.get("preferences").as(Preferences).locale]?
@ -280,40 +300,6 @@ module Invidious::Routes::APIv1
end end
end end
def self.search_suggestions(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
region = env.params.query["region"]?
env.response.content_type = "application/json"
query = env.params.query["q"]?
query ||= ""
begin
headers = HTTP::Headers{":authority" => "suggestqueries.google.com"}
response = YT_POOL.client &.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback", headers).body
body = response[35..-2]
body = JSON.parse(body).as_a
suggestions = body[1].as_a[0..-2]
JSON.build do |json|
json.object do
json.field "query", body[0].as_s
json.field "suggestions" do
json.array do
suggestions.each do |suggestion|
json.string suggestion[0].as_s
end
end
end
end
end
rescue ex
return error_json(500, ex)
end
end
def self.comments(env) def self.comments(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]? locale = LOCALES[env.get("preferences").as(Preferences).locale]?
region = env.params.query["region"]? region = env.params.query["region"]?

24
src/invidious/routes/api_v1/search.cr
View File

@ -1,24 +0,0 @@
module Invidious::Routes::APIv1
def self.channel_search(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
env.response.content_type = "application/json"
ucid = env.params.url["ucid"]
query = env.params.query["q"]?
query ||= ""
page = env.params.query["page"]?.try &.to_i?
page ||= 1
count, search_results = channel_search(query, page, ucid)
JSON.build do |json|
json.array do
search_results.each do |item|
item.to_json(locale, json)
end
end
end
end
end

2
src/invidious/routes/api_v1/video_playback.cr
View File

@ -1,2 +0,0 @@
module Invidious::Routes::APIv1
end

290
src/invidious/routes/video_playback.cr Normal file
View File

@ -0,0 +1,290 @@
module Invidious::Routes::VideoPlayback
# /videoplayback
def self.get_video_playback(env)
locale = LOCALES[env.get("preferences").as(Preferences).locale]?
query_params = env.params.query
fvip = query_params["fvip"]? || "3"
mns = query_params["mn"]?.try &.split(",")
mns ||= [] of String
if query_params["region"]?
region = query_params["region"]
query_params.delete("region")
end
if query_params["host"]? && !query_params["host"].empty?
host = "https://#{query_params["host"]}"
query_params.delete("host")
else
host = "https://r#{fvip}---#{mns.pop}.googlevideo.com"
end
url = "/videoplayback?#{query_params.to_s}"
headers = HTTP::Headers.new
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
client = make_client(URI.parse(host), region)
response = HTTP::Client::Response.new(500)
error = ""
5.times do
begin
response = client.head(url, headers)
if response.headers["Location"]?
location = URI.parse(response.headers["Location"])
env.response.headers["Access-Control-Allow-Origin"] = "*"
new_host = "#{location.scheme}://#{location.host}"
if new_host != host
host = new_host
client.close
client = make_client(URI.parse(new_host), region)
end
url = "#{location.request_target}&host=#{location.host}#{region ? "&region=#{region}" : ""}"
else
break
end
rescue Socket::Addrinfo::Error
if !mns.empty?
mn = mns.pop
end
fvip = "3"
host = "https://r#{fvip}---#{mn}.googlevideo.com"
client = make_client(URI.parse(host), region)
rescue ex
error = ex.message
end
end
if response.status_code >= 400
env.response.content_type = "text/plain"
haltf env, response.status_code
end
if url.includes? "&file=seg.ts"
if CONFIG.disabled?("livestreams")
return error_template(403, "Administrator has disabled this endpoint.")
end
begin
client.get(url, headers) do |response|
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = response.headers["Location"]?
location = URI.parse(location)
location = "#{location.request_target}&host=#{location.host}"
if region
location += "&region=#{region}"
end
return env.redirect location
end
IO.copy(response.body_io, env.response)
end
rescue ex
end
else
if query_params["title"]? && CONFIG.disabled?("downloads") ||
CONFIG.disabled?("dash")
return error_template(403, "Administrator has disabled this endpoint.")
end
content_length = nil
first_chunk = true
range_start, range_end = parse_range(env.request.headers["Range"]?)
chunk_start = range_start
chunk_end = range_end
if !chunk_end || chunk_end - chunk_start > HTTP_CHUNK_SIZE
chunk_end = chunk_start + HTTP_CHUNK_SIZE - 1
end
# TODO: Record bytes written so we can restart after a chunk fails
while true
if !range_end && content_length
range_end = content_length
end
if range_end && chunk_start > range_end
break
end
if range_end && chunk_end > range_end
chunk_end = range_end
end
headers["Range"] = "bytes=#{chunk_start}-#{chunk_end}"
begin
client.get(url, headers) do |response|
if first_chunk
if !env.request.headers["Range"]? && response.status_code == 206
env.response.status_code = 200
else
env.response.status_code = response.status_code
end
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "content-range"
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = response.headers["Location"]?
location = URI.parse(location)
location = "#{location.request_target}&host=#{location.host}#{region ? "&region=#{region}" : ""}"
env.redirect location
break
end
if title = query_params["title"]?
# https://blog.fastmail.com/2011/06/24/download-non-english-filenames/
env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}"
end
if !response.headers.includes_word?("Transfer-Encoding", "chunked")
content_length = response.headers["Content-Range"].split("/")[-1].to_i64
if env.request.headers["Range"]?
env.response.headers["Content-Range"] = "bytes #{range_start}-#{range_end || (content_length - 1)}/#{content_length}"
env.response.content_length = ((range_end.try &.+ 1) || content_length) - range_start
else
env.response.content_length = content_length
end
end
end
proxy_file(response, env)
end
rescue ex
if ex.message != "Error reading socket: Connection reset by peer"
break
else
client.close
client = make_client(URI.parse(host), region)
end
end
chunk_start = chunk_end + 1
chunk_end += HTTP_CHUNK_SIZE
first_chunk = false
end
end
client.close
end
# /videoplayback/*
def self.get_video_playback_greedy(env)
path = env.request.path
path = path.lchop("/videoplayback/")
path = path.rchop("/")
path = path.gsub(/mime\/\w+\/\w+/) do |mimetype|
mimetype = mimetype.split("/")
mimetype[0] + "/" + mimetype[1] + "%2F" + mimetype[2]
end
path = path.split("/")
raw_params = {} of String => Array(String)
path.each_slice(2) do |pair|
key, value = pair
value = URI.decode_www_form(value)
if raw_params[key]?
raw_params[key] << value
else
raw_params[key] = [value]
end
end
query_params = HTTP::Params.new(raw_params)
env.response.headers["Access-Control-Allow-Origin"] = "*"
return env.redirect "/videoplayback?#{query_params}"
end
# /videoplayback/* && /videoplayback/*
def self.options_video_playback(env)
env.response.headers.delete("Content-Type")
env.response.headers["Access-Control-Allow-Origin"] = "*"
env.response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS"
env.response.headers["Access-Control-Allow-Headers"] = "Content-Type, Range"
end
# /latest_version
#
# YouTube /videoplayback links expire after 6 hours,
# so we have a mechanism here to redirect to the latest version
def self.latest_version(env)
if env.params.query["download_widget"]?
download_widget = JSON.parse(env.params.query["download_widget"])
id = download_widget["id"].as_s
title = download_widget["title"].as_s
if label = download_widget["label"]?
return env.redirect "/api/v1/captions/#{id}?label=#{label}&title=#{title}"
else
itag = download_widget["itag"].as_s.to_i
local = "true"
end
end
id ||= env.params.query["id"]?
itag ||= env.params.query["itag"]?.try &.to_i
region = env.params.query["region"]?
local ||= env.params.query["local"]?
local ||= "false"
local = local == "true"
if !id || !itag
haltf env, status_code: 400, response: "TESTING"
end
video = get_video(id, PG_DB, region: region)
fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag }
url = fmt.try &.["url"]?.try &.as_s
if !url
haltf env, status_code: 404
end
url = URI.parse(url).request_target.not_nil! if local
url = "#{url}&title=#{title}" if title
return env.redirect url
end
end
macro define_video_playback_routes
Invidious::Routing.get "/videoplayback", Invidious::Routes::VideoPlayback, :get_video_playback
Invidious::Routing.get "/videoplayback/*", Invidious::Routes::VideoPlayback, :get_video_playback_greedy
Invidious::Routing.options "/videoplayback", Invidious::Routes::VideoPlayback, :options_video_playback
Invidious::Routing.options "/videoplayback/*", Invidious::Routes::VideoPlayback, :options_video_playback
Invidious::Routing.get "/latest_version", Invidious::Routes::VideoPlayback, :latest_version
end