diff --git a/setup.py b/setup.py
index 89e3d43..0c8c99b 100644
--- a/setup.py
+++ b/setup.py
@@ -34,7 +34,7 @@ setup(
python_requires='>=3.5',
install_requires=[
"m3u8>=1.0.0,<2.0.0",
- "requests>=2.13,<3.0",
+ "httpx>=0.17.0,<1.0.0",
],
entry_points={
'console_scripts': [
diff --git a/twitchdl/commands/download.py b/twitchdl/commands/download.py
index 6e711fd..9bc03a6 100644
--- a/twitchdl/commands/download.py
+++ b/twitchdl/commands/download.py
@@ -1,15 +1,15 @@
import asyncio
-from typing import OrderedDict
+import httpx
import m3u8
import os
import re
-import requests
import shutil
import subprocess
import tempfile
from os import path
from pathlib import Path
+from typing import OrderedDict
from urllib.parse import urlparse, urlencode
from twitchdl import twitch, utils
@@ -287,7 +287,7 @@ def _download_video(video_id, args):
else _select_playlist_interactive(playlists))
print_out("Fetching playlist...")
- response = requests.get(playlist_uri)
+ response = httpx.get(playlist_uri)
response.raise_for_status()
playlist = m3u8.loads(response.text)
diff --git a/twitchdl/download.py b/twitchdl/download.py
index 49552b4..a3bf1e8 100644
--- a/twitchdl/download.py
+++ b/twitchdl/download.py
@@ -1,14 +1,5 @@
import os
-import requests
-
-from collections import OrderedDict
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import datetime
-from functools import partial
-from requests.exceptions import RequestException
-from twitchdl.output import print_out
-from twitchdl.utils import format_size, format_duration
-
+import httpx
CHUNK_SIZE = 1024
CONNECT_TIMEOUT = 5
@@ -21,12 +12,12 @@ class DownloadFailed(Exception):
def _download(url, path):
tmp_path = path + ".tmp"
- response = requests.get(url, stream=True, timeout=CONNECT_TIMEOUT)
size = 0
- with open(tmp_path, 'wb') as target:
- for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
- target.write(chunk)
- size += len(chunk)
+ with httpx.stream("GET", url, timeout=CONNECT_TIMEOUT) as response:
+ with open(tmp_path, "wb") as target:
+ for chunk in response.iter_bytes(chunk_size=CHUNK_SIZE):
+ target.write(chunk)
+ size += len(chunk)
os.rename(tmp_path, path)
return size
@@ -41,63 +32,7 @@ def download_file(url, path, retries=RETRY_COUNT):
for _ in range(retries):
try:
return (_download(url, path), from_disk)
- except RequestException:
+ except httpx.RequestError:
pass
raise DownloadFailed(":(")
-
-
-def _print_progress(futures):
- downloaded_count = 0
- downloaded_size = 0
- max_msg_size = 0
- start_time = datetime.now()
- total_count = len(futures)
- current_download_size = 0
- current_downloaded_count = 0
-
- for future in as_completed(futures):
- size, from_disk = future.result()
- downloaded_count += 1
- downloaded_size += size
-
- # If we find something on disk, we don't want to take it in account in
- # the speed calculation
- if not from_disk:
- current_download_size += size
- current_downloaded_count += 1
-
- percentage = 100 * downloaded_count // total_count
- est_total_size = int(total_count * downloaded_size / downloaded_count)
- duration = (datetime.now() - start_time).seconds
- speed = current_download_size // duration if duration else 0
- remaining = (total_count - downloaded_count) * duration / current_downloaded_count \
- if current_downloaded_count else 0
-
- msg = " ".join([
- "Downloaded VOD {}/{}".format(downloaded_count, total_count),
- "({}%)".format(percentage),
- "{}".format(format_size(downloaded_size)),
- "of ~{}".format(format_size(est_total_size)),
- "at {}/s".format(format_size(speed)) if speed > 0 else "",
- "remaining ~{}".format(format_duration(remaining)) if remaining > 0 else "",
- ])
-
- max_msg_size = max(len(msg), max_msg_size)
- print_out("\r" + msg.ljust(max_msg_size), end="")
-
-
-def download_files(base_url, target_dir, vod_paths, max_workers):
- """
- Downloads a list of VODs defined by a common `base_url` and a list of
- `vod_paths`, returning a dict which maps the paths to the downloaded files.
- """
- urls = [base_url + path for path in vod_paths]
- targets = [os.path.join(target_dir, "{:05d}.ts".format(k)) for k, _ in enumerate(vod_paths)]
- partials = (partial(download_file, url, path) for url, path in zip(urls, targets))
-
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- futures = [executor.submit(fn) for fn in partials]
- _print_progress(futures)
-
- return OrderedDict(zip(vod_paths, targets))
diff --git a/twitchdl/twitch.py b/twitchdl/twitch.py
index b974f28..e0ff2ee 100644
--- a/twitchdl/twitch.py
+++ b/twitchdl/twitch.py
@@ -2,9 +2,8 @@
Twitch API access.
"""
-import requests
+import httpx
-from requests.exceptions import HTTPError
from twitchdl import CLIENT_ID
from twitchdl.exceptions import ConsoleError
@@ -18,7 +17,7 @@ class GQLError(Exception):
def authenticated_get(url, params={}, headers={}):
headers['Client-ID'] = CLIENT_ID
- response = requests.get(url, params, headers=headers)
+ response = httpx.get(url, params=params, headers=headers)
if 400 <= response.status_code < 500:
data = response.json()
# TODO: this does not look nice in the console since data["message"]
@@ -33,7 +32,7 @@ def authenticated_get(url, params={}, headers={}):
def authenticated_post(url, data=None, json=None, headers={}):
headers['Client-ID'] = CLIENT_ID
- response = requests.post(url, data=data, json=json, headers=headers)
+ response = httpx.post(url, data=data, json=json, headers=headers)
if response.status_code == 400:
data = response.json()
raise ConsoleError(data["message"])
@@ -330,7 +329,7 @@ def get_access_token(video_id, auth_token=None):
try:
response = gql_query(query, headers=headers)
return response["data"]["videoPlaybackAccessToken"]
- except HTTPError as error:
+ except httpx.HTTPStatusError as error:
# Provide a more useful error message when server returns HTTP 401
# Unauthorized while using a user-provided auth token.
if error.response.status_code == 401:
@@ -351,7 +350,7 @@ def get_playlists(video_id, access_token):
"""
url = "http://usher.twitch.tv/vod/{}".format(video_id)
- response = requests.get(url, params={
+ response = httpx.get(url, params={
"nauth": access_token['value'],
"nauthsig": access_token['signature'],
"allow_audio_only": "true",