Replace requests with httpx, remove unused code

setup.py

@@ -34,7 +34,7 @@ setup(
     python_requires='>=3.5',
     install_requires=[
         "m3u8>=1.0.0,<2.0.0",
-        "requests>=2.13,<3.0",
+        "httpx>=0.17.0,<1.0.0",
     ],
     entry_points={
         'console_scripts': [

twitchdl/commands/download.py

@@ -1,15 +1,15 @@
 import asyncio
-from typing import OrderedDict
+import httpx
 import m3u8
 import os
 import re
-import requests
 import shutil
 import subprocess
 import tempfile
 
 from os import path
 from pathlib import Path
+from typing import OrderedDict
 from urllib.parse import urlparse, urlencode
 
 from twitchdl import twitch, utils
@@ -287,7 +287,7 @@ def _download_video(video_id, args):
             else _select_playlist_interactive(playlists))
 
     print_out("<dim>Fetching playlist...</dim>")
-    response = requests.get(playlist_uri)
+    response = httpx.get(playlist_uri)
     response.raise_for_status()
     playlist = m3u8.loads(response.text)
 

twitchdl/download.py

@@ -1,14 +1,5 @@
 import os
-import requests
+import httpx
-
-from collections import OrderedDict
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import datetime
-from functools import partial
-from requests.exceptions import RequestException
-from twitchdl.output import print_out
-from twitchdl.utils import format_size, format_duration
-
 
 CHUNK_SIZE = 1024
 CONNECT_TIMEOUT = 5
@@ -21,12 +12,12 @@ class DownloadFailed(Exception):
 
 def _download(url, path):
     tmp_path = path + ".tmp"
-    response = requests.get(url, stream=True, timeout=CONNECT_TIMEOUT)
     size = 0
-    with open(tmp_path, 'wb') as target:
+    with httpx.stream("GET", url, timeout=CONNECT_TIMEOUT) as response:
-        for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
+        with open(tmp_path, "wb") as target:
-            target.write(chunk)
+            for chunk in response.iter_bytes(chunk_size=CHUNK_SIZE):
-            size += len(chunk)
+                target.write(chunk)
+                size += len(chunk)
 
     os.rename(tmp_path, path)
     return size
@@ -41,63 +32,7 @@ def download_file(url, path, retries=RETRY_COUNT):
     for _ in range(retries):
         try:
             return (_download(url, path), from_disk)
-        except RequestException:
+        except httpx.RequestError:
             pass
 
     raise DownloadFailed(":(")
-
-
-def _print_progress(futures):
-    downloaded_count = 0
-    downloaded_size = 0
-    max_msg_size = 0
-    start_time = datetime.now()
-    total_count = len(futures)
-    current_download_size = 0
-    current_downloaded_count = 0
-
-    for future in as_completed(futures):
-        size, from_disk = future.result()
-        downloaded_count += 1
-        downloaded_size += size
-
-        # If we find something on disk, we don't want to take it in account in
-        # the speed calculation
-        if not from_disk:
-            current_download_size += size
-            current_downloaded_count += 1
-
-        percentage = 100 * downloaded_count // total_count
-        est_total_size = int(total_count * downloaded_size / downloaded_count)
-        duration = (datetime.now() - start_time).seconds
-        speed = current_download_size // duration if duration else 0
-        remaining = (total_count - downloaded_count) * duration / current_downloaded_count \
-            if current_downloaded_count else 0
-
-        msg = " ".join([
-            "Downloaded VOD {}/{}".format(downloaded_count, total_count),
-            "({}%)".format(percentage),
-            "<cyan>{}</cyan>".format(format_size(downloaded_size)),
-            "of <cyan>~{}</cyan>".format(format_size(est_total_size)),
-            "at <cyan>{}/s</cyan>".format(format_size(speed)) if speed > 0 else "",
-            "remaining <cyan>~{}</cyan>".format(format_duration(remaining)) if remaining > 0 else "",
-        ])
-
-        max_msg_size = max(len(msg), max_msg_size)
-        print_out("\r" + msg.ljust(max_msg_size), end="")
-
-
-def download_files(base_url, target_dir, vod_paths, max_workers):
-    """
-    Downloads a list of VODs defined by a common `base_url` and a list of
-    `vod_paths`, returning a dict which maps the paths to the downloaded files.
-    """
-    urls = [base_url + path for path in vod_paths]
-    targets = [os.path.join(target_dir, "{:05d}.ts".format(k)) for k, _ in enumerate(vod_paths)]
-    partials = (partial(download_file, url, path) for url, path in zip(urls, targets))
-
-    with ThreadPoolExecutor(max_workers=max_workers) as executor:
-        futures = [executor.submit(fn) for fn in partials]
-        _print_progress(futures)
-
-    return OrderedDict(zip(vod_paths, targets))

twitchdl/twitch.py

@@ -2,9 +2,8 @@
 Twitch API access.
 """
 
-import requests
+import httpx
 
-from requests.exceptions import HTTPError
 from twitchdl import CLIENT_ID
 from twitchdl.exceptions import ConsoleError
 
@@ -18,7 +17,7 @@ class GQLError(Exception):
 def authenticated_get(url, params={}, headers={}):
     headers['Client-ID'] = CLIENT_ID
 
-    response = requests.get(url, params, headers=headers)
+    response = httpx.get(url, params=params, headers=headers)
     if 400 <= response.status_code < 500:
         data = response.json()
         # TODO: this does not look nice in the console since data["message"]
@@ -33,7 +32,7 @@ def authenticated_get(url, params={}, headers={}):
 def authenticated_post(url, data=None, json=None, headers={}):
     headers['Client-ID'] = CLIENT_ID
 
-    response = requests.post(url, data=data, json=json, headers=headers)
+    response = httpx.post(url, data=data, json=json, headers=headers)
     if response.status_code == 400:
         data = response.json()
         raise ConsoleError(data["message"])
@@ -330,7 +329,7 @@ def get_access_token(video_id, auth_token=None):
     try:
         response = gql_query(query, headers=headers)
         return response["data"]["videoPlaybackAccessToken"]
-    except HTTPError as error:
+    except httpx.HTTPStatusError as error:
         # Provide a more useful error message when server returns HTTP 401
         # Unauthorized while using a user-provided auth token.
         if error.response.status_code == 401:
@@ -351,7 +350,7 @@ def get_playlists(video_id, access_token):
     """
     url = "http://usher.twitch.tv/vod/{}".format(video_id)
 
-    response = requests.get(url, params={
+    response = httpx.get(url, params={
         "nauth": access_token['value'],
         "nauthsig": access_token['signature'],
         "allow_audio_only": "true",