From f40fd290f78dc0da7d71b865130dcd39b8e621eb Mon Sep 17 00:00:00 2001 From: Ivan Habunek Date: Mon, 15 Aug 2022 07:12:10 +0200 Subject: [PATCH] Replace requests with httpx, remove unused code --- setup.py | 2 +- twitchdl/commands/download.py | 6 +-- twitchdl/download.py | 79 ++++------------------------------- twitchdl/twitch.py | 11 +++-- 4 files changed, 16 insertions(+), 82 deletions(-) diff --git a/setup.py b/setup.py index 89e3d43..0c8c99b 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( python_requires='>=3.5', install_requires=[ "m3u8>=1.0.0,<2.0.0", - "requests>=2.13,<3.0", + "httpx>=0.17.0,<1.0.0", ], entry_points={ 'console_scripts': [ diff --git a/twitchdl/commands/download.py b/twitchdl/commands/download.py index 6e711fd..9bc03a6 100644 --- a/twitchdl/commands/download.py +++ b/twitchdl/commands/download.py @@ -1,15 +1,15 @@ import asyncio -from typing import OrderedDict +import httpx import m3u8 import os import re -import requests import shutil import subprocess import tempfile from os import path from pathlib import Path +from typing import OrderedDict from urllib.parse import urlparse, urlencode from twitchdl import twitch, utils @@ -287,7 +287,7 @@ def _download_video(video_id, args): else _select_playlist_interactive(playlists)) print_out("Fetching playlist...") - response = requests.get(playlist_uri) + response = httpx.get(playlist_uri) response.raise_for_status() playlist = m3u8.loads(response.text) diff --git a/twitchdl/download.py b/twitchdl/download.py index 49552b4..a3bf1e8 100644 --- a/twitchdl/download.py +++ b/twitchdl/download.py @@ -1,14 +1,5 @@ import os -import requests - -from collections import OrderedDict -from concurrent.futures import ThreadPoolExecutor, as_completed -from datetime import datetime -from functools import partial -from requests.exceptions import RequestException -from twitchdl.output import print_out -from twitchdl.utils import format_size, format_duration - +import httpx CHUNK_SIZE = 1024 CONNECT_TIMEOUT = 5 @@ -21,12 +12,12 @@ class DownloadFailed(Exception): def _download(url, path): tmp_path = path + ".tmp" - response = requests.get(url, stream=True, timeout=CONNECT_TIMEOUT) size = 0 - with open(tmp_path, 'wb') as target: - for chunk in response.iter_content(chunk_size=CHUNK_SIZE): - target.write(chunk) - size += len(chunk) + with httpx.stream("GET", url, timeout=CONNECT_TIMEOUT) as response: + with open(tmp_path, "wb") as target: + for chunk in response.iter_bytes(chunk_size=CHUNK_SIZE): + target.write(chunk) + size += len(chunk) os.rename(tmp_path, path) return size @@ -41,63 +32,7 @@ def download_file(url, path, retries=RETRY_COUNT): for _ in range(retries): try: return (_download(url, path), from_disk) - except RequestException: + except httpx.RequestError: pass raise DownloadFailed(":(") - - -def _print_progress(futures): - downloaded_count = 0 - downloaded_size = 0 - max_msg_size = 0 - start_time = datetime.now() - total_count = len(futures) - current_download_size = 0 - current_downloaded_count = 0 - - for future in as_completed(futures): - size, from_disk = future.result() - downloaded_count += 1 - downloaded_size += size - - # If we find something on disk, we don't want to take it in account in - # the speed calculation - if not from_disk: - current_download_size += size - current_downloaded_count += 1 - - percentage = 100 * downloaded_count // total_count - est_total_size = int(total_count * downloaded_size / downloaded_count) - duration = (datetime.now() - start_time).seconds - speed = current_download_size // duration if duration else 0 - remaining = (total_count - downloaded_count) * duration / current_downloaded_count \ - if current_downloaded_count else 0 - - msg = " ".join([ - "Downloaded VOD {}/{}".format(downloaded_count, total_count), - "({}%)".format(percentage), - "{}".format(format_size(downloaded_size)), - "of ~{}".format(format_size(est_total_size)), - "at {}/s".format(format_size(speed)) if speed > 0 else "", - "remaining ~{}".format(format_duration(remaining)) if remaining > 0 else "", - ]) - - max_msg_size = max(len(msg), max_msg_size) - print_out("\r" + msg.ljust(max_msg_size), end="") - - -def download_files(base_url, target_dir, vod_paths, max_workers): - """ - Downloads a list of VODs defined by a common `base_url` and a list of - `vod_paths`, returning a dict which maps the paths to the downloaded files. - """ - urls = [base_url + path for path in vod_paths] - targets = [os.path.join(target_dir, "{:05d}.ts".format(k)) for k, _ in enumerate(vod_paths)] - partials = (partial(download_file, url, path) for url, path in zip(urls, targets)) - - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = [executor.submit(fn) for fn in partials] - _print_progress(futures) - - return OrderedDict(zip(vod_paths, targets)) diff --git a/twitchdl/twitch.py b/twitchdl/twitch.py index b974f28..e0ff2ee 100644 --- a/twitchdl/twitch.py +++ b/twitchdl/twitch.py @@ -2,9 +2,8 @@ Twitch API access. """ -import requests +import httpx -from requests.exceptions import HTTPError from twitchdl import CLIENT_ID from twitchdl.exceptions import ConsoleError @@ -18,7 +17,7 @@ class GQLError(Exception): def authenticated_get(url, params={}, headers={}): headers['Client-ID'] = CLIENT_ID - response = requests.get(url, params, headers=headers) + response = httpx.get(url, params=params, headers=headers) if 400 <= response.status_code < 500: data = response.json() # TODO: this does not look nice in the console since data["message"] @@ -33,7 +32,7 @@ def authenticated_get(url, params={}, headers={}): def authenticated_post(url, data=None, json=None, headers={}): headers['Client-ID'] = CLIENT_ID - response = requests.post(url, data=data, json=json, headers=headers) + response = httpx.post(url, data=data, json=json, headers=headers) if response.status_code == 400: data = response.json() raise ConsoleError(data["message"]) @@ -330,7 +329,7 @@ def get_access_token(video_id, auth_token=None): try: response = gql_query(query, headers=headers) return response["data"]["videoPlaybackAccessToken"] - except HTTPError as error: + except httpx.HTTPStatusError as error: # Provide a more useful error message when server returns HTTP 401 # Unauthorized while using a user-provided auth token. if error.response.status_code == 401: @@ -351,7 +350,7 @@ def get_playlists(video_id, access_token): """ url = "http://usher.twitch.tv/vod/{}".format(video_id) - response = requests.get(url, params={ + response = httpx.get(url, params={ "nauth": access_token['value'], "nauthsig": access_token['signature'], "allow_audio_only": "true",