Better long description

Only die if all retries fail.

CHANGELOG.md

@@ -0,0 +1,42 @@
+Twitch Downloader change log
+============================
+
+1.5.0 (2020-04-10)
+------------------
+
+* Fix video downloads after Twitch deprecated access token access
+* Don't print errors when retrying download, only if all fails
+
+1.4.0 (2019-08-23)
+------------------
+
+* Fix usage of deprecated v3 API
+* Use m3u8 lib for parsing playlists
+* Add `--keep` option not preserve downloaded VODs
+
+1.3.1 (2019-08-13)
+------------------
+
+* No changes, bumped to fix issue with pypi
+
+1.3.0 (2019-08-13)
+------------------
+
+* Add `--sort` and `--offset` options to `videos` command, allows paging (#7)
+* Show video URL in `videos` command output
+
+1.2.0 (2019-07-05)
+------------------
+
+* Add `--format` option to `download` command for specifying the output format (#6)
+* Add `--version` option for printing program version
+
+1.1.0 (2019-06-06)
+------------------
+
+* Allow limiting download by start and end time
+
+1.0.0 (2019-04-30)
+------------------
+
+* Initial release

Makefile

@@ -1,22 +1,15 @@
 default : clean dist
 
 dist :
-	@echo "\nMaking source"
-	@echo "-------------"
-	@python setup.py sdist
-
-	@echo "\nMaking wheel"
-	@echo "-------------"
-	@python setup.py bdist_wheel --universal
-
-	@echo "\nDone."
+	python setup.py sdist --formats=gztar,zip
+	python setup.py bdist_wheel --python-tag=py3
 
 clean :
 	find . -name "*pyc" | xargs rm -rf $1
 	rm -rf build dist MANIFEST htmlcov deb_dist twitch-dl*.tar.gz twitch-dl.1.man
 
 publish :
-	twine upload dist/*
+	twine upload dist/*.tar.gz dist/*.whl
 
 coverage:
 	py.test --cov=toot --cov-report html tests/

setup.cfg

@@ -1,2 +0,0 @@
-[bdist_wheel]
-universal=1

setup.py

@@ -2,27 +2,35 @@
 
 from setuptools import setup
 
+long_description = """
+Quickly download videos from twitch.tv.
+
+Works simliarly to youtube-dl but downloads multiple VODs in parallel which
+makes it faster.
+"""
 
 setup(
     name='twitch-dl',
-    version='0.1.0',
+    version='1.5.0',
     description='Twitch downloader',
-    long_description="A simple script for downloading videos from Twitch",
+    long_description=long_description.strip(),
     author='Ivan Habunek',
     author_email='ivan@habunek.com',
     url='https://github.com/ihabunek/twitch-dl/',
     keywords='twitch vod video download',
     license='GPLv3',
     classifiers=[
-        'Development Status :: 4 - Beta',
+        'Development Status :: 5 - Production/Stable',
         'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
     ],
     packages=['twitchdl'],
     python_requires='>=3.5',
     install_requires=[
+        "m3u8>=0.3.12,<0.4",
         "requests>=2.13,<3.0",
     ],
     entry_points={

stdeb.cfg

@@ -1,3 +1,3 @@
 [DEFAULT]
-X-Python3-Version: >= 3.3
+X-Python3-Version: >= 3.5
 Copyright-File: LICENSE

twitch-dl.1.scd

@@ -36,6 +36,18 @@ Download by ID:
 twitch-dl download 377220226
 ```
 
+Specify output format:
+
+```
+twitch-dl download --format=avi 377220226
+```
+
+Partial download by setting start and end time (hh:mm or hh:mm:ss):
+
+```
+twitch-dl download --start=00:10 --end=02:15 377220226
+```
+
 # SEE ALSO
 
 youtube-dl(1)

twitchdl/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "0.1.0"
+__version__ = "1.5.0"
 
-CLIENT_ID = "miwy5zk23vh2he94san0bzj5ks1r0p"
+CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko"

twitchdl/commands.py

@@ -1,12 +1,16 @@
+import m3u8
 import os
 import pathlib
 import re
+import requests
+import shutil
 import subprocess
 import tempfile
 
-from datetime import datetime
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime
 from functools import partial
+from urllib.parse import urlparse
 
 from twitchdl import twitch
 from twitchdl.download import download_file
@@ -69,12 +73,26 @@ def _print_video(video):
     print_out("<green>{}</green>".format(video["title"]))
     print_out("<cyan>{}</cyan> playing <cyan>{}</cyan>".format(name, video['game']))
     print_out("Published <cyan>{}</cyan>  Length: <cyan>{}</cyan> ".format(published_at, length))
+    print_out("<i>{}</i>".format(video["url"]))
 
 
-def videos(channel_name, **kwargs):
-    videos = twitch.get_channel_videos(channel_name)
+def videos(channel_name, limit, offset, sort, **kwargs):
+    print_out("Looking up user...")
+    user = twitch.get_user(channel_name)
+    if not user:
+        raise ConsoleError("User {} not found.".format(channel_name))
 
-    print("Found {} videos".format(videos["_total"]))
+    print_out("Loading videos...")
+    videos = twitch.get_channel_videos(user["id"], limit, offset, sort)
+    count = len(videos['videos'])
+    if not count:
+        print_out("No videos found")
+        return
+
+    first = offset + 1
+    last = offset + len(videos['videos'])
+    total = videos["_total"]
+    print_out("<yellow>Showing videos {}-{} of {}</yellow>".format(first, last, total))
 
     for video in videos['videos']:
         _print_video(video)
@@ -82,13 +100,14 @@ def videos(channel_name, **kwargs):
 
 def _select_quality(playlists):
     print_out("\nAvailable qualities:")
-    for no, v in playlists.items():
-        print_out("{}) {}".format(no, v[0]))
+    for n, p in enumerate(playlists):
+        name = p.media[0].name if p.media else ""
+        resolution = "x".join(str(r) for r in p.stream_info.resolution)
+        print_out("{}) {} [{}]".format(n + 1, name, resolution))
 
-    keys = list(playlists.keys())
-    no = read_int("Choose quality", min=min(keys), max=max(keys), default=keys[0])
+    no = read_int("Choose quality", min=1, max=len(playlists) + 1, default=1)
 
-    return playlists[no]
+    return playlists[no - 1]
 
 
 def _print_progress(futures):
@@ -113,7 +132,7 @@ def _print_progress(futures):
 
 
 def _download_files(base_url, directory, filenames, max_workers):
-    urls = [base_url.format(f) for f in filenames]
+    urls = [base_url + f for f in filenames]
     paths = ["/".join([directory, f]) for f in filenames]
     partials = (partial(download_file, url, path) for url, path in zip(urls, paths))
 
@@ -158,7 +177,7 @@ def _video_target_filename(video, format):
     return name + "." + format
 
 
-def parse_video_id(video_id):
+def _parse_video_id(video_id):
     """This can be either a integer ID or an URL to the video on twitch."""
     if re.search(r"^\d+$", video_id):
         return int(video_id)
@@ -170,8 +189,36 @@ def parse_video_id(video_id):
     raise ConsoleError("Invalid video ID given, expected integer ID or Twitch URL")
 
 
-def download(video_id, max_workers, format='mkv', **kwargs):
-    video_id = parse_video_id(video_id)
+def _get_files(playlist, start, end):
+    """Extract files for download from playlist."""
+    vod_start = 0
+    for segment in playlist.segments:
+        vod_end = vod_start + segment.duration
+
+        # `vod_end > start` is used here becuase it's better to download a bit
+        # more than a bit less, similar for the end condition
+        start_condition = not start or vod_end > start
+        end_condition = not end or vod_start < end
+
+        if start_condition and end_condition:
+            yield segment.uri
+
+        vod_start = vod_end
+
+
+def _crete_temp_dir(base_uri):
+    """Create a temp dir to store downloads if it doesn't exist."""
+    path = urlparse(base_uri).path
+    directory = '{}/twitch-dl{}'.format(tempfile.gettempdir(), path)
+    pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
+    return directory
+
+
+def download(video_id, max_workers, format='mkv', start=None, end=None, keep=False, **kwargs):
+    video_id = _parse_video_id(video_id)
+
+    if start and end and end <= start:
+        raise ConsoleError("End time must be greater than start time")
 
     print_out("Looking up video...")
     video = twitch.get_video(video_id)
@@ -184,25 +231,36 @@ def download(video_id, max_workers, format='mkv', **kwargs):
 
     print_out("Fetching playlists...")
     playlists = twitch.get_playlists(video_id, access_token)
-    quality, playlist_url = _select_quality(playlists)
+    parsed = m3u8.loads(playlists)
+    selected = _select_quality(parsed.playlists)
 
     print_out("\nFetching playlist...")
-    base_url, filenames = twitch.get_playlist_urls(playlist_url)
+    response = requests.get(selected.uri)
+    response.raise_for_status()
+    playlist = m3u8.loads(response.text)
 
-    # Create a temp dir to store downloads if it doesn't exist
-    directory = '{}/twitch-dl/{}/{}'.format(tempfile.gettempdir(), video_id, quality)
-    pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
-    print_out("Download dir: {}".format(directory))
+    base_uri = re.sub("/[^/]+$", "/", selected.uri)
+    target_dir = _crete_temp_dir(base_uri)
+    filenames = list(_get_files(playlist, start, end))
 
-    print_out("Downloading VODs with {} workers...".format(max_workers))
-    paths = _download_files(base_url, directory, filenames, max_workers)
+    # Save playlists for debugging purposes
+    with open(target_dir + "playlists.m3u8", "w") as f:
+        f.write(playlists)
+    with open(target_dir + "playlist.m3u8", "w") as f:
+        f.write(response.text)
+
+    print_out("\nDownloading {} VODs using {} workers to {}".format(
+        len(filenames), max_workers, target_dir))
+    _download_files(base_uri, target_dir, filenames, max_workers)
 
     print_out("\n\nJoining files...")
     target = _video_target_filename(video, format)
-    _join_vods(directory, paths, target)
+    _join_vods(target_dir, filenames, target)
 
-    print_out("\nDeleting vods...")
-    for path in paths:
-        os.unlink(path)
+    if keep:
+        print_out("\nTemporary files not deleted: {}".format(target_dir))
+    else:
+        print_out("\nDeleting temporary files...")
+        shutil.rmtree(target_dir)
 
-    print_out("\nDownloaded: {}".format(target))
+    print_out("Downloaded: {}".format(target))

twitchdl/console.py

@@ -1,17 +1,37 @@
 # -*- coding: utf-8 -*-
 
-from argparse import ArgumentParser
+import sys
+
+from argparse import ArgumentParser, ArgumentTypeError
 from collections import namedtuple
 
 from twitchdl.exceptions import ConsoleError
 from twitchdl.output import print_err
-from . import commands
+from . import commands, __version__
 
 
 Command = namedtuple("Command", ["name", "description", "arguments"])
 
 CLIENT_WEBSITE = 'https://github.com/ihabunek/twitch-dl'
 
+
+def time(value):
+    """Parse a time string (hh:mm or hh:mm:ss) to number of seconds."""
+    parts = [int(p) for p in value.split(":")]
+
+    if not 2 <= len(parts) <= 3:
+        raise ArgumentTypeError()
+
+    hours = parts[0]
+    minutes = parts[1]
+    seconds = parts[2] if len(parts) > 2 else 0
+
+    if hours < 0 or not (0 <= minutes <= 59) or not (0 <= seconds <= 59):
+        raise ArgumentTypeError()
+
+    return hours * 3600 + minutes * 60 + seconds
+
+
 COMMANDS = [
     Command(
         name="videos",
@@ -21,6 +41,22 @@ COMMANDS = [
                 "help": "channel name",
                 "type": str,
             }),
+            (["-l", "--limit"], {
+                "help": "Number of videos to fetch (default 10, max 100)",
+                "type": int,
+                "default": 10,
+            }),
+            (["-o", "--offset"], {
+                "help": "Offset for pagination of results. (default 0)",
+                "type": int,
+                "default": 0,
+            }),
+            (["-s", "--sort"], {
+                "help": "Sorting order of videos. (default: time)",
+                "type": str,
+                "choices": ["views", "time"],
+                "default": "time",
+            }),
         ],
     ),
     Command(
@@ -32,10 +68,32 @@ COMMANDS = [
                 "type": str,
             }),
             (["-w", "--max_workers"], {
-                "help": "maximal number of threads for downloading vods concurrently (default 5)",
+                "help": "maximal number of threads for downloading vods "
+                        "concurrently (default 5)",
                 "type": int,
                 "default": 20,
             }),
+            (["-s", "--start"], {
+                "help": "Download video from this time (hh:mm or hh:mm:ss)",
+                "type": time,
+                "default": None,
+            }),
+            (["-e", "--end"], {
+                "help": "Download video up to this time (hh:mm or hh:mm:ss)",
+                "type": time,
+                "default": None,
+            }),
+            (["-f", "--format"], {
+                "help": "Video format to convert into, passed to ffmpeg as the "
+                        "target file extension (default: mkv)",
+                "type": str,
+                "default": "mkv",
+            }),
+            (["-k", "--keep"], {
+                "help": "Don't delete downloaded VODs and playlists after merging.",
+                "action": "store_true",
+                "default": False,
+            }),
         ],
     ),
 ]
@@ -58,6 +116,8 @@ def get_parser():
     description = "A script for downloading videos from Twitch"
 
     parser = ArgumentParser(prog='twitch-dl', description=description, epilog=CLIENT_WEBSITE)
+    parser.add_argument("--version", help="show version number", action='store_true')
+
     subparsers = parser.add_subparsers(title="commands")
 
     for command in COMMANDS:
@@ -76,6 +136,10 @@ def main():
     parser = get_parser()
     args = parser.parse_args()
 
+    if args.version:
+        print("twitch-dl v{}".format(__version__))
+        return
+
     if "func" not in args:
         parser.print_help()
         return
@@ -84,3 +148,4 @@ def main():
         args.func(**args.__dict__)
     except ConsoleError as e:
         print_err(e)
+        sys.exit(1)

twitchdl/download.py

@@ -6,6 +6,7 @@ from requests.exceptions import RequestException
 
 CHUNK_SIZE = 1024
 CONNECT_TIMEOUT = 5
+RETRY_COUNT = 5
 
 
 class DownloadFailed(Exception):
@@ -25,14 +26,14 @@ def _download(url, path):
     return size
 
 
-def download_file(url, path, retries=3):
+def download_file(url, path, retries=RETRY_COUNT):
     if os.path.exists(path):
         return 0
 
     for _ in range(retries):
         try:
             return _download(url, path)
-        except RequestException as e:
-            print("Download failed: {}".format(e))
+        except RequestException:
+            pass
 
     raise DownloadFailed(":(")

twitchdl/output.py

@@ -5,6 +5,8 @@ import re
 
 START_CODES = {
     'bold': '\033[1m',
+    'i': '\033[3m',
+    'u': '\033[4m',
     'red': '\033[31m',
     'green': '\033[32m',
     'yellow': '\033[33m',

twitchdl/parse.py

@@ -1,28 +0,0 @@
-import re
-
-from collections import OrderedDict
-
-
-def parse_playlists(data):
-    media_pattern = re.compile(r'^#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="(?P<group>\w+)",NAME="(?P<name>\w+)"')
-
-    playlists = OrderedDict()
-    n = 1
-    name = None
-    for line in data.split():
-        match = re.match(media_pattern, line)
-        if match:
-            name = match.group('name')
-        elif line.startswith('http'):
-            playlists[n] = (name, line)
-            n += 1
-
-    return playlists
-
-
-def parse_playlist(url, data):
-    base_url = re.sub("/[^/]+$", "/{}", url)
-
-    filenames = [line for line in data.split() if re.match(r"\d+\.ts", line)]
-
-    return base_url, filenames

twitchdl/twitch.py

@@ -1,36 +1,67 @@
+"""
+Twitch API access.
+"""
+
 import requests
 
 from twitchdl import CLIENT_ID
-from twitchdl.parse import parse_playlists, parse_playlist
+from twitchdl.exceptions import ConsoleError
 
 
-def authenticated_get(url, params={}):
-    headers = {'Client-ID': CLIENT_ID}
+def authenticated_get(url, params={}, headers={}):
+    headers['Client-ID'] = CLIENT_ID
 
     response = requests.get(url, params, headers=headers)
+    if response.status_code == 400:
+        data = response.json()
+        raise ConsoleError(data["message"])
+
     response.raise_for_status()
 
     return response
 
 
+def kraken_get(url, params={}, headers={}):
+    """
+    Add accept header required by kraken API v5.
+    see: https://discuss.dev.twitch.tv/t/change-in-access-to-deprecated-kraken-twitch-apis/22241
+    """
+    headers["Accept"] = "application/vnd.twitchtv.v5+json"
+    return authenticated_get(url, params, headers)
+
+
+def get_user(login):
+    """
+    https://dev.twitch.tv/docs/api/reference/#get-users
+    """
+    response = authenticated_get("https://api.twitch.tv/helix/users", {
+        "login": login
+    })
+
+    users = response.json()["data"]
+    return users[0] if users else None
+
+
 def get_video(video_id):
     """
     https://dev.twitch.tv/docs/v5/reference/videos#get-video
     """
     url = "https://api.twitch.tv/kraken/videos/%d" % video_id
 
-    return authenticated_get(url).json()
+    return kraken_get(url).json()
 
 
-def get_channel_videos(channel_name, limit=20):
+def get_channel_videos(channel_id, limit, offset, sort):
     """
     https://dev.twitch.tv/docs/v5/reference/channels#get-channel-videos
     """
-    url = "https://api.twitch.tv/kraken/channels/%s/videos" % channel_name
+    url = "https://api.twitch.tv/kraken/channels/{}/videos".format(channel_id)
 
-    return authenticated_get(url, {
+    return kraken_get(url, {
         "broadcast_type": "archive",
         "limit": limit,
+        "offset": offset,
+        "sort": sort,
     }).json()
 
 
@@ -41,6 +72,9 @@ def get_access_token(video_id):
 
 
 def get_playlists(video_id, access_token):
+    """
+    For a given video return a playlist which contains possible video qualities.
+    """
     url = "http://usher.twitch.tv/vod/{}".format(video_id)
 
     response = requests.get(url, params={
@@ -50,16 +84,4 @@ def get_playlists(video_id, access_token):
         "player": "twitchweb",
     })
     response.raise_for_status()
-
-    data = response.content.decode('utf-8')
-
-    return parse_playlists(data)
-
-
-def get_playlist_urls(url):
-    response = requests.get(url)
-    response.raise_for_status()
-
-    data = response.content.decode('utf-8')
-
-    return parse_playlist(url, data)
+    return response.content.decode('utf-8')