Bump version

Distinct VODs took from disk and freshly downloaded

CHANGELOG.md

@@ -1,6 +1,40 @@
 Twitch Downloader change log
 ============================
 
+1.17.0 (2021-12-03)
+-------------------
+
+* Fix speed calculation when resuming download (#75, thanks CroquetteTheThe)
+* Add artist and title metadata to resulting video (#80)
+
+1.16.1 (2021-07-31)
+-------------------
+
+* Fix compat with older versions of python (#71)
+
+1.16.0 (2021-06-09)
+-------------------
+
+* Fix clips download caused by Twitch changes (#64, thanks to all participants)
+
+
+1.15.0 (2021-02-15)
+-------------------
+
+* Add support for new format of clip slug (thanks @Loveangel1337)
+
+1.14.1 (2021-01-14)
+-------------------
+
+* Handle videos which don't exist more gracefully
+
+1.14.0 (2021-01-14)
+-------------------
+
+* Added `info` command for displaying video or clip info (#51)
+* Don't show there are more videos when there aren't (#52, thanks @scottyallen)
+* Fixed Twitch regression for getting the access token (#53)
+
 1.13.1 (2020-11-23)
 -------------------
 

README.md

@@ -79,6 +79,39 @@ pipx install twitch-dl
 Usage
 -----
 
+This section does an overview of available features.
+
+To see a list of available commands run:
+
+```
+twitch-dl --help
+```
+
+And to see description and all arguments for a given command run:
+
+```
+twitch-dl <command> --help
+```
+
+### Print clip or video info
+
+Videos can be referenced by URL or ID:
+
+```
+twitch-dl info 863849735
+twitch-dl info https://www.twitch.tv/videos/863849735
+```
+
+Clips by slug or ID:
+
+```
+twitch-dl info BusyBlushingCattleItsBoshyTime
+twitch-dl info https://www.twitch.tv/bananasaurus_rex/clip/BusyBlushingCattleItsBoshyTime
+```
+
+Shows info about the video or clip as well as download URLs for clips and
+playlist URLs for videos.
+
 ### Listing videos
 
 List recent streams for a given channel:
@@ -87,27 +120,6 @@ List recent streams for a given channel:
 twitch-dl videos bananasaurus_rex
 ```
 
-Yields (trimmed):
-
-```
-Found 33 videos
-
-221837124
-SUPER MARIO ODYSSSEY - Stream #2 / 600,000,000
-Bananasaurus_Rex playing Super Mario Odyssey
-Published 2018-01-24 @ 12:05:25  Length: 3h 40min
-
-221418913
-Dead Space and then SUPER MARIO ODYSSEY PogChamp
-Bananasaurus_Rex playing Dead Space
-Published 2018-01-23 @ 02:40:58  Length: 6h 2min
-
-220783179
-Dead Space | Got my new setup working! rexChamp
-Bananasaurus_Rex playing Dead Space
-Published 2018-01-21 @ 05:47:03  Length: 5h 7min
-```
-
 Use the `--game` option to specify one or more games to show:
 
 ```
@@ -116,7 +128,7 @@ twitch-dl videos --game "doom eternal" --game "cave story" bananasaurus_rex
 
 ### Downloading videos
 
-Download a stream by ID or URL:
+Download a video by ID or URL:
 
 ```
 twitch-dl download 221837124

setup.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-from setuptools import setup
+from setuptools import setup, find_packages
 
 long_description = """
 Quickly download videos from twitch.tv.
@@ -11,7 +11,7 @@ makes it faster.
 
 setup(
     name='twitch-dl',
-    version='1.13.1',
+    version='1.17.0',
     description='Twitch downloader',
     long_description=long_description.strip(),
     author='Ivan Habunek',
@@ -27,7 +27,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
     ],
-    packages=['twitchdl'],
+    packages=find_packages(),
     python_requires='>=3.5',
     install_requires=[
         "m3u8>=0.3.12,<0.4",

tests/test_api.py

@@ -0,0 +1,30 @@
+"""
+These tests depend on the channel having some videos and clips published.
+"""
+
+from twitchdl import twitch
+
+TEST_CHANNEL = "bananasaurus_rex"
+
+
+def test_get_videos():
+    videos = twitch.get_channel_videos(TEST_CHANNEL, 3, "time")
+    assert videos["pageInfo"]
+    assert len(videos["edges"]) > 0
+
+    video_id = videos["edges"][0]["node"]["id"]
+    video = twitch.get_video(video_id)
+    assert video["id"] == video_id
+
+
+def test_get_clips():
+    """
+    This test depends on the channel having some videos published.
+    """
+    clips = twitch.get_channel_clips(TEST_CHANNEL, "all_time", 3)
+    assert clips["pageInfo"]
+    assert len(clips["edges"]) > 0
+
+    clip_slug = clips["edges"][0]["node"]["slug"]
+    clip = twitch.get_clip(clip_slug)
+    assert clip["slug"] == clip_slug

tests/test_patterns.py

@@ -1,10 +1,6 @@
 import pytest
 
-from unittest.mock import patch
-from twitchdl.commands import download
-from collections import namedtuple
-
-Args = namedtuple("args", ["video"])
+from twitchdl.utils import parse_video_identifier, parse_clip_identifier
 
 
 TEST_VIDEO_PATTERNS = [
@@ -22,24 +18,18 @@ TEST_CLIP_PATTERNS = {
     ("HungryProudRadicchioDoggo", "https://clips.twitch.tv/HungryProudRadicchioDoggo"),
     ("HungryProudRadicchioDoggo", "https://www.twitch.tv/bananasaurus_rex/clip/HungryProudRadicchioDoggo?filter=clips&range=7d&sort=time"),
     ("HungryProudRadicchioDoggo", "https://twitch.tv/bananasaurus_rex/clip/HungryProudRadicchioDoggo?filter=clips&range=7d&sort=time"),
+    ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ"),
+    ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ"),
+    ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ?filter=clips&range=7d&sort=time"),
+    ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://www.twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ?filter=clips&range=7d&sort=time"),
 }
 
 
-@patch("twitchdl.commands._download_clip")
-@patch("twitchdl.commands._download_video")
 @pytest.mark.parametrize("expected,input", TEST_VIDEO_PATTERNS)
-def test_video_patterns(video_dl, clip_dl, expected, input):
-    args = Args(video=input)
-    download(args)
-    video_dl.assert_called_once_with(expected, args)
-    clip_dl.assert_not_called()
+def test_video_patterns(expected, input):
+    assert parse_video_identifier(input) == expected
 
 
-@patch("twitchdl.commands._download_clip")
-@patch("twitchdl.commands._download_video")
 @pytest.mark.parametrize("expected,input", TEST_CLIP_PATTERNS)
-def test_clip_patterns(video_dl, clip_dl, expected, input):
-    args = Args(video=input)
-    download(args)
-    clip_dl.assert_called_once_with(expected, args)
-    video_dl.assert_not_called()
+def test_clip_patterns(expected, input):
+    assert parse_clip_identifier(input) == expected

twitchdl/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "1.13.1"
+__version__ = "1.17.0"
 
 CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko"

twitchdl/commands/__init__.py

@@ -0,0 +1,11 @@
+from .clips import clips
+from .download import download
+from .info import info
+from .videos import videos
+
+__all__ = [
+    clips,
+    download,
+    info,
+    videos,
+]

twitchdl/commands/clips.py

@@ -0,0 +1,126 @@
+import re
+
+from os import path
+
+from twitchdl import twitch, utils
+from twitchdl.commands.download import get_clip_authenticated_url
+from twitchdl.download import download_file
+from twitchdl.exceptions import ConsoleError
+from twitchdl.output import print_out, print_clip, print_json
+
+
+def clips(args):
+    if args.json:
+        return _clips_json(args)
+
+    if args.download:
+        return _clips_download(args)
+
+    return _clips_list(args)
+
+
+def _continue():
+    print_out(
+        "\nThere are more clips. "
+        "Press <green><b>Enter</green> to continue, "
+        "<yellow><b>Ctrl+C</yellow> to break."
+    )
+
+    try:
+        input()
+    except KeyboardInterrupt:
+        return False
+
+    return True
+
+
+def _get_game_ids(names):
+    if not names:
+        return []
+
+    game_ids = []
+    for name in names:
+        print_out("<dim>Looking up game '{}'...</dim>".format(name))
+        game_id = twitch.get_game_id(name)
+        if not game_id:
+            raise ConsoleError("Game '{}' not found".format(name))
+        game_ids.append(int(game_id))
+
+    return game_ids
+
+
+def _clips_json(args):
+    clips = twitch.get_channel_clips(args.channel_name, args.period, args.limit)
+    nodes = list(edge["node"] for edge in clips["edges"])
+    print_json(nodes)
+
+
+def _clip_target_filename(clip):
+    url = clip["videoQualities"][0]["sourceURL"]
+    _, ext = path.splitext(url)
+    ext = ext.lstrip(".")
+
+    match = re.search(r"^(\d{4})-(\d{2})-(\d{2})T", clip["createdAt"])
+    date = "".join(match.groups())
+
+    name = "_".join([
+        date,
+        clip["id"],
+        clip["broadcaster"]["login"],
+        utils.slugify(clip["title"]),
+    ])
+
+    return "{}.{}".format(name, ext)
+
+
+def _clips_download(args):
+    downloaded_count = 0
+    generator = twitch.channel_clips_generator(args.channel_name, args.period, 100)
+
+    for clips, _ in generator:
+        for clip in clips["edges"]:
+            clip = clip["node"]
+            url = get_clip_authenticated_url(clip["slug"], "source")
+            target = _clip_target_filename(clip)
+
+            if path.exists(target):
+                print_out("Already downloaded: <green>{}</green>".format(target))
+            else:
+                print_out("Downloading: <yellow>{}</yellow>".format(target))
+                download_file(url, target)
+
+            downloaded_count += 1
+            if args.limit and downloaded_count >= args.limit:
+                return
+
+
+def _clips_list(args):
+    print_out("<dim>Loading clips...</dim>")
+    generator = twitch.channel_clips_generator(args.channel_name, args.period, args.limit)
+
+    first = 1
+
+    for clips, has_more in generator:
+        count = len(clips["edges"]) if "edges" in clips else 0
+        last = first + count - 1
+
+        print_out("-" * 80)
+        print_out("<yellow>Showing clips {}-{} of ??</yellow>".format(first, last))
+
+        for clip in clips["edges"]:
+            print_out()
+            print_clip(clip["node"])
+
+        if not args.pager:
+            print_out(
+                "\n<dim>There are more clips. "
+                "Increase the --limit or use --pager to see the rest.</dim>"
+            )
+            break
+
+        if not has_more or not _continue():
+            break
+
+        first += count
+    else:
+        print_out("<yellow>No clips found</yellow>")

twitchdl/commands/download.py

@@ -7,134 +7,12 @@ import tempfile
 
 from os import path
 from pathlib import Path
-from urllib.parse import urlparse
+from urllib.parse import urlparse, urlencode
 
 from twitchdl import twitch, utils
 from twitchdl.download import download_file, download_files
 from twitchdl.exceptions import ConsoleError
-from twitchdl.output import print_out, print_clip, print_video, print_json
-
-
-def _continue():
-    print_out(
-        "\nThere are more videos. "
-        "Press <green><b>Enter</green> to continue, "
-        "<yellow><b>Ctrl+C</yellow> to break."
-    )
-
-    try:
-        input()
-    except KeyboardInterrupt:
-        return False
-
-    return True
-
-
-def _get_game_ids(names):
-    if not names:
-        return []
-
-    game_ids = []
-    for name in names:
-        print_out("<dim>Looking up game '{}'...</dim>".format(name))
-        game_id = twitch.get_game_id(name)
-        if not game_id:
-            raise ConsoleError("Game '{}' not found".format(name))
-        game_ids.append(int(game_id))
-
-    return game_ids
-
-
-def _clips_json(args):
-    clips = twitch.get_channel_clips(args.channel_name, args.period, args.limit)
-    nodes = list(edge["node"] for edge in clips["edges"])
-    print_json(nodes)
-
-
-def _clips_download(args):
-    generator = twitch.channel_clips_generator(args.channel_name, args.period, 100)
-    for clips, _ in generator:
-        for clip in clips["edges"]:
-            clip = clip["node"]
-            url = clip["videoQualities"][0]["sourceURL"]
-            target = _clip_target_filename(clip)
-            if path.exists(target):
-                print_out("Already downloaded: <green>{}</green>".format(target))
-            else:
-                print_out("Downloading: <yellow>{}</yellow>".format(target))
-                download_file(url, target)
-
-
-def clips(args):
-    if args.json:
-        return _clips_json(args)
-
-    if args.download:
-        return _clips_download(args)
-
-    print_out("<dim>Loading clips...</dim>")
-    generator = twitch.channel_clips_generator(args.channel_name, args.period, args.limit)
-
-    first = 1
-
-    for clips, has_more in generator:
-        count = len(clips["edges"]) if "edges" in clips else 0
-        last = first + count - 1
-
-        print_out("-" * 80)
-        print_out("<yellow>Showing clips {}-{} of ??</yellow>".format(first, last))
-
-        for clip in clips["edges"]:
-            print_clip(clip["node"])
-
-        if not args.pager:
-            print_out(
-                "\n<dim>There are more clips. "
-                "Increase the --limit or use --pager to see the rest.</dim>"
-            )
-            break
-
-        if not has_more or not _continue():
-            break
-
-        first += count
-    else:
-        print_out("<yellow>No clips found</yellow>")
-
-
-def videos(args):
-    game_ids = _get_game_ids(args.game)
-
-    print_out("<dim>Loading videos...</dim>")
-    generator = twitch.channel_videos_generator(
-        args.channel_name, args.limit, args.sort, args.type, game_ids=game_ids)
-
-    first = 1
-
-    for videos, has_more in generator:
-        count = len(videos["edges"]) if "edges" in videos else 0
-        total = videos["totalCount"]
-        last = first + count - 1
-
-        print_out("-" * 80)
-        print_out("<yellow>Showing videos {}-{} of {}</yellow>".format(first, last, total))
-
-        for video in videos["edges"]:
-            print_video(video["node"])
-
-        if not args.pager:
-            print_out(
-                "\n<dim>There are more videos. "
-                "Increase the --limit or use --pager to see the rest.</dim>"
-            )
-            break
-
-        if not has_more or not _continue():
-            break
-
-        first += count
-    else:
-        print_out("<yellow>No videos found</yellow>")
+from twitchdl.output import print_out
 
 
 def _parse_playlists(playlists_m3u8):
@@ -170,14 +48,17 @@ def _select_playlist_interactive(playlists):
     return uri
 
 
-def _join_vods(playlist_path, target, overwrite):
+def _join_vods(playlist_path, target, overwrite, video):
     command = [
         "ffmpeg",
         "-i", playlist_path,
         "-c", "copy",
-        target,
+        "-metadata", "artist={}".format(video["creator"]["displayName"]),
+        "-metadata", "title={}".format(video["title"]),
+        "-metadata", "encoded_by=twitch-dl",
         "-stats",
         "-loglevel", "warning",
+        target,
     ]
 
     if overwrite:
@@ -190,13 +71,13 @@ def _join_vods(playlist_path, target, overwrite):
 
 
 def _video_target_filename(video, format):
-    match = re.search(r"^(\d{4})-(\d{2})-(\d{2})T", video['published_at'])
+    match = re.search(r"^(\d{4})-(\d{2})-(\d{2})T", video['publishedAt'])
     date = "".join(match.groups())
 
     name = "_".join([
         date,
-        video['_id'][1:],
-        video['channel']['name'],
+        video['id'],
+        video['creator']['login'],
         utils.slugify(video['title']),
     ])
 
@@ -214,7 +95,7 @@ def _clip_target_filename(clip):
     name = "_".join([
         date,
         clip["id"],
-        clip["broadcaster"]["channel"]["name"],
+        clip["broadcaster"]["login"],
         utils.slugify(clip["title"]),
     ])
 
@@ -246,52 +127,36 @@ def _crete_temp_dir(base_uri):
     path = urlparse(base_uri).path.lstrip("/")
     temp_dir = Path(tempfile.gettempdir(), "twitch-dl", path)
     temp_dir.mkdir(parents=True, exist_ok=True)
-    return temp_dir
-
-
-VIDEO_PATTERNS = [
-    r"^(?P<id>\d+)?$",
-    r"^https://(www.)?twitch.tv/videos/(?P<id>\d+)(\?.+)?$",
-]
-
-CLIP_PATTERNS = [
-    r"^(?P<slug>[A-Za-z0-9]+)$",
-    r"^https://(www.)?twitch.tv/\w+/clip/(?P<slug>[A-Za-z0-9]+)(\?.+)?$",
-    r"^https://clips.twitch.tv/(?P<slug>[A-Za-z0-9]+)(\?.+)?$",
-]
+    return str(temp_dir)
 
 
 def download(args):
-    for pattern in VIDEO_PATTERNS:
-        match = re.match(pattern, args.video)
-        if match:
-            video_id = match.group('id')
-            return _download_video(video_id, args)
+    video_id = utils.parse_video_identifier(args.video)
+    if video_id:
+        return _download_video(video_id, args)
 
-    for pattern in CLIP_PATTERNS:
-        match = re.match(pattern, args.video)
-        if match:
-            clip_slug = match.group('slug')
-            return _download_clip(clip_slug, args)
+    clip_slug = utils.parse_clip_identifier(args.video)
+    if clip_slug:
+        return _download_clip(clip_slug, args)
 
-    raise ConsoleError("Invalid video: {}".format(args.video))
+    raise ConsoleError("Invalid input: {}".format(args.video))
 
 
-def _get_clip_url(clip, args):
+def _get_clip_url(clip, quality):
     qualities = clip["videoQualities"]
 
     # Quality given as an argument
-    if args.quality:
-        if args.quality == "source":
+    if quality:
+        if quality == "source":
             return qualities[0]["sourceURL"]
 
-        selected_quality = args.quality.rstrip("p")  # allow 720p as well as 720
+        selected_quality = quality.rstrip("p")  # allow 720p as well as 720
         for q in qualities:
             if q["quality"] == selected_quality:
                 return q["sourceURL"]
 
         available = ", ".join([str(q["quality"]) for q in qualities])
-        msg = "Quality '{}' not found. Available qualities are: {}".format(args.quality, available)
+        msg = "Quality '{}' not found. Available qualities are: {}".format(quality, available)
         raise ConsoleError(msg)
 
     # Ask user to select quality
@@ -305,6 +170,23 @@ def _get_clip_url(clip, args):
     return selected_quality["sourceURL"]
 
 
+def get_clip_authenticated_url(slug, quality):
+    print_out("<dim>Fetching access token...</dim>")
+    access_token = twitch.get_clip_access_token(slug)
+
+    if not access_token:
+        raise ConsoleError("Access token not found for slug '{}'".format(slug))
+
+    url = _get_clip_url(access_token, quality)
+
+    query = urlencode({
+        "sig": access_token["playbackAccessToken"]["signature"],
+        "token": access_token["playbackAccessToken"]["value"],
+    })
+
+    return "{}?{}".format(url, query)
+
+
 def _download_clip(slug, args):
     print_out("<dim>Looking up clip...</dim>")
     clip = twitch.get_clip(slug)
@@ -312,6 +194,12 @@ def _download_clip(slug, args):
     if not clip:
         raise ConsoleError("Clip '{}' not found".format(slug))
 
+    print_out("<dim>Fetching access token...</dim>")
+    access_token = twitch.get_clip_access_token(slug)
+
+    if not access_token:
+        raise ConsoleError("Access token not found for slug '{}'".format(slug))
+
     print_out("Found: <green>{}</green> by <yellow>{}</yellow>, playing <blue>{}</blue> ({})".format(
         clip["title"],
         clip["broadcaster"]["displayName"],
@@ -319,7 +207,7 @@ def _download_clip(slug, args):
         utils.format_duration(clip["durationSeconds"])
     ))
 
-    url = _get_clip_url(clip, args)
+    url = get_clip_authenticated_url(slug, args.quality)
     print_out("<dim>Selected URL: {}</dim>".format(url))
 
     target = _clip_target_filename(clip)
@@ -337,8 +225,11 @@ def _download_video(video_id, args):
     print_out("<dim>Looking up video...</dim>")
     video = twitch.get_video(video_id)
 
+    if not video:
+        raise ConsoleError("Video {} not found".format(video_id))
+
     print_out("Found: <blue>{}</blue> by <yellow>{}</yellow>".format(
-        video['title'], video['channel']['display_name']))
+        video['title'], video['creator']['displayName']))
 
     print_out("<dim>Fetching access token...</dim>")
     access_token = twitch.get_access_token(video_id)
@@ -387,7 +278,7 @@ def _download_video(video_id, args):
 
     print_out("\n\nJoining files...")
     target = _video_target_filename(video, args.format)
-    _join_vods(playlist_path, target, args.overwrite)
+    _join_vods(playlist_path, target, args.overwrite, video)
 
     if args.keep:
         print_out("\n<dim>Temporary files not deleted: {}</dim>".format(target_dir))

twitchdl/commands/info.py

@@ -0,0 +1,79 @@
+import m3u8
+
+from twitchdl import utils, twitch
+from twitchdl.exceptions import ConsoleError
+from twitchdl.output import print_video, print_clip, print_json, print_out, print_log
+
+
+def info(args):
+    video_id = utils.parse_video_identifier(args.identifier)
+    if video_id:
+        print_log("Fetching video...")
+        video = twitch.get_video(video_id)
+
+        if not video:
+            raise ConsoleError("Video {} not found".format(video_id))
+
+        print_log("Fetching access token...")
+        access_token = twitch.get_access_token(video_id)
+
+        print_log("Fetching playlists...")
+        playlists = twitch.get_playlists(video_id, access_token)
+
+        if video:
+            if args.json:
+                video_json(video, playlists)
+            else:
+                video_info(video, playlists)
+            return
+
+    clip_slug = utils.parse_clip_identifier(args.identifier)
+    if clip_slug:
+        print_log("Fetching clip...")
+        clip = twitch.get_clip(clip_slug)
+        if not clip:
+            raise ConsoleError("Clip {} not found".format(clip_slug))
+
+        if args.json:
+            print_json(clip)
+        else:
+            clip_info(clip)
+        return
+
+    raise ConsoleError("Invalid input: {}".format(args.identifier))
+
+
+def video_info(video, playlists):
+    print_out()
+    print_video(video)
+
+    print_out()
+    print_out("Playlists:")
+    for p in m3u8.loads(playlists).playlists:
+        print_out("<b>{}</b> {}".format(p.stream_info.video, p.uri))
+
+
+def video_json(video, playlists):
+    playlists = m3u8.loads(playlists).playlists
+
+    video["playlists"] = [
+        {
+            "bandwidth": p.stream_info.bandwidth,
+            "resolution": p.stream_info.resolution,
+            "codecs": p.stream_info.codecs,
+            "video": p.stream_info.video,
+            "uri": p.uri
+        } for p in playlists
+    ]
+
+    print_json(video)
+
+
+def clip_info(clip):
+    print_out()
+    print_clip(clip)
+    print_out()
+    print_out("Download links:")
+
+    for q in clip["videoQualities"]:
+        print_out("<b>{quality}p{frameRate}</b> {sourceURL}".format(**q))

twitchdl/commands/videos.py

@@ -0,0 +1,69 @@
+from twitchdl import twitch
+from twitchdl.exceptions import ConsoleError
+from twitchdl.output import print_out, print_video
+
+
+def _continue():
+    print_out(
+        "\nThere are more videos. "
+        "Press <green><b>Enter</green> to continue, "
+        "<yellow><b>Ctrl+C</yellow> to break."
+    )
+
+    try:
+        input()
+    except KeyboardInterrupt:
+        return False
+
+    return True
+
+
+def _get_game_ids(names):
+    if not names:
+        return []
+
+    game_ids = []
+    for name in names:
+        print_out("<dim>Looking up game '{}'...</dim>".format(name))
+        game_id = twitch.get_game_id(name)
+        if not game_id:
+            raise ConsoleError("Game '{}' not found".format(name))
+        game_ids.append(int(game_id))
+
+    return game_ids
+
+
+def videos(args):
+    game_ids = _get_game_ids(args.game)
+
+    print_out("<dim>Loading videos...</dim>")
+    generator = twitch.channel_videos_generator(
+        args.channel_name, args.limit, args.sort, args.type, game_ids=game_ids)
+
+    first = 1
+
+    for videos, has_more in generator:
+        count = len(videos["edges"]) if "edges" in videos else 0
+        total = videos["totalCount"]
+        last = first + count - 1
+
+        print_out("-" * 80)
+        print_out("<yellow>Showing videos {}-{} of {}</yellow>".format(first, last, total))
+
+        for video in videos["edges"]:
+            print_out()
+            print_video(video["node"])
+
+        if not args.pager and has_more:
+            print_out(
+                "\n<dim>There are more videos. "
+                "Increase the --limit or use --pager to see the rest.</dim>"
+            )
+            break
+
+        if not has_more or not _continue():
+            break
+
+        first += count
+    else:
+        print_out("<yellow>No videos found</yellow>")

twitchdl/console.py

@@ -171,6 +171,21 @@ COMMANDS = [
             })
         ],
     ),
+    Command(
+        name="info",
+        description="Print information for a given Twitch URL, video ID or clip slug",
+        arguments=[
+            (["identifier"], {
+                "help": "identifier",
+                "type": str,
+            }),
+            (["-j", "--json"], {
+                "help": "Show results as JSON",
+                "action": "store_true",
+                "default": False,
+            }),
+        ],
+    )
 ]
 
 COMMON_ARGUMENTS = [
@@ -224,6 +239,9 @@ def main():
     except ConsoleError as e:
         print_err(e)
         sys.exit(1)
+    except KeyboardInterrupt:
+        print_err("Operation canceled")
+        sys.exit(1)
     except GQLError as e:
         print_err(e)
         for err in e.errors:

twitchdl/download.py

@@ -34,11 +34,13 @@ def _download(url, path):
 
 def download_file(url, path, retries=RETRY_COUNT):
     if os.path.exists(path):
-        return os.path.getsize(path)
+        from_disk = True
+        return (os.path.getsize(path), from_disk)
 
+    from_disk = False
     for _ in range(retries):
         try:
-            return _download(url, path)
+            return (_download(url, path), from_disk)
         except RequestException:
             pass
 
@@ -51,17 +53,26 @@ def _print_progress(futures):
     max_msg_size = 0
     start_time = datetime.now()
     total_count = len(futures)
+    current_download_size = 0
+    current_downloaded_count = 0
 
     for future in as_completed(futures):
-        size = future.result()
+        size, from_disk = future.result()
         downloaded_count += 1
         downloaded_size += size
 
+        # If we find something on disk, we don't want to take it in account in
+        # the speed calculation
+        if not from_disk:
+            current_download_size += size
+            current_downloaded_count += 1
+
         percentage = 100 * downloaded_count // total_count
         est_total_size = int(total_count * downloaded_size / downloaded_count)
         duration = (datetime.now() - start_time).seconds
-        speed = downloaded_size // duration if duration else 0
-        remaining = (total_count - downloaded_count) * duration / downloaded_count
+        speed = current_download_size // duration if duration else 0
+        remaining = (total_count - downloaded_count) * duration / current_downloaded_count \
+            if current_downloaded_count else 0
 
         msg = " ".join([
             "Downloaded VOD {}/{}".format(downloaded_count, total_count),
@@ -69,7 +80,7 @@ def _print_progress(futures):
             "<cyan>{}</cyan>".format(format_size(downloaded_size)),
             "of <cyan>~{}</cyan>".format(format_size(est_total_size)),
             "at <cyan>{}/s</cyan>".format(format_size(speed)) if speed > 0 else "",
-            "remaining <cyan>~{}</cyan>".format(format_duration(remaining)) if speed > 0 else "",
+            "remaining <cyan>~{}</cyan>".format(format_duration(remaining)) if remaining > 0 else "",
         ])
 
         max_msg_size = max(len(msg), max_msg_size)

twitchdl/output.py

@@ -62,10 +62,16 @@ def print_err(*args, **kwargs):
     print(*args, file=sys.stderr, **kwargs)
 
 
+def print_log(*args, **kwargs):
+    args = ["<dim>{}</dim>".format(a) for a in args]
+    args = [colorize(a) if USE_ANSI_COLOR else strip_tags(a) for a in args]
+    print(*args, file=sys.stderr, **kwargs)
+
+
 def print_video(video):
     published_at = video["publishedAt"].replace("T", " @ ").replace("Z", "")
     length = utils.format_duration(video["lengthSeconds"])
-    channel = video["creator"]["channel"]["displayName"]
+    channel = video["creator"]["displayName"]
     playing = (
         "playing <blue>{}</blue>".format(video["game"]["name"])
         if video["game"] else ""
@@ -74,7 +80,7 @@ def print_video(video):
     # Can't find URL in video object, strange
     url = "https://www.twitch.tv/videos/{}".format(video["id"])
 
-    print_out("\n<b>{}</b>".format(video["id"]))
+    print_out("<b>Video {}</b>".format(video["id"]))
     print_out("<green>{}</green>".format(video["title"]))
     print_out("<blue>{}</blue> {}".format(channel, playing))
     print_out("Published <blue>{}</blue>  Length: <blue>{}</blue> ".format(published_at, length))
@@ -84,13 +90,13 @@ def print_video(video):
 def print_clip(clip):
     published_at = clip["createdAt"].replace("T", " @ ").replace("Z", "")
     length = utils.format_duration(clip["durationSeconds"])
-    channel = clip["broadcaster"]["channel"]["displayName"]
+    channel = clip["broadcaster"]["displayName"]
     playing = (
         "playing <blue>{}</blue>".format(clip["game"]["name"])
         if clip["game"] else ""
     )
 
-    print_out("\n<b>{}</b>".format(clip["slug"]))
+    print_out("Clip <b>{}</b>".format(clip["slug"]))
     print_out("<green>{}</green>".format(clip["title"]))
     print_out("<blue>{}</blue> {}".format(channel, playing))
     print_out(
@@ -98,3 +104,8 @@ def print_clip(clip):
         "  Length: <blue>{}</blue>"
         "  Views: <blue>{}</blue>".format(published_at, length, clip["viewCount"]))
     print_out("<i>{}</i>".format(clip["url"]))
+
+
+def print_clip_urls(clip):
+    from pprint import pprint
+    pprint(clip)

twitchdl/twitch.py

@@ -42,13 +42,14 @@ def authenticated_post(url, data=None, json=None, headers={}):
     return response
 
 
-def kraken_get(url, params={}, headers={}):
-    """
-    Add accept header required by kraken API v5.
-    see: https://discuss.dev.twitch.tv/t/change-in-access-to-deprecated-kraken-twitch-apis/22241
-    """
-    headers["Accept"] = "application/vnd.twitchtv.v5+json"
-    return authenticated_get(url, params, headers)
+def gql_post(query):
+    url = "https://gql.twitch.tv/gql"
+    response = authenticated_post(url, data=query).json()
+
+    if "errors" in response:
+        raise GQLError(response["errors"])
+
+    return response
 
 
 def gql_query(query):
@@ -61,43 +62,91 @@ def gql_query(query):
     return response
 
 
-def get_video(video_id):
-    """
-    https://dev.twitch.tv/docs/v5/reference/videos#get-video
-    """
-    url = "https://api.twitch.tv/kraken/videos/{}".format(video_id)
+VIDEO_FIELDS = """
+    id
+    title
+    publishedAt
+    broadcastType
+    lengthSeconds
+    game {
+        name
+    }
+    creator {
+        login
+        displayName
+    }
+"""
 
-    return kraken_get(url).json()
+
+CLIP_FIELDS = """
+    id
+    slug
+    title
+    createdAt
+    viewCount
+    durationSeconds
+    url
+    videoQualities {
+        frameRate
+        quality
+        sourceURL
+    }
+    game {
+        id
+        name
+    }
+    broadcaster {
+        displayName
+        login
+    }
+"""
+
+
+def get_video(video_id):
+    query = """
+    {{
+        video(id: "{video_id}") {{
+            {fields}
+        }}
+    }}
+    """
+
+    query = query.format(video_id=video_id, fields=VIDEO_FIELDS)
+
+    response = gql_query(query)
+    return response["data"]["video"]
 
 
 def get_clip(slug):
     query = """
     {{
         clip(slug: "{}") {{
-            id
-            title
-            createdAt
-            durationSeconds
-            game {{
-                name
-            }}
-            broadcaster {{
-                login
-                displayName
-                channel {{
-                  name
-                }}
-            }}
-            videoQualities {{
-                frameRate
-                quality
-                sourceURL
+            {fields}
+        }}
+    }}
+    """
+
+    response = gql_query(query.format(slug, fields=CLIP_FIELDS))
+    return response["data"]["clip"]
+
+
+def get_clip_access_token(slug):
+    query = """
+    {{
+        "operationName": "VideoAccessToken_Clip",
+        "variables": {{
+            "slug": "{slug}"
+        }},
+        "extensions": {{
+            "persistedQuery": {{
+                "version": 1,
+                "sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11"
             }}
         }}
     }}
     """
 
-    response = gql_query(query.format(slug))
+    response = gql_post(query.format(slug=slug).strip())
     return response["data"]["clip"]
 
 
@@ -122,28 +171,7 @@ def get_channel_clips(channel_id, period, limit, after=None):
           edges {{
             cursor
             node {{
-              id
-              slug
-              title
-              createdAt
-              viewCount
-              durationSeconds
-              url
-              videoQualities {{
-                frameRate
-                quality
-                sourceURL
-              }}
-              game {{
-                id
-                name
-              }}
-              broadcaster {{
-                channel {{
-                  name
-                  displayName
-                }}
-              }}
+              {fields}
             }}
           }}
         }}
@@ -151,12 +179,13 @@ def get_channel_clips(channel_id, period, limit, after=None):
     }}
     """
 
-    query = query.format(**{
-        "channel_id": channel_id,
-        "after": after if after else "",
-        "limit": limit,
-        "period": period.upper(),
-    })
+    query = query.format(
+        channel_id=channel_id,
+        after=after if after else "",
+        limit=limit,
+        period=period.upper(),
+        fields=CLIP_FIELDS
+    )
 
     response = gql_query(query)
     user = response["data"]["user"]
@@ -204,19 +233,7 @@ def get_channel_videos(channel_id, limit, sort, type="archive", game_ids=[], aft
                 edges {{
                     cursor
                     node {{
-                        id
-                        title
-                        publishedAt
-                        broadcastType
-                        lengthSeconds
-                        game {{
-                            name
-                        }}
-                        creator {{
-                            channel {{
-                                displayName
-                            }}
-                        }}
+                        {fields}
                     }}
                 }}
             }}
@@ -224,16 +241,21 @@ def get_channel_videos(channel_id, limit, sort, type="archive", game_ids=[], aft
     }}
     """
 
-    query = query.format(**{
-        "channel_id": channel_id,
-        "game_ids": game_ids,
-        "after": after if after else "",
-        "limit": limit,
-        "sort": sort.upper(),
-        "type": type.upper(),
-    })
+    query = query.format(
+        channel_id=channel_id,
+        game_ids=game_ids,
+        after=after if after else "",
+        limit=limit,
+        sort=sort.upper(),
+        type=type.upper(),
+        fields=VIDEO_FIELDS
+    )
 
     response = gql_query(query)
+
+    if not response["data"]["user"]:
+        raise ConsoleError("Channel {} not found".format(channel_id))
+
     return response["data"]["user"]["videos"]
 
 
@@ -256,9 +278,26 @@ def channel_videos_generator(channel_id, limit, sort, type, game_ids=None):
 
 
 def get_access_token(video_id):
-    url = "https://api.twitch.tv/api/vods/{}/access_token".format(video_id)
+    query = """
+    {{
+        videoPlaybackAccessToken(
+            id: {video_id},
+            params: {{
+                platform: "web",
+                playerBackend: "mediaplayer",
+                playerType: "site"
+            }}
+        ) {{
+            signature
+            value
+        }}
+    }}
+    """
 
-    return authenticated_get(url).json()
+    query = query.format(video_id=video_id)
+
+    response = gql_query(query)
+    return response["data"]["videoPlaybackAccessToken"]
 
 
 def get_playlists(video_id, access_token):
@@ -268,8 +307,8 @@ def get_playlists(video_id, access_token):
     url = "http://usher.twitch.tv/vod/{}".format(video_id)
 
     response = requests.get(url, params={
-        "nauth": access_token['token'],
-        "nauthsig": access_token['sig'],
+        "nauth": access_token['value'],
+        "nauthsig": access_token['signature'],
         "allow_source": "true",
         "player": "twitchweb",
     })

twitchdl/utils.py

@@ -61,3 +61,31 @@ def slugify(value):
     value = unicodedata.normalize('NFKC', value)
     value = re_pattern.sub('', value).strip().lower()
     return re_spaces.sub('_', value)
+
+
+VIDEO_PATTERNS = [
+    r"^(?P<id>\d+)?$",
+    r"^https://(www.)?twitch.tv/videos/(?P<id>\d+)(\?.+)?$",
+]
+
+CLIP_PATTERNS = [
+    r"^(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)$",
+    r"^https://(www.)?twitch.tv/\w+/clip/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
+    r"^https://clips.twitch.tv/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
+]
+
+
+def parse_video_identifier(identifier):
+    """Given a video ID or URL returns the video ID, or null if not matched"""
+    for pattern in VIDEO_PATTERNS:
+        match = re.match(pattern, identifier)
+        if match:
+            return match.group("id")
+
+
+def parse_clip_identifier(identifier):
+    """Given a clip slug or URL returns the clip slug, or null if not matched"""
+    for pattern in CLIP_PATTERNS:
+        match = re.match(pattern, identifier)
+        if match:
+            return match.group("slug")