Bump version, changelog

fixes #64

CHANGELOG.md

@@ -1,6 +1,17 @@
 Twitch Downloader change log
 ============================
 
+1.16.0 (2021-06-09)
+-------------------
+
+* Fix clips download caused by Twitch changes (#64, thanks to all participants)
+
+
+1.15.0 (2021-02-15)
+-------------------
+
+* Add support for new format of clip slug (thanks @Loveangel1337)
+
 1.14.1 (2021-01-14)
 -------------------
 

setup.py

@@ -11,7 +11,7 @@ makes it faster.
 
 setup(
     name='twitch-dl',
-    version='1.14.1',
+    version='1.16.0',
     description='Twitch downloader',
     long_description=long_description.strip(),
     author='Ivan Habunek',

tests/test_patterns.py

@@ -1,10 +1,6 @@
 import pytest
 
-from unittest.mock import patch
-from twitchdl.commands import download
-from collections import namedtuple
-
-Args = namedtuple("args", ["video"])
+from twitchdl.utils import parse_video_identifier, parse_clip_identifier
 
 
 TEST_VIDEO_PATTERNS = [
@@ -22,24 +18,18 @@ TEST_CLIP_PATTERNS = {
     ("HungryProudRadicchioDoggo", "https://clips.twitch.tv/HungryProudRadicchioDoggo"),
     ("HungryProudRadicchioDoggo", "https://www.twitch.tv/bananasaurus_rex/clip/HungryProudRadicchioDoggo?filter=clips&range=7d&sort=time"),
     ("HungryProudRadicchioDoggo", "https://twitch.tv/bananasaurus_rex/clip/HungryProudRadicchioDoggo?filter=clips&range=7d&sort=time"),
+    ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ"),
+    ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ"),
+    ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ?filter=clips&range=7d&sort=time"),
+    ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://www.twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ?filter=clips&range=7d&sort=time"),
 }
 
 
-@patch("twitchdl.commands._download_clip")
-@patch("twitchdl.commands._download_video")
 @pytest.mark.parametrize("expected,input", TEST_VIDEO_PATTERNS)
-def test_video_patterns(video_dl, clip_dl, expected, input):
-    args = Args(video=input)
-    download(args)
-    video_dl.assert_called_once_with(expected, args)
-    clip_dl.assert_not_called()
+def test_video_patterns(expected, input):
+    assert parse_video_identifier(input) == expected
 
 
-@patch("twitchdl.commands._download_clip")
-@patch("twitchdl.commands._download_video")
 @pytest.mark.parametrize("expected,input", TEST_CLIP_PATTERNS)
-def test_clip_patterns(video_dl, clip_dl, expected, input):
-    args = Args(video=input)
-    download(args)
-    clip_dl.assert_called_once_with(expected, args)
-    video_dl.assert_not_called()
+def test_clip_patterns(expected, input):
+    assert parse_clip_identifier(input) == expected

twitchdl/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "1.14.1"
+__version__ = "1.16.0"
 
 CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko"

twitchdl/commands/clips.py

@@ -3,11 +3,22 @@ import re
 from os import path
 
 from twitchdl import twitch, utils
+from twitchdl.commands.download import get_clip_authenticated_url
 from twitchdl.download import download_file
 from twitchdl.exceptions import ConsoleError
 from twitchdl.output import print_out, print_clip, print_json
 
 
+def clips(args):
+    if args.json:
+        return _clips_json(args)
+
+    if args.download:
+        return _clips_download(args)
+
+    return _clips_list(args)
+
+
 def _continue():
     print_out(
         "\nThere are more clips. "
@@ -63,26 +74,27 @@ def _clip_target_filename(clip):
 
 
 def _clips_download(args):
+    downloaded_count = 0
     generator = twitch.channel_clips_generator(args.channel_name, args.period, 100)
+
     for clips, _ in generator:
         for clip in clips["edges"]:
             clip = clip["node"]
-            url = clip["videoQualities"][0]["sourceURL"]
+            url = get_clip_authenticated_url(clip["slug"], "source")
             target = _clip_target_filename(clip)
+
             if path.exists(target):
                 print_out("Already downloaded: <green>{}</green>".format(target))
             else:
                 print_out("Downloading: <yellow>{}</yellow>".format(target))
                 download_file(url, target)
 
+            downloaded_count += 1
+            if args.limit and downloaded_count >= args.limit:
+                return
 
-def clips(args):
-    if args.json:
-        return _clips_json(args)
-
-    if args.download:
-        return _clips_download(args)
 
+def _clips_list(args):
     print_out("<dim>Loading clips...</dim>")
     generator = twitch.channel_clips_generator(args.channel_name, args.period, args.limit)
 

twitchdl/commands/download.py

@@ -7,7 +7,7 @@ import tempfile
 
 from os import path
 from pathlib import Path
-from urllib.parse import urlparse
+from urllib.parse import urlparse, urlencode
 
 from twitchdl import twitch, utils
 from twitchdl.download import download_file, download_files
@@ -73,7 +73,7 @@ def _video_target_filename(video, format):
 
     name = "_".join([
         date,
-        video['id'][1:],
+        video['id'],
         video['creator']['login'],
         utils.slugify(video['title']),
     ])
@@ -139,21 +139,21 @@ def download(args):
     raise ConsoleError("Invalid input: {}".format(args.video))
 
 
-def _get_clip_url(clip, args):
+def _get_clip_url(clip, quality):
     qualities = clip["videoQualities"]
 
     # Quality given as an argument
-    if args.quality:
-        if args.quality == "source":
+    if quality:
+        if quality == "source":
             return qualities[0]["sourceURL"]
 
-        selected_quality = args.quality.rstrip("p")  # allow 720p as well as 720
+        selected_quality = quality.rstrip("p")  # allow 720p as well as 720
         for q in qualities:
             if q["quality"] == selected_quality:
                 return q["sourceURL"]
 
         available = ", ".join([str(q["quality"]) for q in qualities])
-        msg = "Quality '{}' not found. Available qualities are: {}".format(args.quality, available)
+        msg = "Quality '{}' not found. Available qualities are: {}".format(quality, available)
         raise ConsoleError(msg)
 
     # Ask user to select quality
@@ -167,6 +167,23 @@ def _get_clip_url(clip, args):
     return selected_quality["sourceURL"]
 
 
+def get_clip_authenticated_url(slug, quality):
+    print_out("<dim>Fetching access token...</dim>")
+    access_token = twitch.get_clip_access_token(slug)
+
+    if not access_token:
+        raise ConsoleError("Access token not found for slug '{}'".format(slug))
+
+    url = _get_clip_url(access_token, quality)
+
+    query = urlencode({
+        "sig": access_token["playbackAccessToken"]["signature"],
+        "token": access_token["playbackAccessToken"]["value"],
+    })
+
+    return "{}?{}".format(url, query)
+
+
 def _download_clip(slug, args):
     print_out("<dim>Looking up clip...</dim>")
     clip = twitch.get_clip(slug)
@@ -174,6 +191,12 @@ def _download_clip(slug, args):
     if not clip:
         raise ConsoleError("Clip '{}' not found".format(slug))
 
+    print_out("<dim>Fetching access token...</dim>")
+    access_token = twitch.get_clip_access_token(slug)
+
+    if not access_token:
+        raise ConsoleError("Access token not found for slug '{}'".format(slug))
+
     print_out("Found: <green>{}</green> by <yellow>{}</yellow>, playing <blue>{}</blue> ({})".format(
         clip["title"],
         clip["broadcaster"]["displayName"],
@@ -181,7 +204,7 @@ def _download_clip(slug, args):
         utils.format_duration(clip["durationSeconds"])
     ))
 
-    url = _get_clip_url(clip, args)
+    url = get_clip_authenticated_url(slug, args.quality)
     print_out("<dim>Selected URL: {}</dim>".format(url))
 
     target = _clip_target_filename(clip)

twitchdl/commands/info.py

@@ -40,7 +40,7 @@ def info(args):
             clip_info(clip)
         return
 
-    raise ConsoleError("Invalid input: {}".format(args.video))
+    raise ConsoleError("Invalid input: {}".format(args.identifier))
 
 
 def video_info(video, playlists):

twitchdl/console.py

@@ -239,6 +239,9 @@ def main():
     except ConsoleError as e:
         print_err(e)
         sys.exit(1)
+    except KeyboardInterrupt:
+        print_err("Operation canceled")
+        sys.exit(1)
     except GQLError as e:
         print_err(e)
         for err in e.errors:

twitchdl/twitch.py

@@ -51,6 +51,16 @@ def kraken_get(url, params={}, headers={}):
     return authenticated_get(url, params, headers)
 
 
+def gql_post(query):
+    url = "https://gql.twitch.tv/gql"
+    response = authenticated_post(url, data=query).json()
+
+    if "errors" in response:
+        raise GQLError(response["errors"])
+
+    return response
+
+
 def gql_query(query):
     url = "https://gql.twitch.tv/gql"
     response = authenticated_post(url, json={"query": query}).json()
@@ -133,6 +143,26 @@ def get_clip(slug):
     return response["data"]["clip"]
 
 
+def get_clip_access_token(slug):
+    query = """
+    {{
+        "operationName": "VideoAccessToken_Clip",
+        "variables": {{
+            "slug": "{slug}"
+        }},
+        "extensions": {{
+            "persistedQuery": {{
+                "version": 1,
+                "sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11"
+            }}
+        }}
+    }}
+    """
+
+    response = gql_post(query.format(slug=slug).strip())
+    return response["data"]["clip"]
+
+
 def get_channel_clips(channel_id, period, limit, after=None):
     """
     List channel clips.
@@ -263,6 +293,10 @@ def get_channel_videos(channel_id, limit, sort, type="archive", game_ids=[], aft
     })
 
     response = gql_query(query)
+
+    if not response["data"]["user"]:
+        raise ConsoleError("Channel {} not found".format(channel_id))
+
     return response["data"]["user"]["videos"]
 
 

twitchdl/utils.py

@@ -69,9 +69,9 @@ VIDEO_PATTERNS = [
 ]
 
 CLIP_PATTERNS = [
-    r"^(?P<slug>[A-Za-z0-9]+)$",
-    r"^https://(www.)?twitch.tv/\w+/clip/(?P<slug>[A-Za-z0-9]+)(\?.+)?$",
-    r"^https://clips.twitch.tv/(?P<slug>[A-Za-z0-9]+)(\?.+)?$",
+    r"^(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)$",
+    r"^https://(www.)?twitch.tv/\w+/clip/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
+    r"^https://clips.twitch.tv/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
 ]