Bump version

CHANGELOG.md

@@ -1,6 +1,11 @@
 Twitch Downloader change log
 ============================
 
+1.13.0 (2020-11-10)
+-------------------
+
+* Added `clips` command for listing and batch downloading clips (#26)
+
 1.12.1 (2020-09-29)
 -------------------
 

README.md

@@ -32,14 +32,14 @@ Run the archive by either:
 a) passing it to python:
 
 ```
-python3 twitch-dl.1.10.2.pyz --help
+python3 twitch-dl.1.13.0.pyz --help
 ```
 
 b) making it executable and invoking it directly (linux specific):
 
 ```
-chmod +x twitch-dl.1.10.2.pyz
-./twitch-dl.1.10.2.pyz --help
+chmod +x twitch-dl.1.13.0.pyz
+./twitch-dl.1.13.0.pyz --help
 ```
 
 Feel free to rename the archive to something more managable, like `twitch-dl`.
@@ -79,6 +79,8 @@ pipx install twitch-dl
 Usage
 -----
 
+### Listing videos
+
 List recent streams for a given channel:
 
 ```
@@ -112,6 +114,8 @@ Use the `--game` option to specify one or more games to show:
 twitch-dl videos --game "doom eternal" --game "cave story" bananasaurus_rex
 ```
 
+### Downloading videos
+
 Download a stream by ID or URL:
 
 ```
@@ -131,6 +135,26 @@ Setting quality to `source` will download the best available quality:
 twitch-dl download -q source 221837124
 ```
 
+### Listing clips
+
+List clips for the given period:
+
+```
+twitch-dl clips bananasaurus_rex --period last_week
+```
+
+Supported periods are: `last_day`, `last_week`, `last_month`, `all_time`.
+
+For listing a large number of clips, it's nice to page them:
+
+```
+twitch-dl clips bananasaurus_rex --period all_time --limit 10 --pager
+```
+
+This will show 10 clips at a time and ask to continue.
+
+### Downloading clips
+
 Download a clip by slug or URL:
 
 ```
@@ -146,6 +170,27 @@ twitch-dl download -q 720 VenomousTameWormHumbleLife
 
 Note that twitch names for clip qualities have no trailing "p".
 
+### Batch downloading clips
+
+It's possible to download all clips for a given period:
+
+```
+twitch-dl clips bananasaurus_rex --period last_week --download
+```
+
+Clips are downloaded in source quality.
+
+A note about clips
+------------------
+
+Currently it doesn't seem to be possible to get a list of clips ordered by time
+of creation, only by view count. Clips with the same view count seem to be
+returned in random order. This can break paging resulting in duplicate clips
+listed or clips missed.
+
+When batch downloading a large number of clips (over 100), it's possible that
+some will be missed.
+
 Temporary files
 ---------------
 

setup.py

@@ -11,7 +11,7 @@ makes it faster.
 
 setup(
     name='twitch-dl',
-    version='1.12.1',
+    version='1.13.0',
     description='Twitch downloader',
     long_description=long_description.strip(),
     author='Ivan Habunek',

twitchdl/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "1.12.1"
+__version__ = "1.13.0"
 
 CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko"

twitchdl/commands.py

@@ -12,7 +12,7 @@ from urllib.parse import urlparse
 from twitchdl import twitch, utils
 from twitchdl.download import download_file, download_files
 from twitchdl.exceptions import ConsoleError
-from twitchdl.output import print_out, print_video
+from twitchdl.output import print_out, print_clip, print_video, print_json
 
 
 def _continue():
@@ -45,6 +45,63 @@ def _get_game_ids(names):
     return game_ids
 
 
+def _clips_json(args):
+    clips = twitch.get_channel_clips(args.channel_name, args.period, args.limit)
+    nodes = list(edge["node"] for edge in clips["edges"])
+    print_json(nodes)
+
+
+def _clips_download(args):
+    generator = twitch.channel_clips_generator(args.channel_name, args.period, 100)
+    for clips, _ in generator:
+        for clip in clips["edges"]:
+            clip = clip["node"]
+            url = clip["videoQualities"][0]["sourceURL"]
+            target = _clip_target_filename(clip)
+            if path.exists(target):
+                print_out("Already downloaded: <green>{}</green>".format(target))
+            else:
+                print_out("Downloading: <yellow>{}</yellow>".format(target))
+                download_file(url, target)
+
+
+def clips(args):
+    if args.json:
+        return _clips_json(args)
+
+    if args.download:
+        return _clips_download(args)
+
+    print_out("<dim>Loading clips...</dim>")
+    generator = twitch.channel_clips_generator(args.channel_name, args.period, args.limit)
+
+    first = 1
+
+    for clips, has_more in generator:
+        count = len(clips["edges"]) if "edges" in clips else 0
+        last = first + count - 1
+
+        print_out("-" * 80)
+        print_out("<yellow>Showing clips {}-{} of ??</yellow>".format(first, last))
+
+        for clip in clips["edges"]:
+            print_clip(clip["node"])
+
+        if not args.pager:
+            print_out(
+                "\n<dim>There are more clips. "
+                "Increase the --limit or use --pager to see the rest.</dim>"
+            )
+            break
+
+        if not has_more or not _continue():
+            break
+
+        first += count
+    else:
+        print_out("<yellow>No clips found</yellow>")
+
+
 def videos(args):
     game_ids = _get_game_ids(args.game)
 
@@ -146,6 +203,24 @@ def _video_target_filename(video, format):
     return name + "." + format
 
 
+def _clip_target_filename(clip):
+    url = clip["videoQualities"][0]["sourceURL"]
+    _, ext = path.splitext(url)
+    ext = ext.lstrip(".")
+
+    match = re.search(r"^(\d{4})-(\d{2})-(\d{2})T", clip["createdAt"])
+    date = "".join(match.groups())
+
+    name = "_".join([
+        date,
+        clip["id"],
+        clip["broadcaster"]["channel"]["name"],
+        utils.slugify(clip["title"]),
+    ])
+
+    return "{}.{}".format(name, ext)
+
+
 def _get_vod_paths(playlist, start, end):
     """Extract unique VOD paths for download from playlist."""
     files = []
@@ -247,18 +322,12 @@ def _download_clip(slug, args):
     url = _get_clip_url(clip, args)
     print_out("<dim>Selected URL: {}</dim>".format(url))
 
-    url_path = urlparse(url).path
-    extension = Path(url_path).suffix
-    filename = "{}_{}{}".format(
-        clip["broadcaster"]["login"],
-        utils.slugify(clip["title"]),
-        extension
-    )
+    target = _clip_target_filename(clip)
 
     print_out("Downloading clip...")
-    download_file(url, filename)
+    download_file(url, target)
 
-    print_out("Downloaded: {}".format(filename))
+    print_out("Downloaded: {}".format(target))
 
 
 def _download_video(video_id, args):

twitchdl/console.py

@@ -84,6 +84,42 @@ COMMANDS = [
             }),
         ],
     ),
+    Command(
+        name="clips",
+        description="List clips",
+        arguments=[
+            (["channel_name"], {
+                "help": "channel name",
+                "type": str,
+            }),
+            (["-l", "--limit"], {
+                "help": "Number of videos to fetch (default 10, max 100)",
+                "type": limit,
+                "default": 10,
+            }),
+            (["-P", "--period"], {
+                "help": "Period from which to return clips. (default: 'all_time')",
+                "type": str,
+                "choices": ["last_day", "last_week", "last_month", "all_time"],
+                "default": "all_time",
+            }),
+            (["-j", "--json"], {
+                "help": "Show results as JSON",
+                "action": "store_true",
+                "default": False,
+            }),
+            (["-p", "--pager"], {
+                "help": "If there are more results than LIMIT, ask to show next page",
+                "action": "store_true",
+                "default": False,
+            }),
+            (["-d", "--download"], {
+                "help": "Download all videos in given period (in source quality)",
+                "action": "store_true",
+                "default": False,
+            }),
+        ],
+    ),
     Command(
         name="download",
         description="Download a video",

twitchdl/output.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+import json
 import sys
 import re
 
@@ -51,6 +52,10 @@ def print_out(*args, **kwargs):
     print(*args, **kwargs)
 
 
+def print_json(data):
+    print(json.dumps(data))
+
+
 def print_err(*args, **kwargs):
     args = ["<red>{}</red>".format(a) for a in args]
     args = [colorize(a) if USE_ANSI_COLOR else strip_tags(a) for a in args]
@@ -62,7 +67,7 @@ def print_video(video):
     length = utils.format_duration(video["lengthSeconds"])
     channel = video["creator"]["channel"]["displayName"]
     playing = (
-        " playing <blue>{}</blue>".format(video["game"]["name"])
+        "playing <blue>{}</blue>".format(video["game"]["name"])
         if video["game"] else ""
     )
 
@@ -74,3 +79,22 @@ def print_video(video):
     print_out("<blue>{}</blue> {}".format(channel, playing))
     print_out("Published <blue>{}</blue>  Length: <blue>{}</blue> ".format(published_at, length))
     print_out("<i>{}</i>".format(url))
+
+
+def print_clip(clip):
+    published_at = clip["createdAt"].replace("T", " @ ").replace("Z", "")
+    length = utils.format_duration(clip["durationSeconds"])
+    channel = clip["broadcaster"]["channel"]["displayName"]
+    playing = (
+        "playing <blue>{}</blue>".format(clip["game"]["name"])
+        if clip["game"] else ""
+    )
+
+    print_out("\n<b>{}</b>".format(clip["slug"]))
+    print_out("<green>{}</green>".format(clip["title"]))
+    print_out("<blue>{}</blue> {}".format(channel, playing))
+    print_out(
+        "Published <blue>{}</blue>"
+        "  Length: <blue>{}</blue>"
+        "  Views: <blue>{}</blue>".format(published_at, length, clip["viewCount"]))
+    print_out("<i>{}</i>".format(clip["url"]))

twitchdl/twitch.py

@@ -96,6 +96,85 @@ def get_clip(slug):
     return response["data"]["clip"]
 
 
+def get_channel_clips(channel_id, period, limit, after=None):
+    """
+    List channel clips.
+
+    At the time of writing this:
+    * filtering by game name returns an error
+    * sorting by anything but VIEWS_DESC or TRENDING returns an error
+    * sorting by VIEWS_DESC and TRENDING returns the same results
+    * there is no totalCount
+    """
+    query = """
+    {{
+      user(login: "{channel_id}") {{
+        clips(first: {limit}, after: "{after}", criteria: {{ period: {period}, sort: VIEWS_DESC }}) {{
+          pageInfo {{
+            hasNextPage
+            hasPreviousPage
+          }}
+          edges {{
+            cursor
+            node {{
+              id
+              slug
+              title
+              createdAt
+              viewCount
+              durationSeconds
+              url
+              videoQualities {{
+                frameRate
+                quality
+                sourceURL
+              }}
+              game {{
+                id
+                name
+              }}
+              broadcaster {{
+                channel {{
+                  name
+                  displayName
+                }}
+              }}
+            }}
+          }}
+        }}
+      }}
+    }}
+    """
+
+    query = query.format(**{
+        "channel_id": channel_id,
+        "after": after if after else "",
+        "limit": limit,
+        "period": period.upper(),
+    })
+
+    response = gql_query(query)
+    return response["data"]["user"]["clips"]
+
+
+def channel_clips_generator(channel_id, period, limit):
+    cursor = ""
+    while True:
+        clips = get_channel_clips(
+            channel_id, period, limit, after=cursor)
+
+        if not clips["edges"]:
+            break
+
+        has_next = clips["pageInfo"]["hasNextPage"]
+        cursor = clips["edges"][-1]["cursor"] if has_next else None
+
+        yield clips, has_next
+
+        if not cursor:
+            break
+
+
 def get_channel_videos(channel_id, limit, sort, type="archive", game_ids=[], after=None):
     query = """
     {{
@@ -139,7 +218,7 @@ def get_channel_videos(channel_id, limit, sort, type="archive", game_ids=[], aft
     query = query.format(**{
         "channel_id": channel_id,
         "game_ids": game_ids,
-        "after": after,
+        "after": after if after else "",
         "limit": limit,
         "sort": sort.upper(),
         "type": type.upper(),
@@ -150,7 +229,7 @@ def get_channel_videos(channel_id, limit, sort, type="archive", game_ids=[], aft
 
 
 def channel_videos_generator(channel_id, limit, sort, type, game_ids=None):
-    cursor = None
+    cursor = ""
     while True:
         videos = get_channel_videos(
             channel_id, limit, sort, type, game_ids=game_ids, after=cursor)