Make clips behave similar to recent videos changes

README.md

@@ -215,6 +215,9 @@ Expands to: `KatLink - Dark Souls III - Dark Souls 3 First playthrough.mkv`
 
 ### Listing clips
 
+Listing clips works similar to listing videos. Shows 10 clips by default. Use
+`--all` to list all in one go or `--pager` to show them in pages.
+
 List clips for the given period:
 
 ```
@@ -223,13 +226,15 @@ twitch-dl clips bananasaurus_rex --period last_week
 
 Supported periods are: `last_day`, `last_week`, `last_month`, `all_time`.
 
-For listing a large number of clips, it's nice to page them:
+Also supports JSON output:
 
 ```
-twitch-dl clips bananasaurus_rex --period all_time --limit 10 --pager
+twitch-dl clips bananasaurus_rex --json --all
 ```
 
-This will show 10 clips at a time and ask to continue.
+Note that this may make multiple requests to the server because each request is
+limited to 100 clips, so it may take a little while. You can use `--debug` to
+log requests.
 
 ### Downloading clips
 

twitchdl/commands/clips.py

@@ -1,5 +1,7 @@
 import re
+import sys
 
+from itertools import islice
 from os import path
 
 from twitchdl import twitch, utils
@@ -10,21 +12,26 @@ from twitchdl.output import print_out, print_clip, print_json
 
 
 def clips(args):
+    # Ignore --limit if --pager or --all are given
+    limit = sys.maxsize if args.all or args.pager else args.limit
+
+    generator = twitch.channel_clips_generator(args.channel_name, args.period, limit)
+
     if args.json:
-        return _clips_json(args)
+        return print_json(list(generator))
 
     if args.download:
-        return _clips_download(args)
+        return _download_clips(generator)
 
-    return _clips_list(args)
+    if args.pager:
+        print(args)
+        return _print_paged(generator, args.pager)
+
+    return _print_all(generator, args)
 
 
 def _continue():
-    print_out(
-        "\nThere are more clips. "
-        "Press <green><b>Enter</green> to continue, "
-        "<yellow><b>Ctrl+C</yellow> to break."
-    )
+    print_out("Press <green><b>Enter</green> to continue, <yellow><b>Ctrl+C</yellow> to break.")
 
     try:
         input()
@@ -34,28 +41,7 @@ def _continue():
     return True
 
 
-def _get_game_ids(names):
-    if not names:
-        return []
-
-    game_ids = []
-    for name in names:
-        print_out("<dim>Looking up game '{}'...</dim>".format(name))
-        game_id = twitch.get_game_id(name)
-        if not game_id:
-            raise ConsoleError("Game '{}' not found".format(name))
-        game_ids.append(int(game_id))
-
-    return game_ids
-
-
-def _clips_json(args):
-    clips = twitch.get_channel_clips(args.channel_name, args.period, args.limit)
-    nodes = list(edge["node"] for edge in clips["edges"])
-    print_json(nodes)
-
-
-def _clip_target_filename(clip):
+def _target_filename(clip):
     url = clip["videoQualities"][0]["sourceURL"]
     _, ext = path.splitext(url)
     ext = ext.lstrip(".")
@@ -73,54 +59,54 @@ def _clip_target_filename(clip):
     return "{}.{}".format(name, ext)
 
 
-def _clips_download(args):
-    downloaded_count = 0
-    generator = twitch.channel_clips_generator(args.channel_name, args.period, 100)
+def _download_clips(generator):
+    for clip in generator:
+        target = _target_filename(clip)
 
-    for clips, _ in generator:
-        for clip in clips["edges"]:
-            clip = clip["node"]
+        if path.exists(target):
+            print_out("Already downloaded: <green>{}</green>".format(target))
+        else:
             url = get_clip_authenticated_url(clip["slug"], "source")
-            target = _clip_target_filename(clip)
-
-            if path.exists(target):
-                print_out("Already downloaded: <green>{}</green>".format(target))
-            else:
-                print_out("Downloading: <yellow>{}</yellow>".format(target))
-                download_file(url, target)
-
-            downloaded_count += 1
-            if args.limit and downloaded_count >= args.limit:
-                return
+            print_out("Downloading: <yellow>{}</yellow>".format(target))
+            download_file(url, target)
 
 
-def _clips_list(args):
-    print_out("<dim>Loading clips...</dim>")
-    generator = twitch.channel_clips_generator(args.channel_name, args.period, args.limit)
+def _print_all(generator, args):
+    for clip in generator:
+        print_out()
+        print_clip(clip)
+
+
+    if not args.all:
+        print_out(
+            "\n<dim>There may be more clips. " +
+            "Increase the --limit, use --all or --pager to see the rest.</dim>"
+        )
+
+
+def _print_paged(generator, page_size):
+    iterator = iter(generator)
+    page = list(islice(iterator, page_size))
 
     first = 1
+    last = first + len(page) - 1
 
-    for clips, has_more in generator:
-        count = len(clips["edges"]) if "edges" in clips else 0
-        last = first + count - 1
+    while True:
+        print_out("-" * 80)
+
+        print_out()
+        for clip in page:
+            print_clip(clip)
+            print_out()
+
+        last = first + len(page) - 1
 
         print_out("-" * 80)
-        print_out("<yellow>Showing clips {}-{} of ??</yellow>".format(first, last))
+        print_out("<yellow>Clips {}-{}</yellow>".format(first, last))
 
-        for clip in clips["edges"]:
-            print_out()
-            print_clip(clip["node"])
+        first = first + len(page)
+        last = first + 1
 
-        if not args.pager:
-            print_out(
-                "\n<dim>There are more clips. "
-                "Increase the --limit or use --pager to see the rest.</dim>"
-            )
+        page = list(islice(iterator, page_size))
+        if not page or not _continue():
             break
-
-        if not has_more or not _continue():
-            break
-
-        first += count
-    else:
-        print_out("<yellow>No clips found</yellow>")

twitchdl/console.py

@@ -34,19 +34,6 @@ def time(value):
     return hours * 3600 + minutes * 60 + seconds
 
 
-def limit(value):
-    """Validates the number of videos to fetch."""
-    try:
-        value = int(value)
-    except ValueError:
-        raise ArgumentTypeError("must be an integer")
-
-    if not 1 <= int(value) <= 100:
-        raise ArgumentTypeError("must be between 1 and 100")
-
-    return value
-
-
 def pos_integer(value):
     try:
         value = int(value)
@@ -118,9 +105,14 @@ COMMANDS = [
             }),
             (["-l", "--limit"], {
                 "help": "Number of videos to fetch (default 10, max 100)",
-                "type": limit,
+                "type": pos_integer,
                 "default": 10,
             }),
+            (["-a", "--all"], {
+                "help": "Fetch all videos, overrides --limit",
+                "action": "store_true",
+                "default": False,
+            }),
             (["-P", "--period"], {
                 "help": "Period from which to return clips. (default: 'all_time')",
                 "type": str,
@@ -133,9 +125,10 @@ COMMANDS = [
                 "default": False,
             }),
             (["-p", "--pager"], {
-                "help": "If there are more results than LIMIT, ask to show next page",
-                "action": "store_true",
-                "default": False,
+                "help": "Number of clips to show per page. Disabled by default.",
+                "type": pos_integer,
+                "nargs": "?",
+                "const": 10,
             }),
             (["-d", "--download"], {
                 "help": "Download all videos in given period (in source quality)",

twitchdl/twitch.py

@@ -196,6 +196,28 @@ def get_channel_clips(channel_id, period, limit, after=None):
 
 
 def channel_clips_generator(channel_id, period, limit):
+    def _generator(clips, limit):
+        for clip in clips["edges"]:
+            if limit < 1:
+                return
+            yield clip["node"]
+            limit -= 1
+
+        has_next = clips["pageInfo"]["hasNextPage"]
+        if limit < 1 or not has_next:
+            return
+
+        req_limit = min(limit, 100)
+        cursor = clips["edges"][-1]["cursor"]
+        clips = get_channel_clips(channel_id, period, req_limit, cursor)
+        yield from _generator(clips, limit)
+
+    req_limit = min(limit, 100)
+    clips = get_channel_clips(channel_id, period, req_limit)
+    return _generator(clips, limit)
+
+
+def channel_clips_generator_old(channel_id, period, limit):
     cursor = ""
     while True:
         clips = get_channel_clips(