Reorganize paging and limiting video list

twitchdl/commands/videos.py

@@ -1,22 +1,51 @@
+import sys
+
 from twitchdl import twitch
 from twitchdl.exceptions import ConsoleError
-from twitchdl.output import print_out, print_video, print_json
+from twitchdl.output import print_out, print_paged_videos, print_video, print_json
 
 
-def _continue():
+def videos(args):
+    game_ids = _get_game_ids(args.game)
+    # Ignore --limit if --pager or --all are given
+    max_videos = sys.maxsize if args.all or args.pager else args.limit
+
+    total_count, generator = twitch.channel_videos_generator(
+        args.channel_name, max_videos, args.sort, args.type, game_ids=game_ids)
+
+    if args.json:
+        videos = list(generator)
+        print_json({
+            "count": len(videos),
+            "totalCount": total_count,
+            "videos": videos
+        })
+        return
+
+    if total_count == 0:
+        print_out("<yellow>No videos found</yellow>")
+        return
+
+    if args.pager:
+        print_paged_videos(generator, args.pager, total_count)
+        return
+
+    count = 0
+    for video in generator:
+        print_out()
+        print_video(video)
+        count += 1
+
+    print_out()
+    print_out("-" * 80)
+    print_out("<yellow>Videos {}-{} of {}</yellow>".format(1, count, total_count))
+
+    if total_count > count:
+        print_out()
         print_out(
-        "\nThere are more videos. "
-        "Press <green><b>Enter</green> to continue, "
-        "<yellow><b>Ctrl+C</yellow> to break."
+            "<dim>There are more videos. Increase the --limit, use --all or --pager to see the rest.</dim>"
         )
 
-    try:
-        input()
-    except KeyboardInterrupt:
-        return False
-
-    return True
-
 
 def _get_game_ids(names):
     if not names:
@@ -31,47 +60,3 @@ def _get_game_ids(names):
         game_ids.append(int(game_id))
 
     return game_ids
-
-
-def _videos_json(generator):
-    print_json([video["edges"] for video, has_more in generator][0])
-
-
-def videos(args):
-    game_ids = _get_game_ids(args.game)
-
-    generator = twitch.channel_videos_generator(
-        args.channel_name, args.limit, args.sort, args.type, game_ids=game_ids)
-
-    if args.json:
-        return _videos_json(generator)
-
-    print_out("<dim>Loading videos...</dim>")
-
-    first = 1
-
-    for videos, has_more in generator:
-        count = len(videos["edges"]) if "edges" in videos else 0
-        total = videos["totalCount"]
-        last = first + count - 1
-
-        print_out("-" * 80)
-        print_out("<yellow>Showing videos {}-{} of {}</yellow>".format(first, last, total))
-
-        for video in videos["edges"]:
-            print_out()
-            print_video(video["node"])
-
-        if not args.pager and has_more:
-            print_out(
-                "\n<dim>There are more videos. "
-                "Increase the --limit or use --pager to see the rest.</dim>"
-            )
-            break
-
-        if not has_more or not _continue():
-            break
-
-        first += count
-    else:
-        print_out("<yellow>No videos found</yellow>")

twitchdl/console.py

@@ -47,6 +47,18 @@ def limit(value):
     return value
 
 
+def pos_integer(value):
+    try:
+        value = int(value)
+    except ValueError:
+        raise ArgumentTypeError("must be an integer")
+
+    if value < 1:
+        raise ArgumentTypeError("must be positive")
+
+    return value
+
+
 COMMANDS = [
     Command(
         name="videos",
@@ -62,10 +74,15 @@ COMMANDS = [
                 "type": str,
             }),
             (["-l", "--limit"], {
-                "help": "Number of videos to fetch (default 10, max 100)",
-                "type": limit,
+                "help": "Number of videos to fetch (default 10)",
+                "type": pos_integer,
                 "default": 10,
             }),
+            (["-a", "--all"], {
+                "help": "Fetch all videos, overrides --limit",
+                "action": "store_true",
+                "default": False,
+            }),
             (["-s", "--sort"], {
                 "help": "Sorting order of videos. (default: time)",
                 "type": str,
@@ -79,14 +96,15 @@ COMMANDS = [
                 "default": "archive",
             }),
             (["-j", "--json"], {
-                "help": "Show results as JSON",
+                "help": "Show results as JSON. Ignores --pager.",
                 "action": "store_true",
                 "default": False,
             }),
             (["-p", "--pager"], {
-                "help": "If there are more results than LIMIT, ask to show next page",
-                "action": "store_true",
-                "default": False,
+                "help": "Number of videos to show per page. Disabled by default.",
+                "type": pos_integer,
+                "nargs": "?",
+                "const": 10,
             }),
         ],
     ),

twitchdl/output.py

@@ -4,6 +4,7 @@ import json
 import sys
 import re
 
+from itertools import islice
 from twitchdl import utils
 
 
@@ -88,6 +89,34 @@ def print_video(video):
     print_out("<i>{}</i>".format(url))
 
 
+def print_paged_videos(generator, page_size, total_count):
+    iterator = iter(generator)
+    page = list(islice(iterator, page_size))
+
+    first = 1
+    last = first + len(page) - 1
+
+    while True:
+        print_out("-" * 80)
+
+        print_out()
+        for video in page:
+            print_video(video)
+            print_out()
+
+        last = first + len(page) - 1
+
+        print_out("-" * 80)
+        print_out("<yellow>Videos {}-{} of {}</yellow>".format(first, last, total_count))
+
+        first = first + len(page)
+        last = first + 1
+
+        page = list(islice(iterator, page_size))
+        if not page or not _continue():
+            break
+
+
 def print_clip(clip):
     published_at = clip["createdAt"].replace("T", " @ ").replace("Z", "")
     length = utils.format_duration(clip["durationSeconds"])
@@ -105,3 +134,14 @@ def print_clip(clip):
         "  Length: <blue>{}</blue>"
         "  Views: <blue>{}</blue>".format(published_at, length, clip["viewCount"]))
     print_out("<i>{}</i>".format(clip["url"]))
+
+
+def _continue():
+    print_out("Press <green><b>Enter</green> to continue, <yellow><b>Ctrl+C</yellow> to break.")
+
+    try:
+        input()
+    except KeyboardInterrupt:
+        return False
+
+    return True

twitchdl/twitch.py

@@ -259,22 +259,26 @@ def get_channel_videos(channel_id, limit, sort, type="archive", game_ids=[], aft
     return response["data"]["user"]["videos"]
 
 
-def channel_videos_generator(channel_id, limit, sort, type, game_ids=None):
-    cursor = ""
-    while True:
-        videos = get_channel_videos(
-            channel_id, limit, sort, type, game_ids=game_ids, after=cursor)
-
-        if not videos["edges"]:
-            break
+def channel_videos_generator(channel_id, max_videos, sort, type, game_ids=None):
+    def _generator(videos, max_videos):
+        for video in videos["edges"]:
+            if max_videos < 1:
+                return
+            yield video["node"]
+            max_videos -= 1
 
         has_next = videos["pageInfo"]["hasNextPage"]
-        cursor = videos["edges"][-1]["cursor"] if has_next else None
+        if max_videos < 1 or not has_next:
+            return
 
-        yield videos, has_next
+        limit = min(max_videos, 100)
+        cursor = videos["edges"][-1]["cursor"]
+        videos = get_channel_videos(channel_id, limit, sort, type, game_ids, cursor)
+        yield from _generator(videos, max_videos)
 
-        if not cursor:
-            break
+    limit = min(max_videos, 100)
+    videos = get_channel_videos(channel_id, limit, sort, type, game_ids)
+    return videos["totalCount"], _generator(videos, max_videos)
 
 
 def get_access_token(video_id):