Bump version

issue #111

CHANGELOG.md

@@ -3,6 +3,16 @@ twitch-dl changelog
 
 <!-- Do not edit. This file is automatically generated from changelog.yaml.-->
 
+### [2.1.0 (2022-11-20)](https://github.com/ihabunek/twitch-dl/releases/tag/2.1.0)
+
+* Add chapter list to `info` command
+* Add `--chapter` option to `download` command for downloading a single chapter
+
+### [2.0.1 (2022-09-09)](https://github.com/ihabunek/twitch-dl/releases/tag/2.0.1)
+
+* Fix an issue where a temp vod file would be renamed while still being open,
+  which caused an exception on Windows (#111)
+
 ### [2.0.0 (2022-08-18)](https://github.com/ihabunek/twitch-dl/releases/tag/2.0.0)
 
 This release switches from using `requests` to `httpx` for making http requests,

changelog.yaml

@@ -1,3 +1,14 @@
+2.1.0:
+  date: 2022-11-20
+  changes:
+    - "Add chapter list to `info` command"
+    - "Add `--chapter` option to `download` command for downloading a single chapter"
+
+2.0.1:
+  date: 2022-09-09
+  changes:
+    - "Fix an issue where a temp vod file would be renamed while still being open, which caused an exception on Windows (#111)"
+
 2.0.0:
   date: 2022-08-18
   description: |

docs/changelog.md

@@ -3,6 +3,16 @@ twitch-dl changelog
 
 <!-- Do not edit. This file is automatically generated from changelog.yaml.-->
 
+### [2.1.0 (2022-11-20)](https://github.com/ihabunek/twitch-dl/releases/tag/2.1.0)
+
+* Add chapter list to `info` command
+* Add `--chapter` option to `download` command for downloading a single chapter
+
+### [2.0.1 (2022-09-09)](https://github.com/ihabunek/twitch-dl/releases/tag/2.0.1)
+
+* Fix an issue where a temp vod file would be renamed while still being open,
+  which caused an exception on Windows (#111)
+
 ### [2.0.0 (2022-08-18)](https://github.com/ihabunek/twitch-dl/releases/tag/2.0.0)
 
 This release switches from using `requests` to `httpx` for making http requests,

docs/commands/download.md

@@ -84,6 +84,11 @@ twitch-dl download <videos> [FLAGS] [OPTIONS]
     <td class="code">-r, --rate-limit</td>
     <td>Limit the maximum download speed in bytes per second. Use &#x27;k&#x27; and &#x27;m&#x27; suffixes for kbps and mbps.</td>
 </tr>
+
+<tr>
+    <td class="code">-c, --chapter</td>
+    <td>Download a single chapter of the video. Specify the chapter number or use the flag without a number to display a chapter select prompt.</td>
+</tr>
 </tbody>
 </table>
 

setup.py

@@ -11,7 +11,7 @@ makes it faster.
 
 setup(
     name="twitch-dl",
-    version="2.0.0",
+    version="2.1.0",
     description="Twitch downloader",
     long_description=long_description.strip(),
     author="Ivan Habunek",
@@ -31,7 +31,7 @@ setup(
     packages=find_packages(),
     python_requires=">=3.7",
     install_requires=[
-        "m3u8>=1.0.0,<2.0.0",
+        "m3u8>=1.0.0,<4.0.0",
         "httpx>=0.17.0,<1.0.0",
     ],
     entry_points={

tests/test_api.py

@@ -2,7 +2,10 @@
 These tests depend on the channel having some videos and clips published.
 """
 
+import httpx
+import m3u8
 from twitchdl import twitch
+from twitchdl.commands.download import _parse_playlists, get_clip_authenticated_url
 
 TEST_CHANNEL = "bananasaurus_rex"
 
@@ -16,6 +19,21 @@ def test_get_videos():
     video = twitch.get_video(video_id)
     assert video["id"] == video_id
 
+    access_token = twitch.get_access_token(video_id)
+    assert "signature" in access_token
+    assert "value" in access_token
+
+    playlists = twitch.get_playlists(video_id, access_token)
+    assert playlists.startswith("#EXTM3U")
+
+    name, res, url = next(_parse_playlists(playlists))
+    playlist = httpx.get(url).text
+    assert playlist.startswith("#EXTM3U")
+
+    playlist = m3u8.loads(playlist)
+    vod_path = playlist.segments[0].uri
+    assert vod_path == "0.ts"
+
 
 def test_get_clips():
     """
@@ -25,6 +43,8 @@ def test_get_clips():
     assert clips["pageInfo"]
     assert len(clips["edges"]) > 0
 
-    clip_slug = clips["edges"][0]["node"]["slug"]
-    clip = twitch.get_clip(clip_slug)
-    assert clip["slug"] == clip_slug
+    slug = clips["edges"][0]["node"]["slug"]
+    clip = twitch.get_clip(slug)
+    assert clip["slug"] == slug
+
+    assert get_clip_authenticated_url(slug, "source")

twitchdl/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "2.0.0"
+__version__ = "2.1.0"
 
 CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko"

twitchdl/commands/download.py

@@ -9,7 +9,7 @@ import tempfile
 
 from os import path
 from pathlib import Path
-from typing import OrderedDict
+from typing import List, Optional, OrderedDict
 from urllib.parse import urlparse, urlencode
 
 from twitchdl import twitch, utils
@@ -51,9 +51,9 @@ def _select_playlist_interactive(playlists):
     print_out("\nAvailable qualities:")
     for n, (name, resolution, uri) in enumerate(playlists):
         if resolution:
-            print_out("{}) {} [{}]".format(n + 1, name, resolution))
+            print_out("{}) <b>{}</b> <dim>({})</dim>".format(n + 1, name, resolution))
         else:
-            print_out("{}) {}".format(n + 1, name))
+            print_out("{}) <b>{}</b>".format(n + 1, name))
 
     no = utils.read_int("Choose quality", min=1, max=len(playlists) + 1, default=1)
     _, _, uri = playlists[no - 1]
@@ -137,7 +137,7 @@ def _clip_target_filename(clip, args):
         raise ConsoleError("Invalid key {} used in --output. Supported keys are: {}".format(e, supported))
 
 
-def _get_vod_paths(playlist, start, end):
+def _get_vod_paths(playlist, start: Optional[int], end: Optional[int]) -> List[str]:
     """Extract unique VOD paths for download from playlist."""
     files = []
     vod_start = 0
@@ -157,7 +157,7 @@ def _get_vod_paths(playlist, start, end):
     return files
 
 
-def _crete_temp_dir(base_uri):
+def _crete_temp_dir(base_uri: str) -> str:
     """Create a temp dir to store downloads if it doesn't exist."""
     path = urlparse(base_uri).path.lstrip("/")
     temp_dir = Path(tempfile.gettempdir(), "twitch-dl", path)
@@ -166,11 +166,11 @@ def _crete_temp_dir(base_uri):
 
 
 def download(args):
-    for video in args.videos:
-        download_one(video, args)
+    for video_id in args.videos:
+        download_one(video_id, args)
 
 
-def download_one(video, args):
+def download_one(video: str, args):
     video_id = utils.parse_video_identifier(video)
     if video_id:
         return _download_video(video_id, args)
@@ -227,7 +227,7 @@ def get_clip_authenticated_url(slug, quality):
     return "{}?{}".format(url, query)
 
 
-def _download_clip(slug, args):
+def _download_clip(slug: str, args) -> None:
     print_out("<dim>Looking up clip...</dim>")
     clip = twitch.get_clip(slug)
     game = clip["game"]["name"] if clip["game"] else "Unknown"
@@ -260,7 +260,7 @@ def _download_clip(slug, args):
     print_out("Downloaded: <blue>{}</blue>".format(target))
 
 
-def _download_video(video_id, args):
+def _download_video(video_id, args) -> None:
     if args.start and args.end and args.end <= args.start:
         raise ConsoleError("End time must be greater than start time")
 
@@ -282,6 +282,9 @@ def _download_video(video_id, args):
             raise ConsoleError("Aborted")
         args.overwrite = True
 
+    # Chapter select or manual offset
+    start, end = _determine_time_range(video_id, args)
+
     print_out("<dim>Fetching access token...</dim>")
     access_token = twitch.get_access_token(video_id, auth_token=args.auth_token)
 
@@ -298,7 +301,7 @@ def _download_video(video_id, args):
 
     base_uri = re.sub("/[^/]+$", "/", playlist_uri)
     target_dir = _crete_temp_dir(base_uri)
-    vod_paths = _get_vod_paths(playlist, args.start, args.end)
+    vod_paths = _get_vod_paths(playlist, start, end)
 
     # Save playlists for debugging purposes
     with open(path.join(target_dir, "playlists.m3u8"), "w") as f:
@@ -341,3 +344,40 @@ def _download_video(video_id, args):
         shutil.rmtree(target_dir)
 
     print_out("\nDownloaded: <green>{}</green>".format(target))
+
+
+def _determine_time_range(video_id, args):
+    if args.start or args.end:
+        return args.start, args.end
+
+    if args.chapter is not None:
+        print_out("<dim>Fetching chapters...</dim>")
+        chapters = twitch.get_video_chapters(video_id)
+
+        if not chapters:
+            raise ConsoleError("This video has no chapters")
+
+        if args.chapter == 0:
+            chapter = _choose_chapter_interactive(chapters)
+        else:
+            try:
+                chapter = chapters[args.chapter - 1]
+            except IndexError:
+                raise ConsoleError(f"Chapter {args.chapter} does not exist. This video has {len(chapters)} chapters.")
+
+        print_out(f'Chapter selected: <blue>{chapter["description"]}</blue>\n')
+        start = chapter["positionMilliseconds"] // 1000
+        duration = chapter["durationMilliseconds"] // 1000
+        return start, start + duration
+
+    return None, None
+
+
+def _choose_chapter_interactive(chapters):
+    print_out("\nChapters:")
+    for index, chapter in enumerate(chapters):
+        duration = utils.format_time(chapter["durationMilliseconds"] // 1000)
+        print_out(f'{index + 1}) <b>{chapter["description"]}</b> <dim>({duration})</dim>')
+    index = utils.read_int("Select a chapter", 1, len(chapters))
+    chapter = chapters[index - 1]
+    return chapter

twitchdl/commands/info.py

@@ -20,12 +20,14 @@ def info(args):
         print_log("Fetching playlists...")
         playlists = twitch.get_playlists(video_id, access_token)
 
-        if video:
-            if args.json:
-                video_json(video, playlists)
-            else:
-                video_info(video, playlists)
-            return
+        print_log("Fetching chapters...")
+        chapters = twitch.get_video_chapters(video_id)
+
+        if args.json:
+            video_json(video, playlists, chapters)
+        else:
+            video_info(video, playlists, chapters)
+        return
 
     clip_slug = utils.parse_clip_identifier(args.video)
     if clip_slug:
@@ -43,7 +45,7 @@ def info(args):
     raise ConsoleError("Invalid input: {}".format(args.video))
 
 
-def video_info(video, playlists):
+def video_info(video, playlists, chapters):
     print_out()
     print_video(video)
 
@@ -52,8 +54,16 @@ def video_info(video, playlists):
     for p in m3u8.loads(playlists).playlists:
         print_out("<b>{}</b> {}".format(p.stream_info.video, p.uri))
 
+    if chapters:
+        print_out()
+        print_out("Chapters:")
+        for chapter in chapters:
+            start = utils.format_time(chapter["positionMilliseconds"] // 1000, force_hours=True)
+            duration = utils.format_time(chapter["durationMilliseconds"] // 1000)
+            print_out(f'{start} <b>{chapter["description"]}</b> ({duration})')
 
-def video_json(video, playlists):
+
+def video_json(video, playlists, chapters):
     playlists = m3u8.loads(playlists).playlists
 
     video["playlists"] = [
@@ -66,6 +76,8 @@ def video_json(video, playlists):
         } for p in playlists
     ]
 
+    video["chapters"] = chapters
+
     print_json(video)
 
 

twitchdl/console.py

@@ -5,7 +5,7 @@ import sys
 import re
 
 from argparse import ArgumentParser, ArgumentTypeError
-from collections import namedtuple
+from typing import NamedTuple, List, Tuple, Any, Dict
 
 from twitchdl.exceptions import ConsoleError
 from twitchdl.output import print_err
@@ -13,12 +13,19 @@ from twitchdl.twitch import GQLError
 from . import commands, __version__
 
 
-Command = namedtuple("Command", ["name", "description", "arguments"])
-
-CLIENT_WEBSITE = 'https://github.com/ihabunek/twitch-dl'
+Argument = Tuple[List[str], Dict[str, Any]]
 
 
-def time(value):
+class Command(NamedTuple):
+    name: str
+    description: str
+    arguments: List[Argument]
+
+
+CLIENT_WEBSITE = "https://twitch-dl.bezdomni.net/"
+
+
+def time(value: str) -> int:
     """Parse a time string (hh:mm or hh:mm:ss) to number of seconds."""
     parts = [int(p) for p in value.split(":")]
 
@@ -35,19 +42,19 @@ def time(value):
     return hours * 3600 + minutes * 60 + seconds
 
 
-def pos_integer(value):
+def pos_integer(value: str) -> int:
     try:
-        value = int(value)
+        parsed = int(value)
     except ValueError:
         raise ArgumentTypeError("must be an integer")
 
-    if value < 1:
+    if parsed < 1:
         raise ArgumentTypeError("must be positive")
 
-    return value
+    return parsed
 
 
-def rate(value):
+def rate(value: str) -> int:
     match = re.search(r"^([0-9]+)(k|m|)$", value, flags=re.IGNORECASE)
 
     if not match:
@@ -226,6 +233,13 @@ COMMANDS = [
                         "Use 'k' and 'm' suffixes for kbps and mbps.",
                 "type": rate,
             }),
+            (["-c", "--chapter"], {
+                "help": "Download a single chapter of the video. Specify the chapter number or "
+                        "use the flag without a number to display a chapter select prompt.",
+                "type": int,
+                "nargs": "?",
+                "const": 0
+            }),
         ],
     ),
     Command(

twitchdl/download.py

@@ -10,7 +10,7 @@ class DownloadFailed(Exception):
     pass
 
 
-def _download(url, path):
+def _download(url: str, path: str):
     tmp_path = path + ".tmp"
     size = 0
     with httpx.stream("GET", url, timeout=CONNECT_TIMEOUT) as response:
@@ -23,7 +23,7 @@ def _download(url, path):
     return size
 
 
-def download_file(url, path, retries=RETRY_COUNT):
+def download_file(url: str, path: str, retries: int = RETRY_COUNT):
     if os.path.exists(path):
         from_disk = True
         return (os.path.getsize(path), from_disk)

twitchdl/http.py

@@ -55,7 +55,7 @@ class TokenBucket:
 
 class EndlessTokenBucket:
     """Used when download speed is not limited."""
-    def advance(self, size):
+    def advance(self, size: int):
         pass
 
 
@@ -83,7 +83,7 @@ async def download(
                 token_bucket.advance(size)
                 progress.advance(task_id, size)
             progress.end(task_id)
-        os.rename(tmp_target, target)
+    os.rename(tmp_target, target)
 
 
 async def download_with_retries(

twitchdl/output.py

@@ -6,6 +6,7 @@ import re
 
 from itertools import islice
 from twitchdl import utils
+from typing import Any, Match
 
 
 START_CODES = {
@@ -29,26 +30,26 @@ END_PATTERN = "</(" + "|".join(START_CODES.keys()) + ")>"
 USE_ANSI_COLOR = "--no-color" not in sys.argv
 
 
-def start_code(match):
+def start_code(match: Match[str]) -> str:
     name = match.group(1)
     return START_CODES[name]
 
 
-def colorize(text):
+def colorize(text: str) -> str:
     text = re.sub(START_PATTERN, start_code, text)
     text = re.sub(END_PATTERN, END_CODE, text)
 
     return text
 
 
-def strip_tags(text):
+def strip_tags(text: str) -> str:
     text = re.sub(START_PATTERN, '', text)
     text = re.sub(END_PATTERN, '', text)
 
     return text
 
 
-def truncate(string, length):
+def truncate(string: str, length: int) -> str:
     if len(string) > length:
         return string[:length - 1] + "…"
 
@@ -60,7 +61,7 @@ def print_out(*args, **kwargs):
     print(*args, **kwargs)
 
 
-def print_json(data):
+def print_json(data: Any):
     print(json.dumps(data))
 
 

twitchdl/twitch.py

@@ -3,7 +3,9 @@ Twitch API access.
 """
 
 import httpx
+import json
 
+from typing import Dict
 from twitchdl import CLIENT_ID
 from twitchdl.exceptions import ConsoleError
 
@@ -14,21 +16,6 @@ class GQLError(Exception):
         self.errors = errors
 
 
-def authenticated_get(url, params={}, headers={}):
-    headers['Client-ID'] = CLIENT_ID
-
-    response = httpx.get(url, params=params, headers=headers)
-    if 400 <= response.status_code < 500:
-        data = response.json()
-        # TODO: this does not look nice in the console since data["message"]
-        # can contain a JSON encoded object.
-        raise ConsoleError(data["message"])
-
-    response.raise_for_status()
-
-    return response
-
-
 def authenticated_post(url, data=None, json=None, headers={}):
     headers['Client-ID'] = CLIENT_ID
 
@@ -52,7 +39,7 @@ def gql_post(query):
     return response
 
 
-def gql_query(query, headers={}):
+def gql_query(query: str, headers: Dict[str, str] = {}):
     url = "https://gql.twitch.tv/gql"
     response = authenticated_post(url, json={"query": query}, headers=headers).json()
 
@@ -374,3 +361,32 @@ def get_game_id(name):
     game = response["data"]["game"]
     if game:
         return game["id"]
+
+
+def get_video_chapters(video_id):
+    query = {
+        "operationName": "VideoPlayer_ChapterSelectButtonVideo",
+        "variables":
+        {
+            "includePrivate": False,
+            "videoID": video_id
+        },
+        "extensions":
+        {
+            "persistedQuery":
+            {
+                "version": 1,
+                "sha256Hash": "8d2793384aac3773beab5e59bd5d6f585aedb923d292800119e03d40cd0f9b41"
+            }
+        }
+    }
+
+    response = gql_post(json.dumps(query))
+    return list(_chapter_nodes(response["data"]["video"]["moments"]))
+
+
+def _chapter_nodes(collection):
+    for edge in collection["edges"]:
+        node = edge["node"]
+        del node["moments"]
+        yield node

twitchdl/utils.py

@@ -40,26 +40,29 @@ def format_duration(total_seconds):
     return "{} sec".format(seconds)
 
 
-def format_time(total_seconds):
+def format_time(total_seconds, force_hours=False):
     total_seconds = int(total_seconds)
     hours = total_seconds // 3600
     remainder = total_seconds % 3600
     minutes = remainder // 60
     seconds = total_seconds % 60
 
-    if hours:
+    if hours or force_hours:
         return f"{hours:02}:{minutes:02}:{seconds:02}"
 
     return f"{minutes:02}:{seconds:02}"
 
 
-def read_int(msg, min, max, default):
-    msg = msg + " [default {}]: ".format(default)
+def read_int(msg, min, max, default=None):
+    if default:
+        msg = msg + f" [default {default}]"
+
+    msg += ": "
 
     while True:
         try:
             val = input(msg)
-            if not val:
+            if default and not val:
                 return default
             if min <= int(val) <= max:
                 return int(val)