WIP, BROKEN

Better support for "enhanced broadcast" streams

issue #154

.github/workflows/test.yml

@@ -10,9 +10,9 @@ jobs:
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies

CHANGELOG.md

@@ -3,6 +3,21 @@ twitch-dl changelog
 
 <!-- Do not edit. This file is automatically generated from changelog.yaml.-->
 
+### [2.3.0 (2024-04-27)](https://github.com/ihabunek/twitch-dl/releases/tag/2.3.0)
+
+* Show more playlist data when choosing quality
+* Improve detection of 'source' quality for Twitch Enhanced Broadcast Streams
+  (#154)
+
+### [2.2.4 (2024-04-25)](https://github.com/ihabunek/twitch-dl/releases/tag/2.2.4)
+
+* Add m dot url support to video and clip regexes (thanks @localnerve)
+
+### [2.2.3 (2024-04-24)](https://github.com/ihabunek/twitch-dl/releases/tag/2.2.3)
+
+* Respect --dry-run option when downloading videos
+* Add automated tests on github actions
+
 ### [2.2.2 (2024-04-23)](https://github.com/ihabunek/twitch-dl/releases/tag/2.2.2)
 
 * Fix more compat issues Python < 3.10 (#152)

Makefile

@@ -7,7 +7,7 @@ dist:
 
 clean :
 	find . -name "*pyc" | xargs rm -rf $1
-	rm -rf build dist bundle MANIFEST htmlcov deb_dist twitch-dl.*.pyz twitch-dl.1.man twitch_dl.egg-info
+	rm -rf build dist book bundle MANIFEST htmlcov deb_dist twitch-dl.*.pyz twitch-dl.1.man twitch_dl.egg-info
 
 bundle:
 	mkdir bundle
@@ -24,7 +24,7 @@ publish :
 	twine upload dist/*.tar.gz dist/*.whl
 
 coverage:
-	py.test --cov=toot --cov-report html tests/
+	pytest --cov=twitchdl --cov-report html tests/
 
 man:
 	scdoc < twitch-dl.1.scd > twitch-dl.1.man

changelog.yaml

@@ -1,3 +1,20 @@
+2.3.0:
+  date: 2024-04-27
+  changes:
+    - "Show more playlist data when choosing quality"
+    - "Improve detection of 'source' quality for Twitch Enhanced Broadcast Streams (#154)"
+
+2.2.4:
+  date: 2024-04-25
+  changes:
+    - "Add m dot url support to video and clip regexes (thanks @localnerve)"
+
+2.2.3:
+  date: 2024-04-24
+  changes:
+    - "Respect --dry-run option when downloading videos"
+    - "Add automated tests on github actions"
+
 2.2.2:
   date: 2024-04-23
   changes:

docs/changelog.md

@@ -3,6 +3,21 @@ twitch-dl changelog
 
 <!-- Do not edit. This file is automatically generated from changelog.yaml.-->
 
+### [2.3.0 (2024-04-27)](https://github.com/ihabunek/twitch-dl/releases/tag/2.3.0)
+
+* Show more playlist data when choosing quality
+* Improve detection of 'source' quality for Twitch Enhanced Broadcast Streams
+  (#154)
+
+### [2.2.4 (2024-04-25)](https://github.com/ihabunek/twitch-dl/releases/tag/2.2.4)
+
+* Add m dot url support to video and clip regexes (thanks @localnerve)
+
+### [2.2.3 (2024-04-24)](https://github.com/ihabunek/twitch-dl/releases/tag/2.2.3)
+
+* Respect --dry-run option when downloading videos
+* Add automated tests on github actions
+
 ### [2.2.2 (2024-04-23)](https://github.com/ihabunek/twitch-dl/releases/tag/2.2.2)
 
 * Fix more compat issues Python < 3.10 (#152)

pyproject.toml

@@ -22,7 +22,7 @@ classifiers = [
 dependencies = [
     "click>=8.0.0,<9.0.0",
     "httpx>=0.17.0,<1.0.0",
-    "m3u8>=1.0.0,<4.0.0",
+    "m3u8>=3.0.0,<5.0.0",
 ]
 
 [tool.setuptools]
@@ -56,7 +56,6 @@ test = [
 twitch-dl = "twitchdl.cli:cli"
 
 [tool.pyright]
-include = ["twitchdl"]
 typeCheckingMode = "strict"
 pythonVersion = "3.8"
 

tests/test_api.py

@@ -3,9 +3,12 @@ These tests depend on the channel having some videos and clips published.
 """
 
 import httpx
+import pytest
 
 from twitchdl import twitch
 from twitchdl.commands.download import get_clip_authenticated_url
+from twitchdl.commands.videos import get_game_ids
+from twitchdl.exceptions import ConsoleError
 from twitchdl.playlists import enumerate_vods, load_m3u8, parse_playlists
 
 TEST_CHANNEL = "bananasaurus_rex"
@@ -53,3 +56,15 @@ def test_get_clips():
     assert clip["slug"] == slug
 
     assert get_clip_authenticated_url(slug, "source")
+
+
+def test_get_games():
+    assert get_game_ids([]) == []
+    assert get_game_ids(["Bioshock"]) == ["15866"]
+    assert get_game_ids(["Bioshock", "Portal"]) == ["15866", "6187"]
+
+
+def test_get_games_not_found():
+    with pytest.raises(ConsoleError) as ex:
+        get_game_ids(["the game which does not exist"])
+    assert str(ex.value) == "Game 'the game which does not exist' not found"

tests/test_cli.py

@@ -0,0 +1,156 @@
+import json
+
+import pytest
+from click.testing import CliRunner, Result
+
+from twitchdl import cli
+
+
+@pytest.fixture(scope="session")
+def runner():
+    return CliRunner(mix_stderr=False)
+
+
+def assert_ok(result: Result):
+    if result.exit_code != 0:
+        raise AssertionError(
+            f"Command failed with exit code {result.exit_code}\nStderr: {result.stderr}"
+        )
+
+
+def test_info_video(runner: CliRunner):
+    result = runner.invoke(cli.info, ["2090131595"])
+    assert_ok(result)
+
+    assert "Frost Fatales 2024 Day 1" in result.stdout
+    assert "frozenflygone playing Tomb Raider" in result.stdout
+
+
+def test_info_video_json(runner: CliRunner):
+    result = runner.invoke(cli.info, ["2090131595", "--json"])
+    assert_ok(result)
+
+    video = json.loads(result.stdout)
+    assert video["title"] == "Frost Fatales 2024 Day 1"
+    assert video["game"] == {"id": "2770", "name": "Tomb Raider"}
+    assert video["creator"] == {"login": "frozenflygone", "displayName": "frozenflygone"}
+
+
+def test_info_clip(runner: CliRunner):
+    result = runner.invoke(cli.info, ["PoisedTalentedPuddingChefFrank"])
+    assert_ok(result)
+
+    assert "AGDQ Crashes during Bioshock run" in result.stdout
+    assert "GamesDoneQuick playing BioShock" in result.stdout
+
+
+def test_info_clip_json(runner: CliRunner):
+    result = runner.invoke(cli.info, ["PoisedTalentedPuddingChefFrank", "--json"])
+    assert_ok(result)
+
+    clip = json.loads(result.stdout)
+    assert clip["slug"] == "PoisedTalentedPuddingChefFrank"
+    assert clip["title"] == "AGDQ Crashes during Bioshock run"
+    assert clip["game"] == {"id": "15866", "name": "BioShock"}
+    assert clip["broadcaster"] == {"displayName": "GamesDoneQuick", "login": "gamesdonequick"}
+
+
+def test_info_not_found(runner: CliRunner):
+    result = runner.invoke(cli.info, ["banana"])
+    assert result.exit_code == 1
+    assert "Clip banana not found" in result.stderr
+
+    result = runner.invoke(cli.info, ["12345"])
+    assert result.exit_code == 1
+    assert "Video 12345 not found" in result.stderr
+
+    result = runner.invoke(cli.info, [""])
+    assert result.exit_code == 1
+    assert "Invalid input" in result.stderr
+
+
+def test_download_clip(runner: CliRunner):
+    result = runner.invoke(
+        cli.download,
+        [
+            "PoisedTalentedPuddingChefFrank",
+            "-q",
+            "source",
+            "--dry-run",
+        ],
+    )
+    assert_ok(result)
+    assert (
+        "Found: AGDQ Crashes during Bioshock run by GamesDoneQuick, playing BioShock (30 sec)"
+        in result.stdout
+    )
+    assert (
+        "Target: 2020-01-10_3099545841_gamesdonequick_agdq_crashes_during_bioshock_run.mp4"
+        in result.stdout
+    )
+    assert "Dry run, clip not downloaded." in result.stdout
+
+
+def test_download_video(runner: CliRunner):
+    result = runner.invoke(
+        cli.download,
+        [
+            "2090131595",
+            "-q",
+            "source",
+            "--dry-run",
+        ],
+    )
+    assert_ok(result)
+    assert "Found: Frost Fatales 2024 Day 1 by frozenflygone" in result.stdout
+    assert (
+        "Output: 2024-03-14_2090131595_frozenflygone_frost_fatales_2024_day_1.mkv" in result.stdout
+    )
+    assert "Dry run, video not downloaded." in result.stdout
+
+
+def test_videos(runner: CliRunner):
+    result = runner.invoke(cli.videos, ["gamesdonequick", "--json"])
+    assert_ok(result)
+    videos = json.loads(result.stdout)
+
+    assert videos["count"] == 10
+    assert videos["totalCount"] > 0
+    video = videos["videos"][0]
+
+    result = runner.invoke(cli.videos, "gamesdonequick")
+    assert_ok(result)
+
+    assert f"Video {video['id']}" in result.stdout
+    assert video["title"] in result.stdout
+
+    result = runner.invoke(cli.videos, ["gamesdonequick", "--compact"])
+    assert_ok(result)
+
+    assert video["id"] in result.stdout
+    assert video["title"][:60] in result.stdout
+
+
+def test_videos_channel_not_found(runner: CliRunner):
+    result = runner.invoke(cli.videos, ["doesnotexisthopefully"])
+    assert result.exit_code == 1
+    assert result.stderr.strip() == "Error: Channel doesnotexisthopefully not found"
+
+
+def test_clips(runner: CliRunner):
+    result = runner.invoke(cli.clips, ["gamesdonequick", "--json"])
+    assert_ok(result)
+    clips = json.loads(result.stdout)
+    clip = clips[0]
+
+    result = runner.invoke(cli.clips, "gamesdonequick")
+    assert_ok(result)
+
+    assert f"Clip {clip['slug']}" in result.stdout
+    assert clip["title"] in result.stdout
+
+    result = runner.invoke(cli.clips, ["gamesdonequick", "--compact"])
+    assert_ok(result)
+
+    assert clip["slug"] in result.stdout
+    assert clip["title"][:60] in result.stdout

tests/test_patterns.py

@@ -1,35 +1,38 @@
 import pytest
 
-from twitchdl.utils import parse_video_identifier, parse_clip_identifier
-
+from twitchdl.utils import parse_clip_identifier, parse_video_identifier
 
 TEST_VIDEO_PATTERNS = [
     ("702689313", "702689313"),
     ("702689313", "https://twitch.tv/videos/702689313"),
     ("702689313", "https://www.twitch.tv/videos/702689313"),
+    ("702689313", "https://m.twitch.tv/videos/702689313"),
 ]
 
 TEST_CLIP_PATTERNS = {
     ("AbrasivePlayfulMangoMau5", "AbrasivePlayfulMangoMau5"),
     ("AbrasivePlayfulMangoMau5", "https://clips.twitch.tv/AbrasivePlayfulMangoMau5"),
     ("AbrasivePlayfulMangoMau5", "https://www.twitch.tv/dracul1nx/clip/AbrasivePlayfulMangoMau5"),
+    ("AbrasivePlayfulMangoMau5", "https://m.twitch.tv/dracul1nx/clip/AbrasivePlayfulMangoMau5"),
     ("AbrasivePlayfulMangoMau5", "https://twitch.tv/dracul1nx/clip/AbrasivePlayfulMangoMau5"),
     ("HungryProudRadicchioDoggo", "HungryProudRadicchioDoggo"),
     ("HungryProudRadicchioDoggo", "https://clips.twitch.tv/HungryProudRadicchioDoggo"),
     ("HungryProudRadicchioDoggo", "https://www.twitch.tv/bananasaurus_rex/clip/HungryProudRadicchioDoggo?filter=clips&range=7d&sort=time"),
+    ("HungryProudRadicchioDoggo", "https://m.twitch.tv/bananasaurus_rex/clip/HungryProudRadicchioDoggo?filter=clips&range=7d&sort=time"),
     ("HungryProudRadicchioDoggo", "https://twitch.tv/bananasaurus_rex/clip/HungryProudRadicchioDoggo?filter=clips&range=7d&sort=time"),
     ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ"),
     ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ"),
     ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ?filter=clips&range=7d&sort=time"),
     ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://www.twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ?filter=clips&range=7d&sort=time"),
+    ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://m.twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ?filter=clips&range=7d&sort=time"),
 }
 
 
 @pytest.mark.parametrize("expected,input", TEST_VIDEO_PATTERNS)
-def test_video_patterns(expected, input):
+def test_video_patterns(expected: str, input: str):
     assert parse_video_identifier(input) == expected
 
 
 @pytest.mark.parametrize("expected,input", TEST_CLIP_PATTERNS)
-def test_clip_patterns(expected, input):
+def test_clip_patterns(expected: str, input: str):
     assert parse_clip_identifier(input) == expected

tests/test_progress.py

@@ -23,26 +23,31 @@ def test_downloaded():
     assert progress.progress_perc == 0
 
     progress.advance(1, 100)
+    progress._recalculate()
     assert progress.downloaded == 100
     assert progress.progress_bytes == 100
     assert progress.progress_perc == 11
 
     progress.advance(2, 200)
+    progress._recalculate()
     assert progress.downloaded == 300
     assert progress.progress_bytes == 300
     assert progress.progress_perc == 33
 
     progress.advance(3, 150)
+    progress._recalculate()
     assert progress.downloaded == 450
     assert progress.progress_bytes == 450
     assert progress.progress_perc == 50
 
     progress.advance(1, 50)
+    progress._recalculate()
     assert progress.downloaded == 500
     assert progress.progress_bytes == 500
     assert progress.progress_perc == 55
 
     progress.abort(2)
+    progress._recalculate()
     assert progress.downloaded == 500
     assert progress.progress_bytes == 300
     assert progress.progress_perc == 33
@@ -52,6 +57,7 @@ def test_downloaded():
     progress.advance(1, 150)
     progress.advance(2, 300)
     progress.advance(3, 150)
+    progress._recalculate()
 
     assert progress.downloaded == 1100
     assert progress.progress_bytes == 900
@@ -71,12 +77,15 @@ def test_estimated_total():
     assert progress.estimated_total is None
 
     progress.start(1, 12000)
+    progress._recalculate()
     assert progress.estimated_total == 12000 * 3
 
     progress.start(2, 11000)
+    progress._recalculate()
     assert progress.estimated_total == 11500 * 3
 
     progress.start(3, 10000)
+    progress._recalculate()
     assert progress.estimated_total == 11000 * 3
 
 

twitchdl/cli.py

@@ -91,7 +91,9 @@ def cli(ctx: click.Context, color: bool, debug: bool):
     ctx.color = color
 
     if debug:
-        logging.basicConfig(level=logging.INFO)
+        logging.basicConfig(level=logging.DEBUG)
+        logging.getLogger("httpx").setLevel(logging.WARN)
+        logging.getLogger("httpcore").setLevel(logging.WARN)
 
 
 @cli.command()

twitchdl/commands/download.py

@@ -7,7 +7,7 @@ import subprocess
 import tempfile
 from os import path
 from pathlib import Path
-from typing import Dict, List
+from typing import Dict, List, Optional
 from urllib.parse import urlencode, urlparse
 
 import click
@@ -30,6 +30,10 @@ from twitchdl.twitch import Chapter, Clip, ClipAccessToken, Video
 
 
 def download(ids: List[str], args: DownloadOptions):
+    if not ids:
+        print_log("No IDs to downlad given")
+        return
+
     for video_id in ids:
         download_one(video_id, args)
 
@@ -156,7 +160,7 @@ def _crete_temp_dir(base_uri: str) -> str:
     return str(temp_dir)
 
 
-def _get_clip_url(access_token: ClipAccessToken, quality: str) -> str:
+def _get_clip_url(access_token: ClipAccessToken, quality: Optional[str]) -> str:
     qualities = access_token["videoQualities"]
 
     # Quality given as an argument
@@ -184,7 +188,7 @@ def _get_clip_url(access_token: ClipAccessToken, quality: str) -> str:
     return selected_quality["sourceURL"]
 
 
-def get_clip_authenticated_url(slug: str, quality: str):
+def get_clip_authenticated_url(slug: str, quality: Optional[str]):
     print_log("Fetching access token...")
     access_token = twitch.get_clip_access_token(slug)
 
@@ -273,8 +277,13 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
     vods_m3u8 = load_m3u8(vods_text)
     vods = enumerate_vods(vods_m3u8, start, end)
 
+    if args.dry_run:
+        click.echo("Dry run, video not downloaded.")
+        return
+
     base_uri = re.sub("/[^/]+$", "/", playlist.url)
-    target_dir = _crete_temp_dir(base_uri)
+    target_dir = f".twitch_dl_{video_id}_{playlist.group_id}"
+    os.makedirs(target_dir, exist_ok=True)
 
     # Save playlists for debugging purposes
     with open(path.join(target_dir, "playlists.m3u8"), "w") as f:
@@ -288,7 +297,7 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
     targets = [os.path.join(target_dir, f"{vod.index:05d}.ts") for vod in vods]
     asyncio.run(download_all(sources, targets, args.max_workers, rate_limit=args.rate_limit))
 
-    join_playlist = make_join_playlist(vods_m3u8, vods, targets)
+    join_playlist = make_join_playlist(vods, targets)
     join_playlist_path = path.join(target_dir, "playlist_downloaded.m3u8")
     join_playlist.dump(join_playlist_path)  # type: ignore
     click.echo()

twitchdl/commands/videos.py

@@ -20,7 +20,7 @@ def videos(
     sort: twitch.VideosSort,
     type: twitch.VideosType,
 ):
-    game_ids = _get_game_ids(games)
+    game_ids = get_game_ids(games)
 
     # Set different defaults for limit for compact display
     limit = limit or (40 if compact else 10)
@@ -67,16 +67,13 @@ def videos(
         )
 
 
-def _get_game_ids(names: List[str]) -> List[str]:
-    if not names:
-        return []
+def get_game_ids(names: List[str]) -> List[str]:
+    return [get_game_id(name) for name in names]
 
-    game_ids = []
-    for name in names:
-        print_log(f"Looking up game '{name}'...")
-        game_id = twitch.get_game_id(name)
-        if not game_id:
-            raise ConsoleError(f"Game '{name}' not found")
-        game_ids.append(int(game_id))
 
-    return game_ids
+def get_game_id(name: str) -> str:
+    print_log(f"Looking up game '{name}'...")
+    game_id = twitch.get_game_id(name)
+    if not game_id:
+        raise ConsoleError(f"Game '{name}' not found")
+    return game_id

twitchdl/output.py

@@ -25,15 +25,24 @@ def print_log(message: Any):
     click.secho(message, err=True, dim=True)
 
 
+def visual_len(text: str):
+    return len(click.unstyle(text))
+
+
+def ljust(text: str, width: int):
+    diff = width - visual_len(text)
+    return text + (" " * diff) if diff > 0 else text
+
+
 def print_table(headers: List[str], data: List[List[str]]):
-    widths = [[len(cell) for cell in row] for row in data + [headers]]
+    widths = [[visual_len(cell) for cell in row] for row in data + [headers]]
     widths = [max(width) for width in zip(*widths)]
     underlines = ["-" * width for width in widths]
 
     def print_row(row: List[str]):
         for idx, cell in enumerate(row):
             width = widths[idx]
-            click.echo(cell.ljust(width), nl=False)
+            click.echo(ljust(cell, width), nl=False)
             click.echo("  ", nl=False)
         click.echo()
 

twitchdl/playlists.py

@@ -3,20 +3,23 @@ Parse and manipulate m3u8 playlists.
 """
 
 from dataclasses import dataclass
-from typing import Generator, List, Optional, OrderedDict
+from os.path import basename
+from typing import Generator, List, Optional
 
 import click
 import m3u8
 
 from twitchdl import utils
-from twitchdl.output import bold, dim
+from twitchdl.output import bold, dim, print_table
 
 
 @dataclass
 class Playlist:
     name: str
+    group_id: str
     resolution: Optional[str]
     url: str
+    is_source: bool
 
 
 @dataclass
@@ -34,21 +37,17 @@ def parse_playlists(playlists_m3u8: str) -> List[Playlist]:
         document = load_m3u8(source)
 
         for p in document.playlists:
-            from pprint import pp
+            resolution = (
+                "x".join(str(r) for r in p.stream_info.resolution)
+                if p.stream_info.resolution
+                else None
+            )
 
-            pp(p.__dict__)
-            pp(p.stream_info.__dict__)
-            if p.stream_info.resolution:
-                name = p.media[0].name
-                resolution = "x".join(str(r) for r in p.stream_info.resolution)
-            else:
-                name = p.media[0].group_id
-                resolution = None
+            media = p.media[0]
+            is_source = media.group_id == "chunked"
+            yield Playlist(media.name, media.group_id, resolution, p.uri, is_source)
 
-            yield Playlist(name, resolution, p.uri)
-
-    # Move audio to bottom, it has no resolution
-    return sorted(_parse(playlists_m3u8), key=lambda p: p.resolution is None)
+    return list(_parse(playlists_m3u8))
 
 
 def load_m3u8(playlist_m3u8: str) -> m3u8.M3U8:
@@ -80,23 +79,15 @@ def enumerate_vods(
     return vods
 
 
-def make_join_playlist(
-    playlist: m3u8.M3U8,
-    vods: List[Vod],
-    targets: List[str],
-) -> m3u8.Playlist:
+def make_join_playlist(vods: List[Vod], targets: List[str]) -> m3u8.Playlist:
     """
     Make a modified playlist which references downloaded VODs
     Keep only the downloaded segments and skip the rest
     """
-    org_segments = playlist.segments.copy()
+    playlist = m3u8.M3U8()
 
-    path_map = OrderedDict(zip([v.path for v in vods], targets))
-    playlist.segments.clear()
-    for segment in org_segments:
-        if segment.uri in path_map:
-            segment.uri = path_map[segment.uri]
-            playlist.segments.append(segment)
+    for vod, target in zip(vods, targets):
+        playlist.add_segment(m3u8.Segment(uri=basename(target), duration=vod.duration))
 
     return playlist
 
@@ -111,10 +102,13 @@ def select_playlist(playlists: List[Playlist], quality: Optional[str]) -> Playli
 
 def select_playlist_by_name(playlists: List[Playlist], quality: str) -> Playlist:
     if quality == "source":
-        return playlists[0]
+        for playlist in playlists:
+            if playlist.is_source:
+                return playlist
+        raise click.ClickException("Source quality not found, please report an issue on github.")
 
     for playlist in playlists:
-        if playlist.name == quality:
+        if playlist.name == quality or playlist.group_id == quality:
             return playlist
 
     available = ", ".join([p.name for p in playlists])
@@ -123,13 +117,47 @@ def select_playlist_by_name(playlists: List[Playlist], quality: str) -> Playlist
 
 
 def select_playlist_interactive(playlists: List[Playlist]) -> Playlist:
-    click.echo("\nAvailable qualities:")
-    for n, playlist in enumerate(playlists):
-        if playlist.resolution:
-            click.echo(f"{n + 1}) {bold(playlist.name)} {dim(f'({playlist.resolution})')}")
-        else:
-            click.echo(f"{n + 1}) {bold(playlist.name)}")
+    playlists = sorted(playlists, key=_playlist_key)
+    headers = ["#", "Name", "Group ID", "Resolution"]
 
-    no = utils.read_int("Choose quality", min=1, max=len(playlists) + 1, default=1)
+    rows = [
+        [
+            f"{n + 1})",
+            bold(playlist.name),
+            dim(playlist.group_id),
+            dim(playlist.resolution or ""),
+        ]
+        for n, playlist in enumerate(playlists)
+    ]
+
+    click.echo()
+    print_table(headers, rows)
+
+    default = 1
+    for index, playlist in enumerate(playlists):
+        if playlist.is_source:
+            default = index + 1
+
+    no = utils.read_int("\nChoose quality", min=1, max=len(playlists) + 1, default=default)
     playlist = playlists[no - 1]
     return playlist
+
+
+MAX = 1_000_000
+
+
+def _playlist_key(playlist: Playlist) -> int:
+    """Attempt to sort playlists so that source quality is on top, audio only
+    is on bottom and others are sorted descending by resolution."""
+    if playlist.is_source:
+        return 0
+
+    if playlist.group_id == "audio_only":
+        return MAX
+
+    try:
+        return MAX - int(playlist.name.split("p")[0])
+    except Exception:
+        pass
+
+    return MAX

twitchdl/progress.py

@@ -1,7 +1,7 @@
 import logging
 import time
 from collections import deque
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from statistics import mean
 from typing import Deque, Dict, NamedTuple, Optional
 
@@ -31,28 +31,25 @@ class Sample(NamedTuple):
     timestamp: float
 
 
-@dataclass
 class Progress:
-    vod_count: int
-    downloaded: int = 0
-    estimated_total: Optional[int] = None
-    last_printed: float = field(default_factory=time.time)
-    progress_bytes: int = 0
-    progress_perc: int = 0
-    remaining_time: Optional[int] = None
-    speed: Optional[float] = None
-    start_time: float = field(default_factory=time.time)
-    tasks: Dict[TaskId, Task] = field(default_factory=dict)
-    vod_downloaded_count: int = 0
-    samples: Deque[Sample] = field(default_factory=lambda: deque(maxlen=100))
+    def __init__(self, vod_count: int):
+        self.downloaded: int = 0
+        self.estimated_total: Optional[int] = None
+        self.last_printed: Optional[float] = None
+        self.progress_bytes: int = 0
+        self.progress_perc: int = 0
+        self.remaining_time: Optional[int] = None
+        self.samples: Deque[Sample] = deque(maxlen=1000)
+        self.speed: Optional[float] = None
+        self.tasks: Dict[TaskId, Task] = {}
+        self.vod_count = vod_count
+        self.vod_downloaded_count: int = 0
 
     def start(self, task_id: int, size: int):
         if task_id in self.tasks:
             raise ValueError(f"Task {task_id}: cannot start, already started")
 
         self.tasks[task_id] = Task(task_id, size)
-        self._calculate_total()
-        self._calculate_progress()
         self.print()
 
     def advance(self, task_id: int, size: int):
@@ -63,7 +60,6 @@ class Progress:
         self.progress_bytes += size
         self.tasks[task_id].advance(size)
         self.samples.append(Sample(self.downloaded, time.time()))
-        self._calculate_progress()
         self.print()
 
     def already_downloaded(self, task_id: int, size: int):
@@ -73,8 +69,6 @@ class Progress:
         self.tasks[task_id] = Task(task_id, size)
         self.progress_bytes += size
         self.vod_downloaded_count += 1
-        self._calculate_total()
-        self._calculate_progress()
         self.print()
 
     def abort(self, task_id: int):
@@ -83,9 +77,6 @@ class Progress:
 
         del self.tasks[task_id]
         self.progress_bytes = sum(t.downloaded for t in self.tasks.values())
-
-        self._calculate_total()
-        self._calculate_progress()
         self.print()
 
     def end(self, task_id: int):
@@ -101,12 +92,10 @@ class Progress:
         self.vod_downloaded_count += 1
         self.print()
 
-    def _calculate_total(self):
+    def _recalculate(self):
         self.estimated_total = (
             int(mean(t.size for t in self.tasks.values()) * self.vod_count) if self.tasks else None
         )
-
-    def _calculate_progress(self):
         self.speed = self._calculate_speed()
         self.progress_perc = (
             int(100 * self.progress_bytes / self.estimated_total) if self.estimated_total else 0
@@ -133,9 +122,11 @@ class Progress:
         now = time.time()
 
         # Don't print more often than 10 times per second
-        if now - self.last_printed < 0.1:
+        if self.last_printed and now - self.last_printed < 0.1:
             return
 
+        self._recalculate()
+
         click.echo(f"\rDownloaded {self.vod_downloaded_count}/{self.vod_count} VODs", nl=False)
         click.secho(f" {self.progress_perc}%", fg="blue", nl=False)
 

twitchdl/twitch.py

@@ -3,7 +3,9 @@ Twitch API access.
 """
 
 import json
-from typing import Dict, Generator, List, Literal, Mapping, Optional, Tuple, TypedDict
+import logging
+import time
+from typing import Any, Dict, Generator, List, Literal, Mapping, Optional, Tuple, TypedDict, Union
 
 import click
 import httpx
@@ -87,10 +89,22 @@ class GQLError(click.ClickException):
         super().__init__(message)
 
 
-def authenticated_post(url, data=None, json=None, headers={}):
-    headers["Client-ID"] = CLIENT_ID
+Content = Union[str, bytes]
+Headers = Dict[str, str]
 
-    response = httpx.post(url, data=data, json=json, headers=headers)
+
+def authenticated_post(
+    url: str,
+    *,
+    json: Any = None,
+    content: Optional[Content] = None,
+    auth_token: Optional[str] = None,
+):
+    headers = {"Client-ID": CLIENT_ID}
+    if auth_token is not None:
+        headers["authorization"] = f"OAuth {auth_token}"
+
+    response = request("POST", url, content=content, json=json, headers=headers)
     if response.status_code == 400:
         data = response.json()
         raise ConsoleError(data["message"])
@@ -100,16 +114,50 @@ def authenticated_post(url, data=None, json=None, headers={}):
     return response
 
 
+def request(
+    method: str,
+    url: str,
+    json: Any = None,
+    content: Optional[Content] = None,
+    headers: Optional[Mapping[str, str]] = None,
+):
+    with httpx.Client() as client:
+        request = client.build_request(method, url, json=json, content=content, headers=headers)
+        log_request(request)
+        start = time.time()
+        response = client.send(request)
+        duration = time.time() - start
+        log_response(response, duration)
+        return response
+
+
+logger = logging.getLogger(__name__)
+
+
+def log_request(request: httpx.Request):
+    logger.debug(f"--> {request.method} {request.url}")
+    if request.content:
+        logger.debug(f"--> {request.content}")
+
+
+def log_response(response: httpx.Response, duration: float):
+    request = response.request
+    duration_ms = int(1000 * duration)
+    logger.debug(f"<-- {request.method} {request.url} HTTP {response.status_code} {duration_ms}ms")
+    if response.content:
+        logger.debug(f"<-- {response.content}")
+
+
 def gql_post(query: str):
     url = "https://gql.twitch.tv/gql"
-    response = authenticated_post(url, data=query)
+    response = authenticated_post(url, content=query)
     gql_raise_on_error(response)
     return response.json()
 
 
-def gql_query(query: str, headers: Dict[str, str] = {}):
+def gql_query(query: str, auth_token: Optional[str] = None):
     url = "https://gql.twitch.tv/gql"
-    response = authenticated_post(url, json={"query": query}, headers=headers)
+    response = authenticated_post(url, json={"query": query}, auth_token=auth_token)
     gql_raise_on_error(response)
     return response.json()
 
@@ -303,6 +351,7 @@ def get_channel_videos(
     after: Optional[str] = None,
 ):
     game_ids = game_ids or []
+    game_ids_str = f"[{','.join(game_ids)}]"
 
     query = f"""
     {{
@@ -313,7 +362,7 @@ def get_channel_videos(
                 sort: {sort.upper()},
                 after: "{after or ''}",
                 options: {{
-                    gameIDs: {game_ids}
+                    gameIDs: {game_ids_str}
                 }}
             ) {{
                 totalCount
@@ -386,12 +435,8 @@ def get_access_token(video_id: str, auth_token: Optional[str] = None) -> AccessT
     }}
     """
 
-    headers: Mapping[str, str] = {}
-    if auth_token is not None:
-        headers["authorization"] = f"OAuth {auth_token}"
-
     try:
-        response = gql_query(query, headers=headers)
+        response = gql_query(query, auth_token=auth_token)
         return response["data"]["videoPlaybackAccessToken"]
     except httpx.HTTPStatusError as error:
         # Provide a more useful error message when server returns HTTP 401

twitchdl/utils.py

@@ -84,13 +84,13 @@ def titlify(value: str) -> str:
 
 VIDEO_PATTERNS = [
     r"^(?P<id>\d+)?$",
-    r"^https://(www.)?twitch.tv/videos/(?P<id>\d+)(\?.+)?$",
+    r"^https://(www\.|m\.)?twitch\.tv/videos/(?P<id>\d+)(\?.+)?$",
 ]
 
 CLIP_PATTERNS = [
     r"^(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)$",
-    r"^https://(www.)?twitch.tv/\w+/clip/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
-    r"^https://clips.twitch.tv/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
+    r"^https://(www\.|m\.)?twitch\.tv/\w+/clip/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
+    r"^https://clips\.twitch\.tv/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
 ]
 
 

video_before.json

@@ -1,203 +0,0 @@
-{
-  "id": "2115833882",
-  "title": "Games I Feel Like Speedrun Marathon !newyt !Slender !Merch",
-  "description": null,
-  "publishedAt": "2024-04-10T05:01:12Z",
-  "broadcastType": "ARCHIVE",
-  "lengthSeconds": 30163,
-  "game": {
-    "id": "21063",
-    "name": "Saw"
-  },
-  "creator": {
-    "login": "ecdycis",
-    "displayName": "Ecdycis"
-  },
-  "playlists": [
-    {
-      "bandwidth": 6395968,
-      "resolution": [
-        1920,
-        1080
-      ],
-      "codecs": "avc1.64002A,mp4a.40.2",
-      "video": "chunked",
-      "uri": "https://d2nvs31859zcd8.cloudfront.net/3104d817048588f268fa_ecdycis_43996750059_1712725267/chunked/index-muted-GKGIUOE29B.m3u8"
-    },
-    {
-      "bandwidth": 3430341,
-      "resolution": [
-        1280,
-        720
-      ],
-      "codecs": "avc1.4D0020,mp4a.40.2",
-      "video": "720p60",
-      "uri": "https://d2nvs31859zcd8.cloudfront.net/3104d817048588f268fa_ecdycis_43996750059_1712725267/720p60/index-muted-GKGIUOE29B.m3u8"
-    },
-    {
-      "bandwidth": 1448243,
-      "resolution": [
-        852,
-        480
-      ],
-      "codecs": "avc1.4D001F,mp4a.40.2",
-      "video": "480p30",
-      "uri": "https://d2nvs31859zcd8.cloudfront.net/3104d817048588f268fa_ecdycis_43996750059_1712725267/480p30/index-muted-GKGIUOE29B.m3u8"
-    },
-    {
-      "bandwidth": 215544,
-      "resolution": null,
-      "codecs": "mp4a.40.2",
-      "video": "audio_only",
-      "uri": "https://d2nvs31859zcd8.cloudfront.net/3104d817048588f268fa_ecdycis_43996750059_1712725267/audio_only/index-muted-GKGIUOE29B.m3u8"
-    },
-    {
-      "bandwidth": 709051,
-      "resolution": [
-        640,
-        360
-      ],
-      "codecs": "avc1.4D001E,mp4a.40.2",
-      "video": "360p30",
-      "uri": "https://d2nvs31859zcd8.cloudfront.net/3104d817048588f268fa_ecdycis_43996750059_1712725267/360p30/index-muted-GKGIUOE29B.m3u8"
-    },
-    {
-      "bandwidth": 288844,
-      "resolution": [
-        284,
-        160
-      ],
-      "codecs": "avc1.4D000C,mp4a.40.2",
-      "video": "160p30",
-      "uri": "https://d2nvs31859zcd8.cloudfront.net/3104d817048588f268fa_ecdycis_43996750059_1712725267/160p30/index-muted-GKGIUOE29B.m3u8"
-    }
-  ],
-  "chapters": [
-    {
-      "id": "6049e803f47a0b9cf64ce95adcf3d96d",
-      "durationMilliseconds": 4891000,
-      "positionMilliseconds": 0,
-      "type": "GAME_CHANGE",
-      "description": "Saw",
-      "subDescription": "",
-      "thumbnailURL": "",
-      "video": {
-        "id": "2115833882",
-        "lengthSeconds": 30163,
-        "__typename": "Video"
-      },
-      "__typename": "VideoMoment",
-      "game": {
-        "id": "21063",
-        "displayName": "Saw",
-        "boxArtURL": "https://static-cdn.jtvnw.net/ttv-boxart/21063_IGDB-40x53.jpg",
-        "__typename": "Game"
-      }
-    },
-    {
-      "id": "c3be4ce5a7bae802d7a6df02baf62a85",
-      "durationMilliseconds": 6235000,
-      "positionMilliseconds": 4891000,
-      "type": "GAME_CHANGE",
-      "description": "Resident Evil 7: Biohazard",
-      "subDescription": "",
-      "thumbnailURL": "",
-      "video": {
-        "id": "2115833882",
-        "lengthSeconds": 30163,
-        "__typename": "Video"
-      },
-      "__typename": "VideoMoment",
-      "game": {
-        "id": "492934",
-        "displayName": "Resident Evil 7: Biohazard",
-        "boxArtURL": "https://static-cdn.jtvnw.net/ttv-boxart/492934_IGDB-40x53.jpg",
-        "__typename": "Game"
-      }
-    },
-    {
-      "id": "bd584f17bc1f91fc11ed14dcec5e4742",
-      "durationMilliseconds": 3401000,
-      "positionMilliseconds": 11126000,
-      "type": "GAME_CHANGE",
-      "description": "Silent Hill: Homecoming",
-      "subDescription": "",
-      "thumbnailURL": "",
-      "video": {
-        "id": "2115833882",
-        "lengthSeconds": 30163,
-        "__typename": "Video"
-      },
-      "__typename": "VideoMoment",
-      "game": {
-        "id": "18864",
-        "displayName": "Silent Hill: Homecoming",
-        "boxArtURL": "https://static-cdn.jtvnw.net/ttv-boxart/18864_IGDB-40x53.jpg",
-        "__typename": "Game"
-      }
-    },
-    {
-      "id": "9f4be8bc7b5bf398213bc602a4b39c4d",
-      "durationMilliseconds": 3490000,
-      "positionMilliseconds": 14527000,
-      "type": "GAME_CHANGE",
-      "description": "Silent Hill 2",
-      "subDescription": "",
-      "thumbnailURL": "",
-      "video": {
-        "id": "2115833882",
-        "lengthSeconds": 30163,
-        "__typename": "Video"
-      },
-      "__typename": "VideoMoment",
-      "game": {
-        "id": "9891",
-        "displayName": "Silent Hill 2",
-        "boxArtURL": "https://static-cdn.jtvnw.net/ttv-boxart/9891_IGDB-40x53.jpg",
-        "__typename": "Game"
-      }
-    },
-    {
-      "id": "7a42d537681decc10660c33f9071a37f",
-      "durationMilliseconds": 7241000,
-      "positionMilliseconds": 18017000,
-      "type": "GAME_CHANGE",
-      "description": "The Darkness",
-      "subDescription": "",
-      "thumbnailURL": "",
-      "video": {
-        "id": "2115833882",
-        "lengthSeconds": 30163,
-        "__typename": "Video"
-      },
-      "__typename": "VideoMoment",
-      "game": {
-        "id": "8448",
-        "displayName": "The Darkness",
-        "boxArtURL": "https://static-cdn.jtvnw.net/ttv-boxart/8448_IGDB-40x53.jpg",
-        "__typename": "Game"
-      }
-    },
-    {
-      "id": "64418186f436d80b1359d2e6686222ef",
-      "durationMilliseconds": 4905000,
-      "positionMilliseconds": 25258000,
-      "type": "GAME_CHANGE",
-      "description": "Silent Hill 4: The Room",
-      "subDescription": "",
-      "thumbnailURL": "",
-      "video": {
-        "id": "2115833882",
-        "lengthSeconds": 30163,
-        "__typename": "Video"
-      },
-      "__typename": "VideoMoment",
-      "game": {
-        "id": "5804",
-        "displayName": "Silent Hill 4: The Room",
-        "boxArtURL": "https://static-cdn.jtvnw.net/ttv-boxart/5804_IGDB-40x53.jpg",
-        "__typename": "Game"
-      }
-    }
-  ]
-}