Trim line when printing table, simplify code

Looks nicer

.github/workflows/test.yml

@@ -0,0 +1,27 @@
+name: Run tests
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[test]"
+      - name: Run tests
+        run: |
+          pytest
+      - name: Validate minimum required version
+        run: |
+          vermin --no-tips twitchdl

.gitignore

@@ -15,3 +15,5 @@ tmp/
 /*.pyz
 /pyrightconfig.json
 /book
+*.mp4
+*.mkv

CHANGELOG.md

@@ -3,6 +3,37 @@ twitch-dl changelog
 
 <!-- Do not edit. This file is automatically generated from changelog.yaml.-->
 
+### [2.5.0 (2024-08-30)](https://github.com/ihabunek/twitch-dl/releases/tag/2.5.0)
+
+* Add support for HD video qualities (#163)
+
+### [2.4.0 (2024-08-30)](https://github.com/ihabunek/twitch-dl/releases/tag/2.4.0)
+
+* Add `clips --target-dir` option. Use in conjunction with `--download` to
+  specify target directory.
+* Fix a crash when downloading clips (#160)
+* Handle video URLs which contain the channel name (#162)
+* Don't stop downloading clips if one download fails
+
+### [2.3.1 (2024-05-19)](https://github.com/ihabunek/twitch-dl/releases/tag/2.3.1)
+
+* Fix fetching access token (#155, thanks @KryptonicDragon)
+
+### [2.3.0 (2024-04-27)](https://github.com/ihabunek/twitch-dl/releases/tag/2.3.0)
+
+* Show more playlist data when choosing quality
+* Improve detection of 'source' quality for Twitch Enhanced Broadcast Streams
+  (#154)
+
+### [2.2.4 (2024-04-25)](https://github.com/ihabunek/twitch-dl/releases/tag/2.2.4)
+
+* Add m dot url support to video and clip regexes (thanks @localnerve)
+
+### [2.2.3 (2024-04-24)](https://github.com/ihabunek/twitch-dl/releases/tag/2.2.3)
+
+* Respect --dry-run option when downloading videos
+* Add automated tests on github actions
+
 ### [2.2.2 (2024-04-23)](https://github.com/ihabunek/twitch-dl/releases/tag/2.2.2)
 
 * Fix more compat issues Python < 3.10 (#152)

Makefile

@@ -7,7 +7,7 @@ dist:
 
 clean :
 	find . -name "*pyc" | xargs rm -rf $1
-	rm -rf build dist bundle MANIFEST htmlcov deb_dist twitch-dl.*.pyz twitch-dl.1.man twitch_dl.egg-info
+	rm -rf build dist book bundle MANIFEST htmlcov deb_dist twitch-dl.*.pyz twitch-dl.1.man twitch_dl.egg-info
 
 bundle:
 	mkdir bundle
@@ -24,7 +24,7 @@ publish :
 	twine upload dist/*.tar.gz dist/*.whl
 
 coverage:
-	py.test --cov=toot --cov-report html tests/
+	pytest --cov=twitchdl --cov-report html tests/
 
 man:
 	scdoc < twitch-dl.1.scd > twitch-dl.1.man

changelog.yaml

@@ -1,3 +1,38 @@
+2.5.0:
+  date: 2024-08-30
+  changes:
+    - "Add support for HD video qualities (#163)"
+
+2.4.0:
+  date: 2024-08-30
+  changes:
+    - "Add `clips --target-dir` option. Use in conjunction with `--download` to specify target directory."
+    - "Fix a crash when downloading clips (#160)"
+    - "Handle video URLs which contain the channel name (#162)"
+    - "Don't stop downloading clips if one download fails"
+
+2.3.1:
+  date: 2024-05-19
+  changes:
+    - "Fix fetching access token (#155, thanks @KryptonicDragon)"
+
+2.3.0:
+  date: 2024-04-27
+  changes:
+    - "Show more playlist data when choosing quality"
+    - "Improve detection of 'source' quality for Twitch Enhanced Broadcast Streams (#154)"
+
+2.2.4:
+  date: 2024-04-25
+  changes:
+    - "Add m dot url support to video and clip regexes (thanks @localnerve)"
+
+2.2.3:
+  date: 2024-04-24
+  changes:
+    - "Respect --dry-run option when downloading videos"
+    - "Add automated tests on github actions"
+
 2.2.2:
   date: 2024-04-23
   changes:

docs/changelog.md

@@ -3,6 +3,37 @@ twitch-dl changelog
 
 <!-- Do not edit. This file is automatically generated from changelog.yaml.-->
 
+### [2.5.0 (2024-08-30)](https://github.com/ihabunek/twitch-dl/releases/tag/2.5.0)
+
+* Add support for HD video qualities (#163)
+
+### [2.4.0 (2024-08-30)](https://github.com/ihabunek/twitch-dl/releases/tag/2.4.0)
+
+* Add `clips --target-dir` option. Use in conjunction with `--download` to
+  specify target directory.
+* Fix a crash when downloading clips (#160)
+* Handle video URLs which contain the channel name (#162)
+* Don't stop downloading clips if one download fails
+
+### [2.3.1 (2024-05-19)](https://github.com/ihabunek/twitch-dl/releases/tag/2.3.1)
+
+* Fix fetching access token (#155, thanks @KryptonicDragon)
+
+### [2.3.0 (2024-04-27)](https://github.com/ihabunek/twitch-dl/releases/tag/2.3.0)
+
+* Show more playlist data when choosing quality
+* Improve detection of 'source' quality for Twitch Enhanced Broadcast Streams
+  (#154)
+
+### [2.2.4 (2024-04-25)](https://github.com/ihabunek/twitch-dl/releases/tag/2.2.4)
+
+* Add m dot url support to video and clip regexes (thanks @localnerve)
+
+### [2.2.3 (2024-04-24)](https://github.com/ihabunek/twitch-dl/releases/tag/2.2.3)
+
+* Respect --dry-run option when downloading videos
+* Add automated tests on github actions
+
 ### [2.2.2 (2024-04-23)](https://github.com/ihabunek/twitch-dl/releases/tag/2.2.2)
 
 * Fix more compat issues Python < 3.10 (#152)

docs/commands/clips.md

@@ -43,6 +43,11 @@ twitch-dl clips [OPTIONS] CHANNEL_NAME
     <td>Period from which to return clips Possible values: <code>last_day</code>, <code>last_week</code>, <code>last_month</code>, <code>all_time</code>. [default: <code>all_time</code>]</td>
 </tr>
 
+<tr>
+    <td class="code">-t, --target-dir</td>
+    <td>Target directory when downloading clips [default: <code>.</code>]</td>
+</tr>
+
 <tr>
     <td class="code">--json</td>
     <td>Print data as JSON rather than human readable text</td>

pyproject.toml

@@ -22,7 +22,7 @@ classifiers = [
 dependencies = [
     "click>=8.0.0,<9.0.0",
     "httpx>=0.17.0,<1.0.0",
-    "m3u8>=1.0.0,<4.0.0",
+    "m3u8>=3.0.0,<7.0.0",
 ]
 
 [tool.setuptools]
@@ -43,6 +43,11 @@ dev = [
     "vermin",
 ]
 
+test = [
+    "pytest",
+    "vermin",
+]
+
 [project.urls]
 "Homepage" = "https://twitch-dl.bezdomni.net/"
 "Source" = "https://github.com/ihabunek/twitch-dl"
@@ -51,7 +56,6 @@ dev = [
 twitch-dl = "twitchdl.cli:cli"
 
 [tool.pyright]
-include = ["twitchdl"]
 typeCheckingMode = "strict"
 pythonVersion = "3.8"
 

tests/test_api.py

@@ -3,9 +3,12 @@ These tests depend on the channel having some videos and clips published.
 """
 
 import httpx
+import pytest
 
 from twitchdl import twitch
 from twitchdl.commands.download import get_clip_authenticated_url
+from twitchdl.commands.videos import get_game_ids
+from twitchdl.exceptions import ConsoleError
 from twitchdl.playlists import enumerate_vods, load_m3u8, parse_playlists
 
 TEST_CHANNEL = "bananasaurus_rex"
@@ -53,3 +56,15 @@ def test_get_clips():
     assert clip["slug"] == slug
 
     assert get_clip_authenticated_url(slug, "source")
+
+
+def test_get_games():
+    assert get_game_ids([]) == []
+    assert get_game_ids(["Bioshock"]) == ["15866"]
+    assert get_game_ids(["Bioshock", "Portal"]) == ["15866", "6187"]
+
+
+def test_get_games_not_found():
+    with pytest.raises(ConsoleError) as ex:
+        get_game_ids(["the game which does not exist"])
+    assert str(ex.value) == "Game 'the game which does not exist' not found"

tests/test_cli.py

@@ -0,0 +1,156 @@
+import json
+
+import pytest
+from click.testing import CliRunner, Result
+
+from twitchdl import cli
+
+
+@pytest.fixture(scope="session")
+def runner():
+    return CliRunner(mix_stderr=False)
+
+
+def assert_ok(result: Result):
+    if result.exit_code != 0:
+        raise AssertionError(
+            f"Command failed with exit code {result.exit_code}\nStderr: {result.stderr}"
+        )
+
+
+def test_info_video(runner: CliRunner):
+    result = runner.invoke(cli.info, ["2090131595"])
+    assert_ok(result)
+
+    assert "Frost Fatales 2024 Day 1" in result.stdout
+    assert "frozenflygone playing Tomb Raider" in result.stdout
+
+
+def test_info_video_json(runner: CliRunner):
+    result = runner.invoke(cli.info, ["2090131595", "--json"])
+    assert_ok(result)
+
+    video = json.loads(result.stdout)
+    assert video["title"] == "Frost Fatales 2024 Day 1"
+    assert video["game"] == {"id": "2770", "name": "Tomb Raider"}
+    assert video["creator"] == {"login": "frozenflygone", "displayName": "frozenflygone"}
+
+
+def test_info_clip(runner: CliRunner):
+    result = runner.invoke(cli.info, ["PoisedTalentedPuddingChefFrank"])
+    assert_ok(result)
+
+    assert "AGDQ Crashes during Bioshock run" in result.stdout
+    assert "GamesDoneQuick playing BioShock" in result.stdout
+
+
+def test_info_clip_json(runner: CliRunner):
+    result = runner.invoke(cli.info, ["PoisedTalentedPuddingChefFrank", "--json"])
+    assert_ok(result)
+
+    clip = json.loads(result.stdout)
+    assert clip["slug"] == "PoisedTalentedPuddingChefFrank"
+    assert clip["title"] == "AGDQ Crashes during Bioshock run"
+    assert clip["game"] == {"id": "15866", "name": "BioShock"}
+    assert clip["broadcaster"] == {"displayName": "GamesDoneQuick", "login": "gamesdonequick"}
+
+
+def test_info_not_found(runner: CliRunner):
+    result = runner.invoke(cli.info, ["banana"])
+    assert result.exit_code == 1
+    assert "Clip banana not found" in result.stderr
+
+    result = runner.invoke(cli.info, ["12345"])
+    assert result.exit_code == 1
+    assert "Video 12345 not found" in result.stderr
+
+    result = runner.invoke(cli.info, [""])
+    assert result.exit_code == 1
+    assert "Invalid input" in result.stderr
+
+
+def test_download_clip(runner: CliRunner):
+    result = runner.invoke(
+        cli.download,
+        [
+            "PoisedTalentedPuddingChefFrank",
+            "-q",
+            "source",
+            "--dry-run",
+        ],
+    )
+    assert_ok(result)
+    assert (
+        "Found: AGDQ Crashes during Bioshock run by GamesDoneQuick, playing BioShock (30 sec)"
+        in result.stdout
+    )
+    assert (
+        "Target: 2020-01-10_3099545841_gamesdonequick_agdq_crashes_during_bioshock_run.mp4"
+        in result.stdout
+    )
+    assert "Dry run, clip not downloaded." in result.stdout
+
+
+def test_download_video(runner: CliRunner):
+    result = runner.invoke(
+        cli.download,
+        [
+            "2090131595",
+            "-q",
+            "source",
+            "--dry-run",
+        ],
+    )
+    assert_ok(result)
+    assert "Found: Frost Fatales 2024 Day 1 by frozenflygone" in result.stdout
+    assert (
+        "Output: 2024-03-14_2090131595_frozenflygone_frost_fatales_2024_day_1.mkv" in result.stdout
+    )
+    assert "Dry run, video not downloaded." in result.stdout
+
+
+def test_videos(runner: CliRunner):
+    result = runner.invoke(cli.videos, ["gamesdonequick", "--json"])
+    assert_ok(result)
+    videos = json.loads(result.stdout)
+
+    assert videos["count"] == 10
+    assert videos["totalCount"] > 0
+    video = videos["videos"][0]
+
+    result = runner.invoke(cli.videos, "gamesdonequick")
+    assert_ok(result)
+
+    assert f"Video {video['id']}" in result.stdout
+    assert video["title"] in result.stdout
+
+    result = runner.invoke(cli.videos, ["gamesdonequick", "--compact"])
+    assert_ok(result)
+
+    assert video["id"] in result.stdout
+    assert video["title"][:60] in result.stdout
+
+
+def test_videos_channel_not_found(runner: CliRunner):
+    result = runner.invoke(cli.videos, ["doesnotexisthopefully"])
+    assert result.exit_code == 1
+    assert result.stderr.strip() == "Error: Channel doesnotexisthopefully not found"
+
+
+def test_clips(runner: CliRunner):
+    result = runner.invoke(cli.clips, ["gamesdonequick", "--json"])
+    assert_ok(result)
+    clips = json.loads(result.stdout)
+    clip = clips[0]
+
+    result = runner.invoke(cli.clips, "gamesdonequick")
+    assert_ok(result)
+
+    assert f"Clip {clip['slug']}" in result.stdout
+    assert clip["title"] in result.stdout
+
+    result = runner.invoke(cli.clips, ["gamesdonequick", "--compact"])
+    assert_ok(result)
+
+    assert clip["slug"] in result.stdout
+    assert clip["title"][:60] in result.stdout

tests/test_patterns.py

@@ -1,35 +1,39 @@
 import pytest
 
-from twitchdl.utils import parse_video_identifier, parse_clip_identifier
-
+from twitchdl.utils import parse_clip_identifier, parse_video_identifier
 
 TEST_VIDEO_PATTERNS = [
     ("702689313", "702689313"),
     ("702689313", "https://twitch.tv/videos/702689313"),
     ("702689313", "https://www.twitch.tv/videos/702689313"),
+    ("702689313", "https://m.twitch.tv/videos/702689313"),
+    ("2223719525", "https://www.twitch.tv/r0dn3y/video/2223719525"),
 ]
 
 TEST_CLIP_PATTERNS = {
     ("AbrasivePlayfulMangoMau5", "AbrasivePlayfulMangoMau5"),
     ("AbrasivePlayfulMangoMau5", "https://clips.twitch.tv/AbrasivePlayfulMangoMau5"),
     ("AbrasivePlayfulMangoMau5", "https://www.twitch.tv/dracul1nx/clip/AbrasivePlayfulMangoMau5"),
+    ("AbrasivePlayfulMangoMau5", "https://m.twitch.tv/dracul1nx/clip/AbrasivePlayfulMangoMau5"),
     ("AbrasivePlayfulMangoMau5", "https://twitch.tv/dracul1nx/clip/AbrasivePlayfulMangoMau5"),
     ("HungryProudRadicchioDoggo", "HungryProudRadicchioDoggo"),
     ("HungryProudRadicchioDoggo", "https://clips.twitch.tv/HungryProudRadicchioDoggo"),
     ("HungryProudRadicchioDoggo", "https://www.twitch.tv/bananasaurus_rex/clip/HungryProudRadicchioDoggo?filter=clips&range=7d&sort=time"),
+    ("HungryProudRadicchioDoggo", "https://m.twitch.tv/bananasaurus_rex/clip/HungryProudRadicchioDoggo?filter=clips&range=7d&sort=time"),
     ("HungryProudRadicchioDoggo", "https://twitch.tv/bananasaurus_rex/clip/HungryProudRadicchioDoggo?filter=clips&range=7d&sort=time"),
     ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ"),
     ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ"),
     ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ?filter=clips&range=7d&sort=time"),
     ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://www.twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ?filter=clips&range=7d&sort=time"),
+    ("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://m.twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ?filter=clips&range=7d&sort=time"),
 }
 
 
 @pytest.mark.parametrize("expected,input", TEST_VIDEO_PATTERNS)
-def test_video_patterns(expected, input):
+def test_video_patterns(expected: str, input: str):
     assert parse_video_identifier(input) == expected
 
 
 @pytest.mark.parametrize("expected,input", TEST_CLIP_PATTERNS)
-def test_clip_patterns(expected, input):
+def test_clip_patterns(expected: str, input: str):
     assert parse_clip_identifier(input) == expected

tests/test_progress.py

@@ -8,8 +8,8 @@ def test_initial_values():
     assert progress.progress_perc == 0
     assert progress.remaining_time is None
     assert progress.speed is None
-    assert progress.vod_count == 10
-    assert progress.vod_downloaded_count == 0
+    assert progress.file_count == 10
+    assert progress.downloaded_count == 0
 
 
 def test_downloaded():
@@ -23,26 +23,31 @@ def test_downloaded():
     assert progress.progress_perc == 0
 
     progress.advance(1, 100)
+    progress._recalculate()
     assert progress.downloaded == 100
     assert progress.progress_bytes == 100
     assert progress.progress_perc == 11
 
     progress.advance(2, 200)
+    progress._recalculate()
     assert progress.downloaded == 300
     assert progress.progress_bytes == 300
     assert progress.progress_perc == 33
 
     progress.advance(3, 150)
+    progress._recalculate()
     assert progress.downloaded == 450
     assert progress.progress_bytes == 450
     assert progress.progress_perc == 50
 
     progress.advance(1, 50)
+    progress._recalculate()
     assert progress.downloaded == 500
     assert progress.progress_bytes == 500
     assert progress.progress_perc == 55
 
     progress.abort(2)
+    progress._recalculate()
     assert progress.downloaded == 500
     assert progress.progress_bytes == 300
     assert progress.progress_perc == 33
@@ -52,6 +57,7 @@ def test_downloaded():
     progress.advance(1, 150)
     progress.advance(2, 300)
     progress.advance(3, 150)
+    progress._recalculate()
 
     assert progress.downloaded == 1100
     assert progress.progress_bytes == 900
@@ -71,12 +77,15 @@ def test_estimated_total():
     assert progress.estimated_total is None
 
     progress.start(1, 12000)
+    progress._recalculate()
     assert progress.estimated_total == 12000 * 3
 
     progress.start(2, 11000)
+    progress._recalculate()
     assert progress.estimated_total == 11500 * 3
 
     progress.start(3, 10000)
+    progress._recalculate()
     assert progress.estimated_total == 11000 * 3
 
 
@@ -87,16 +96,16 @@ def test_vod_downloaded_count():
     progress.start(2, 100)
     progress.start(3, 100)
 
-    assert progress.vod_downloaded_count == 0
+    assert progress.downloaded_count == 0
 
     progress.advance(1, 100)
     progress.end(1)
-    assert progress.vod_downloaded_count == 1
+    assert progress.downloaded_count == 1
 
     progress.advance(2, 100)
     progress.end(2)
-    assert progress.vod_downloaded_count == 2
+    assert progress.downloaded_count == 2
 
     progress.advance(3, 100)
     progress.end(3)
-    assert progress.vod_downloaded_count == 3
+    assert progress.downloaded_count == 3

twitchdl/cli.py

@@ -2,12 +2,14 @@ import logging
 import platform
 import re
 import sys
-from typing import Optional
+from pathlib import Path
+from typing import Optional, Tuple
 
 import click
 
 from twitchdl import __version__
 from twitchdl.entities import DownloadOptions
+from twitchdl.naming import DEFAULT_OUTPUT_TEMPLATE
 from twitchdl.twitch import ClipsPeriod, VideosSort, VideosType
 
 # Tweak the Click context
@@ -79,11 +81,12 @@ def validate_rate(_ctx: click.Context, _param: click.Parameter, value: str) -> O
 
 
 @click.group(context_settings=CONTEXT)
-@click.option("--debug/--no-debug", default=False, help="Log debug info to stderr")
+@click.option("--debug/--no-debug", default=False, help="Enable debug logging to stderr")
+@click.option("--verbose/--no-verbose", default=False, help="More verbose debug logging")
 @click.option("--color/--no-color", default=sys.stdout.isatty(), help="Use ANSI color in output")
 @click.version_option(package_name="twitch-dl")
 @click.pass_context
-def cli(ctx: click.Context, color: bool, debug: bool):
+def cli(ctx: click.Context, color: bool, debug: bool, verbose: bool):
     """twitch-dl - twitch.tv downloader
 
     https://twitch-dl.bezdomni.net/
@@ -91,7 +94,9 @@ def cli(ctx: click.Context, color: bool, debug: bool):
     ctx.color = color
 
     if debug:
-        logging.basicConfig(level=logging.INFO)
+        logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO)
+        logging.getLogger("httpx").setLevel(logging.WARN)
+        logging.getLogger("httpcore").setLevel(logging.WARN)
 
 
 @cli.command()
@@ -137,6 +142,18 @@ def cli(ctx: click.Context, color: bool, debug: bool):
     default="all_time",
     type=click.Choice(["last_day", "last_week", "last_month", "all_time"]),
 )
+@click.option(
+    "-t",
+    "--target-dir",
+    help="Target directory when downloading clips",
+    type=click.Path(
+        file_okay=False,
+        readable=False,
+        writable=True,
+        path_type=Path,
+    ),
+    default=Path(),
+)
 @json_option
 def clips(
     channel_name: str,
@@ -147,10 +164,14 @@ def clips(
     limit: Optional[int],
     pager: Optional[int],
     period: ClipsPeriod,
+    target_dir: Path,
 ):
     """List or download clips for given CHANNEL_NAME."""
     from twitchdl.commands.clips import clips
 
+    if not target_dir.exists():
+        target_dir.mkdir(parents=True, exist_ok=True)
+
     clips(
         channel_name,
         all=all,
@@ -160,6 +181,7 @@ def clips(
         limit=limit,
         pager=pager,
         period=period,
+        target_dir=target_dir,
     )
 
 
@@ -227,7 +249,7 @@ def clips(
     "-o",
     "--output",
     help="Output file name template. See docs for details.",
-    default="{date}_{id}_{channel_login}_{title_slug}.{format}",
+    default=DEFAULT_OUTPUT_TEMPLATE,
 )
 @click.option(
     "-q",
@@ -255,7 +277,7 @@ def clips(
     default=5,
 )
 def download(
-    ids: tuple[str, ...],
+    ids: Tuple[str, ...],
     auth_token: Optional[str],
     chapter: Optional[int],
     concat: bool,
@@ -374,7 +396,7 @@ def videos(
     channel_name: str,
     all: bool,
     compact: bool,
-    games_tuple: tuple[str, ...],
+    games_tuple: Tuple[str, ...],
     json: bool,
     limit: Optional[int],
     pager: Optional[int],

twitchdl/commands/clips.py

@@ -1,13 +1,15 @@
 import re
 import sys
 from os import path
-from typing import Callable, Generator, Optional
+from pathlib import Path
+from typing import Callable, Generator, List, Optional
 
 import click
 
 from twitchdl import twitch, utils
 from twitchdl.commands.download import get_clip_authenticated_url
-from twitchdl.download import download_file
+from twitchdl.entities import VideoQuality
+from twitchdl.http import download_file
 from twitchdl.output import green, print_clip, print_clip_compact, print_json, print_paged, yellow
 from twitchdl.twitch import Clip, ClipsPeriod
 
@@ -22,6 +24,7 @@ def clips(
     limit: Optional[int] = None,
     pager: Optional[int] = None,
     period: ClipsPeriod = "all_time",
+    target_dir: Path = Path(),
 ):
     # Set different defaults for limit for compact display
     default_limit = 40 if compact else 10
@@ -35,7 +38,7 @@ def clips(
         return print_json(list(generator))
 
     if download:
-        return _download_clips(generator)
+        return _download_clips(target_dir, generator)
 
     print_fn = print_clip_compact if compact else print_clip
 
@@ -45,8 +48,8 @@ def clips(
     return _print_all(generator, print_fn, all)
 
 
-def _target_filename(clip: Clip):
-    url = clip["videoQualities"][0]["sourceURL"]
+def _target_filename(clip: Clip, video_qualities: List[VideoQuality]):
+    url = video_qualities[0]["sourceURL"]
     _, ext = path.splitext(url)
     ext = ext.lstrip(".")
 
@@ -67,16 +70,27 @@ def _target_filename(clip: Clip):
     return f"{name}.{ext}"
 
 
-def _download_clips(generator: Generator[Clip, None, None]):
-    for clip in generator:
-        target = _target_filename(clip)
+def _download_clips(target_dir: Path, generator: Generator[Clip, None, None]):
+    if not target_dir.exists():
+        target_dir.mkdir(parents=True, exist_ok=True)
 
-        if path.exists(target):
+    for clip in generator:
+        # videoQualities can be null in some circumstances, see:
+        # https://github.com/ihabunek/twitch-dl/issues/160
+        if not clip["videoQualities"]:
+            continue
+
+        target = target_dir / _target_filename(clip, clip["videoQualities"])
+
+        if target.exists():
             click.echo(f"Already downloaded: {green(target)}")
         else:
-            url = get_clip_authenticated_url(clip["slug"], "source")
-            click.echo(f"Downloading: {yellow(target)}")
-            download_file(url, target)
+            try:
+                url = get_clip_authenticated_url(clip["slug"], "source")
+                click.echo(f"Downloading: {yellow(target)}")
+                download_file(url, target)
+            except Exception as ex:
+                click.secho(ex, err=True, fg="red")
 
 
 def _print_all(

twitchdl/commands/download.py

@@ -1,35 +1,39 @@
 import asyncio
-import os
 import platform
 import re
+import shlex
 import shutil
 import subprocess
 import tempfile
-from os import path
 from pathlib import Path
-from typing import Dict, List
+from typing import List, Optional
 from urllib.parse import urlencode, urlparse
 
 import click
 import httpx
 
 from twitchdl import twitch, utils
-from twitchdl.download import download_file
 from twitchdl.entities import DownloadOptions
 from twitchdl.exceptions import ConsoleError
-from twitchdl.http import download_all
+from twitchdl.http import download_all, download_file
+from twitchdl.naming import clip_filename, video_filename
 from twitchdl.output import blue, bold, green, print_log, yellow
 from twitchdl.playlists import (
     enumerate_vods,
+    get_init_sections,
     load_m3u8,
     make_join_playlist,
     parse_playlists,
     select_playlist,
 )
-from twitchdl.twitch import Chapter, Clip, ClipAccessToken, Video
+from twitchdl.twitch import Chapter, ClipAccessToken, Video
 
 
 def download(ids: List[str], args: DownloadOptions):
+    if not ids:
+        print_log("No IDs to downlad given")
+        return
+
     for video_id in ids:
         download_one(video_id, args)
 
@@ -46,14 +50,14 @@ def download_one(video: str, args: DownloadOptions):
     raise ConsoleError(f"Invalid input: {video}")
 
 
-def _join_vods(playlist_path: str, target: str, overwrite: bool, video: Video):
+def _join_vods(playlist_path: Path, target: Path, overwrite: bool, video: Video):
     description = video["description"] or ""
     description = description.strip()
 
-    command = [
+    command: List[str] = [
         "ffmpeg",
         "-i",
-        playlist_path,
+        str(playlist_path),
         "-c",
         "copy",
         "-metadata",
@@ -73,15 +77,15 @@ def _join_vods(playlist_path: str, target: str, overwrite: bool, video: Video):
     if overwrite:
         command.append("-y")
 
-    click.secho(f"{' '.join(command)}", dim=True)
+    click.secho(f"{shlex.join(command)}", dim=True)
     result = subprocess.run(command)
     if result.returncode != 0:
         raise ConsoleError("Joining files failed")
 
 
-def _concat_vods(vod_paths: List[str], target: str):
+def _concat_vods(vod_paths: List[Path], target: Path):
     tool = "type" if platform.system() == "Windows" else "cat"
-    command = [tool] + vod_paths
+    command = [tool] + [str(p) for p in vod_paths]
 
     with open(target, "wb") as target_file:
         result = subprocess.run(command, stdout=target_file)
@@ -89,74 +93,15 @@ def _concat_vods(vod_paths: List[str], target: str):
             raise ConsoleError(f"Joining files failed: {result.stderr}")
 
 
-def get_video_placeholders(video: Video, format: str) -> Dict[str, str]:
-    date, time = video["publishedAt"].split("T")
-    game = video["game"]["name"] if video["game"] else "Unknown"
-
-    return {
-        "channel": video["creator"]["displayName"],
-        "channel_login": video["creator"]["login"],
-        "date": date,
-        "datetime": video["publishedAt"],
-        "format": format,
-        "game": game,
-        "game_slug": utils.slugify(game),
-        "id": video["id"],
-        "time": time,
-        "title": utils.titlify(video["title"]),
-        "title_slug": utils.slugify(video["title"]),
-    }
-
-
-def _video_target_filename(video: Video, args: DownloadOptions):
-    subs = get_video_placeholders(video, args.format)
-
-    try:
-        return args.output.format(**subs)
-    except KeyError as e:
-        supported = ", ".join(subs.keys())
-        raise ConsoleError(f"Invalid key {e} used in --output. Supported keys are: {supported}")
-
-
-def _clip_target_filename(clip: Clip, args: DownloadOptions):
-    date, time = clip["createdAt"].split("T")
-    game = clip["game"]["name"] if clip["game"] else "Unknown"
-
-    url = clip["videoQualities"][0]["sourceURL"]
-    _, ext = path.splitext(url)
-    ext = ext.lstrip(".")
-
-    subs = {
-        "channel": clip["broadcaster"]["displayName"],
-        "channel_login": clip["broadcaster"]["login"],
-        "date": date,
-        "datetime": clip["createdAt"],
-        "format": ext,
-        "game": game,
-        "game_slug": utils.slugify(game),
-        "id": clip["id"],
-        "slug": clip["slug"],
-        "time": time,
-        "title": utils.titlify(clip["title"]),
-        "title_slug": utils.slugify(clip["title"]),
-    }
-
-    try:
-        return args.output.format(**subs)
-    except KeyError as e:
-        supported = ", ".join(subs.keys())
-        raise ConsoleError(f"Invalid key {e} used in --output. Supported keys are: {supported}")
-
-
-def _crete_temp_dir(base_uri: str) -> str:
+def _crete_temp_dir(base_uri: str) -> Path:
     """Create a temp dir to store downloads if it doesn't exist."""
     path = urlparse(base_uri).path.lstrip("/")
     temp_dir = Path(tempfile.gettempdir(), "twitch-dl", path)
     temp_dir.mkdir(parents=True, exist_ok=True)
-    return str(temp_dir)
+    return temp_dir
 
 
-def _get_clip_url(access_token: ClipAccessToken, quality: str) -> str:
+def _get_clip_url(access_token: ClipAccessToken, quality: Optional[str]) -> str:
     qualities = access_token["videoQualities"]
 
     # Quality given as an argument
@@ -184,7 +129,7 @@ def _get_clip_url(access_token: ClipAccessToken, quality: str) -> str:
     return selected_quality["sourceURL"]
 
 
-def get_clip_authenticated_url(slug: str, quality: str):
+def get_clip_authenticated_url(slug: str, quality: Optional[str]):
     print_log("Fetching access token...")
     access_token = twitch.get_clip_access_token(slug)
 
@@ -216,10 +161,10 @@ def _download_clip(slug: str, args: DownloadOptions) -> None:
     duration = utils.format_duration(clip["durationSeconds"])
     click.echo(f"Found: {green(title)} by {yellow(user)}, playing {blue(game)} ({duration})")
 
-    target = _clip_target_filename(clip, args)
+    target = Path(clip_filename(clip, args.output))
     click.echo(f"Target: {blue(target)}")
 
-    if not args.overwrite and path.exists(target):
+    if not args.overwrite and target.exists():
         response = click.prompt("File exists. Overwrite? [Y/n]", default="Y", show_default=False)
         if response.lower().strip() != "y":
             raise click.Abort()
@@ -248,10 +193,10 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
 
     click.echo(f"Found: {blue(video['title'])} by {yellow(video['creator']['displayName'])}")
 
-    target = _video_target_filename(video, args)
+    target = Path(video_filename(video, args.format, args.output))
     click.echo(f"Output: {blue(target)}")
 
-    if not args.overwrite and path.exists(target):
+    if not args.overwrite and target.exists():
         response = click.prompt("File exists. Overwrite? [Y/n]", default="Y", show_default=False)
         if response.lower().strip() != "y":
             raise click.Abort()
@@ -273,23 +218,40 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
     vods_m3u8 = load_m3u8(vods_text)
     vods = enumerate_vods(vods_m3u8, start, end)
 
+    if args.dry_run:
+        click.echo("Dry run, video not downloaded.")
+        return
+
     base_uri = re.sub("/[^/]+$", "/", playlist.url)
     target_dir = _crete_temp_dir(base_uri)
 
     # Save playlists for debugging purposes
-    with open(path.join(target_dir, "playlists.m3u8"), "w") as f:
+    with open(target_dir / "playlists.m3u8", "w") as f:
         f.write(playlists_text)
-    with open(path.join(target_dir, "playlist.m3u8"), "w") as f:
+    with open(target_dir / "playlist.m3u8", "w") as f:
         f.write(vods_text)
 
-    click.echo(f"\nDownloading {len(vods)} VODs using {args.max_workers} workers to {target_dir}")
+    init_sections = get_init_sections(vods_m3u8)
+    for uri in init_sections:
+        print_log(f"Downloading init section {uri}...")
+        download_file(f"{base_uri}{uri}", target_dir / uri)
+
+    print_log(f"Downloading {len(vods)} VODs using {args.max_workers} workers to {target_dir}")
 
     sources = [base_uri + vod.path for vod in vods]
-    targets = [os.path.join(target_dir, f"{vod.index:05d}.ts") for vod in vods]
-    asyncio.run(download_all(sources, targets, args.max_workers, rate_limit=args.rate_limit))
+    targets = [target_dir / f"{vod.index:05d}.ts" for vod in vods]
+
+    asyncio.run(
+        download_all(
+            zip(sources, targets),
+            args.max_workers,
+            rate_limit=args.rate_limit,
+            count=len(vods),
+        )
+    )
 
     join_playlist = make_join_playlist(vods_m3u8, vods, targets)
-    join_playlist_path = path.join(target_dir, "playlist_downloaded.m3u8")
+    join_playlist_path = target_dir / "playlist_downloaded.m3u8"
     join_playlist.dump(join_playlist_path)  # type: ignore
     click.echo()
 
@@ -308,12 +270,12 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
     click.echo()
 
     if args.keep:
-        click.echo(f"Temporary files not deleted: {target_dir}")
+        click.echo(f"Temporary files not deleted: {yellow(target_dir)}")
     else:
         print_log("Deleting temporary files...")
         shutil.rmtree(target_dir)
 
-    click.echo(f"\nDownloaded: {green(target)}")
+    click.echo(f"Downloaded: {green(target)}")
 
 
 def http_get(url: str) -> str:

twitchdl/commands/info.py

@@ -4,9 +4,10 @@ import click
 import m3u8
 
 from twitchdl import twitch, utils
-from twitchdl.commands.download import get_video_placeholders
 from twitchdl.exceptions import ConsoleError
-from twitchdl.output import bold, print_clip, print_json, print_log, print_table, print_video
+from twitchdl.naming import video_placeholders
+from twitchdl.output import bold, dim, print_clip, print_json, print_log, print_table, print_video
+from twitchdl.playlists import parse_playlists
 from twitchdl.twitch import Chapter, Clip, Video
 
 
@@ -50,13 +51,23 @@ def info(id: str, *, json: bool = False):
     raise ConsoleError(f"Invalid input: {id}")
 
 
-def video_info(video: Video, playlists, chapters: List[Chapter]):
+def video_info(video: Video, playlists: str, chapters: List[Chapter]):
     click.echo()
     print_video(video)
 
-    click.echo("Playlists:")
-    for p in m3u8.loads(playlists).playlists:
-        click.echo(f"{bold(p.stream_info.video)} {p.uri}")
+    click.echo("Playlists:\n")
+
+    playlist_headers = ["Name", "Group", "Resolution", "URL"]
+    playlist_data = [
+        [
+            f"{p.name} {dim('source')}" if p.is_source else p.name,
+            p.group_id,
+            f"{p.resolution}",
+            p.url,
+        ]
+        for p in parse_playlists(playlists)
+    ]
+    print_table(playlist_headers, playlist_data)
 
     if chapters:
         click.echo()
@@ -66,13 +77,13 @@ def video_info(video: Video, playlists, chapters: List[Chapter]):
             duration = utils.format_time(chapter["durationMilliseconds"] // 1000)
             click.echo(f'{start} {bold(chapter["description"])} ({duration})')
 
-    placeholders = get_video_placeholders(video, format="mkv")
+    placeholders = video_placeholders(video, format="mkv")
     placeholders = [[f"{{{k}}}", v] for k, v in placeholders.items()]
     click.echo("")
     print_table(["Placeholder", "Value"], placeholders)
 
 
-def video_json(video, playlists, chapters):
+def video_json(video: Video, playlists: str, chapters: List[Chapter]):
     playlists = m3u8.loads(playlists).playlists
 
     video["playlists"] = [
@@ -97,5 +108,8 @@ def clip_info(clip: Clip):
     click.echo()
     click.echo("Download links:")
 
-    for q in clip["videoQualities"]:
-        click.echo(f"{bold(q['quality'])} [{q['frameRate']} fps] {q['sourceURL']}")
+    if clip["videoQualities"]:
+        for q in clip["videoQualities"]:
+            click.echo(f"{bold(q['quality'])} [{q['frameRate']} fps] {q['sourceURL']}")
+    else:
+        click.echo("No download URLs found")

twitchdl/commands/videos.py

@@ -20,7 +20,7 @@ def videos(
     sort: twitch.VideosSort,
     type: twitch.VideosType,
 ):
-    game_ids = _get_game_ids(games)
+    game_ids = get_game_ids(games)
 
     # Set different defaults for limit for compact display
     limit = limit or (40 if compact else 10)
@@ -67,16 +67,13 @@ def videos(
         )
 
 
-def _get_game_ids(names: List[str]) -> List[str]:
-    if not names:
-        return []
+def get_game_ids(names: List[str]) -> List[str]:
+    return [get_game_id(name) for name in names]
 
-    game_ids = []
-    for name in names:
-        print_log(f"Looking up game '{name}'...")
-        game_id = twitch.get_game_id(name)
-        if not game_id:
-            raise ConsoleError(f"Game '{name}' not found")
-        game_ids.append(int(game_id))
 
-    return game_ids
+def get_game_id(name: str) -> str:
+    print_log(f"Looking up game '{name}'...")
+    game_id = twitch.get_game_id(name)
+    if not game_id:
+        raise ConsoleError(f"Game '{name}' not found")
+    return game_id

twitchdl/download.py

@@ -1,37 +0,0 @@
-import os
-
-import httpx
-
-from twitchdl.exceptions import ConsoleError
-
-CHUNK_SIZE = 1024
-CONNECT_TIMEOUT = 5
-RETRY_COUNT = 5
-
-
-def _download(url: str, path: str):
-    tmp_path = path + ".tmp"
-    size = 0
-    with httpx.stream("GET", url, timeout=CONNECT_TIMEOUT) as response:
-        with open(tmp_path, "wb") as target:
-            for chunk in response.iter_bytes(chunk_size=CHUNK_SIZE):
-                target.write(chunk)
-                size += len(chunk)
-
-    os.rename(tmp_path, path)
-    return size
-
-
-def download_file(url: str, path: str, retries: int = RETRY_COUNT):
-    if os.path.exists(path):
-        from_disk = True
-        return (os.path.getsize(path), from_disk)
-
-    from_disk = False
-    for _ in range(retries):
-        try:
-            return (_download(url, path), from_disk)
-        except httpx.RequestError:
-            pass
-
-    raise ConsoleError(f"Failed downloading after {retries} attempts: {url}")

twitchdl/entities.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Any, Mapping, Optional
+from typing import Any, List, Literal, Mapping, Optional, TypedDict
 
 
 @dataclass
@@ -20,6 +20,73 @@ class DownloadOptions:
     max_workers: int
 
 
+ClipsPeriod = Literal["last_day", "last_week", "last_month", "all_time"]
+VideosSort = Literal["views", "time"]
+VideosType = Literal["archive", "highlight", "upload"]
+
+
+class AccessToken(TypedDict):
+    signature: str
+    value: str
+
+
+class User(TypedDict):
+    login: str
+    displayName: str
+
+
+class Game(TypedDict):
+    id: str
+    name: str
+
+
+class VideoQuality(TypedDict):
+    frameRate: str
+    quality: str
+    sourceURL: str
+
+
+class ClipAccessToken(TypedDict):
+    id: str
+    playbackAccessToken: AccessToken
+    videoQualities: List[VideoQuality]
+
+
+class Clip(TypedDict):
+    id: str
+    slug: str
+    title: str
+    createdAt: str
+    viewCount: int
+    durationSeconds: int
+    url: str
+    videoQualities: Optional[List[VideoQuality]]
+    game: Game
+    broadcaster: User
+
+
+class Video(TypedDict):
+    id: str
+    title: str
+    description: str
+    publishedAt: str
+    broadcastType: str
+    lengthSeconds: int
+    game: Game
+    creator: User
+
+
+class Chapter(TypedDict):
+    id: str
+    durationMilliseconds: int
+    positionMilliseconds: int
+    type: str
+    description: str
+    subDescription: str
+    thumbnailURL: str
+    game: Game
+
+
 # Type for annotating decoded JSON
 # TODO: make data classes for common structs
 Data = Mapping[str, Any]

twitchdl/http.py

@@ -3,10 +3,12 @@ import logging
 import os
 import time
 from abc import ABC, abstractmethod
-from typing import List, Optional
+from pathlib import Path
+from typing import Iterable, Optional, Tuple
 
 import httpx
 
+from twitchdl.exceptions import ConsoleError
 from twitchdl.progress import Progress
 
 logger = logging.getLogger(__name__)
@@ -71,7 +73,7 @@ async def download(
     client: httpx.AsyncClient,
     task_id: int,
     source: str,
-    target: str,
+    target: Path,
     progress: Progress,
     token_bucket: TokenBucket,
 ):
@@ -96,12 +98,12 @@ async def download_with_retries(
     semaphore: asyncio.Semaphore,
     task_id: int,
     source: str,
-    target: str,
+    target: Path,
     progress: Progress,
     token_bucket: TokenBucket,
 ):
     async with semaphore:
-        if os.path.exists(target):
+        if target.exists():
             size = os.path.getsize(target)
             progress.already_downloaded(task_id, size)
             return
@@ -119,13 +121,13 @@ async def download_with_retries(
 
 
 async def download_all(
-    sources: List[str],
-    targets: List[str],
+    source_targets: Iterable[Tuple[str, Path]],
     workers: int,
     *,
+    count: Optional[int] = None,
     rate_limit: Optional[int] = None,
 ):
-    progress = Progress(len(sources))
+    progress = Progress(count)
     token_bucket = LimitingTokenBucket(rate_limit) if rate_limit else EndlessTokenBucket()
     async with httpx.AsyncClient(timeout=TIMEOUT) as client:
         semaphore = asyncio.Semaphore(workers)
@@ -139,6 +141,36 @@ async def download_all(
                 progress,
                 token_bucket,
             )
-            for task_id, (source, target) in enumerate(zip(sources, targets))
+            for task_id, (source, target) in enumerate(source_targets)
         ]
         await asyncio.gather(*tasks)
+
+
+def download_file(url: str, target: Path, retries: int = RETRY_COUNT) -> None:
+    """Download URL to given target path with retries"""
+    error_message = ""
+    for r in range(retries):
+        try:
+            retry_info = f" (retry {r})" if r > 0 else ""
+            logger.info(f"Downloading {url} to {target}{retry_info}")
+            return _do_download_file(url, target)
+        except httpx.HTTPStatusError as ex:
+            logger.error(ex)
+            error_message = f"Server responded with HTTP {ex.response.status_code}"
+        except httpx.RequestError as ex:
+            logger.error(ex)
+            error_message = str(ex)
+
+    raise ConsoleError(f"Failed downloading after {retries} attempts: {error_message}")
+
+
+def _do_download_file(url: str, target: Path) -> None:
+    tmp_path = Path(str(target) + ".tmp")
+
+    with httpx.stream("GET", url, timeout=TIMEOUT, follow_redirects=True) as response:
+        response.raise_for_status()
+        with open(tmp_path, "wb") as f:
+            for chunk in response.iter_bytes(chunk_size=CHUNK_SIZE):
+                f.write(chunk)
+
+    os.rename(tmp_path, target)

twitchdl/naming.py

@@ -0,0 +1,72 @@
+import os
+from typing import Dict
+
+from twitchdl import utils
+from twitchdl.entities import Clip, Video
+from twitchdl.exceptions import ConsoleError
+
+DEFAULT_OUTPUT_TEMPLATE = "{date}_{id}_{channel_login}_{title_slug}.{format}"
+
+
+def video_filename(video: Video, format: str, output: str) -> str:
+    subs = video_placeholders(video, format)
+    return _format(output, subs)
+
+
+def video_placeholders(video: Video, format: str) -> Dict[str, str]:
+    date, time = video["publishedAt"].split("T")
+    game = video["game"]["name"] if video["game"] else "Unknown"
+
+    return {
+        "channel": video["creator"]["displayName"],
+        "channel_login": video["creator"]["login"],
+        "date": date,
+        "datetime": video["publishedAt"],
+        "format": format,
+        "game": game,
+        "game_slug": utils.slugify(game),
+        "id": video["id"],
+        "time": time,
+        "title": utils.titlify(video["title"]),
+        "title_slug": utils.slugify(video["title"]),
+    }
+
+
+def clip_filename(clip: Clip, output: str):
+    subs = clip_placeholders(clip)
+    return _format(output, subs)
+
+
+def clip_placeholders(clip: Clip) -> Dict[str, str]:
+    date, time = clip["createdAt"].split("T")
+    game = clip["game"]["name"] if clip["game"] else "Unknown"
+
+    if clip["videoQualities"]:
+        url = clip["videoQualities"][0]["sourceURL"]
+        _, ext = os.path.splitext(url)
+        ext = ext.lstrip(".")
+    else:
+        ext = "mp4"
+
+    return {
+        "channel": clip["broadcaster"]["displayName"],
+        "channel_login": clip["broadcaster"]["login"],
+        "date": date,
+        "datetime": clip["createdAt"],
+        "format": ext,
+        "game": game,
+        "game_slug": utils.slugify(game),
+        "id": clip["id"],
+        "slug": clip["slug"],
+        "time": time,
+        "title": utils.titlify(clip["title"]),
+        "title_slug": utils.slugify(clip["title"]),
+    }
+
+
+def _format(output: str, subs: Dict[str, str]) -> str:
+    try:
+        return output.format(**subs)
+    except KeyError as e:
+        supported = ", ".join(subs.keys())
+        raise ConsoleError(f"Invalid key {e} used in --output. Supported keys are: {supported}")

twitchdl/output.py

@@ -1,15 +1,21 @@
 import json
+import sys
 from itertools import islice
 from typing import Any, Callable, Generator, List, Optional, TypeVar
 
 import click
 
 from twitchdl import utils
-from twitchdl.twitch import Clip, Video
+from twitchdl.entities import Clip, Video
 
 T = TypeVar("T")
 
 
+def clear_line():
+    sys.stdout.write("\033[1K")
+    sys.stdout.write("\r")
+
+
 def truncate(string: str, length: int) -> str:
     if len(string) > length:
         return string[: length - 1] + "…"
@@ -25,17 +31,23 @@ def print_log(message: Any):
     click.secho(message, err=True, dim=True)
 
 
+def visual_len(text: str):
+    return len(click.unstyle(text))
+
+
+def ljust(text: str, width: int):
+    diff = width - visual_len(text)
+    return text + (" " * diff) if diff > 0 else text
+
+
 def print_table(headers: List[str], data: List[List[str]]):
-    widths = [[len(cell) for cell in row] for row in data + [headers]]
+    widths = [[visual_len(cell) for cell in row] for row in data + [headers]]
     widths = [max(width) for width in zip(*widths)]
     underlines = ["-" * width for width in widths]
 
     def print_row(row: List[str]):
-        for idx, cell in enumerate(row):
-            width = widths[idx]
-            click.echo(cell.ljust(width), nl=False)
-            click.echo("  ", nl=False)
-        click.echo()
+        parts = (ljust(cell, widths[idx]) for idx, cell in enumerate(row))
+        click.echo("  ".join(parts).strip())
 
     print_row(headers)
     print_row(underlines)
@@ -93,11 +105,12 @@ def print_video(video: Video):
     if channel or playing:
         click.echo(" ".join([channel, playing]))
 
-    if video["description"]:
-        click.echo(f"Description: {video['description']}")
-
     click.echo(f"Published {blue(published_at)}  Length: {blue(length)} ")
     click.secho(url, italic=True)
+
+    if video["description"]:
+        click.echo(f"\nDescription:\n{video['description']}")
+
     click.echo()
 
 

twitchdl/playlists.py

@@ -3,20 +3,23 @@ Parse and manipulate m3u8 playlists.
 """
 
 from dataclasses import dataclass
-from typing import Generator, List, Optional, OrderedDict
+from pathlib import Path
+from typing import Generator, List, Optional, OrderedDict, Set
 
 import click
 import m3u8
 
 from twitchdl import utils
-from twitchdl.output import bold, dim
+from twitchdl.output import bold, dim, print_table
 
 
 @dataclass
 class Playlist:
     name: str
+    group_id: str
     resolution: Optional[str]
     url: str
+    is_source: bool
 
 
 @dataclass
@@ -29,22 +32,22 @@ class Vod:
     """Segment duration in seconds"""
 
 
-def parse_playlists(playlists_m3u8: str):
+def parse_playlists(playlists_m3u8: str) -> List[Playlist]:
     def _parse(source: str) -> Generator[Playlist, None, None]:
         document = load_m3u8(source)
 
         for p in document.playlists:
-            if p.stream_info.resolution:
-                name = p.media[0].name
-                resolution = "x".join(str(r) for r in p.stream_info.resolution)
-            else:
-                name = p.media[0].group_id
-                resolution = None
+            resolution = (
+                "x".join(str(r) for r in p.stream_info.resolution)
+                if p.stream_info.resolution
+                else None
+            )
 
-            yield Playlist(name, resolution, p.uri)
+            media = p.media[0]
+            is_source = media.group_id == "chunked"
+            yield Playlist(media.name, media.group_id, resolution, p.uri, is_source)
 
-    # Move audio to bottom, it has no resolution
-    return sorted(_parse(playlists_m3u8), key=lambda p: p.resolution is None)
+    return list(_parse(playlists_m3u8))
 
 
 def load_m3u8(playlist_m3u8: str) -> m3u8.M3U8:
@@ -79,7 +82,7 @@ def enumerate_vods(
 def make_join_playlist(
     playlist: m3u8.M3U8,
     vods: List[Vod],
-    targets: List[str],
+    targets: List[Path],
 ) -> m3u8.Playlist:
     """
     Make a modified playlist which references downloaded VODs
@@ -91,7 +94,7 @@ def make_join_playlist(
     playlist.segments.clear()
     for segment in org_segments:
         if segment.uri in path_map:
-            segment.uri = path_map[segment.uri]
+            segment.uri = str(path_map[segment.uri].name)
             playlist.segments.append(segment)
 
     return playlist
@@ -107,10 +110,13 @@ def select_playlist(playlists: List[Playlist], quality: Optional[str]) -> Playli
 
 def select_playlist_by_name(playlists: List[Playlist], quality: str) -> Playlist:
     if quality == "source":
-        return playlists[0]
+        for playlist in playlists:
+            if playlist.is_source:
+                return playlist
+        raise click.ClickException("Source quality not found, please report an issue on github.")
 
     for playlist in playlists:
-        if playlist.name == quality:
+        if playlist.name == quality or playlist.group_id == quality:
             return playlist
 
     available = ", ".join([p.name for p in playlists])
@@ -119,13 +125,56 @@ def select_playlist_by_name(playlists: List[Playlist], quality: str) -> Playlist
 
 
 def select_playlist_interactive(playlists: List[Playlist]) -> Playlist:
-    click.echo("\nAvailable qualities:")
-    for n, playlist in enumerate(playlists):
-        if playlist.resolution:
-            click.echo(f"{n + 1}) {bold(playlist.name)} {dim(f'({playlist.resolution})')}")
-        else:
-            click.echo(f"{n + 1}) {bold(playlist.name)}")
+    playlists = sorted(playlists, key=_playlist_key)
+    headers = ["#", "Name", "Group ID", "Resolution"]
 
-    no = utils.read_int("Choose quality", min=1, max=len(playlists) + 1, default=1)
+    rows = [
+        [
+            f"{n + 1})",
+            bold(playlist.name),
+            dim(playlist.group_id),
+            dim(playlist.resolution or ""),
+        ]
+        for n, playlist in enumerate(playlists)
+    ]
+
+    click.echo()
+    print_table(headers, rows)
+
+    default = 1
+    for index, playlist in enumerate(playlists):
+        if playlist.is_source:
+            default = index + 1
+
+    no = utils.read_int("\nChoose quality", min=1, max=len(playlists) + 1, default=default)
     playlist = playlists[no - 1]
     return playlist
+
+
+MAX = 1_000_000
+
+
+def _playlist_key(playlist: Playlist) -> int:
+    """Attempt to sort playlists so that source quality is on top, audio only
+    is on bottom and others are sorted descending by resolution."""
+    if playlist.is_source:
+        return 0
+
+    if playlist.group_id == "audio_only":
+        return MAX
+
+    try:
+        return MAX - int(playlist.name.split("p")[0])
+    except Exception:
+        pass
+
+    return MAX
+
+
+def get_init_sections(playlist: m3u8.M3U8) -> Set[str]:
+    # TODO: we're ignoring initi_section.base_uri and bytes
+    return set(
+        segment.init_section.uri
+        for segment in playlist.segments
+        if segment.init_section is not None
+    )

twitchdl/progress.py

@@ -1,13 +1,13 @@
 import logging
 import time
 from collections import deque
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from statistics import mean
 from typing import Deque, Dict, NamedTuple, Optional
 
 import click
 
-from twitchdl.output import blue
+from twitchdl.output import blue, clear_line
 from twitchdl.utils import format_size, format_time
 
 logger = logging.getLogger(__name__)
@@ -31,28 +31,25 @@ class Sample(NamedTuple):
     timestamp: float
 
 
-@dataclass
 class Progress:
-    vod_count: int
-    downloaded: int = 0
-    estimated_total: Optional[int] = None
-    last_printed: float = field(default_factory=time.time)
-    progress_bytes: int = 0
-    progress_perc: int = 0
-    remaining_time: Optional[int] = None
-    speed: Optional[float] = None
-    start_time: float = field(default_factory=time.time)
-    tasks: Dict[TaskId, Task] = field(default_factory=dict)
-    vod_downloaded_count: int = 0
-    samples: Deque[Sample] = field(default_factory=lambda: deque(maxlen=100))
+    def __init__(self, file_count: Optional[int] = None):
+        self.downloaded: int = 0
+        self.estimated_total: Optional[int] = None
+        self.last_printed: Optional[float] = None
+        self.progress_bytes: int = 0
+        self.progress_perc: int = 0
+        self.remaining_time: Optional[int] = None
+        self.samples: Deque[Sample] = deque(maxlen=1000)
+        self.speed: Optional[float] = None
+        self.tasks: Dict[TaskId, Task] = {}
+        self.file_count = file_count
+        self.downloaded_count: int = 0
 
     def start(self, task_id: int, size: int):
         if task_id in self.tasks:
             raise ValueError(f"Task {task_id}: cannot start, already started")
 
         self.tasks[task_id] = Task(task_id, size)
-        self._calculate_total()
-        self._calculate_progress()
         self.print()
 
     def advance(self, task_id: int, size: int):
@@ -63,7 +60,6 @@ class Progress:
         self.progress_bytes += size
         self.tasks[task_id].advance(size)
         self.samples.append(Sample(self.downloaded, time.time()))
-        self._calculate_progress()
         self.print()
 
     def already_downloaded(self, task_id: int, size: int):
@@ -72,9 +68,7 @@ class Progress:
 
         self.tasks[task_id] = Task(task_id, size)
         self.progress_bytes += size
-        self.vod_downloaded_count += 1
-        self._calculate_total()
-        self._calculate_progress()
+        self.downloaded_count += 1
         self.print()
 
     def abort(self, task_id: int):
@@ -83,9 +77,6 @@ class Progress:
 
         del self.tasks[task_id]
         self.progress_bytes = sum(t.downloaded for t in self.tasks.values())
-
-        self._calculate_total()
-        self._calculate_progress()
         self.print()
 
     def end(self, task_id: int):
@@ -98,15 +89,15 @@ class Progress:
                 f"Taks {task_id} ended with {task.downloaded}b downloaded, expected {task.size}b."
             )
 
-        self.vod_downloaded_count += 1
+        self.downloaded_count += 1
         self.print()
 
-    def _calculate_total(self):
-        self.estimated_total = (
-            int(mean(t.size for t in self.tasks.values()) * self.vod_count) if self.tasks else None
-        )
+    def _recalculate(self):
+        if self.tasks and self.file_count:
+            self.estimated_total = int(mean(t.size for t in self.tasks.values()) * self.file_count)
+        else:
+            self.estimated_total = None
 
-    def _calculate_progress(self):
         self.speed = self._calculate_speed()
         self.progress_perc = (
             int(100 * self.progress_bytes / self.estimated_total) if self.estimated_total else 0
@@ -133,10 +124,14 @@ class Progress:
         now = time.time()
 
         # Don't print more often than 10 times per second
-        if now - self.last_printed < 0.1:
+        if self.last_printed and now - self.last_printed < 0.1:
             return
 
-        click.echo(f"\rDownloaded {self.vod_downloaded_count}/{self.vod_count} VODs", nl=False)
+        self._recalculate()
+
+        clear_line()
+        total_label = f"/{self.file_count}" if self.file_count else ""
+        click.echo(f"Downloaded {self.downloaded_count}{total_label} VODs", nl=False)
         click.secho(f" {self.progress_perc}%", fg="blue", nl=False)
 
         if self.estimated_total is not None:
@@ -150,6 +145,4 @@ class Progress:
         if self.remaining_time is not None:
             click.echo(f" ETA {blue(format_time(self.remaining_time))}", nl=False)
 
-        click.echo("    ", nl=False)
-
         self.last_printed = now

twitchdl/twitch.py

@@ -2,81 +2,28 @@
 Twitch API access.
 """
 
-import json
-from typing import Dict, Generator, List, Literal, Mapping, Optional, TypedDict
+import logging
+import random
+import time
+from typing import Any, Dict, Generator, List, Mapping, Optional, Tuple, Union
 
 import click
 import httpx
 
 from twitchdl import CLIENT_ID
-from twitchdl.entities import Data
+from twitchdl.entities import (
+    AccessToken,
+    Chapter,
+    Clip,
+    ClipAccessToken,
+    ClipsPeriod,
+    Data,
+    Video,
+    VideosSort,
+    VideosType,
+)
 from twitchdl.exceptions import ConsoleError
-
-ClipsPeriod = Literal["last_day", "last_week", "last_month", "all_time"]
-VideosSort = Literal["views", "time"]
-VideosType = Literal["archive", "highlight", "upload"]
-
-
-class AccessToken(TypedDict):
-    signature: str
-    value: str
-
-
-class User(TypedDict):
-    login: str
-    displayName: str
-
-
-class Game(TypedDict):
-    id: str
-    name: str
-
-
-class VideoQuality(TypedDict):
-    frameRate: str
-    quality: str
-    sourceURL: str
-
-
-class ClipAccessToken(TypedDict):
-    id: str
-    playbackAccessToken: AccessToken
-    videoQualities: List[VideoQuality]
-
-
-class Clip(TypedDict):
-    id: str
-    slug: str
-    title: str
-    createdAt: str
-    viewCount: int
-    durationSeconds: int
-    url: str
-    videoQualities: List[VideoQuality]
-    game: Game
-    broadcaster: User
-
-
-class Video(TypedDict):
-    id: str
-    title: str
-    description: str
-    publishedAt: str
-    broadcastType: str
-    lengthSeconds: int
-    game: Game
-    creator: User
-
-
-class Chapter(TypedDict):
-    id: str
-    durationMilliseconds: int
-    positionMilliseconds: int
-    type: str
-    description: str
-    subDescription: str
-    thumbnailURL: str
-    game: Game
+from twitchdl.utils import format_size
 
 
 class GQLError(click.ClickException):
@@ -87,10 +34,22 @@ class GQLError(click.ClickException):
         super().__init__(message)
 
 
-def authenticated_post(url, data=None, json=None, headers={}):
-    headers["Client-ID"] = CLIENT_ID
+Content = Union[str, bytes]
+Headers = Dict[str, str]
 
-    response = httpx.post(url, data=data, json=json, headers=headers)
+
+def authenticated_post(
+    url: str,
+    *,
+    json: Any = None,
+    content: Optional[Content] = None,
+    auth_token: Optional[str] = None,
+):
+    headers = {"Client-ID": CLIENT_ID}
+    if auth_token is not None:
+        headers["authorization"] = f"OAuth {auth_token}"
+
+    response = request("POST", url, content=content, json=json, headers=headers)
     if response.status_code == 400:
         data = response.json()
         raise ConsoleError(data["message"])
@@ -100,16 +59,51 @@ def authenticated_post(url, data=None, json=None, headers={}):
     return response
 
 
-def gql_post(query: str):
+def request(
+    method: str,
+    url: str,
+    json: Any = None,
+    content: Optional[Content] = None,
+    headers: Optional[Mapping[str, str]] = None,
+):
+    with httpx.Client() as client:
+        request = client.build_request(method, url, json=json, content=content, headers=headers)
+        log_request(request)
+        start = time.time()
+        response = client.send(request)
+        duration = time.time() - start
+        log_response(response, duration)
+        return response
+
+
+logger = logging.getLogger(__name__)
+
+
+def log_request(request: httpx.Request):
+    logger.info(f"--> {request.method} {request.url}")
+    if request.content:
+        logger.debug(f"--> {request.content}")
+
+
+def log_response(response: httpx.Response, duration_seconds: float):
+    request = response.request
+    duration = f"{int(1000 * duration_seconds)}ms"
+    size = format_size(len(response.content))
+    logger.info(f"<-- {request.method} {request.url} HTTP {response.status_code} {duration} {size}")
+    if response.content:
+        logger.debug(f"<-- {response.content}")
+
+
+def gql_persisted_query(query: Data):
     url = "https://gql.twitch.tv/gql"
-    response = authenticated_post(url, data=query)
+    response = authenticated_post(url, json=query)
     gql_raise_on_error(response)
     return response.json()
 
 
-def gql_query(query: str, headers: Dict[str, str] = {}):
+def gql_query(query: str, auth_token: Optional[str] = None):
     url = "https://gql.twitch.tv/gql"
-    response = authenticated_post(url, json={"query": query}, headers=headers)
+    response = authenticated_post(url, json={"query": query}, auth_token=auth_token)
     gql_raise_on_error(response)
     return response.json()
 
@@ -190,22 +184,18 @@ def get_clip(slug: str) -> Optional[Clip]:
 
 
 def get_clip_access_token(slug: str) -> ClipAccessToken:
-    query = f"""
-    {{
+    query = {
         "operationName": "VideoAccessToken_Clip",
-        "variables": {{
-            "slug": "{slug}"
-        }},
-        "extensions": {{
-            "persistedQuery": {{
+        "variables": {"slug": slug},
+        "extensions": {
+            "persistedQuery": {
                 "version": 1,
-                "sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11"
-            }}
-        }}
-    }}
-    """
+                "sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11",
+            }
+        },
+    }
 
-    response = gql_post(query.strip())
+    response = gql_persisted_query(query)
     return response["data"]["clip"]
 
 
@@ -277,23 +267,6 @@ def channel_clips_generator(
     return _generator(clips, limit)
 
 
-def channel_clips_generator_old(channel_id: str, period: ClipsPeriod, limit: int):
-    cursor = ""
-    while True:
-        clips = get_channel_clips(channel_id, period, limit, after=cursor)
-
-        if not clips["edges"]:
-            break
-
-        has_next = clips["pageInfo"]["hasNextPage"]
-        cursor = clips["edges"][-1]["cursor"] if has_next else None
-
-        yield clips, has_next
-
-        if not cursor:
-            break
-
-
 def get_channel_videos(
     channel_id: str,
     limit: int,
@@ -303,6 +276,7 @@ def get_channel_videos(
     after: Optional[str] = None,
 ):
     game_ids = game_ids or []
+    game_ids_str = f"[{','.join(game_ids)}]"
 
     query = f"""
     {{
@@ -313,7 +287,7 @@ def get_channel_videos(
                 sort: {sort.upper()},
                 after: "{after or ''}",
                 options: {{
-                    gameIDs: {game_ids}
+                    gameIDs: {game_ids_str}
                 }}
             ) {{
                 totalCount
@@ -345,7 +319,7 @@ def channel_videos_generator(
     sort: VideosSort,
     type: VideosType,
     game_ids: Optional[List[str]] = None,
-) -> tuple[int, Generator[Video, None, None]]:
+) -> Tuple[int, Generator[Video, None, None]]:
     game_ids = game_ids or []
 
     def _generator(videos: Data, max_videos: int) -> Generator[Video, None, None]:
@@ -373,7 +347,7 @@ def get_access_token(video_id: str, auth_token: Optional[str] = None) -> AccessT
     query = f"""
     {{
         videoPlaybackAccessToken(
-            id: {video_id},
+            id: "{video_id}",
             params: {{
                 platform: "web",
                 playerBackend: "mediaplayer",
@@ -386,12 +360,8 @@ def get_access_token(video_id: str, auth_token: Optional[str] = None) -> AccessT
     }}
     """
 
-    headers: Mapping[str, str] = {}
-    if auth_token is not None:
-        headers["authorization"] = f"OAuth {auth_token}"
-
     try:
-        response = gql_query(query, headers=headers)
+        response = gql_query(query, auth_token=auth_token)
         return response["data"]["videoPlaybackAccessToken"]
     except httpx.HTTPStatusError as error:
         # Provide a more useful error message when server returns HTTP 401
@@ -422,8 +392,12 @@ def get_playlists(video_id: str, access_token: AccessToken) -> str:
             "allow_audio_only": "true",
             "allow_source": "true",
             "player": "twitchweb",
+            "platform": "web",
+            "supported_codecs": "av1,h265,h264",
+            "p": random.randint(1000000, 10000000),
         },
     )
+
     response.raise_for_status()
     return response.content.decode("utf-8")
 
@@ -458,7 +432,7 @@ def get_video_chapters(video_id: str) -> List[Chapter]:
         },
     }
 
-    response = gql_post(json.dumps(query))
+    response = gql_persisted_query(query)
     return list(_chapter_nodes(response["data"]["video"]["moments"]))
 
 

twitchdl/utils.py

@@ -84,13 +84,14 @@ def titlify(value: str) -> str:
 
 VIDEO_PATTERNS = [
     r"^(?P<id>\d+)?$",
-    r"^https://(www.)?twitch.tv/videos/(?P<id>\d+)(\?.+)?$",
+    r"^https://(www\.|m\.)?twitch\.tv/videos/(?P<id>\d+)(\?.+)?$",
+    r"^https://(www\.|m\.)?twitch\.tv/\w+/video/(?P<id>\d+)(\?.+)?$",
 ]
 
 CLIP_PATTERNS = [
     r"^(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)$",
-    r"^https://(www.)?twitch.tv/\w+/clip/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
-    r"^https://clips.twitch.tv/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
+    r"^https://(www\.|m\.)?twitch\.tv/\w+/clip/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
+    r"^https://clips\.twitch\.tv/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
 ]