wip

.gitignore

@@ -15,3 +15,4 @@ tmp/
 /*.pyz
 /pyrightconfig.json
 /book
+/*.mkv

CHANGELOG.md

@@ -3,6 +3,10 @@ twitch-dl changelog
 
 <!-- Do not edit. This file is automatically generated from changelog.yaml.-->
 
+### [2.3.1 (2024-05-19)](https://github.com/ihabunek/twitch-dl/releases/tag/2.3.1)
+
+* Fix fetching access token (#155, thanks @KryptonicDragon)
+
 ### [2.3.0 (2024-04-27)](https://github.com/ihabunek/twitch-dl/releases/tag/2.3.0)
 
 * Show more playlist data when choosing quality

changelog.yaml

@@ -1,3 +1,8 @@
+2.3.1:
+  date: 2024-05-19
+  changes:
+    - "Fix fetching access token (#155, thanks @KryptonicDragon)"
+
 2.3.0:
   date: 2024-04-27
   changes:

docs/changelog.md

@@ -3,6 +3,10 @@ twitch-dl changelog
 
 <!-- Do not edit. This file is automatically generated from changelog.yaml.-->
 
+### [2.3.1 (2024-05-19)](https://github.com/ihabunek/twitch-dl/releases/tag/2.3.1)
+
+* Fix fetching access token (#155, thanks @KryptonicDragon)
+
 ### [2.3.0 (2024-04-27)](https://github.com/ihabunek/twitch-dl/releases/tag/2.3.0)
 
 * Show more playlist data when choosing quality

tests/test_api.py

@@ -9,7 +9,7 @@ from twitchdl import twitch
 from twitchdl.commands.download import get_clip_authenticated_url
 from twitchdl.commands.videos import get_game_ids
 from twitchdl.exceptions import ConsoleError
-from twitchdl.playlists import enumerate_vods, load_m3u8, parse_playlists
+from twitchdl.playlists import parse_playlists
 
 TEST_CHANNEL = "bananasaurus_rex"
 
@@ -37,10 +37,6 @@ def test_get_videos():
     playlist_txt = httpx.get(playlist_url).text
     assert playlist_txt.startswith("#EXTM3U")
 
-    playlist_m3u8 = load_m3u8(playlist_txt)
-    vods = enumerate_vods(playlist_m3u8)
-    assert vods[0].path == "0.ts"
-
 
 def test_get_clips():
     """

tests/test_download.py

@@ -0,0 +1,90 @@
+from decimal import Decimal
+
+from twitchdl.commands.download import filter_vods
+from twitchdl.playlists import Vod
+
+VODS = [
+    Vod(index=1, path="1.ts", duration=Decimal("10.0")),
+    Vod(index=2, path="2.ts", duration=Decimal("10.0")),
+    Vod(index=3, path="3.ts", duration=Decimal("10.0")),
+    Vod(index=4, path="4.ts", duration=Decimal("10.0")),
+    Vod(index=5, path="5.ts", duration=Decimal("10.0")),
+    Vod(index=6, path="6.ts", duration=Decimal("10.0")),
+    Vod(index=7, path="7.ts", duration=Decimal("10.0")),
+    Vod(index=8, path="8.ts", duration=Decimal("10.0")),
+    Vod(index=9, path="9.ts", duration=Decimal("10.0")),
+    Vod(index=10, path="10.ts", duration=Decimal("3.15")),
+]
+
+
+def test_filter_vods_no_start_no_end():
+    vods, start_offset, duration = filter_vods(VODS, None, None)
+    assert vods == VODS
+    assert start_offset == Decimal("0")
+    assert duration == Decimal("93.15")
+
+
+def test_filter_vods_start():
+    # Zero offset
+    vods, start_offset, duration = filter_vods(VODS, 0, None)
+    assert [v.index for v in vods] == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+    assert start_offset == Decimal("0")
+    assert duration == Decimal("93.15")
+
+    # Mid-vod
+    vods, start_offset, duration = filter_vods(VODS, 13, None)
+    assert [v.index for v in vods] == [2, 3, 4, 5, 6, 7, 8, 9, 10]
+    assert start_offset == Decimal("3.0")
+    assert duration == Decimal("80.15")
+
+    # Between vods
+    vods, start_offset, duration = filter_vods(VODS, 50, None)
+    assert [v.index for v in vods] == [6, 7, 8, 9, 10]
+    assert start_offset == Decimal("0")
+    assert duration == Decimal("43.15")
+
+    # Close to end
+    vods, start_offset, duration = filter_vods(VODS, 93, None)
+    assert [v.index for v in vods] == [10]
+    assert start_offset == Decimal("3.0")
+    assert duration == Decimal("0.15")
+
+
+def test_filter_vods_end():
+    # Zero offset
+    vods, start_offset, duration = filter_vods(VODS, 0, None)
+    assert [v.index for v in vods] == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+    assert start_offset == Decimal("0")
+    assert duration == Decimal("93.15")
+
+    # Mid-vod
+    vods, start_offset, duration = filter_vods(VODS, None, 56)
+    assert [v.index for v in vods] == [1, 2, 3, 4, 5, 6]
+    assert start_offset == Decimal("0")
+    assert duration == Decimal("56")
+
+    # Between vods
+    vods, start_offset, duration = filter_vods(VODS, None, 30)
+    assert [v.index for v in vods] == [1, 2, 3]
+    assert start_offset == Decimal("0")
+    assert duration == Decimal("30")
+
+
+def test_filter_vods_start_end():
+    # Zero offset
+    vods, start_offset, duration = filter_vods(VODS, 0, 0)
+    assert [v.index for v in vods] == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+    assert start_offset == Decimal("0")
+    assert duration == Decimal("93.15")
+
+    # Mid-vod
+    vods, start_offset, duration = filter_vods(VODS, 32, 56)
+    assert [v.index for v in vods] == [4, 5, 6]
+    assert start_offset == Decimal("2")
+    assert duration == Decimal("24")
+
+    # Between vods
+    vods, start_offset, duration = filter_vods(VODS, 20, 60)
+    assert [v.index for v in vods] == [3, 4, 5, 6]
+    assert start_offset == Decimal("0")
+    assert duration == Decimal("40")

twitchdl/commands/download.py

@@ -5,9 +5,10 @@ import re
 import shutil
 import subprocess
 import tempfile
+from decimal import Decimal
 from os import path
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Tuple
 from urllib.parse import urlencode, urlparse
 
 import click
@@ -20,10 +21,11 @@ from twitchdl.exceptions import ConsoleError
 from twitchdl.http import download_all
 from twitchdl.output import blue, bold, green, print_log, yellow
 from twitchdl.playlists import (
-    enumerate_vods,
+    Vod,
     load_m3u8,
     make_join_playlist,
     parse_playlists,
+    parse_vods,
     select_playlist,
 )
 from twitchdl.twitch import Chapter, Clip, ClipAccessToken, Video
@@ -50,7 +52,14 @@ def download_one(video: str, args: DownloadOptions):
     raise ConsoleError(f"Invalid input: {video}")
 
 
-def _join_vods(playlist_path: str, target: str, overwrite: bool, video: Video):
+def _join_vods(
+    playlist_path: str,
+    target: str,
+    overwrite: bool,
+    video: Video,
+    start_offset: int,
+    duration: int,
+):
     description = video["description"] or ""
     description = description.strip()
 
@@ -60,6 +69,10 @@ def _join_vods(playlist_path: str, target: str, overwrite: bool, video: Video):
         playlist_path,
         "-c",
         "copy",
+        "-ss",
+        str(start_offset),
+        "-to",
+        str(duration),
         "-metadata",
         f"artist={video['creator']['displayName']}",
         "-metadata",
@@ -73,11 +86,11 @@ def _join_vods(playlist_path: str, target: str, overwrite: bool, video: Video):
         "warning",
         f"file:{target}",
     ]
-
     if overwrite:
         command.append("-y")
 
     click.secho(f"{' '.join(command)}", dim=True)
+
     result = subprocess.run(command)
     if result.returncode != 0:
         raise ConsoleError("Joining files failed")
@@ -275,15 +288,15 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
     print_log("Fetching playlist...")
     vods_text = http_get(playlist.url)
     vods_m3u8 = load_m3u8(vods_text)
-    vods = enumerate_vods(vods_m3u8, start, end)
+    all_vods = parse_vods(vods_m3u8)
+    vods, start_offset, duration = filter_vods(all_vods, start, end)
 
     if args.dry_run:
         click.echo("Dry run, video not downloaded.")
         return
 
     base_uri = re.sub("/[^/]+$", "/", playlist.url)
-    target_dir = f".twitch_dl_{video_id}_{playlist.group_id}"
-    os.makedirs(target_dir, exist_ok=True)
+    target_dir = _crete_temp_dir(base_uri)
 
     # Save playlists for debugging purposes
     with open(path.join(target_dir, "playlists.m3u8"), "w") as f:
@@ -297,7 +310,7 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
     targets = [os.path.join(target_dir, f"{vod.index:05d}.ts") for vod in vods]
     asyncio.run(download_all(sources, targets, args.max_workers, rate_limit=args.rate_limit))
 
-    join_playlist = make_join_playlist(vods, targets)
+    join_playlist = make_join_playlist(vods_m3u8, vods, targets)
     join_playlist_path = path.join(target_dir, "playlist_downloaded.m3u8")
     join_playlist.dump(join_playlist_path)  # type: ignore
     click.echo()
@@ -312,7 +325,7 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
         _concat_vods(targets, target)
     else:
         print_log("Joining files...")
-        _join_vods(join_playlist_path, target, args.overwrite, video)
+        _join_vods(join_playlist_path, target, args.overwrite, video, start_offset, duration)
 
     click.echo()
 
@@ -325,6 +338,33 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
     click.echo(f"\nDownloaded: {green(target)}")
 
 
+def filter_vods(
+    vods: List[Vod], start: Optional[int], end: Optional[int]
+) -> Tuple[List[Vod], Decimal, Decimal]:
+    vod_start = Decimal(0)
+    start_offset = Decimal(0)
+    end_offset = Decimal(0)
+    filtered_vods: List[Vod] = []
+
+    for vod in vods:
+        vod_end = vod_start + vod.duration
+
+        if (not start or vod_end > start) and (not end or vod_start < end):
+            filtered_vods.append(vod)
+
+        if start and start > vod_start and start < vod_end:
+            start_offset = start - vod_start
+
+        if end and end > vod_start and end < vod_end:
+            end_offset = vod_end - end
+
+        vod_start = vod_end
+
+    filtered_vod_duration = sum(v.duration for v in filtered_vods)
+    duration = filtered_vod_duration - start_offset - end_offset
+    return filtered_vods, start_offset, duration
+
+
 def http_get(url: str) -> str:
     response = httpx.get(url)
     response.raise_for_status()

twitchdl/output.py

@@ -1,4 +1,5 @@
 import json
+import sys
 from itertools import islice
 from typing import Any, Callable, Generator, List, Optional, TypeVar
 
@@ -10,6 +11,11 @@ from twitchdl.twitch import Clip, Video
 T = TypeVar("T")
 
 
+def clear_line():
+    sys.stdout.write("\033[1K")
+    sys.stdout.write("\r")
+
+
 def truncate(string: str, length: int) -> str:
     if len(string) > length:
         return string[: length - 1] + "…"

twitchdl/playlists.py

@@ -3,8 +3,8 @@ Parse and manipulate m3u8 playlists.
 """
 
 from dataclasses import dataclass
-from os.path import basename
-from typing import Generator, List, Optional
+from decimal import Decimal
+from typing import Generator, List, Optional, OrderedDict
 
 import click
 import m3u8
@@ -28,7 +28,7 @@ class Vod:
     """Ordinal number of the VOD in the playlist"""
     path: str
     """Path part of the VOD URL"""
-    duration: int
+    duration: Decimal
     """Segment duration in seconds"""
 
 
@@ -54,40 +54,30 @@ def load_m3u8(playlist_m3u8: str) -> m3u8.M3U8:
     return m3u8.loads(playlist_m3u8)
 
 
-def enumerate_vods(
-    document: m3u8.M3U8,
-    start: Optional[int] = None,
-    end: Optional[int] = None,
-) -> List[Vod]:
-    """Extract VODs for download from document."""
-    vods = []
-    vod_start = 0
-
-    for index, segment in enumerate(document.segments):
-        vod_end = vod_start + segment.duration
-
-        # `vod_end > start` is used here becuase it's better to download a bit
-        # more than a bit less, similar for the end condition
-        start_condition = not start or vod_end > start
-        end_condition = not end or vod_start < end
-
-        if start_condition and end_condition:
-            vods.append(Vod(index, segment.uri, segment.duration))
-
-        vod_start = vod_end
-
-    return vods
+def parse_vods(document: m3u8.M3U8) -> List[Vod]:
+    return [
+        Vod(index, segment.uri, Decimal(segment.duration))
+        for index, segment in enumerate(document.segments)
+    ]
 
 
-def make_join_playlist(vods: List[Vod], targets: List[str]) -> m3u8.Playlist:
+def make_join_playlist(
+    playlist: m3u8.M3U8,
+    vods: List[Vod],
+    targets: List[str],
+) -> m3u8.Playlist:
     """
     Make a modified playlist which references downloaded VODs
     Keep only the downloaded segments and skip the rest
     """
-    playlist = m3u8.M3U8()
+    org_segments = playlist.segments.copy()
 
-    for vod, target in zip(vods, targets):
-        playlist.add_segment(m3u8.Segment(uri=basename(target), duration=vod.duration))
+    path_map = OrderedDict(zip([v.path for v in vods], targets))
+    playlist.segments.clear()
+    for segment in org_segments:
+        if segment.uri in path_map:
+            segment.uri = path_map[segment.uri]
+            playlist.segments.append(segment)
 
     return playlist
 

twitchdl/progress.py

@@ -7,7 +7,7 @@ from typing import Deque, Dict, NamedTuple, Optional
 
 import click
 
-from twitchdl.output import blue
+from twitchdl.output import blue, clear_line
 from twitchdl.utils import format_size, format_time
 
 logger = logging.getLogger(__name__)
@@ -121,13 +121,14 @@ class Progress:
     def print(self):
         now = time.time()
 
-        # Don't print more often than 10 times per second
-        if self.last_printed and now - self.last_printed < 0.1:
+        # Don't print more often than 5 times per second
+        if self.last_printed and now - self.last_printed < 0.2:
             return
 
         self._recalculate()
 
-        click.echo(f"\rDownloaded {self.vod_downloaded_count}/{self.vod_count} VODs", nl=False)
+        clear_line()
+        click.echo(f"Downloaded {self.vod_downloaded_count}/{self.vod_count} VODs", nl=False)
         click.secho(f" {self.progress_perc}%", fg="blue", nl=False)
 
         if self.estimated_total is not None:
@@ -141,6 +142,4 @@ class Progress:
         if self.remaining_time is not None:
             click.echo(f" ETA {blue(format_time(self.remaining_time))}", nl=False)
 
-        click.echo("    ", nl=False)
-
         self.last_printed = now

twitchdl/twitch.py

@@ -422,7 +422,7 @@ def get_access_token(video_id: str, auth_token: Optional[str] = None) -> AccessT
     query = f"""
     {{
         videoPlaybackAccessToken(
-            id: {video_id},
+            id: "{video_id}",
             params: {{
                 platform: "web",
                 playerBackend: "mediaplayer",