WIP, BROKEN

.gitignore

@@ -15,4 +15,3 @@ tmp/
 /*.pyz
 /pyrightconfig.json
 /book
-/*.mkv

CHANGELOG.md

@@ -3,10 +3,6 @@ twitch-dl changelog
 
 <!-- Do not edit. This file is automatically generated from changelog.yaml.-->
 
-### [2.3.1 (2024-05-19)](https://github.com/ihabunek/twitch-dl/releases/tag/2.3.1)
-
-* Fix fetching access token (#155, thanks @KryptonicDragon)
-
 ### [2.3.0 (2024-04-27)](https://github.com/ihabunek/twitch-dl/releases/tag/2.3.0)
 
 * Show more playlist data when choosing quality

changelog.yaml

@@ -1,8 +1,3 @@
-2.3.1:
-  date: 2024-05-19
-  changes:
-    - "Fix fetching access token (#155, thanks @KryptonicDragon)"
-
 2.3.0:
   date: 2024-04-27
   changes:

docs/changelog.md

@@ -3,10 +3,6 @@ twitch-dl changelog
 
 <!-- Do not edit. This file is automatically generated from changelog.yaml.-->
 
-### [2.3.1 (2024-05-19)](https://github.com/ihabunek/twitch-dl/releases/tag/2.3.1)
-
-* Fix fetching access token (#155, thanks @KryptonicDragon)
-
 ### [2.3.0 (2024-04-27)](https://github.com/ihabunek/twitch-dl/releases/tag/2.3.0)
 
 * Show more playlist data when choosing quality

tests/test_api.py

@@ -9,7 +9,7 @@ from twitchdl import twitch
 from twitchdl.commands.download import get_clip_authenticated_url
 from twitchdl.commands.videos import get_game_ids
 from twitchdl.exceptions import ConsoleError
-from twitchdl.playlists import parse_playlists
+from twitchdl.playlists import enumerate_vods, load_m3u8, parse_playlists
 
 TEST_CHANNEL = "bananasaurus_rex"
 
@@ -37,6 +37,10 @@ def test_get_videos():
     playlist_txt = httpx.get(playlist_url).text
     assert playlist_txt.startswith("#EXTM3U")
 
+    playlist_m3u8 = load_m3u8(playlist_txt)
+    vods = enumerate_vods(playlist_m3u8)
+    assert vods[0].path == "0.ts"
+
 
 def test_get_clips():
     """

tests/test_download.py

@@ -1,90 +0,0 @@
-from decimal import Decimal
-
-from twitchdl.commands.download import filter_vods
-from twitchdl.playlists import Vod
-
-VODS = [
-    Vod(index=1, path="1.ts", duration=Decimal("10.0")),
-    Vod(index=2, path="2.ts", duration=Decimal("10.0")),
-    Vod(index=3, path="3.ts", duration=Decimal("10.0")),
-    Vod(index=4, path="4.ts", duration=Decimal("10.0")),
-    Vod(index=5, path="5.ts", duration=Decimal("10.0")),
-    Vod(index=6, path="6.ts", duration=Decimal("10.0")),
-    Vod(index=7, path="7.ts", duration=Decimal("10.0")),
-    Vod(index=8, path="8.ts", duration=Decimal("10.0")),
-    Vod(index=9, path="9.ts", duration=Decimal("10.0")),
-    Vod(index=10, path="10.ts", duration=Decimal("3.15")),
-]
-
-
-def test_filter_vods_no_start_no_end():
-    vods, start_offset, duration = filter_vods(VODS, None, None)
-    assert vods == VODS
-    assert start_offset == Decimal("0")
-    assert duration == Decimal("93.15")
-
-
-def test_filter_vods_start():
-    # Zero offset
-    vods, start_offset, duration = filter_vods(VODS, 0, None)
-    assert [v.index for v in vods] == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-    assert start_offset == Decimal("0")
-    assert duration == Decimal("93.15")
-
-    # Mid-vod
-    vods, start_offset, duration = filter_vods(VODS, 13, None)
-    assert [v.index for v in vods] == [2, 3, 4, 5, 6, 7, 8, 9, 10]
-    assert start_offset == Decimal("3.0")
-    assert duration == Decimal("80.15")
-
-    # Between vods
-    vods, start_offset, duration = filter_vods(VODS, 50, None)
-    assert [v.index for v in vods] == [6, 7, 8, 9, 10]
-    assert start_offset == Decimal("0")
-    assert duration == Decimal("43.15")
-
-    # Close to end
-    vods, start_offset, duration = filter_vods(VODS, 93, None)
-    assert [v.index for v in vods] == [10]
-    assert start_offset == Decimal("3.0")
-    assert duration == Decimal("0.15")
-
-
-def test_filter_vods_end():
-    # Zero offset
-    vods, start_offset, duration = filter_vods(VODS, 0, None)
-    assert [v.index for v in vods] == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-    assert start_offset == Decimal("0")
-    assert duration == Decimal("93.15")
-
-    # Mid-vod
-    vods, start_offset, duration = filter_vods(VODS, None, 56)
-    assert [v.index for v in vods] == [1, 2, 3, 4, 5, 6]
-    assert start_offset == Decimal("0")
-    assert duration == Decimal("56")
-
-    # Between vods
-    vods, start_offset, duration = filter_vods(VODS, None, 30)
-    assert [v.index for v in vods] == [1, 2, 3]
-    assert start_offset == Decimal("0")
-    assert duration == Decimal("30")
-
-
-def test_filter_vods_start_end():
-    # Zero offset
-    vods, start_offset, duration = filter_vods(VODS, 0, 0)
-    assert [v.index for v in vods] == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-    assert start_offset == Decimal("0")
-    assert duration == Decimal("93.15")
-
-    # Mid-vod
-    vods, start_offset, duration = filter_vods(VODS, 32, 56)
-    assert [v.index for v in vods] == [4, 5, 6]
-    assert start_offset == Decimal("2")
-    assert duration == Decimal("24")
-
-    # Between vods
-    vods, start_offset, duration = filter_vods(VODS, 20, 60)
-    assert [v.index for v in vods] == [3, 4, 5, 6]
-    assert start_offset == Decimal("0")
-    assert duration == Decimal("40")

twitchdl/commands/download.py

@@ -5,10 +5,9 @@ import re
 import shutil
 import subprocess
 import tempfile
-from decimal import Decimal
 from os import path
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional
 from urllib.parse import urlencode, urlparse
 
 import click
@@ -21,11 +20,10 @@ from twitchdl.exceptions import ConsoleError
 from twitchdl.http import download_all
 from twitchdl.output import blue, bold, green, print_log, yellow
 from twitchdl.playlists import (
-    Vod,
+    enumerate_vods,
     load_m3u8,
     make_join_playlist,
     parse_playlists,
-    parse_vods,
     select_playlist,
 )
 from twitchdl.twitch import Chapter, Clip, ClipAccessToken, Video
@@ -52,14 +50,7 @@ def download_one(video: str, args: DownloadOptions):
     raise ConsoleError(f"Invalid input: {video}")
 
 
-def _join_vods(
-    playlist_path: str,
-    target: str,
-    overwrite: bool,
-    video: Video,
-    start_offset: int,
-    duration: int,
-):
+def _join_vods(playlist_path: str, target: str, overwrite: bool, video: Video):
     description = video["description"] or ""
     description = description.strip()
 
@@ -69,10 +60,6 @@ def _join_vods(
         playlist_path,
         "-c",
         "copy",
-        "-ss",
-        str(start_offset),
-        "-to",
-        str(duration),
         "-metadata",
         f"artist={video['creator']['displayName']}",
         "-metadata",
@@ -86,11 +73,11 @@ def _join_vods(
         "warning",
         f"file:{target}",
     ]
+
     if overwrite:
         command.append("-y")
 
     click.secho(f"{' '.join(command)}", dim=True)
-
     result = subprocess.run(command)
     if result.returncode != 0:
         raise ConsoleError("Joining files failed")
@@ -288,15 +275,15 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
     print_log("Fetching playlist...")
     vods_text = http_get(playlist.url)
     vods_m3u8 = load_m3u8(vods_text)
-    all_vods = parse_vods(vods_m3u8)
-    vods, start_offset, duration = filter_vods(all_vods, start, end)
+    vods = enumerate_vods(vods_m3u8, start, end)
 
     if args.dry_run:
         click.echo("Dry run, video not downloaded.")
         return
 
     base_uri = re.sub("/[^/]+$", "/", playlist.url)
-    target_dir = _crete_temp_dir(base_uri)
+    target_dir = f".twitch_dl_{video_id}_{playlist.group_id}"
+    os.makedirs(target_dir, exist_ok=True)
 
     # Save playlists for debugging purposes
     with open(path.join(target_dir, "playlists.m3u8"), "w") as f:
@@ -310,7 +297,7 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
     targets = [os.path.join(target_dir, f"{vod.index:05d}.ts") for vod in vods]
     asyncio.run(download_all(sources, targets, args.max_workers, rate_limit=args.rate_limit))
 
-    join_playlist = make_join_playlist(vods_m3u8, vods, targets)
+    join_playlist = make_join_playlist(vods, targets)
     join_playlist_path = path.join(target_dir, "playlist_downloaded.m3u8")
     join_playlist.dump(join_playlist_path)  # type: ignore
     click.echo()
@@ -325,7 +312,7 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
         _concat_vods(targets, target)
     else:
         print_log("Joining files...")
-        _join_vods(join_playlist_path, target, args.overwrite, video, start_offset, duration)
+        _join_vods(join_playlist_path, target, args.overwrite, video)
 
     click.echo()
 
@@ -338,33 +325,6 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
     click.echo(f"\nDownloaded: {green(target)}")
 
 
-def filter_vods(
-    vods: List[Vod], start: Optional[int], end: Optional[int]
-) -> Tuple[List[Vod], Decimal, Decimal]:
-    vod_start = Decimal(0)
-    start_offset = Decimal(0)
-    end_offset = Decimal(0)
-    filtered_vods: List[Vod] = []
-
-    for vod in vods:
-        vod_end = vod_start + vod.duration
-
-        if (not start or vod_end > start) and (not end or vod_start < end):
-            filtered_vods.append(vod)
-
-        if start and start > vod_start and start < vod_end:
-            start_offset = start - vod_start
-
-        if end and end > vod_start and end < vod_end:
-            end_offset = vod_end - end
-
-        vod_start = vod_end
-
-    filtered_vod_duration = sum(v.duration for v in filtered_vods)
-    duration = filtered_vod_duration - start_offset - end_offset
-    return filtered_vods, start_offset, duration
-
-
 def http_get(url: str) -> str:
     response = httpx.get(url)
     response.raise_for_status()

twitchdl/output.py

@@ -1,5 +1,4 @@
 import json
-import sys
 from itertools import islice
 from typing import Any, Callable, Generator, List, Optional, TypeVar
 
@@ -11,11 +10,6 @@ from twitchdl.twitch import Clip, Video
 T = TypeVar("T")
 
 
-def clear_line():
-    sys.stdout.write("\033[1K")
-    sys.stdout.write("\r")
-
-
 def truncate(string: str, length: int) -> str:
     if len(string) > length:
         return string[: length - 1] + "…"

twitchdl/playlists.py

@@ -3,8 +3,8 @@ Parse and manipulate m3u8 playlists.
 """
 
 from dataclasses import dataclass
-from decimal import Decimal
-from typing import Generator, List, Optional, OrderedDict
+from os.path import basename
+from typing import Generator, List, Optional
 
 import click
 import m3u8
@@ -28,7 +28,7 @@ class Vod:
     """Ordinal number of the VOD in the playlist"""
     path: str
     """Path part of the VOD URL"""
-    duration: Decimal
+    duration: int
     """Segment duration in seconds"""
 
 
@@ -54,30 +54,40 @@ def load_m3u8(playlist_m3u8: str) -> m3u8.M3U8:
     return m3u8.loads(playlist_m3u8)
 
 
-def parse_vods(document: m3u8.M3U8) -> List[Vod]:
-    return [
-        Vod(index, segment.uri, Decimal(segment.duration))
-        for index, segment in enumerate(document.segments)
-    ]
+def enumerate_vods(
+    document: m3u8.M3U8,
+    start: Optional[int] = None,
+    end: Optional[int] = None,
+) -> List[Vod]:
+    """Extract VODs for download from document."""
+    vods = []
+    vod_start = 0
+
+    for index, segment in enumerate(document.segments):
+        vod_end = vod_start + segment.duration
+
+        # `vod_end > start` is used here becuase it's better to download a bit
+        # more than a bit less, similar for the end condition
+        start_condition = not start or vod_end > start
+        end_condition = not end or vod_start < end
+
+        if start_condition and end_condition:
+            vods.append(Vod(index, segment.uri, segment.duration))
+
+        vod_start = vod_end
+
+    return vods
 
 
-def make_join_playlist(
-    playlist: m3u8.M3U8,
-    vods: List[Vod],
-    targets: List[str],
-) -> m3u8.Playlist:
+def make_join_playlist(vods: List[Vod], targets: List[str]) -> m3u8.Playlist:
     """
     Make a modified playlist which references downloaded VODs
     Keep only the downloaded segments and skip the rest
     """
-    org_segments = playlist.segments.copy()
+    playlist = m3u8.M3U8()
 
-    path_map = OrderedDict(zip([v.path for v in vods], targets))
-    playlist.segments.clear()
-    for segment in org_segments:
-        if segment.uri in path_map:
-            segment.uri = path_map[segment.uri]
-            playlist.segments.append(segment)
+    for vod, target in zip(vods, targets):
+        playlist.add_segment(m3u8.Segment(uri=basename(target), duration=vod.duration))
 
     return playlist
 

twitchdl/progress.py

@@ -7,7 +7,7 @@ from typing import Deque, Dict, NamedTuple, Optional
 
 import click
 
-from twitchdl.output import blue, clear_line
+from twitchdl.output import blue
 from twitchdl.utils import format_size, format_time
 
 logger = logging.getLogger(__name__)
@@ -121,14 +121,13 @@ class Progress:
     def print(self):
         now = time.time()
 
-        # Don't print more often than 5 times per second
-        if self.last_printed and now - self.last_printed < 0.2:
+        # Don't print more often than 10 times per second
+        if self.last_printed and now - self.last_printed < 0.1:
             return
 
         self._recalculate()
 
-        clear_line()
-        click.echo(f"Downloaded {self.vod_downloaded_count}/{self.vod_count} VODs", nl=False)
+        click.echo(f"\rDownloaded {self.vod_downloaded_count}/{self.vod_count} VODs", nl=False)
         click.secho(f" {self.progress_perc}%", fg="blue", nl=False)
 
         if self.estimated_total is not None:
@@ -142,4 +141,6 @@ class Progress:
         if self.remaining_time is not None:
             click.echo(f" ETA {blue(format_time(self.remaining_time))}", nl=False)
 
+        click.echo("    ", nl=False)
+
         self.last_printed = now

twitchdl/twitch.py

@@ -422,7 +422,7 @@ def get_access_token(video_id: str, auth_token: Optional[str] = None) -> AccessT
     query = f"""
     {{
         videoPlaybackAccessToken(
-            id: "{video_id}",
+            id: {video_id},
             params: {{
                 platform: "web",
                 playerBackend: "mediaplayer",