WIP, BROKEN

pyproject.toml

@@ -22,7 +22,7 @@ classifiers = [
 dependencies = [
     "click>=8.0.0,<9.0.0",
     "httpx>=0.17.0,<1.0.0",
-    "m3u8>=1.0.0,<4.0.0",
+    "m3u8>=3.0.0,<5.0.0",
 ]
 
 [tool.setuptools]

tests/test_progress.py

@@ -23,26 +23,31 @@ def test_downloaded():
     assert progress.progress_perc == 0
 
     progress.advance(1, 100)
+    progress._recalculate()
     assert progress.downloaded == 100
     assert progress.progress_bytes == 100
     assert progress.progress_perc == 11
 
     progress.advance(2, 200)
+    progress._recalculate()
     assert progress.downloaded == 300
     assert progress.progress_bytes == 300
     assert progress.progress_perc == 33
 
     progress.advance(3, 150)
+    progress._recalculate()
     assert progress.downloaded == 450
     assert progress.progress_bytes == 450
     assert progress.progress_perc == 50
 
     progress.advance(1, 50)
+    progress._recalculate()
     assert progress.downloaded == 500
     assert progress.progress_bytes == 500
     assert progress.progress_perc == 55
 
     progress.abort(2)
+    progress._recalculate()
     assert progress.downloaded == 500
     assert progress.progress_bytes == 300
     assert progress.progress_perc == 33
@@ -52,6 +57,7 @@ def test_downloaded():
     progress.advance(1, 150)
     progress.advance(2, 300)
     progress.advance(3, 150)
+    progress._recalculate()
 
     assert progress.downloaded == 1100
     assert progress.progress_bytes == 900
@@ -71,12 +77,15 @@ def test_estimated_total():
     assert progress.estimated_total is None
 
     progress.start(1, 12000)
+    progress._recalculate()
     assert progress.estimated_total == 12000 * 3
 
     progress.start(2, 11000)
+    progress._recalculate()
     assert progress.estimated_total == 11500 * 3
 
     progress.start(3, 10000)
+    progress._recalculate()
     assert progress.estimated_total == 11000 * 3
 
 

twitchdl/commands/download.py

@@ -7,7 +7,7 @@ import subprocess
 import tempfile
 from os import path
 from pathlib import Path
-from typing import Dict, List
+from typing import Dict, List, Optional
 from urllib.parse import urlencode, urlparse
 
 import click
@@ -30,6 +30,10 @@ from twitchdl.twitch import Chapter, Clip, ClipAccessToken, Video
 
 
 def download(ids: List[str], args: DownloadOptions):
+    if not ids:
+        print_log("No IDs to downlad given")
+        return
+
     for video_id in ids:
         download_one(video_id, args)
 
@@ -156,7 +160,7 @@ def _crete_temp_dir(base_uri: str) -> str:
     return str(temp_dir)
 
 
-def _get_clip_url(access_token: ClipAccessToken, quality: str) -> str:
+def _get_clip_url(access_token: ClipAccessToken, quality: Optional[str]) -> str:
     qualities = access_token["videoQualities"]
 
     # Quality given as an argument
@@ -184,7 +188,7 @@ def _get_clip_url(access_token: ClipAccessToken, quality: str) -> str:
     return selected_quality["sourceURL"]
 
 
-def get_clip_authenticated_url(slug: str, quality: str):
+def get_clip_authenticated_url(slug: str, quality: Optional[str]):
     print_log("Fetching access token...")
     access_token = twitch.get_clip_access_token(slug)
 
@@ -278,7 +282,8 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
         return
 
     base_uri = re.sub("/[^/]+$", "/", playlist.url)
-    target_dir = _crete_temp_dir(base_uri)
+    target_dir = f".twitch_dl_{video_id}_{playlist.group_id}"
+    os.makedirs(target_dir, exist_ok=True)
 
     # Save playlists for debugging purposes
     with open(path.join(target_dir, "playlists.m3u8"), "w") as f:
@@ -292,7 +297,7 @@ def _download_video(video_id: str, args: DownloadOptions) -> None:
     targets = [os.path.join(target_dir, f"{vod.index:05d}.ts") for vod in vods]
     asyncio.run(download_all(sources, targets, args.max_workers, rate_limit=args.rate_limit))
 
-    join_playlist = make_join_playlist(vods_m3u8, vods, targets)
+    join_playlist = make_join_playlist(vods, targets)
     join_playlist_path = path.join(target_dir, "playlist_downloaded.m3u8")
     join_playlist.dump(join_playlist_path)  # type: ignore
     click.echo()

twitchdl/playlists.py

@@ -3,7 +3,8 @@ Parse and manipulate m3u8 playlists.
 """
 
 from dataclasses import dataclass
-from typing import Generator, List, Optional, OrderedDict
+from os.path import basename
+from typing import Generator, List, Optional
 
 import click
 import m3u8
@@ -78,23 +79,15 @@ def enumerate_vods(
     return vods
 
 
-def make_join_playlist(
-    playlist: m3u8.M3U8,
-    vods: List[Vod],
-    targets: List[str],
-) -> m3u8.Playlist:
+def make_join_playlist(vods: List[Vod], targets: List[str]) -> m3u8.Playlist:
     """
     Make a modified playlist which references downloaded VODs
     Keep only the downloaded segments and skip the rest
     """
-    org_segments = playlist.segments.copy()
+    playlist = m3u8.M3U8()
 
-    path_map = OrderedDict(zip([v.path for v in vods], targets))
-    playlist.segments.clear()
-    for segment in org_segments:
-        if segment.uri in path_map:
-            segment.uri = path_map[segment.uri]
-            playlist.segments.append(segment)
+    for vod, target in zip(vods, targets):
+        playlist.add_segment(m3u8.Segment(uri=basename(target), duration=vod.duration))
 
     return playlist
 

twitchdl/progress.py

@@ -1,7 +1,7 @@
 import logging
 import time
 from collections import deque
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from statistics import mean
 from typing import Deque, Dict, NamedTuple, Optional
 
@@ -31,28 +31,25 @@ class Sample(NamedTuple):
     timestamp: float
 
 
-@dataclass
 class Progress:
-    vod_count: int
-    downloaded: int = 0
-    estimated_total: Optional[int] = None
-    last_printed: float = field(default_factory=time.time)
-    progress_bytes: int = 0
-    progress_perc: int = 0
-    remaining_time: Optional[int] = None
-    speed: Optional[float] = None
-    start_time: float = field(default_factory=time.time)
-    tasks: Dict[TaskId, Task] = field(default_factory=dict)
-    vod_downloaded_count: int = 0
-    samples: Deque[Sample] = field(default_factory=lambda: deque(maxlen=100))
+    def __init__(self, vod_count: int):
+        self.downloaded: int = 0
+        self.estimated_total: Optional[int] = None
+        self.last_printed: Optional[float] = None
+        self.progress_bytes: int = 0
+        self.progress_perc: int = 0
+        self.remaining_time: Optional[int] = None
+        self.samples: Deque[Sample] = deque(maxlen=1000)
+        self.speed: Optional[float] = None
+        self.tasks: Dict[TaskId, Task] = {}
+        self.vod_count = vod_count
+        self.vod_downloaded_count: int = 0
 
     def start(self, task_id: int, size: int):
         if task_id in self.tasks:
             raise ValueError(f"Task {task_id}: cannot start, already started")
 
         self.tasks[task_id] = Task(task_id, size)
-        self._calculate_total()
-        self._calculate_progress()
         self.print()
 
     def advance(self, task_id: int, size: int):
@@ -63,7 +60,6 @@ class Progress:
         self.progress_bytes += size
         self.tasks[task_id].advance(size)
         self.samples.append(Sample(self.downloaded, time.time()))
-        self._calculate_progress()
         self.print()
 
     def already_downloaded(self, task_id: int, size: int):
@@ -73,8 +69,6 @@ class Progress:
         self.tasks[task_id] = Task(task_id, size)
         self.progress_bytes += size
         self.vod_downloaded_count += 1
-        self._calculate_total()
-        self._calculate_progress()
         self.print()
 
     def abort(self, task_id: int):
@@ -83,9 +77,6 @@ class Progress:
 
         del self.tasks[task_id]
         self.progress_bytes = sum(t.downloaded for t in self.tasks.values())
-
-        self._calculate_total()
-        self._calculate_progress()
         self.print()
 
     def end(self, task_id: int):
@@ -101,12 +92,10 @@ class Progress:
         self.vod_downloaded_count += 1
         self.print()
 
-    def _calculate_total(self):
+    def _recalculate(self):
         self.estimated_total = (
             int(mean(t.size for t in self.tasks.values()) * self.vod_count) if self.tasks else None
         )
-
-    def _calculate_progress(self):
         self.speed = self._calculate_speed()
         self.progress_perc = (
             int(100 * self.progress_bytes / self.estimated_total) if self.estimated_total else 0
@@ -133,9 +122,11 @@ class Progress:
         now = time.time()
 
         # Don't print more often than 10 times per second
-        if now - self.last_printed < 0.1:
+        if self.last_printed and now - self.last_printed < 0.1:
             return
 
+        self._recalculate()
+
         click.echo(f"\rDownloaded {self.vod_downloaded_count}/{self.vod_count} VODs", nl=False)
         click.secho(f" {self.progress_perc}%", fg="blue", nl=False)