Compare commits

..

13 Commits

12 changed files with 432 additions and 160 deletions

View File

@ -3,6 +3,18 @@ twitch-dl changelog
<!-- Do not edit. This file is automatically generated from changelog.yaml.-->
**1.20.0 (2022-02-25)**
* Add `--json` option to `videos` command (#92, thanks @miff2000)
* Add `--all` option to `videos` and `clips` commands to list all clips or
videos in one go.
* Modify how `--pager` works, will make multiple requests if needed to show all
available items, ignoring `--limit`.
**1.19.0 (2022-02-05)**
* Add support for downloading audio only (#10)
**1.18.1 (2022-02-05)**
* Fix issues with output formats (#87, #89)

View File

@ -114,18 +114,42 @@ playlist URLs for videos.
### Listing videos
List recent streams for a given channel:
List recent channel videos (10 by default):
```
twitch-dl videos bananasaurus_rex
```
Use the `--game` option to specify one or more games to show:
Limit to videos of one or more games:
```
twitch-dl videos --game "doom eternal" --game "cave story" bananasaurus_rex
```
List all channel videos at once:
```
twitch-dl videos bananasaurus_rex --all
```
List all channel videos in pages of 10:
```
twitch-dl videos bananasaurus_rex --pager
```
Page size can be adjusted by passing number of items per page:
```
twitch-dl videos bananasaurus_rex --pager 5
```
Returns all videos as a JSON list. Useful for scripting.
```
twitch-dl videos bananasaurus_rex --json --all
```
### Downloading videos
Download a video by ID or URL:
@ -147,6 +171,12 @@ Setting quality to `source` will download the best available quality:
twitch-dl download -q source 221837124
```
Setting quality to `audio_only` will download only audio:
```
twitch-dl download -q audio_only 221837124
```
### Overriding file name
The target filename can be defined by passing the `--output` option followed by
@ -185,6 +215,9 @@ Expands to: `KatLink - Dark Souls III - Dark Souls 3 First playthrough.mkv`
### Listing clips
Listing clips works similar to listing videos. Shows 10 clips by default. Use
`--all` to list all in one go or `--pager` to show them in pages.
List clips for the given period:
```
@ -193,13 +226,15 @@ twitch-dl clips bananasaurus_rex --period last_week
Supported periods are: `last_day`, `last_week`, `last_month`, `all_time`.
For listing a large number of clips, it's nice to page them:
Also supports JSON output:
```
twitch-dl clips bananasaurus_rex --period all_time --limit 10 --pager
twitch-dl clips bananasaurus_rex --json --all
```
This will show 10 clips at a time and ask to continue.
Note that this may make multiple requests to the server because each request is
limited to 100 clips, so it may take a little while. You can use `--debug` to
log requests.
### Downloading clips
@ -264,6 +299,6 @@ make man
License
-------
Copyright 2018-2020 Ivan Habunek <ivan@habunek.com>
Copyright 2018-2022 Ivan Habunek <ivan@habunek.com>
Licensed under the GPLv3: http://www.gnu.org/licenses/gpl-3.0.html

View File

@ -1,3 +1,15 @@
1.20.0:
date: 2022-02-25
changes:
- "Add `--json` option to `videos` command (#92, thanks @miff2000)"
- "Add `--all` option to `videos` and `clips` commands to list all clips or videos in one go."
- "Modify how `--pager` works, will make multiple requests if needed to show all available items, ignoring `--limit`."
1.19.0:
date: 2022-02-05
changes:
- "Add support for downloading audio only (#10)"
1.18.1:
date: 2022-02-05
changes:

139
download.py Executable file
View File

@ -0,0 +1,139 @@
#!/usr/bin/env python3
import asyncio
import httpx
import m3u8
import os
import re
import requests
from rich.console import Console
from rich.progress import Progress, TaskID, TransferSpeedColumn
from twitchdl import twitch
from typing import List
console = Console()
# WORKER_POOL_SIZE = 5
CHUNK_SIZE = 1024 * 256
# CONNECT_TIMEOUT = 5
# RETRY_COUNT = 5
class Bucket:
capacity: int
rate: int
content_kb: int = 0
KB = 1024
MB = 1024 * KB
DEFAULT_CAPACITY = 10 * MB
DEFAULT_RATE = 1 * MB
def __init__(self, /, *, capacity=1 * MB, rate=100 * KB):
self.capacity = capacity
self.rate = rate
class Downloader:
downloaded: int = 0
downloaded_vod_count: int = 0
progress: Progress
total_task_id: TaskID
vod_count: int
def __init__(self, worker_count: int):
self.worker_count = worker_count
async def run(self, sources, targets):
if len(sources) != len(targets):
raise ValueError(f"Got {len(sources)} sources but {len(targets)} targets.")
self.vod_count = len(sources)
columns = [*Progress.get_default_columns(), TransferSpeedColumn()]
with Progress(*columns, console=console) as progress:
self.progress = progress
self.total_task_id = self.progress.add_task("Total")
await self.download(sources, targets)
for task in self.progress.tasks:
if task.id != self.total_task_id:
self.progress.remove_task(task.id)
console.print("[chartreuse3]Done.[/chartreuse3]")
def on_init(self, filename: str) -> TaskID:
return self.progress.add_task(filename)
def on_start(self, task_id: TaskID, size: int):
self.downloaded_vod_count += 1
self.downloaded += size
estimated_total = int(self.downloaded * self.vod_count / self.downloaded_vod_count)
self.progress.update(self.total_task_id, total=estimated_total)
self.progress.update(task_id, total=size)
self.progress.start_task(task_id)
def on_progress(self, task_id: TaskID, chunk_size: int):
self.progress.update(self.total_task_id, advance=chunk_size)
self.progress.update(task_id, advance=chunk_size)
def on_end(self, task_id: TaskID):
async def remove_task_after(task_id, delay):
await asyncio.sleep(delay)
self.progress.remove_task(task_id)
asyncio.create_task(remove_task_after(task_id, 1))
async def download_one(
self,
client: httpx.AsyncClient,
semaphore: asyncio.Semaphore,
source: str,
target: str,
):
async with semaphore:
with open(target, "wb") as f:
# TODO: handle failure (retries etc)
task_id = self.on_init(os.path.basename(target))
async with client.stream("GET", source) as response:
size = int(response.headers.get("content-length"))
self.on_start(task_id, size)
async for chunk in response.aiter_bytes(chunk_size=CHUNK_SIZE):
f.write(chunk)
self.on_progress(task_id, len(chunk))
self.on_end(task_id)
async def download(self, sources: List[str], targets: List[str]):
async with httpx.AsyncClient() as client:
semaphore = asyncio.Semaphore(self.worker_count)
tasks = [self.download_one(client, semaphore, source, target)
for source, target in zip(sources, targets)]
await asyncio.gather(*tasks)
if __name__ == "__main__":
videos = twitch.get_channel_videos("bananasaurus_rex", 1, "time")
video_id = videos["edges"][0]["node"]["id"]
from twitchdl.commands.download import _get_vod_paths
console.print("[grey53]Fetching access token...[/grey53]")
access_token = twitch.get_access_token(video_id)
console.print("[grey53]Fetching playlists...[/grey53]")
playlists = twitch.get_playlists(video_id, access_token)
playlist_uri = m3u8.loads(playlists).playlists[-1].uri
console.print("[grey53]Fetching playlist...[/grey53]")
playlist = requests.get(playlist_uri).text
vods = _get_vod_paths(m3u8.loads(playlist), None, None)
base_uri = re.sub("/[^/]+$", "/", playlist_uri)
urls = ["".join([base_uri, vod]) for vod in vods][:3]
targets = [f"tmp/{os.path.basename(url).zfill(8)}" for url in urls]
try:
print("Starting download using 3 workers")
d = Downloader(3)
asyncio.run(d.run(urls, targets))
except KeyboardInterrupt:
console.print("[bold red]Aborted.[/bold red]")

View File

@ -11,7 +11,7 @@ makes it faster.
setup(
name='twitch-dl',
version='1.18.1',
version='1.20.0',
description='Twitch downloader',
long_description=long_description.strip(),
author='Ivan Habunek',

View File

@ -1,3 +1,3 @@
__version__ = "1.18.1"
__version__ = "1.20.0"
CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko"

View File

@ -1,5 +1,7 @@
import re
import sys
from itertools import islice
from os import path
from twitchdl import twitch, utils
@ -10,21 +12,26 @@ from twitchdl.output import print_out, print_clip, print_json
def clips(args):
# Ignore --limit if --pager or --all are given
limit = sys.maxsize if args.all or args.pager else args.limit
generator = twitch.channel_clips_generator(args.channel_name, args.period, limit)
if args.json:
return _clips_json(args)
return print_json(list(generator))
if args.download:
return _clips_download(args)
return _download_clips(generator)
return _clips_list(args)
if args.pager:
print(args)
return _print_paged(generator, args.pager)
return _print_all(generator, args)
def _continue():
print_out(
"\nThere are more clips. "
"Press <green><b>Enter</green> to continue, "
"<yellow><b>Ctrl+C</yellow> to break."
)
print_out("Press <green><b>Enter</green> to continue, <yellow><b>Ctrl+C</yellow> to break.")
try:
input()
@ -34,28 +41,7 @@ def _continue():
return True
def _get_game_ids(names):
if not names:
return []
game_ids = []
for name in names:
print_out("<dim>Looking up game '{}'...</dim>".format(name))
game_id = twitch.get_game_id(name)
if not game_id:
raise ConsoleError("Game '{}' not found".format(name))
game_ids.append(int(game_id))
return game_ids
def _clips_json(args):
clips = twitch.get_channel_clips(args.channel_name, args.period, args.limit)
nodes = list(edge["node"] for edge in clips["edges"])
print_json(nodes)
def _clip_target_filename(clip):
def _target_filename(clip):
url = clip["videoQualities"][0]["sourceURL"]
_, ext = path.splitext(url)
ext = ext.lstrip(".")
@ -73,54 +59,54 @@ def _clip_target_filename(clip):
return "{}.{}".format(name, ext)
def _clips_download(args):
downloaded_count = 0
generator = twitch.channel_clips_generator(args.channel_name, args.period, 100)
def _download_clips(generator):
for clip in generator:
target = _target_filename(clip)
for clips, _ in generator:
for clip in clips["edges"]:
clip = clip["node"]
if path.exists(target):
print_out("Already downloaded: <green>{}</green>".format(target))
else:
url = get_clip_authenticated_url(clip["slug"], "source")
target = _clip_target_filename(clip)
if path.exists(target):
print_out("Already downloaded: <green>{}</green>".format(target))
else:
print_out("Downloading: <yellow>{}</yellow>".format(target))
download_file(url, target)
downloaded_count += 1
if args.limit and downloaded_count >= args.limit:
return
print_out("Downloading: <yellow>{}</yellow>".format(target))
download_file(url, target)
def _clips_list(args):
print_out("<dim>Loading clips...</dim>")
generator = twitch.channel_clips_generator(args.channel_name, args.period, args.limit)
def _print_all(generator, args):
for clip in generator:
print_out()
print_clip(clip)
if not args.all:
print_out(
"\n<dim>There may be more clips. " +
"Increase the --limit, use --all or --pager to see the rest.</dim>"
)
def _print_paged(generator, page_size):
iterator = iter(generator)
page = list(islice(iterator, page_size))
first = 1
last = first + len(page) - 1
for clips, has_more in generator:
count = len(clips["edges"]) if "edges" in clips else 0
last = first + count - 1
while True:
print_out("-" * 80)
print_out()
for clip in page:
print_clip(clip)
print_out()
last = first + len(page) - 1
print_out("-" * 80)
print_out("<yellow>Showing clips {}-{} of ??</yellow>".format(first, last))
print_out("<yellow>Clips {}-{}</yellow>".format(first, last))
for clip in clips["edges"]:
print_out()
print_clip(clip["node"])
first = first + len(page)
last = first + 1
if not args.pager:
print_out(
"\n<dim>There are more clips. "
"Increase the --limit or use --pager to see the rest.</dim>"
)
page = list(islice(iterator, page_size))
if not page or not _continue():
break
if not has_more or not _continue():
break
first += count
else:
print_out("<yellow>No clips found</yellow>")

View File

@ -18,10 +18,15 @@ from twitchdl.output import print_out
def _parse_playlists(playlists_m3u8):
playlists = m3u8.loads(playlists_m3u8)
for p in playlists.playlists:
name = p.media[0].name if p.media else ""
resolution = "x".join(str(r) for r in p.stream_info.resolution)
yield name, resolution, p.uri
for p in sorted(playlists.playlists, key=lambda p: p.stream_info.resolution is None):
if p.stream_info.resolution:
name = p.media[0].name
description = "x".join(str(r) for r in p.stream_info.resolution)
else:
name = p.media[0].group_id
description = None
yield name, description, p.uri
def _get_playlist_by_name(playlists, quality):
@ -41,7 +46,10 @@ def _get_playlist_by_name(playlists, quality):
def _select_playlist_interactive(playlists):
print_out("\nAvailable qualities:")
for n, (name, resolution, uri) in enumerate(playlists):
print_out("{}) {} [{}]".format(n + 1, name, resolution))
if resolution:
print_out("{}) {} [{}]".format(n + 1, name, resolution))
else:
print_out("{}) {}".format(n + 1, name))
no = utils.read_int("Choose quality", min=1, max=len(playlists) + 1, default=1)
_, _, uri = playlists[no - 1]

View File

@ -1,21 +1,50 @@
import sys
from twitchdl import twitch
from twitchdl.exceptions import ConsoleError
from twitchdl.output import print_out, print_video
from twitchdl.output import print_out, print_paged_videos, print_video, print_json
def _continue():
print_out(
"\nThere are more videos. "
"Press <green><b>Enter</green> to continue, "
"<yellow><b>Ctrl+C</yellow> to break."
)
def videos(args):
game_ids = _get_game_ids(args.game)
# Ignore --limit if --pager or --all are given
max_videos = sys.maxsize if args.all or args.pager else args.limit
try:
input()
except KeyboardInterrupt:
return False
total_count, generator = twitch.channel_videos_generator(
args.channel_name, max_videos, args.sort, args.type, game_ids=game_ids)
return True
if args.json:
videos = list(generator)
print_json({
"count": len(videos),
"totalCount": total_count,
"videos": videos
})
return
if total_count == 0:
print_out("<yellow>No videos found</yellow>")
return
if args.pager:
print_paged_videos(generator, args.pager, total_count)
return
count = 0
for video in generator:
print_out()
print_video(video)
count += 1
print_out()
print_out("-" * 80)
print_out("<yellow>Videos {}-{} of {}</yellow>".format(1, count, total_count))
if total_count > count:
print_out()
print_out(
"<dim>There are more videos. Increase the --limit, use --all or --pager to see the rest.</dim>"
)
def _get_game_ids(names):
@ -31,39 +60,3 @@ def _get_game_ids(names):
game_ids.append(int(game_id))
return game_ids
def videos(args):
game_ids = _get_game_ids(args.game)
print_out("<dim>Loading videos...</dim>")
generator = twitch.channel_videos_generator(
args.channel_name, args.limit, args.sort, args.type, game_ids=game_ids)
first = 1
for videos, has_more in generator:
count = len(videos["edges"]) if "edges" in videos else 0
total = videos["totalCount"]
last = first + count - 1
print_out("-" * 80)
print_out("<yellow>Showing videos {}-{} of {}</yellow>".format(first, last, total))
for video in videos["edges"]:
print_out()
print_video(video["node"])
if not args.pager and has_more:
print_out(
"\n<dim>There are more videos. "
"Increase the --limit or use --pager to see the rest.</dim>"
)
break
if not has_more or not _continue():
break
first += count
else:
print_out("<yellow>No videos found</yellow>")

View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
import logging
import sys
from argparse import ArgumentParser, ArgumentTypeError
@ -33,15 +34,14 @@ def time(value):
return hours * 3600 + minutes * 60 + seconds
def limit(value):
"""Validates the number of videos to fetch."""
def pos_integer(value):
try:
value = int(value)
except ValueError:
raise ArgumentTypeError("must be an integer")
if not 1 <= int(value) <= 100:
raise ArgumentTypeError("must be between 1 and 100")
if value < 1:
raise ArgumentTypeError("must be positive")
return value
@ -61,10 +61,15 @@ COMMANDS = [
"type": str,
}),
(["-l", "--limit"], {
"help": "Number of videos to fetch (default 10, max 100)",
"type": limit,
"help": "Number of videos to fetch (default 10)",
"type": pos_integer,
"default": 10,
}),
(["-a", "--all"], {
"help": "Fetch all videos, overrides --limit",
"action": "store_true",
"default": False,
}),
(["-s", "--sort"], {
"help": "Sorting order of videos. (default: time)",
"type": str,
@ -77,11 +82,17 @@ COMMANDS = [
"choices": ["archive", "highlight", "upload"],
"default": "archive",
}),
(["-p", "--pager"], {
"help": "If there are more results than LIMIT, ask to show next page",
(["-j", "--json"], {
"help": "Show results as JSON. Ignores --pager.",
"action": "store_true",
"default": False,
}),
(["-p", "--pager"], {
"help": "Number of videos to show per page. Disabled by default.",
"type": pos_integer,
"nargs": "?",
"const": 10,
}),
],
),
Command(
@ -94,9 +105,14 @@ COMMANDS = [
}),
(["-l", "--limit"], {
"help": "Number of videos to fetch (default 10, max 100)",
"type": limit,
"type": pos_integer,
"default": 10,
}),
(["-a", "--all"], {
"help": "Fetch all videos, overrides --limit",
"action": "store_true",
"default": False,
}),
(["-P", "--period"], {
"help": "Period from which to return clips. (default: 'all_time')",
"type": str,
@ -109,9 +125,10 @@ COMMANDS = [
"default": False,
}),
(["-p", "--pager"], {
"help": "If there are more results than LIMIT, ask to show next page",
"action": "store_true",
"default": False,
"help": "Number of clips to show per page. Disabled by default.",
"type": pos_integer,
"nargs": "?",
"const": 10,
}),
(["-d", "--download"], {
"help": "Download all videos in given period (in source quality)",
@ -231,6 +248,9 @@ def main():
parser = get_parser()
args = parser.parse_args()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
if args.version:
print("twitch-dl v{}".format(__version__))
return

View File

@ -4,6 +4,7 @@ import json
import sys
import re
from itertools import islice
from twitchdl import utils
@ -71,23 +72,51 @@ def print_log(*args, **kwargs):
def print_video(video):
published_at = video["publishedAt"].replace("T", " @ ").replace("Z", "")
length = utils.format_duration(video["lengthSeconds"])
channel = video["creator"]["displayName"]
playing = (
"playing <blue>{}</blue>".format(video["game"]["name"])
if video["game"] else ""
)
channel = "<blue>{}</blue>".format(video["creator"]["displayName"]) if video["creator"] else ""
playing = "playing <blue>{}</blue>".format(video["game"]["name"]) if video["game"] else ""
# Can't find URL in video object, strange
url = "https://www.twitch.tv/videos/{}".format(video["id"])
print_out("<b>Video {}</b>".format(video["id"]))
print_out("<green>{}</green>".format(video["title"]))
print_out("<blue>{}</blue> {}".format(channel, playing))
if channel or playing:
print_out(" ".join([channel, playing]))
print_out("Published <blue>{}</blue> Length: <blue>{}</blue> ".format(published_at, length))
print_out("<i>{}</i>".format(url))
def print_paged_videos(generator, page_size, total_count):
iterator = iter(generator)
page = list(islice(iterator, page_size))
first = 1
last = first + len(page) - 1
while True:
print_out("-" * 80)
print_out()
for video in page:
print_video(video)
print_out()
last = first + len(page) - 1
print_out("-" * 80)
print_out("<yellow>Videos {}-{} of {}</yellow>".format(first, last, total_count))
first = first + len(page)
last = first + 1
page = list(islice(iterator, page_size))
if not page or not _continue():
break
def print_clip(clip):
published_at = clip["createdAt"].replace("T", " @ ").replace("Z", "")
length = utils.format_duration(clip["durationSeconds"])
@ -105,3 +134,14 @@ def print_clip(clip):
" Length: <blue>{}</blue>"
" Views: <blue>{}</blue>".format(published_at, length, clip["viewCount"]))
print_out("<i>{}</i>".format(clip["url"]))
def _continue():
print_out("Press <green><b>Enter</green> to continue, <yellow><b>Ctrl+C</yellow> to break.")
try:
input()
except KeyboardInterrupt:
return False
return True

View File

@ -196,6 +196,28 @@ def get_channel_clips(channel_id, period, limit, after=None):
def channel_clips_generator(channel_id, period, limit):
def _generator(clips, limit):
for clip in clips["edges"]:
if limit < 1:
return
yield clip["node"]
limit -= 1
has_next = clips["pageInfo"]["hasNextPage"]
if limit < 1 or not has_next:
return
req_limit = min(limit, 100)
cursor = clips["edges"][-1]["cursor"]
clips = get_channel_clips(channel_id, period, req_limit, cursor)
yield from _generator(clips, limit)
req_limit = min(limit, 100)
clips = get_channel_clips(channel_id, period, req_limit)
return _generator(clips, limit)
def channel_clips_generator_old(channel_id, period, limit):
cursor = ""
while True:
clips = get_channel_clips(
@ -259,22 +281,26 @@ def get_channel_videos(channel_id, limit, sort, type="archive", game_ids=[], aft
return response["data"]["user"]["videos"]
def channel_videos_generator(channel_id, limit, sort, type, game_ids=None):
cursor = ""
while True:
videos = get_channel_videos(
channel_id, limit, sort, type, game_ids=game_ids, after=cursor)
if not videos["edges"]:
break
def channel_videos_generator(channel_id, max_videos, sort, type, game_ids=None):
def _generator(videos, max_videos):
for video in videos["edges"]:
if max_videos < 1:
return
yield video["node"]
max_videos -= 1
has_next = videos["pageInfo"]["hasNextPage"]
cursor = videos["edges"][-1]["cursor"] if has_next else None
if max_videos < 1 or not has_next:
return
yield videos, has_next
limit = min(max_videos, 100)
cursor = videos["edges"][-1]["cursor"]
videos = get_channel_videos(channel_id, limit, sort, type, game_ids, cursor)
yield from _generator(videos, max_videos)
if not cursor:
break
limit = min(max_videos, 100)
videos = get_channel_videos(channel_id, limit, sort, type, game_ids)
return videos["totalCount"], _generator(videos, max_videos)
def get_access_token(video_id):
@ -309,6 +335,7 @@ def get_playlists(video_id, access_token):
response = requests.get(url, params={
"nauth": access_token['value'],
"nauthsig": access_token['signature'],
"allow_audio_only": "true",
"allow_source": "true",
"player": "twitchweb",
})