Compare commits

..

67 Commits

Author SHA1 Message Date
c09dd15a0c wip 2022-08-04 14:39:06 +02:00
c06ba2248d Bump version 2022-02-25 19:23:08 +01:00
a5039be657 Update changelog 2022-02-25 19:20:53 +01:00
6fa3bd568d Make clips behave similar to recent videos changes 2022-02-25 19:10:48 +01:00
ef059f3dbd Extend docs for the videos command 2022-02-25 17:54:28 +01:00
fbe4a17ff0 Add changelog 2022-02-23 19:30:28 +01:00
1fc5ef6bd1 Reorganize paging and limiting video list 2022-02-23 19:07:27 +01:00
b73ab58432 Fix video output when creator is not set 2022-02-23 19:07:04 +01:00
52651e62c8 Enable logging via --debug flag 2022-02-23 16:24:52 +01:00
4928188055 Add support for JSON output to videos command 2022-02-23 16:02:28 +01:00
b4c31b04e1 Bump copyright year 2022-02-05 15:12:01 +01:00
7ad574d103 Bump version 2022-02-05 15:08:51 +01:00
2fddb0c6a4 Add support for downloading audio only 2022-02-05 14:37:52 +01:00
fbc227017d Bump version, add changelog 2022-02-05 09:52:06 +01:00
b8aaa0c24c Fix issue with downloading clips with no game set
issue #78
2022-02-05 09:50:17 +01:00
0af472528d Remove unused function 2022-02-05 09:50:17 +01:00
fe9b69e1d4 Remove debian packaging stuff 2022-02-05 09:43:45 +01:00
fb73fe07c5 Add clip slug to formatting options 2022-02-05 09:36:50 +01:00
7eb50a0fa1 Fix output formatting when game is not set
issue #87
2022-02-05 09:34:28 +01:00
6ebe263f33 Fix issue with colons in file name
ffmpeg uses colons to define protocols, by prefixing the target with
file: this ambiguity is avoided.

issue #89
2022-02-05 09:26:10 +01:00
15654291d7 Remove double new line 2022-01-25 08:37:37 +01:00
28b4c6146b Add script for tagging versions, yaml changelog 2022-01-25 08:35:50 +01:00
310010363f Bump version, add changelog 2022-01-25 08:22:35 +01:00
4fd532f05d Ask overwrite confirmation early
Ask whether to overwrite before starting download, this way once
download starts, there will be no more prompts.
2022-01-25 08:08:17 +01:00
8156b18b37 Tweak colors 2022-01-25 07:59:54 +01:00
4ff4c8763c Ignore pyright config file 2022-01-24 10:11:16 +01:00
b24bc0eb29 Add option to define target filename 2022-01-24 09:06:02 +01:00
c5b5c49058 Delete bundles with make clean 2022-01-19 08:20:57 +01:00
b67ccc9dde Bump version, add changelog 2022-01-19 08:04:56 +01:00
192c2925b7 Allow newer versions of m3u8 2022-01-19 08:01:18 +01:00
2f977be161 Bump version 2021-12-03 10:26:17 +01:00
928c6d64cf Add metadata to the encoded video file 2021-12-03 10:25:04 +01:00
dd1f4e0d26 Better computation of speed
Distinct VODs took from disk and freshly downloaded
2021-09-16 08:29:33 +02:00
caabe3138c Remove old kraken requests 2021-09-16 08:01:31 +02:00
9c3cf11635 Dedupe clip fields 2021-09-16 07:59:07 +02:00
2f51b3821b Handle video not found gracefully 2021-09-16 07:58:43 +02:00
62092ee25f Add some tests 2021-09-16 07:57:55 +02:00
6f86aea493 Bump version, changelog 2021-07-31 11:44:56 +02:00
e3f66bda43 Fix compat with older versions of python
path.join doesn't handle PosixPath instances in older versions of python
which causes breakage.

fixes #71
2021-07-31 11:41:45 +02:00
5c3cebd0f3 Bump version, changelog 2021-06-09 15:10:39 +02:00
a49dcab419 Fix clips download by fetching access token
fixes #64
2021-06-09 15:08:40 +02:00
0dd04a7e2d Extract clips list method 2021-05-18 14:25:49 +02:00
5bd0747dde Respect --limit when downloading clips 2021-05-18 14:23:56 +02:00
63c2aff334 Handle keyboard interrupt 2021-05-18 14:00:50 +02:00
e95b430eec Remove unused namedtuple 2021-05-18 13:54:17 +02:00
8c582c600e Fix duplicate named test function 2021-05-18 13:52:34 +02:00
c0c5cbf2a8 Don't break if channel not found 2021-03-22 07:42:07 +01:00
3f143b0c84 Fix file naming not to strip first digit of id
fixes #60
2021-03-22 07:42:07 +01:00
2242af05fc Add changelog, bump version 2021-02-15 14:35:20 +01:00
9c901a21d9 Fix reference to invalid argument 2021-02-15 14:33:31 +01:00
270f53c3c1 Fix tests 2021-02-15 14:33:25 +01:00
e12dba26b4 Add dash as an allowed char, set length to 16. 2021-02-15 14:23:13 +01:00
3a61e61226 Add underscore as an allow char for clip extra ID 2021-02-15 14:23:13 +01:00
8ddfad51bc Update clip identifier patterns
New clips ID naming convention. ex: GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ
2021-02-15 14:23:13 +01:00
d152cbff09 Update tests with new clip naming
New clips ID naming convention. ex: GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ
2021-02-15 14:23:13 +01:00
e0681ab53c Bump version, changelog 2021-01-14 22:25:22 +01:00
728e631623 Handle video not found in info 2021-01-14 22:23:40 +01:00
9d19acbe6d Re-apply fix lost to a merge conflict 2021-01-14 22:16:17 +01:00
baeaedaa54 Fix setup.py to detect new package 2021-01-14 22:14:53 +01:00
9e2bbd7e39 Bump version, changelog, readme 2021-01-14 22:14:53 +01:00
dbee7cdc52 Use GraphQL to fetch access token
issue #53
2021-01-14 22:14:52 +01:00
548a9350ba Add twitch-dl info command 2021-01-14 22:14:52 +01:00
2380dc5a35 Split up commands 2021-01-14 22:14:50 +01:00
a7340f178f Fix bug in videos command saying there are more videos when there aren’t
When listing videos, if --pager was not specified and all of the videos
for a channel were listed, a message would still print out saying
“There are more videos. Increase the --limit or use --pager to see the
rest."

This checks to see if there are actually more videos before printing
that error message.
2021-01-11 00:49:57 +01:00
838611b834 Bump version 2020-11-23 16:08:15 +01:00
cf8d13e80e Fix clip download due to missing fields in query
fixes #45
2020-11-23 16:06:09 +01:00
6108b15587 Handle channel not found in clips 2020-11-10 12:40:11 +01:00
25 changed files with 1401 additions and 416 deletions

1
.gitignore vendored
View File

@ -13,3 +13,4 @@ tmp/
/twitch-dl.1.man
/bundle
/*.pyz
/pyrightconfig.json

View File

@ -1,18 +1,76 @@
Twitch Downloader change log
============================
twitch-dl changelog
===================
1.13.0 (2020-11-10)
-------------------
<!-- Do not edit. This file is automatically generated from changelog.yaml.-->
**1.20.0 (2022-02-25)**
* Add `--json` option to `videos` command (#92, thanks @miff2000)
* Add `--all` option to `videos` and `clips` commands to list all clips or
videos in one go.
* Modify how `--pager` works, will make multiple requests if needed to show all
available items, ignoring `--limit`.
**1.19.0 (2022-02-05)**
* Add support for downloading audio only (#10)
**1.18.1 (2022-02-05)**
* Fix issues with output formats (#87, #89)
* Fix issues when downloading clip with no game set (#78)
* Add option to use clip slug in `--output` format
**1.18.0 (2022-01-25)**
* Add `--output` option to `download` command which allows setting output file
template (#70)
* Ask to overwrite before downloading to avoid later prompt
**1.17.1 (2022-01-19)**
* Upgrade m3u8 lib to 1.0.0+
**1.17.0 (2021-12-03)**
* Fix speed calculation when resuming download (#75, thanks CroquetteTheThe)
* Add artist and title metadata to resulting video (#80)
**1.16.1 (2021-07-31)**
* Fix compat with older versions of python (#71)
**1.16.0 (2021-06-09)**
* Fix clips download caused by Twitch changes (#64, thanks to all participants)
**1.15.0 (2021-02-15)**
* Add support for new format of clip slug (thanks @Loveangel1337)
**1.14.1 (2021-01-14)**
* Handle videos which don't exist more gracefully
**1.14.0 (2021-01-14)**
* Added `info` command for displaying video or clip info (#51)
* Don't show there are more videos when there aren't (#52, thanks @scottyallen)
* Fixed Twitch regression for getting the access token (#53)
**1.13.1 (2020-11-23)**
* Fixed clip download issue (#45)
**1.13.0 (2020-11-10)**
* Added `clips` command for listing and batch downloading clips (#26)
1.12.1 (2020-09-29)
-------------------
**1.12.1 (2020-09-29)**
* Fix bug introduced in previous version which broke joining
1.12.0 (2020-09-29)
-------------------
**1.12.0 (2020-09-29)**
* Added `source` as alias for best available quality (#33)
* Added `--no-join` option to `download` to skip ffmpeg join (#36)
@ -20,43 +78,37 @@ Twitch Downloader change log
for confirmation (#37)
* Added `--pager` option to `videos`, don't page by default (#30)
1.11.0 (2020-09-03)
-------------------
**1.11.0 (2020-09-03)**
* Make downloading more robust, fixes issues with some VODs (#35)
* Bundle twitch-dl to a standalone archive, simplifying installation, see
installation instructions in README
1.10.2 (2020-08-11)
-------------------
**1.10.2 (2020-08-11)**
* Fix version number displayed by `twitch-dl --version` (#29)
1.10.1 (2020-08-09)
-------------------
**1.10.1 (2020-08-09)**
* Fix videos incorrectly identified as clips (#28)
* Make download command work with video URLs lacking "www" before "twitch.tv"
* Make download command work with video URLs lacking 'www' before 'twitch.tv'
* Print an error when video or clip is not found instead of an exception trace
1.10.0 (2020-08-07)
-------------------
**1.10.0 (2020-08-07)**
* Add `--quality` option to `download` command, allows specifying the video
quality to download. In this case, twitch-dl will require no user input. (#22)
* Fix download of clips which contain numbers in their slug (#24)
* Fix URL to video displayed by `videos` command (it was missing /videos/)
1.9.0 (2020-06-10)
------------------
**1.9.0 (2020-06-10)**
* **Breaking**: wrongly named `--max_workers` option changed to `--max-workers`.
The shorthand option `-w` remains the same.
* Fix bug where `videos` command would crash if there was no game info (#21)
* Allow unicode characters in filenames, no longer strips e.g. cyrillic script
1.8.0 (2020-05-17)
------------------
**1.8.0 (2020-05-17)**
* Fix videos command (#18)
* **Breaking**: `videos` command no longer takes the `--offset` parameter due to
@ -64,58 +116,50 @@ Twitch Downloader change log
* Add paging to `videos` command to replace offset
* Add `--game` option to `videos` command to filter by game
1.7.0 (2020-04-25)
------------------
**1.7.0 (2020-04-25)**
* Support for specifying broadcast type when listing videos (#13)
1.6.0 (2020-04-11)
------------------
**1.6.0 (2020-04-11)**
* Support for downloading clips (#15)
1.5.1 (2020-04-11)
------------------
**1.5.1 (2020-04-11)**
* Fix VOD naming issue (#12)
* Nice console output while downloading
1.5.0 (2020-04-10)
------------------
**1.5.0 (2020-04-10)**
* Fix video downloads after Twitch deprecated access token access
* Don't print errors when retrying download, only if all fails
1.4.0 (2019-08-23)
------------------
**1.4.0 (2019-08-23)**
* Fix usage of deprecated v3 API
* Use m3u8 lib for parsing playlists
* Add `--keep` option not preserve downloaded VODs
1.3.1 (2019-08-13)
------------------
**1.3.1 (2019-08-13)**
* No changes, bumped to fix issue with pypi
1.3.0 (2019-08-13)
------------------
**1.3.0 (2019-08-13)**
* Add `--sort` and `--offset` options to `videos` command, allows paging (#7)
* Show video URL in `videos` command output
1.2.0 (2019-07-05)
------------------
**1.2.0 (2019-07-05)**
* Add `--format` option to `download` command for specifying the output format (#6)
* Add `--format` option to `download` command for specifying the output format
(#6)
* Add `--version` option for printing program version
1.1.0 (2019-06-06)
------------------
**1.1.0 (2019-06-06)**
* Allow limiting download by start and end time
1.0.0 (2019-04-30)
------------------
**1.0.0 (2019-04-30)**
* Initial release

View File

@ -6,7 +6,7 @@ dist :
clean :
find . -name "*pyc" | xargs rm -rf $1
rm -rf build dist bundle MANIFEST htmlcov deb_dist twitch-dl*.tar.gz twitch-dl.1.man
rm -rf build dist bundle MANIFEST htmlcov deb_dist twitch-dl.*.pyz twitch-dl.1.man
bundle:
mkdir bundle
@ -25,11 +25,11 @@ publish :
coverage:
py.test --cov=toot --cov-report html tests/
deb:
@python setup.py --command-packages=stdeb.command bdist_deb
man:
scdoc < twitch-dl.1.scd > twitch-dl.1.man
test:
pytest
changelog:
./scripts/generate_changelog > CHANGELOG.md

139
README.md
View File

@ -79,44 +79,80 @@ pipx install twitch-dl
Usage
-----
This section does an overview of available features.
To see a list of available commands run:
```
twitch-dl --help
```
And to see description and all arguments for a given command run:
```
twitch-dl <command> --help
```
### Print clip or video info
Videos can be referenced by URL or ID:
```
twitch-dl info 863849735
twitch-dl info https://www.twitch.tv/videos/863849735
```
Clips by slug or ID:
```
twitch-dl info BusyBlushingCattleItsBoshyTime
twitch-dl info https://www.twitch.tv/bananasaurus_rex/clip/BusyBlushingCattleItsBoshyTime
```
Shows info about the video or clip as well as download URLs for clips and
playlist URLs for videos.
### Listing videos
List recent streams for a given channel:
List recent channel videos (10 by default):
```
twitch-dl videos bananasaurus_rex
```
Yields (trimmed):
```
Found 33 videos
221837124
SUPER MARIO ODYSSSEY - Stream #2 / 600,000,000
Bananasaurus_Rex playing Super Mario Odyssey
Published 2018-01-24 @ 12:05:25 Length: 3h 40min
221418913
Dead Space and then SUPER MARIO ODYSSEY PogChamp
Bananasaurus_Rex playing Dead Space
Published 2018-01-23 @ 02:40:58 Length: 6h 2min
220783179
Dead Space | Got my new setup working! rexChamp
Bananasaurus_Rex playing Dead Space
Published 2018-01-21 @ 05:47:03 Length: 5h 7min
```
Use the `--game` option to specify one or more games to show:
Limit to videos of one or more games:
```
twitch-dl videos --game "doom eternal" --game "cave story" bananasaurus_rex
```
List all channel videos at once:
```
twitch-dl videos bananasaurus_rex --all
```
List all channel videos in pages of 10:
```
twitch-dl videos bananasaurus_rex --pager
```
Page size can be adjusted by passing number of items per page:
```
twitch-dl videos bananasaurus_rex --pager 5
```
Returns all videos as a JSON list. Useful for scripting.
```
twitch-dl videos bananasaurus_rex --json --all
```
### Downloading videos
Download a stream by ID or URL:
Download a video by ID or URL:
```
twitch-dl download 221837124
@ -135,8 +171,53 @@ Setting quality to `source` will download the best available quality:
twitch-dl download -q source 221837124
```
Setting quality to `audio_only` will download only audio:
```
twitch-dl download -q audio_only 221837124
```
### Overriding file name
The target filename can be defined by passing the `--output` option followed by
the desired file name, e.g. `--output strim.mkv`.
The filename uses
[Python format string syntax](https://docs.python.org/3/library/string.html#format-string-syntax)
and may contain placeholders in curly braces which will be replaced with
relevant information tied to the downloaded video.
The supported placeholders are:
| Placeholder | Description | Example |
| ----------------- | ------------------------------ | ------------------------------ |
| `{id}` | Video ID | 1255522958 |
| `{title}` | Video title | Dark Souls 3 First playthrough |
| `{title_slug}` | Slugified video title | dark_souls_3_first_playthrough |
| `{datetime}` | Video date and time | 2022-01-07T04:00:27Z |
| `{date}` | Video date | 2022-01-07 |
| `{time}` | Video time | 04:00:27Z |
| `{channel}` | Channel name | KatLink |
| `{channel_login}` | Channel login | katlink |
| `{format}` | File extension, see `--format` | mkv |
| `{game}` | Game name | Dark Souls III |
| `{game_slug}` | Slugified game name | dark_souls_iii |
| `{slug}` | Clip slug (clips only) | AbrasivePlacidCatDxAbomb |
A couple of examples:
Pattern: `"{date}_{id}_{channel_login}_{title_slug}.{format}"`<br />
Expands to: `2022-01-07_1255522958_katlink_dark_souls_3_first_playthrough.mkv`<br />
*This is the default.*
Pattern: `"{channel} - {game} - {title}.{format}"`<br />
Expands to: `KatLink - Dark Souls III - Dark Souls 3 First playthrough.mkv`
### Listing clips
Listing clips works similar to listing videos. Shows 10 clips by default. Use
`--all` to list all in one go or `--pager` to show them in pages.
List clips for the given period:
```
@ -145,13 +226,15 @@ twitch-dl clips bananasaurus_rex --period last_week
Supported periods are: `last_day`, `last_week`, `last_month`, `all_time`.
For listing a large number of clips, it's nice to page them:
Also supports JSON output:
```
twitch-dl clips bananasaurus_rex --period all_time --limit 10 --pager
twitch-dl clips bananasaurus_rex --json --all
```
This will show 10 clips at a time and ask to continue.
Note that this may make multiple requests to the server because each request is
limited to 100 clips, so it may take a little while. You can use `--debug` to
log requests.
### Downloading clips
@ -216,6 +299,6 @@ make man
License
-------
Copyright 2018-2020 Ivan Habunek <ivan@habunek.com>
Copyright 2018-2022 Ivan Habunek <ivan@habunek.com>
Licensed under the GPLv3: http://www.gnu.org/licenses/gpl-3.0.html

181
changelog.yaml Normal file
View File

@ -0,0 +1,181 @@
1.20.0:
date: 2022-02-25
changes:
- "Add `--json` option to `videos` command (#92, thanks @miff2000)"
- "Add `--all` option to `videos` and `clips` commands to list all clips or videos in one go."
- "Modify how `--pager` works, will make multiple requests if needed to show all available items, ignoring `--limit`."
1.19.0:
date: 2022-02-05
changes:
- "Add support for downloading audio only (#10)"
1.18.1:
date: 2022-02-05
changes:
- "Fix issues with output formats (#87, #89)"
- "Fix issues when downloading clip with no game set (#78)"
- "Add option to use clip slug in `--output` format"
1.18.0:
date: 2022-01-25
changes:
- "Add `--output` option to `download` command which allows setting output file template (#70)"
- "Ask to overwrite before downloading to avoid later prompt"
1.17.1:
date: 2022-01-19
changes:
- "Upgrade m3u8 lib to 1.0.0+"
1.17.0:
date: 2021-12-03
changes:
- "Fix speed calculation when resuming download (#75, thanks CroquetteTheThe)"
- "Add artist and title metadata to resulting video (#80)"
1.16.1:
date: 2021-07-31
changes:
- "Fix compat with older versions of python (#71)"
1.16.0:
date: 2021-06-09
changes:
- "Fix clips download caused by Twitch changes (#64, thanks to all participants)"
1.15.0:
date: 2021-02-15
changes:
- "Add support for new format of clip slug (thanks @Loveangel1337)"
1.14.1:
date: 2021-01-14
changes:
- "Handle videos which don't exist more gracefully"
1.14.0:
date: 2021-01-14
changes:
- "Added `info` command for displaying video or clip info (#51)"
- "Don't show there are more videos when there aren't (#52, thanks @scottyallen)"
- "Fixed Twitch regression for getting the access token (#53)"
1.13.1:
date: 2020-11-23
changes:
- "Fixed clip download issue (#45)"
1.13.0:
date: 2020-11-10
changes:
- "Added `clips` command for listing and batch downloading clips (#26)"
1.12.1:
date: 2020-09-29
changes:
- "Fix bug introduced in previous version which broke joining"
1.12.0:
date: 2020-09-29
changes:
- "Added `source` as alias for best available quality (#33)"
- "Added `--no-join` option to `download` to skip ffmpeg join (#36)"
- "Added `--overwrite` option to `download` to overwrite target without prompting for confirmation (#37)"
- "Added `--pager` option to `videos`, don't page by default (#30)"
1.11.0:
date: 2020-09-03
changes:
- "Make downloading more robust, fixes issues with some VODs (#35)"
- "Bundle twitch-dl to a standalone archive, simplifying installation, see installation instructions in README"
1.10.2:
date: 2020-08-11
changes:
- "Fix version number displayed by `twitch-dl --version` (#29)"
1.10.1:
date: 2020-08-09
changes:
- "Fix videos incorrectly identified as clips (#28)"
- "Make download command work with video URLs lacking 'www' before 'twitch.tv'"
- "Print an error when video or clip is not found instead of an exception trace"
1.10.0:
date: 2020-08-07
changes:
- "Add `--quality` option to `download` command, allows specifying the video quality to download. In this case, twitch-dl will require no user input. (#22)"
- "Fix download of clips which contain numbers in their slug (#24)"
- "Fix URL to video displayed by `videos` command (it was missing /videos/)"
1.9.0:
date: 2020-06-10
changes:
- "**Breaking**: wrongly named `--max_workers` option changed to `--max-workers`. The shorthand option `-w` remains the same."
- "Fix bug where `videos` command would crash if there was no game info (#21)"
- "Allow unicode characters in filenames, no longer strips e.g. cyrillic script"
1.8.0:
date: 2020-05-17
changes:
- "Fix videos command (#18)"
- "**Breaking**: `videos` command no longer takes the `--offset` parameter due to API changes"
- "Add paging to `videos` command to replace offset"
- "Add `--game` option to `videos` command to filter by game"
1.7.0:
date: 2020-04-25
changes:
- "Support for specifying broadcast type when listing videos (#13)"
1.6.0:
date: 2020-04-11
changes:
- "Support for downloading clips (#15)"
1.5.1:
date: 2020-04-11
changes:
- "Fix VOD naming issue (#12)"
- "Nice console output while downloading"
1.5.0:
date: 2020-04-10
changes:
- "Fix video downloads after Twitch deprecated access token access"
- "Don't print errors when retrying download, only if all fails"
1.4.0:
date: 2019-08-23
changes:
- "Fix usage of deprecated v3 API"
- "Use m3u8 lib for parsing playlists"
- "Add `--keep` option not preserve downloaded VODs"
1.3.1:
date: 2019-08-13
changes:
- "No changes, bumped to fix issue with pypi"
1.3.0:
date: 2019-08-13
changes:
- "Add `--sort` and `--offset` options to `videos` command, allows paging (#7)"
- "Show video URL in `videos` command output"
1.2.0:
date: 2019-07-05
changes:
- "Add `--format` option to `download` command for specifying the output format (#6)"
- "Add `--version` option for printing program version"
1.1.0:
date: 2019-06-06
changes:
- "Allow limiting download by start and end time"
1.0.0:
date: 2019-04-30
changes:
- "Initial release"

139
download.py Executable file
View File

@ -0,0 +1,139 @@
#!/usr/bin/env python3
import asyncio
import httpx
import m3u8
import os
import re
import requests
from rich.console import Console
from rich.progress import Progress, TaskID, TransferSpeedColumn
from twitchdl import twitch
from typing import List
console = Console()
# WORKER_POOL_SIZE = 5
CHUNK_SIZE = 1024 * 256
# CONNECT_TIMEOUT = 5
# RETRY_COUNT = 5
class Bucket:
capacity: int
rate: int
content_kb: int = 0
KB = 1024
MB = 1024 * KB
DEFAULT_CAPACITY = 10 * MB
DEFAULT_RATE = 1 * MB
def __init__(self, /, *, capacity=1 * MB, rate=100 * KB):
self.capacity = capacity
self.rate = rate
class Downloader:
downloaded: int = 0
downloaded_vod_count: int = 0
progress: Progress
total_task_id: TaskID
vod_count: int
def __init__(self, worker_count: int):
self.worker_count = worker_count
async def run(self, sources, targets):
if len(sources) != len(targets):
raise ValueError(f"Got {len(sources)} sources but {len(targets)} targets.")
self.vod_count = len(sources)
columns = [*Progress.get_default_columns(), TransferSpeedColumn()]
with Progress(*columns, console=console) as progress:
self.progress = progress
self.total_task_id = self.progress.add_task("Total")
await self.download(sources, targets)
for task in self.progress.tasks:
if task.id != self.total_task_id:
self.progress.remove_task(task.id)
console.print("[chartreuse3]Done.[/chartreuse3]")
def on_init(self, filename: str) -> TaskID:
return self.progress.add_task(filename)
def on_start(self, task_id: TaskID, size: int):
self.downloaded_vod_count += 1
self.downloaded += size
estimated_total = int(self.downloaded * self.vod_count / self.downloaded_vod_count)
self.progress.update(self.total_task_id, total=estimated_total)
self.progress.update(task_id, total=size)
self.progress.start_task(task_id)
def on_progress(self, task_id: TaskID, chunk_size: int):
self.progress.update(self.total_task_id, advance=chunk_size)
self.progress.update(task_id, advance=chunk_size)
def on_end(self, task_id: TaskID):
async def remove_task_after(task_id, delay):
await asyncio.sleep(delay)
self.progress.remove_task(task_id)
asyncio.create_task(remove_task_after(task_id, 1))
async def download_one(
self,
client: httpx.AsyncClient,
semaphore: asyncio.Semaphore,
source: str,
target: str,
):
async with semaphore:
with open(target, "wb") as f:
# TODO: handle failure (retries etc)
task_id = self.on_init(os.path.basename(target))
async with client.stream("GET", source) as response:
size = int(response.headers.get("content-length"))
self.on_start(task_id, size)
async for chunk in response.aiter_bytes(chunk_size=CHUNK_SIZE):
f.write(chunk)
self.on_progress(task_id, len(chunk))
self.on_end(task_id)
async def download(self, sources: List[str], targets: List[str]):
async with httpx.AsyncClient() as client:
semaphore = asyncio.Semaphore(self.worker_count)
tasks = [self.download_one(client, semaphore, source, target)
for source, target in zip(sources, targets)]
await asyncio.gather(*tasks)
if __name__ == "__main__":
videos = twitch.get_channel_videos("bananasaurus_rex", 1, "time")
video_id = videos["edges"][0]["node"]["id"]
from twitchdl.commands.download import _get_vod_paths
console.print("[grey53]Fetching access token...[/grey53]")
access_token = twitch.get_access_token(video_id)
console.print("[grey53]Fetching playlists...[/grey53]")
playlists = twitch.get_playlists(video_id, access_token)
playlist_uri = m3u8.loads(playlists).playlists[-1].uri
console.print("[grey53]Fetching playlist...[/grey53]")
playlist = requests.get(playlist_uri).text
vods = _get_vod_paths(m3u8.loads(playlist), None, None)
base_uri = re.sub("/[^/]+$", "/", playlist_uri)
urls = ["".join([base_uri, vod]) for vod in vods][:3]
targets = [f"tmp/{os.path.basename(url).zfill(8)}" for url in urls]
try:
print("Starting download using 3 workers")
d = Downloader(3)
asyncio.run(d.run(urls, targets))
except KeyboardInterrupt:
console.print("[bold red]Aborted.[/bold red]")

View File

@ -1,3 +1,4 @@
pytest
twine
wheel
pyyaml

33
scripts/generate_changelog Executable file
View File

@ -0,0 +1,33 @@
#!/usr/bin/env python3
"""
Generates a more user-readable changelog from changelog.yaml.
"""
import textwrap
import yaml
with open("changelog.yaml", "r") as f:
data = yaml.safe_load(f)
print("twitch-dl changelog")
print("===================")
print()
print("<!-- Do not edit. This file is automatically generated from changelog.yaml.-->")
print()
for version in data.keys():
date = data[version]["date"]
changes = data[version]["changes"]
print(f"**{version} ({date})**")
print()
for c in changes:
lines = textwrap.wrap(c, 78)
initial = True
for line in lines:
if initial:
print("* " + line)
initial = False
else:
print(" " + line)
print()

68
scripts/tag_version Executable file
View File

@ -0,0 +1,68 @@
#!/usr/bin/env python3
"""
Creates an annotated git tag for a given version number.
The tag will include the version number and changes for given version.
Usage: tag_version [version]
"""
import subprocess
import sys
import textwrap
import yaml
import twitchdl
from datetime import date
from os import path
from pkg_resources import get_distribution
path = path.join(path.dirname(path.dirname(path.abspath(__file__))), "changelog.yaml")
with open(path, "r") as f:
changelog = yaml.safe_load(f)
if len(sys.argv) != 2:
print("Wrong argument count", file=sys.stderr)
sys.exit(1)
version = sys.argv[1]
changelog_item = changelog.get(version)
if not changelog_item:
print(f"Version `{version}` not found in changelog.", file=sys.stderr)
sys.exit(1)
if twitchdl.__version__ != version:
print(f"twitchdl.__version__ is `{twitchdl.__version__}`, expected {version}.", file=sys.stderr)
sys.exit(1)
dist_version = get_distribution('twitch-dl').version
if dist_version != version:
print(f"Version in setup.py is `{dist_version}`, expected {version}.", file=sys.stderr)
sys.exit(1)
release_date = changelog_item["date"]
changes = changelog_item["changes"]
if not isinstance(release_date, date):
print(f"Release date not set for version `{version}` in the changelog.", file=sys.stderr)
sys.exit(1)
commit_message = f"twitch-dl {version}\n\n"
for c in changes:
lines = textwrap.wrap(c, 70)
initial = True
for line in lines:
lead = " *" if initial else " "
initial = False
commit_message += f"{lead} {line}\n"
proc = subprocess.run(["git", "tag", "-a", version, "-m", commit_message])
if proc.returncode != 0:
sys.exit(1)
print()
print(commit_message)
print()
print(f"Version {version} tagged \\o/")

View File

@ -1,6 +1,6 @@
#!/usr/bin/env python
from setuptools import setup
from setuptools import setup, find_packages
long_description = """
Quickly download videos from twitch.tv.
@ -11,7 +11,7 @@ makes it faster.
setup(
name='twitch-dl',
version='1.13.0',
version='1.20.0',
description='Twitch downloader',
long_description=long_description.strip(),
author='Ivan Habunek',
@ -27,10 +27,10 @@ setup(
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
],
packages=['twitchdl'],
packages=find_packages(),
python_requires='>=3.5',
install_requires=[
"m3u8>=0.3.12,<0.4",
"m3u8>=1.0.0,<2.0.0",
"requests>=2.13,<3.0",
],
entry_points={

View File

@ -1,3 +0,0 @@
[DEFAULT]
X-Python3-Version: >= 3.5
Copyright-File: LICENSE

30
tests/test_api.py Normal file
View File

@ -0,0 +1,30 @@
"""
These tests depend on the channel having some videos and clips published.
"""
from twitchdl import twitch
TEST_CHANNEL = "bananasaurus_rex"
def test_get_videos():
videos = twitch.get_channel_videos(TEST_CHANNEL, 3, "time")
assert videos["pageInfo"]
assert len(videos["edges"]) > 0
video_id = videos["edges"][0]["node"]["id"]
video = twitch.get_video(video_id)
assert video["id"] == video_id
def test_get_clips():
"""
This test depends on the channel having some videos published.
"""
clips = twitch.get_channel_clips(TEST_CHANNEL, "all_time", 3)
assert clips["pageInfo"]
assert len(clips["edges"]) > 0
clip_slug = clips["edges"][0]["node"]["slug"]
clip = twitch.get_clip(clip_slug)
assert clip["slug"] == clip_slug

View File

@ -1,10 +1,6 @@
import pytest
from unittest.mock import patch
from twitchdl.commands import download
from collections import namedtuple
Args = namedtuple("args", ["video"])
from twitchdl.utils import parse_video_identifier, parse_clip_identifier
TEST_VIDEO_PATTERNS = [
@ -22,24 +18,18 @@ TEST_CLIP_PATTERNS = {
("HungryProudRadicchioDoggo", "https://clips.twitch.tv/HungryProudRadicchioDoggo"),
("HungryProudRadicchioDoggo", "https://www.twitch.tv/bananasaurus_rex/clip/HungryProudRadicchioDoggo?filter=clips&range=7d&sort=time"),
("HungryProudRadicchioDoggo", "https://twitch.tv/bananasaurus_rex/clip/HungryProudRadicchioDoggo?filter=clips&range=7d&sort=time"),
("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ"),
("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ"),
("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ?filter=clips&range=7d&sort=time"),
("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://www.twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ?filter=clips&range=7d&sort=time"),
}
@patch("twitchdl.commands._download_clip")
@patch("twitchdl.commands._download_video")
@pytest.mark.parametrize("expected,input", TEST_VIDEO_PATTERNS)
def test_video_patterns(video_dl, clip_dl, expected, input):
args = Args(video=input)
download(args)
video_dl.assert_called_once_with(expected, args)
clip_dl.assert_not_called()
def test_video_patterns(expected, input):
assert parse_video_identifier(input) == expected
@patch("twitchdl.commands._download_clip")
@patch("twitchdl.commands._download_video")
@pytest.mark.parametrize("expected,input", TEST_CLIP_PATTERNS)
def test_clip_patterns(video_dl, clip_dl, expected, input):
args = Args(video=input)
download(args)
clip_dl.assert_called_once_with(expected, args)
video_dl.assert_not_called()
def test_clip_patterns(expected, input):
assert parse_clip_identifier(input) == expected

13
tests/test_utils.py Normal file
View File

@ -0,0 +1,13 @@
from twitchdl.utils import titlify, slugify
def test_titlify():
assert titlify("Foo Bar Baz.") == "Foo Bar Baz."
assert titlify("Foo (Bar) [Baz]") == "Foo (Bar) [Baz]"
assert titlify("Foo@{} Bar Baz!\"#$%&/=?*+'🔪") == "Foo Bar Baz"
def test_slugify():
assert slugify("Foo Bar Baz") == "foo_bar_baz"
assert slugify(" Foo Bar Baz ") == "foo_bar_baz"
assert slugify("Foo@{}[] Bar Baz!\"#$%&/()=?*+'🔪") == "foo_bar_baz"

View File

@ -1,3 +1,3 @@
__version__ = "1.13.0"
__version__ = "1.20.0"
CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko"

View File

@ -0,0 +1,11 @@
from .clips import clips
from .download import download
from .info import info
from .videos import videos
__all__ = [
clips,
download,
info,
videos,
]

112
twitchdl/commands/clips.py Normal file
View File

@ -0,0 +1,112 @@
import re
import sys
from itertools import islice
from os import path
from twitchdl import twitch, utils
from twitchdl.commands.download import get_clip_authenticated_url
from twitchdl.download import download_file
from twitchdl.exceptions import ConsoleError
from twitchdl.output import print_out, print_clip, print_json
def clips(args):
# Ignore --limit if --pager or --all are given
limit = sys.maxsize if args.all or args.pager else args.limit
generator = twitch.channel_clips_generator(args.channel_name, args.period, limit)
if args.json:
return print_json(list(generator))
if args.download:
return _download_clips(generator)
if args.pager:
print(args)
return _print_paged(generator, args.pager)
return _print_all(generator, args)
def _continue():
print_out("Press <green><b>Enter</green> to continue, <yellow><b>Ctrl+C</yellow> to break.")
try:
input()
except KeyboardInterrupt:
return False
return True
def _target_filename(clip):
url = clip["videoQualities"][0]["sourceURL"]
_, ext = path.splitext(url)
ext = ext.lstrip(".")
match = re.search(r"^(\d{4})-(\d{2})-(\d{2})T", clip["createdAt"])
date = "".join(match.groups())
name = "_".join([
date,
clip["id"],
clip["broadcaster"]["login"],
utils.slugify(clip["title"]),
])
return "{}.{}".format(name, ext)
def _download_clips(generator):
for clip in generator:
target = _target_filename(clip)
if path.exists(target):
print_out("Already downloaded: <green>{}</green>".format(target))
else:
url = get_clip_authenticated_url(clip["slug"], "source")
print_out("Downloading: <yellow>{}</yellow>".format(target))
download_file(url, target)
def _print_all(generator, args):
for clip in generator:
print_out()
print_clip(clip)
if not args.all:
print_out(
"\n<dim>There may be more clips. " +
"Increase the --limit, use --all or --pager to see the rest.</dim>"
)
def _print_paged(generator, page_size):
iterator = iter(generator)
page = list(islice(iterator, page_size))
first = 1
last = first + len(page) - 1
while True:
print_out("-" * 80)
print_out()
for clip in page:
print_clip(clip)
print_out()
last = first + len(page) - 1
print_out("-" * 80)
print_out("<yellow>Clips {}-{}</yellow>".format(first, last))
first = first + len(page)
last = first + 1
page = list(islice(iterator, page_size))
if not page or not _continue():
break

View File

@ -7,143 +7,26 @@ import tempfile
from os import path
from pathlib import Path
from urllib.parse import urlparse
from urllib.parse import urlparse, urlencode
from twitchdl import twitch, utils
from twitchdl.download import download_file, download_files
from twitchdl.exceptions import ConsoleError
from twitchdl.output import print_out, print_clip, print_video, print_json
def _continue():
print_out(
"\nThere are more videos. "
"Press <green><b>Enter</green> to continue, "
"<yellow><b>Ctrl+C</yellow> to break."
)
try:
input()
except KeyboardInterrupt:
return False
return True
def _get_game_ids(names):
if not names:
return []
game_ids = []
for name in names:
print_out("<dim>Looking up game '{}'...</dim>".format(name))
game_id = twitch.get_game_id(name)
if not game_id:
raise ConsoleError("Game '{}' not found".format(name))
game_ids.append(int(game_id))
return game_ids
def _clips_json(args):
clips = twitch.get_channel_clips(args.channel_name, args.period, args.limit)
nodes = list(edge["node"] for edge in clips["edges"])
print_json(nodes)
def _clips_download(args):
generator = twitch.channel_clips_generator(args.channel_name, args.period, 100)
for clips, _ in generator:
for clip in clips["edges"]:
clip = clip["node"]
url = clip["videoQualities"][0]["sourceURL"]
target = _clip_target_filename(clip)
if path.exists(target):
print_out("Already downloaded: <green>{}</green>".format(target))
else:
print_out("Downloading: <yellow>{}</yellow>".format(target))
download_file(url, target)
def clips(args):
if args.json:
return _clips_json(args)
if args.download:
return _clips_download(args)
print_out("<dim>Loading clips...</dim>")
generator = twitch.channel_clips_generator(args.channel_name, args.period, args.limit)
first = 1
for clips, has_more in generator:
count = len(clips["edges"]) if "edges" in clips else 0
last = first + count - 1
print_out("-" * 80)
print_out("<yellow>Showing clips {}-{} of ??</yellow>".format(first, last))
for clip in clips["edges"]:
print_clip(clip["node"])
if not args.pager:
print_out(
"\n<dim>There are more clips. "
"Increase the --limit or use --pager to see the rest.</dim>"
)
break
if not has_more or not _continue():
break
first += count
else:
print_out("<yellow>No clips found</yellow>")
def videos(args):
game_ids = _get_game_ids(args.game)
print_out("<dim>Loading videos...</dim>")
generator = twitch.channel_videos_generator(
args.channel_name, args.limit, args.sort, args.type, game_ids=game_ids)
first = 1
for videos, has_more in generator:
count = len(videos["edges"]) if "edges" in videos else 0
total = videos["totalCount"]
last = first + count - 1
print_out("-" * 80)
print_out("<yellow>Showing videos {}-{} of {}</yellow>".format(first, last, total))
for video in videos["edges"]:
print_video(video["node"])
if not args.pager:
print_out(
"\n<dim>There are more videos. "
"Increase the --limit or use --pager to see the rest.</dim>"
)
break
if not has_more or not _continue():
break
first += count
else:
print_out("<yellow>No videos found</yellow>")
from twitchdl.output import print_out
def _parse_playlists(playlists_m3u8):
playlists = m3u8.loads(playlists_m3u8)
for p in playlists.playlists:
name = p.media[0].name if p.media else ""
resolution = "x".join(str(r) for r in p.stream_info.resolution)
yield name, resolution, p.uri
for p in sorted(playlists.playlists, key=lambda p: p.stream_info.resolution is None):
if p.stream_info.resolution:
name = p.media[0].name
description = "x".join(str(r) for r in p.stream_info.resolution)
else:
name = p.media[0].group_id
description = None
yield name, description, p.uri
def _get_playlist_by_name(playlists, quality):
@ -163,21 +46,27 @@ def _get_playlist_by_name(playlists, quality):
def _select_playlist_interactive(playlists):
print_out("\nAvailable qualities:")
for n, (name, resolution, uri) in enumerate(playlists):
print_out("{}) {} [{}]".format(n + 1, name, resolution))
if resolution:
print_out("{}) {} [{}]".format(n + 1, name, resolution))
else:
print_out("{}) {}".format(n + 1, name))
no = utils.read_int("Choose quality", min=1, max=len(playlists) + 1, default=1)
_, _, uri = playlists[no - 1]
return uri
def _join_vods(playlist_path, target, overwrite):
def _join_vods(playlist_path, target, overwrite, video):
command = [
"ffmpeg",
"-i", playlist_path,
"-c", "copy",
target,
"-metadata", "artist={}".format(video["creator"]["displayName"]),
"-metadata", "title={}".format(video["title"]),
"-metadata", "encoded_by=twitch-dl",
"-stats",
"-loglevel", "warning",
"file:{}".format(target),
]
if overwrite:
@ -189,36 +78,59 @@ def _join_vods(playlist_path, target, overwrite):
raise ConsoleError("Joining files failed")
def _video_target_filename(video, format):
match = re.search(r"^(\d{4})-(\d{2})-(\d{2})T", video['published_at'])
date = "".join(match.groups())
def _video_target_filename(video, args):
date, time = video['publishedAt'].split("T")
game = video["game"]["name"] if video["game"] else "Unknown"
name = "_".join([
date,
video['_id'][1:],
video['channel']['name'],
utils.slugify(video['title']),
])
subs = {
"channel": video["creator"]["displayName"],
"channel_login": video["creator"]["login"],
"date": date,
"datetime": video["publishedAt"],
"format": args.format,
"game": game,
"game_slug": utils.slugify(game),
"id": video["id"],
"time": time,
"title": utils.titlify(video["title"]),
"title_slug": utils.slugify(video["title"]),
}
return name + "." + format
try:
return args.output.format(**subs)
except KeyError as e:
supported = ", ".join(subs.keys())
raise ConsoleError("Invalid key {} used in --output. Supported keys are: {}".format(e, supported))
def _clip_target_filename(clip):
def _clip_target_filename(clip, args):
date, time = clip["createdAt"].split("T")
game = clip["game"]["name"] if clip["game"] else "Unknown"
url = clip["videoQualities"][0]["sourceURL"]
_, ext = path.splitext(url)
ext = ext.lstrip(".")
match = re.search(r"^(\d{4})-(\d{2})-(\d{2})T", clip["createdAt"])
date = "".join(match.groups())
subs = {
"channel": clip["broadcaster"]["displayName"],
"channel_login": clip["broadcaster"]["login"],
"date": date,
"datetime": clip["createdAt"],
"format": ext,
"game": game,
"game_slug": utils.slugify(game),
"id": clip["id"],
"slug": clip["slug"],
"time": time,
"title": utils.titlify(clip["title"]),
"title_slug": utils.slugify(clip["title"]),
}
name = "_".join([
date,
clip["id"],
clip["broadcaster"]["channel"]["name"],
utils.slugify(clip["title"]),
])
return "{}.{}".format(name, ext)
try:
return args.output.format(**subs)
except KeyError as e:
supported = ", ".join(subs.keys())
raise ConsoleError("Invalid key {} used in --output. Supported keys are: {}".format(e, supported))
def _get_vod_paths(playlist, start, end):
@ -246,52 +158,36 @@ def _crete_temp_dir(base_uri):
path = urlparse(base_uri).path.lstrip("/")
temp_dir = Path(tempfile.gettempdir(), "twitch-dl", path)
temp_dir.mkdir(parents=True, exist_ok=True)
return temp_dir
VIDEO_PATTERNS = [
r"^(?P<id>\d+)?$",
r"^https://(www.)?twitch.tv/videos/(?P<id>\d+)(\?.+)?$",
]
CLIP_PATTERNS = [
r"^(?P<slug>[A-Za-z0-9]+)$",
r"^https://(www.)?twitch.tv/\w+/clip/(?P<slug>[A-Za-z0-9]+)(\?.+)?$",
r"^https://clips.twitch.tv/(?P<slug>[A-Za-z0-9]+)(\?.+)?$",
]
return str(temp_dir)
def download(args):
for pattern in VIDEO_PATTERNS:
match = re.match(pattern, args.video)
if match:
video_id = match.group('id')
return _download_video(video_id, args)
video_id = utils.parse_video_identifier(args.video)
if video_id:
return _download_video(video_id, args)
for pattern in CLIP_PATTERNS:
match = re.match(pattern, args.video)
if match:
clip_slug = match.group('slug')
return _download_clip(clip_slug, args)
clip_slug = utils.parse_clip_identifier(args.video)
if clip_slug:
return _download_clip(clip_slug, args)
raise ConsoleError("Invalid video: {}".format(args.video))
raise ConsoleError("Invalid input: {}".format(args.video))
def _get_clip_url(clip, args):
def _get_clip_url(clip, quality):
qualities = clip["videoQualities"]
# Quality given as an argument
if args.quality:
if args.quality == "source":
if quality:
if quality == "source":
return qualities[0]["sourceURL"]
selected_quality = args.quality.rstrip("p") # allow 720p as well as 720
selected_quality = quality.rstrip("p") # allow 720p as well as 720
for q in qualities:
if q["quality"] == selected_quality:
return q["sourceURL"]
available = ", ".join([str(q["quality"]) for q in qualities])
msg = "Quality '{}' not found. Available qualities are: {}".format(args.quality, available)
msg = "Quality '{}' not found. Available qualities are: {}".format(quality, available)
raise ConsoleError(msg)
# Ask user to select quality
@ -305,9 +201,27 @@ def _get_clip_url(clip, args):
return selected_quality["sourceURL"]
def get_clip_authenticated_url(slug, quality):
print_out("<dim>Fetching access token...</dim>")
access_token = twitch.get_clip_access_token(slug)
if not access_token:
raise ConsoleError("Access token not found for slug '{}'".format(slug))
url = _get_clip_url(access_token, quality)
query = urlencode({
"sig": access_token["playbackAccessToken"]["signature"],
"token": access_token["playbackAccessToken"]["value"],
})
return "{}?{}".format(url, query)
def _download_clip(slug, args):
print_out("<dim>Looking up clip...</dim>")
clip = twitch.get_clip(slug)
game = clip["game"]["name"] if clip["game"] else "Unknown"
if not clip:
raise ConsoleError("Clip '{}' not found".format(slug))
@ -315,19 +229,26 @@ def _download_clip(slug, args):
print_out("Found: <green>{}</green> by <yellow>{}</yellow>, playing <blue>{}</blue> ({})".format(
clip["title"],
clip["broadcaster"]["displayName"],
clip["game"]["name"],
game,
utils.format_duration(clip["durationSeconds"])
))
url = _get_clip_url(clip, args)
target = _clip_target_filename(clip, args)
print_out("Target: <blue>{}</blue>".format(target))
if not args.overwrite and path.exists(target):
response = input("File exists. Overwrite? [Y/n]: ")
if response.lower().strip() not in ["", "y"]:
raise ConsoleError("Aborted")
args.overwrite = True
url = get_clip_authenticated_url(slug, args.quality)
print_out("<dim>Selected URL: {}</dim>".format(url))
target = _clip_target_filename(clip)
print_out("Downloading clip...")
print_out("<dim>Downloading clip...</dim>")
download_file(url, target)
print_out("Downloaded: {}".format(target))
print_out("Downloaded: <blue>{}</blue>".format(target))
def _download_video(video_id, args):
@ -337,8 +258,20 @@ def _download_video(video_id, args):
print_out("<dim>Looking up video...</dim>")
video = twitch.get_video(video_id)
if not video:
raise ConsoleError("Video {} not found".format(video_id))
print_out("Found: <blue>{}</blue> by <yellow>{}</yellow>".format(
video['title'], video['channel']['display_name']))
video['title'], video['creator']['displayName']))
target = _video_target_filename(video, args)
print_out("Output: <blue>{}</blue>".format(target))
if not args.overwrite and path.exists(target):
response = input("File exists. Overwrite? [Y/n]: ")
if response.lower().strip() not in ["", "y"]:
raise ConsoleError("Aborted")
args.overwrite = True
print_out("<dim>Fetching access token...</dim>")
access_token = twitch.get_access_token(video_id)
@ -386,8 +319,7 @@ def _download_video(video_id, args):
return
print_out("\n\nJoining files...")
target = _video_target_filename(video, args.format)
_join_vods(playlist_path, target, args.overwrite)
_join_vods(playlist_path, target, args.overwrite, video)
if args.keep:
print_out("\n<dim>Temporary files not deleted: {}</dim>".format(target_dir))

79
twitchdl/commands/info.py Normal file
View File

@ -0,0 +1,79 @@
import m3u8
from twitchdl import utils, twitch
from twitchdl.exceptions import ConsoleError
from twitchdl.output import print_video, print_clip, print_json, print_out, print_log
def info(args):
video_id = utils.parse_video_identifier(args.identifier)
if video_id:
print_log("Fetching video...")
video = twitch.get_video(video_id)
if not video:
raise ConsoleError("Video {} not found".format(video_id))
print_log("Fetching access token...")
access_token = twitch.get_access_token(video_id)
print_log("Fetching playlists...")
playlists = twitch.get_playlists(video_id, access_token)
if video:
if args.json:
video_json(video, playlists)
else:
video_info(video, playlists)
return
clip_slug = utils.parse_clip_identifier(args.identifier)
if clip_slug:
print_log("Fetching clip...")
clip = twitch.get_clip(clip_slug)
if not clip:
raise ConsoleError("Clip {} not found".format(clip_slug))
if args.json:
print_json(clip)
else:
clip_info(clip)
return
raise ConsoleError("Invalid input: {}".format(args.identifier))
def video_info(video, playlists):
print_out()
print_video(video)
print_out()
print_out("Playlists:")
for p in m3u8.loads(playlists).playlists:
print_out("<b>{}</b> {}".format(p.stream_info.video, p.uri))
def video_json(video, playlists):
playlists = m3u8.loads(playlists).playlists
video["playlists"] = [
{
"bandwidth": p.stream_info.bandwidth,
"resolution": p.stream_info.resolution,
"codecs": p.stream_info.codecs,
"video": p.stream_info.video,
"uri": p.uri
} for p in playlists
]
print_json(video)
def clip_info(clip):
print_out()
print_clip(clip)
print_out()
print_out("Download links:")
for q in clip["videoQualities"]:
print_out("<b>{quality}p{frameRate}</b> {sourceURL}".format(**q))

View File

@ -0,0 +1,62 @@
import sys
from twitchdl import twitch
from twitchdl.exceptions import ConsoleError
from twitchdl.output import print_out, print_paged_videos, print_video, print_json
def videos(args):
game_ids = _get_game_ids(args.game)
# Ignore --limit if --pager or --all are given
max_videos = sys.maxsize if args.all or args.pager else args.limit
total_count, generator = twitch.channel_videos_generator(
args.channel_name, max_videos, args.sort, args.type, game_ids=game_ids)
if args.json:
videos = list(generator)
print_json({
"count": len(videos),
"totalCount": total_count,
"videos": videos
})
return
if total_count == 0:
print_out("<yellow>No videos found</yellow>")
return
if args.pager:
print_paged_videos(generator, args.pager, total_count)
return
count = 0
for video in generator:
print_out()
print_video(video)
count += 1
print_out()
print_out("-" * 80)
print_out("<yellow>Videos {}-{} of {}</yellow>".format(1, count, total_count))
if total_count > count:
print_out()
print_out(
"<dim>There are more videos. Increase the --limit, use --all or --pager to see the rest.</dim>"
)
def _get_game_ids(names):
if not names:
return []
game_ids = []
for name in names:
print_out("<dim>Looking up game '{}'...</dim>".format(name))
game_id = twitch.get_game_id(name)
if not game_id:
raise ConsoleError("Game '{}' not found".format(name))
game_ids.append(int(game_id))
return game_ids

View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
import logging
import sys
from argparse import ArgumentParser, ArgumentTypeError
@ -33,15 +34,14 @@ def time(value):
return hours * 3600 + minutes * 60 + seconds
def limit(value):
"""Validates the number of videos to fetch."""
def pos_integer(value):
try:
value = int(value)
except ValueError:
raise ArgumentTypeError("must be an integer")
if not 1 <= int(value) <= 100:
raise ArgumentTypeError("must be between 1 and 100")
if value < 1:
raise ArgumentTypeError("must be positive")
return value
@ -61,10 +61,15 @@ COMMANDS = [
"type": str,
}),
(["-l", "--limit"], {
"help": "Number of videos to fetch (default 10, max 100)",
"type": limit,
"help": "Number of videos to fetch (default 10)",
"type": pos_integer,
"default": 10,
}),
(["-a", "--all"], {
"help": "Fetch all videos, overrides --limit",
"action": "store_true",
"default": False,
}),
(["-s", "--sort"], {
"help": "Sorting order of videos. (default: time)",
"type": str,
@ -77,11 +82,17 @@ COMMANDS = [
"choices": ["archive", "highlight", "upload"],
"default": "archive",
}),
(["-p", "--pager"], {
"help": "If there are more results than LIMIT, ask to show next page",
(["-j", "--json"], {
"help": "Show results as JSON. Ignores --pager.",
"action": "store_true",
"default": False,
}),
(["-p", "--pager"], {
"help": "Number of videos to show per page. Disabled by default.",
"type": pos_integer,
"nargs": "?",
"const": 10,
}),
],
),
Command(
@ -94,9 +105,14 @@ COMMANDS = [
}),
(["-l", "--limit"], {
"help": "Number of videos to fetch (default 10, max 100)",
"type": limit,
"type": pos_integer,
"default": 10,
}),
(["-a", "--all"], {
"help": "Fetch all videos, overrides --limit",
"action": "store_true",
"default": False,
}),
(["-P", "--period"], {
"help": "Period from which to return clips. (default: 'all_time')",
"type": str,
@ -109,9 +125,10 @@ COMMANDS = [
"default": False,
}),
(["-p", "--pager"], {
"help": "If there are more results than LIMIT, ask to show next page",
"action": "store_true",
"default": False,
"help": "Number of clips to show per page. Disabled by default.",
"type": pos_integer,
"nargs": "?",
"const": 10,
}),
(["-d", "--download"], {
"help": "Download all videos in given period (in source quality)",
@ -168,9 +185,29 @@ COMMANDS = [
"help": "Overwrite the target file if it already exists without prompting.",
"action": "store_true",
"default": False,
}),
(["-o", "--output"], {
"help": "Output file name template. See docs for details.",
"type": str,
"default": "{date}_{id}_{channel_login}_{title_slug}.{format}"
})
],
),
Command(
name="info",
description="Print information for a given Twitch URL, video ID or clip slug",
arguments=[
(["identifier"], {
"help": "identifier",
"type": str,
}),
(["-j", "--json"], {
"help": "Show results as JSON",
"action": "store_true",
"default": False,
}),
],
)
]
COMMON_ARGUMENTS = [
@ -211,6 +248,9 @@ def main():
parser = get_parser()
args = parser.parse_args()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
if args.version:
print("twitch-dl v{}".format(__version__))
return
@ -224,6 +264,9 @@ def main():
except ConsoleError as e:
print_err(e)
sys.exit(1)
except KeyboardInterrupt:
print_err("Operation canceled")
sys.exit(1)
except GQLError as e:
print_err(e)
for err in e.errors:

View File

@ -34,11 +34,13 @@ def _download(url, path):
def download_file(url, path, retries=RETRY_COUNT):
if os.path.exists(path):
return os.path.getsize(path)
from_disk = True
return (os.path.getsize(path), from_disk)
from_disk = False
for _ in range(retries):
try:
return _download(url, path)
return (_download(url, path), from_disk)
except RequestException:
pass
@ -51,17 +53,26 @@ def _print_progress(futures):
max_msg_size = 0
start_time = datetime.now()
total_count = len(futures)
current_download_size = 0
current_downloaded_count = 0
for future in as_completed(futures):
size = future.result()
size, from_disk = future.result()
downloaded_count += 1
downloaded_size += size
# If we find something on disk, we don't want to take it in account in
# the speed calculation
if not from_disk:
current_download_size += size
current_downloaded_count += 1
percentage = 100 * downloaded_count // total_count
est_total_size = int(total_count * downloaded_size / downloaded_count)
duration = (datetime.now() - start_time).seconds
speed = downloaded_size // duration if duration else 0
remaining = (total_count - downloaded_count) * duration / downloaded_count
speed = current_download_size // duration if duration else 0
remaining = (total_count - downloaded_count) * duration / current_downloaded_count \
if current_downloaded_count else 0
msg = " ".join([
"Downloaded VOD {}/{}".format(downloaded_count, total_count),
@ -69,7 +80,7 @@ def _print_progress(futures):
"<cyan>{}</cyan>".format(format_size(downloaded_size)),
"of <cyan>~{}</cyan>".format(format_size(est_total_size)),
"at <cyan>{}/s</cyan>".format(format_size(speed)) if speed > 0 else "",
"remaining <cyan>~{}</cyan>".format(format_duration(remaining)) if speed > 0 else "",
"remaining <cyan>~{}</cyan>".format(format_duration(remaining)) if remaining > 0 else "",
])
max_msg_size = max(len(msg), max_msg_size)

View File

@ -4,6 +4,7 @@ import json
import sys
import re
from itertools import islice
from twitchdl import utils
@ -62,35 +63,70 @@ def print_err(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)
def print_log(*args, **kwargs):
args = ["<dim>{}</dim>".format(a) for a in args]
args = [colorize(a) if USE_ANSI_COLOR else strip_tags(a) for a in args]
print(*args, file=sys.stderr, **kwargs)
def print_video(video):
published_at = video["publishedAt"].replace("T", " @ ").replace("Z", "")
length = utils.format_duration(video["lengthSeconds"])
channel = video["creator"]["channel"]["displayName"]
playing = (
"playing <blue>{}</blue>".format(video["game"]["name"])
if video["game"] else ""
)
channel = "<blue>{}</blue>".format(video["creator"]["displayName"]) if video["creator"] else ""
playing = "playing <blue>{}</blue>".format(video["game"]["name"]) if video["game"] else ""
# Can't find URL in video object, strange
url = "https://www.twitch.tv/videos/{}".format(video["id"])
print_out("\n<b>{}</b>".format(video["id"]))
print_out("<b>Video {}</b>".format(video["id"]))
print_out("<green>{}</green>".format(video["title"]))
print_out("<blue>{}</blue> {}".format(channel, playing))
if channel or playing:
print_out(" ".join([channel, playing]))
print_out("Published <blue>{}</blue> Length: <blue>{}</blue> ".format(published_at, length))
print_out("<i>{}</i>".format(url))
def print_paged_videos(generator, page_size, total_count):
iterator = iter(generator)
page = list(islice(iterator, page_size))
first = 1
last = first + len(page) - 1
while True:
print_out("-" * 80)
print_out()
for video in page:
print_video(video)
print_out()
last = first + len(page) - 1
print_out("-" * 80)
print_out("<yellow>Videos {}-{} of {}</yellow>".format(first, last, total_count))
first = first + len(page)
last = first + 1
page = list(islice(iterator, page_size))
if not page or not _continue():
break
def print_clip(clip):
published_at = clip["createdAt"].replace("T", " @ ").replace("Z", "")
length = utils.format_duration(clip["durationSeconds"])
channel = clip["broadcaster"]["channel"]["displayName"]
channel = clip["broadcaster"]["displayName"]
playing = (
"playing <blue>{}</blue>".format(clip["game"]["name"])
if clip["game"] else ""
)
print_out("\n<b>{}</b>".format(clip["slug"]))
print_out("Clip <b>{}</b>".format(clip["slug"]))
print_out("<green>{}</green>".format(clip["title"]))
print_out("<blue>{}</blue> {}".format(channel, playing))
print_out(
@ -98,3 +134,14 @@ def print_clip(clip):
" Length: <blue>{}</blue>"
" Views: <blue>{}</blue>".format(published_at, length, clip["viewCount"]))
print_out("<i>{}</i>".format(clip["url"]))
def _continue():
print_out("Press <green><b>Enter</green> to continue, <yellow><b>Ctrl+C</yellow> to break.")
try:
input()
except KeyboardInterrupt:
return False
return True

View File

@ -42,13 +42,14 @@ def authenticated_post(url, data=None, json=None, headers={}):
return response
def kraken_get(url, params={}, headers={}):
"""
Add accept header required by kraken API v5.
see: https://discuss.dev.twitch.tv/t/change-in-access-to-deprecated-kraken-twitch-apis/22241
"""
headers["Accept"] = "application/vnd.twitchtv.v5+json"
return authenticated_get(url, params, headers)
def gql_post(query):
url = "https://gql.twitch.tv/gql"
response = authenticated_post(url, data=query).json()
if "errors" in response:
raise GQLError(response["errors"])
return response
def gql_query(query):
@ -61,38 +62,91 @@ def gql_query(query):
return response
def get_video(video_id):
"""
https://dev.twitch.tv/docs/v5/reference/videos#get-video
"""
url = "https://api.twitch.tv/kraken/videos/{}".format(video_id)
VIDEO_FIELDS = """
id
title
publishedAt
broadcastType
lengthSeconds
game {
name
}
creator {
login
displayName
}
"""
return kraken_get(url).json()
CLIP_FIELDS = """
id
slug
title
createdAt
viewCount
durationSeconds
url
videoQualities {
frameRate
quality
sourceURL
}
game {
id
name
}
broadcaster {
displayName
login
}
"""
def get_video(video_id):
query = """
{{
video(id: "{video_id}") {{
{fields}
}}
}}
"""
query = query.format(video_id=video_id, fields=VIDEO_FIELDS)
response = gql_query(query)
return response["data"]["video"]
def get_clip(slug):
query = """
{{
clip(slug: "{}") {{
title
durationSeconds
game {{
name
}}
broadcaster {{
login
displayName
}}
videoQualities {{
frameRate
quality
sourceURL
{fields}
}}
}}
"""
response = gql_query(query.format(slug, fields=CLIP_FIELDS))
return response["data"]["clip"]
def get_clip_access_token(slug):
query = """
{{
"operationName": "VideoAccessToken_Clip",
"variables": {{
"slug": "{slug}"
}},
"extensions": {{
"persistedQuery": {{
"version": 1,
"sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11"
}}
}}
}}
"""
response = gql_query(query.format(slug))
response = gql_post(query.format(slug=slug).strip())
return response["data"]["clip"]
@ -117,28 +171,7 @@ def get_channel_clips(channel_id, period, limit, after=None):
edges {{
cursor
node {{
id
slug
title
createdAt
viewCount
durationSeconds
url
videoQualities {{
frameRate
quality
sourceURL
}}
game {{
id
name
}}
broadcaster {{
channel {{
name
displayName
}}
}}
{fields}
}}
}}
}}
@ -146,18 +179,45 @@ def get_channel_clips(channel_id, period, limit, after=None):
}}
"""
query = query.format(**{
"channel_id": channel_id,
"after": after if after else "",
"limit": limit,
"period": period.upper(),
})
query = query.format(
channel_id=channel_id,
after=after if after else "",
limit=limit,
period=period.upper(),
fields=CLIP_FIELDS
)
response = gql_query(query)
user = response["data"]["user"]
if not user:
raise ConsoleError("Channel {} not found".format(channel_id))
return response["data"]["user"]["clips"]
def channel_clips_generator(channel_id, period, limit):
def _generator(clips, limit):
for clip in clips["edges"]:
if limit < 1:
return
yield clip["node"]
limit -= 1
has_next = clips["pageInfo"]["hasNextPage"]
if limit < 1 or not has_next:
return
req_limit = min(limit, 100)
cursor = clips["edges"][-1]["cursor"]
clips = get_channel_clips(channel_id, period, req_limit, cursor)
yield from _generator(clips, limit)
req_limit = min(limit, 100)
clips = get_channel_clips(channel_id, period, req_limit)
return _generator(clips, limit)
def channel_clips_generator_old(channel_id, period, limit):
cursor = ""
while True:
clips = get_channel_clips(
@ -195,19 +255,7 @@ def get_channel_videos(channel_id, limit, sort, type="archive", game_ids=[], aft
edges {{
cursor
node {{
id
title
publishedAt
broadcastType
lengthSeconds
game {{
name
}}
creator {{
channel {{
displayName
}}
}}
{fields}
}}
}}
}}
@ -215,41 +263,67 @@ def get_channel_videos(channel_id, limit, sort, type="archive", game_ids=[], aft
}}
"""
query = query.format(**{
"channel_id": channel_id,
"game_ids": game_ids,
"after": after if after else "",
"limit": limit,
"sort": sort.upper(),
"type": type.upper(),
})
query = query.format(
channel_id=channel_id,
game_ids=game_ids,
after=after if after else "",
limit=limit,
sort=sort.upper(),
type=type.upper(),
fields=VIDEO_FIELDS
)
response = gql_query(query)
if not response["data"]["user"]:
raise ConsoleError("Channel {} not found".format(channel_id))
return response["data"]["user"]["videos"]
def channel_videos_generator(channel_id, limit, sort, type, game_ids=None):
cursor = ""
while True:
videos = get_channel_videos(
channel_id, limit, sort, type, game_ids=game_ids, after=cursor)
if not videos["edges"]:
break
def channel_videos_generator(channel_id, max_videos, sort, type, game_ids=None):
def _generator(videos, max_videos):
for video in videos["edges"]:
if max_videos < 1:
return
yield video["node"]
max_videos -= 1
has_next = videos["pageInfo"]["hasNextPage"]
cursor = videos["edges"][-1]["cursor"] if has_next else None
if max_videos < 1 or not has_next:
return
yield videos, has_next
limit = min(max_videos, 100)
cursor = videos["edges"][-1]["cursor"]
videos = get_channel_videos(channel_id, limit, sort, type, game_ids, cursor)
yield from _generator(videos, max_videos)
if not cursor:
break
limit = min(max_videos, 100)
videos = get_channel_videos(channel_id, limit, sort, type, game_ids)
return videos["totalCount"], _generator(videos, max_videos)
def get_access_token(video_id):
url = "https://api.twitch.tv/api/vods/{}/access_token".format(video_id)
query = """
{{
videoPlaybackAccessToken(
id: {video_id},
params: {{
platform: "web",
playerBackend: "mediaplayer",
playerType: "site"
}}
) {{
signature
value
}}
}}
"""
return authenticated_get(url).json()
query = query.format(video_id=video_id)
response = gql_query(query)
return response["data"]["videoPlaybackAccessToken"]
def get_playlists(video_id, access_token):
@ -259,8 +333,9 @@ def get_playlists(video_id, access_token):
url = "http://usher.twitch.tv/vod/{}".format(video_id)
response = requests.get(url, params={
"nauth": access_token['token'],
"nauthsig": access_token['sig'],
"nauth": access_token['value'],
"nauthsig": access_token['signature'],
"allow_audio_only": "true",
"allow_source": "true",
"player": "twitchweb",
})

View File

@ -55,9 +55,42 @@ def read_int(msg, min, max, default):
def slugify(value):
re_pattern = re.compile(r'[^\w\s-]', flags=re.U)
re_spaces = re.compile(r'[-\s]+', flags=re.U)
value = str(value)
value = unicodedata.normalize('NFKC', value)
value = re_pattern.sub('', value).strip().lower()
return re_spaces.sub('_', value)
value = unicodedata.normalize('NFKC', str(value))
value = re.sub(r'[^\w\s_-]', '', value)
value = re.sub(r'[\s_-]+', '_', value)
return value.strip("_").lower()
def titlify(value):
value = unicodedata.normalize('NFKC', str(value))
value = re.sub(r'[^\w\s\[\]().-]', '', value)
value = re.sub(r'\s+', ' ', value)
return value.strip()
VIDEO_PATTERNS = [
r"^(?P<id>\d+)?$",
r"^https://(www.)?twitch.tv/videos/(?P<id>\d+)(\?.+)?$",
]
CLIP_PATTERNS = [
r"^(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)$",
r"^https://(www.)?twitch.tv/\w+/clip/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
r"^https://clips.twitch.tv/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
]
def parse_video_identifier(identifier):
"""Given a video ID or URL returns the video ID, or null if not matched"""
for pattern in VIDEO_PATTERNS:
match = re.match(pattern, identifier)
if match:
return match.group("id")
def parse_clip_identifier(identifier):
"""Given a clip slug or URL returns the clip slug, or null if not matched"""
for pattern in CLIP_PATTERNS:
match = re.match(pattern, identifier)
if match:
return match.group("slug")