Compare commits

..

149 Commits

Author SHA1 Message Date
c06ba2248d Bump version 2022-02-25 19:23:08 +01:00
a5039be657 Update changelog 2022-02-25 19:20:53 +01:00
6fa3bd568d Make clips behave similar to recent videos changes 2022-02-25 19:10:48 +01:00
ef059f3dbd Extend docs for the videos command 2022-02-25 17:54:28 +01:00
fbe4a17ff0 Add changelog 2022-02-23 19:30:28 +01:00
1fc5ef6bd1 Reorganize paging and limiting video list 2022-02-23 19:07:27 +01:00
b73ab58432 Fix video output when creator is not set 2022-02-23 19:07:04 +01:00
52651e62c8 Enable logging via --debug flag 2022-02-23 16:24:52 +01:00
4928188055 Add support for JSON output to videos command 2022-02-23 16:02:28 +01:00
b4c31b04e1 Bump copyright year 2022-02-05 15:12:01 +01:00
7ad574d103 Bump version 2022-02-05 15:08:51 +01:00
2fddb0c6a4 Add support for downloading audio only 2022-02-05 14:37:52 +01:00
fbc227017d Bump version, add changelog 2022-02-05 09:52:06 +01:00
b8aaa0c24c Fix issue with downloading clips with no game set
issue #78
2022-02-05 09:50:17 +01:00
0af472528d Remove unused function 2022-02-05 09:50:17 +01:00
fe9b69e1d4 Remove debian packaging stuff 2022-02-05 09:43:45 +01:00
fb73fe07c5 Add clip slug to formatting options 2022-02-05 09:36:50 +01:00
7eb50a0fa1 Fix output formatting when game is not set
issue #87
2022-02-05 09:34:28 +01:00
6ebe263f33 Fix issue with colons in file name
ffmpeg uses colons to define protocols, by prefixing the target with
file: this ambiguity is avoided.

issue #89
2022-02-05 09:26:10 +01:00
15654291d7 Remove double new line 2022-01-25 08:37:37 +01:00
28b4c6146b Add script for tagging versions, yaml changelog 2022-01-25 08:35:50 +01:00
310010363f Bump version, add changelog 2022-01-25 08:22:35 +01:00
4fd532f05d Ask overwrite confirmation early
Ask whether to overwrite before starting download, this way once
download starts, there will be no more prompts.
2022-01-25 08:08:17 +01:00
8156b18b37 Tweak colors 2022-01-25 07:59:54 +01:00
4ff4c8763c Ignore pyright config file 2022-01-24 10:11:16 +01:00
b24bc0eb29 Add option to define target filename 2022-01-24 09:06:02 +01:00
c5b5c49058 Delete bundles with make clean 2022-01-19 08:20:57 +01:00
b67ccc9dde Bump version, add changelog 2022-01-19 08:04:56 +01:00
192c2925b7 Allow newer versions of m3u8 2022-01-19 08:01:18 +01:00
2f977be161 Bump version 2021-12-03 10:26:17 +01:00
928c6d64cf Add metadata to the encoded video file 2021-12-03 10:25:04 +01:00
dd1f4e0d26 Better computation of speed
Distinct VODs took from disk and freshly downloaded
2021-09-16 08:29:33 +02:00
caabe3138c Remove old kraken requests 2021-09-16 08:01:31 +02:00
9c3cf11635 Dedupe clip fields 2021-09-16 07:59:07 +02:00
2f51b3821b Handle video not found gracefully 2021-09-16 07:58:43 +02:00
62092ee25f Add some tests 2021-09-16 07:57:55 +02:00
6f86aea493 Bump version, changelog 2021-07-31 11:44:56 +02:00
e3f66bda43 Fix compat with older versions of python
path.join doesn't handle PosixPath instances in older versions of python
which causes breakage.

fixes #71
2021-07-31 11:41:45 +02:00
5c3cebd0f3 Bump version, changelog 2021-06-09 15:10:39 +02:00
a49dcab419 Fix clips download by fetching access token
fixes #64
2021-06-09 15:08:40 +02:00
0dd04a7e2d Extract clips list method 2021-05-18 14:25:49 +02:00
5bd0747dde Respect --limit when downloading clips 2021-05-18 14:23:56 +02:00
63c2aff334 Handle keyboard interrupt 2021-05-18 14:00:50 +02:00
e95b430eec Remove unused namedtuple 2021-05-18 13:54:17 +02:00
8c582c600e Fix duplicate named test function 2021-05-18 13:52:34 +02:00
c0c5cbf2a8 Don't break if channel not found 2021-03-22 07:42:07 +01:00
3f143b0c84 Fix file naming not to strip first digit of id
fixes #60
2021-03-22 07:42:07 +01:00
2242af05fc Add changelog, bump version 2021-02-15 14:35:20 +01:00
9c901a21d9 Fix reference to invalid argument 2021-02-15 14:33:31 +01:00
270f53c3c1 Fix tests 2021-02-15 14:33:25 +01:00
e12dba26b4 Add dash as an allowed char, set length to 16. 2021-02-15 14:23:13 +01:00
3a61e61226 Add underscore as an allow char for clip extra ID 2021-02-15 14:23:13 +01:00
8ddfad51bc Update clip identifier patterns
New clips ID naming convention. ex: GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ
2021-02-15 14:23:13 +01:00
d152cbff09 Update tests with new clip naming
New clips ID naming convention. ex: GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ
2021-02-15 14:23:13 +01:00
e0681ab53c Bump version, changelog 2021-01-14 22:25:22 +01:00
728e631623 Handle video not found in info 2021-01-14 22:23:40 +01:00
9d19acbe6d Re-apply fix lost to a merge conflict 2021-01-14 22:16:17 +01:00
baeaedaa54 Fix setup.py to detect new package 2021-01-14 22:14:53 +01:00
9e2bbd7e39 Bump version, changelog, readme 2021-01-14 22:14:53 +01:00
dbee7cdc52 Use GraphQL to fetch access token
issue #53
2021-01-14 22:14:52 +01:00
548a9350ba Add twitch-dl info command 2021-01-14 22:14:52 +01:00
2380dc5a35 Split up commands 2021-01-14 22:14:50 +01:00
a7340f178f Fix bug in videos command saying there are more videos when there aren’t
When listing videos, if --pager was not specified and all of the videos
for a channel were listed, a message would still print out saying
“There are more videos. Increase the --limit or use --pager to see the
rest."

This checks to see if there are actually more videos before printing
that error message.
2021-01-11 00:49:57 +01:00
838611b834 Bump version 2020-11-23 16:08:15 +01:00
cf8d13e80e Fix clip download due to missing fields in query
fixes #45
2020-11-23 16:06:09 +01:00
6108b15587 Handle channel not found in clips 2020-11-10 12:40:11 +01:00
bb2de4af96 Bump version 2020-11-10 10:48:21 +01:00
678bf11a8a Fix print_clip when game is not set 2020-11-10 10:44:18 +01:00
69de08e2ba Update readme 2020-11-10 10:44:09 +01:00
1cb4598a71 Add downloading clips 2020-11-10 09:22:07 +01:00
02e4cdaff6 Add listing clips 2020-11-10 08:30:01 +01:00
c2e9ab9382 Bump version 2020-09-29 12:22:24 +02:00
739460bed1 Remove extraneous parameter 2020-09-29 12:21:13 +02:00
c92ba5e183 Bump version 2020-09-29 11:28:35 +02:00
f4ab045cdc Add --pager option, don't page by default
issue #30
2020-09-29 11:28:35 +02:00
cdc0495cf1 Document temp file shenanigans 2020-09-29 11:28:34 +02:00
c510d5aae4 Add --overwrite option
issue #37
2020-09-29 11:28:03 +02:00
2abc56213e Add --no-join option to skip ffmpeg join
issue #36
2020-09-29 11:27:49 +02:00
bb16108572 Add source quality alias
issue #33
2020-09-29 08:26:40 +02:00
b982cba566 Bump version, changelog 2020-09-03 12:56:57 +02:00
f4d442c118 Document installation instructions 2020-09-03 12:56:08 +02:00
eecd098f18 Add www before twitch.tv since that's what twitch uses 2020-09-03 12:55:45 +02:00
041689bee9 Construct paths using path libs
Fixes issues with paths on windows.

issue #35
2020-09-03 12:26:28 +02:00
a245ffb6a4 Fix issue with partial downloads
When using --start or --end, only keep the segments which have been
downloaded, and skip the rest.
2020-09-03 11:59:44 +02:00
ac37f179ef Improve bundle command
* directly save to desired file name
* add version number to file name
* remove __pycache__ folders before bundling
* compress the archive
2020-09-03 11:09:38 +02:00
5b200a2cb7 Add a splash of color 2020-09-03 10:38:29 +02:00
bf2a4558f4 Improve VOD joining logic
Instead of creating a file list, create a modified playlist which
references the downloaded files, and give this as input to ffmpeg. Since
ffmpeg handles M3U8 playlists, this means options such as
`EXT-X-BYTERANGE` are supported.

issue #35
2020-09-03 10:38:29 +02:00
772faa5901 Log ffmpeg command and handle errors better 2020-09-03 10:38:28 +02:00
04ddadef26 Add bundling with zipapp 2020-08-13 13:47:03 +02:00
bbed398cf6 Fix version number in init file, bump version 2020-08-11 18:10:21 +02:00
d8863ac695 Bump version, changelog 2020-08-09 12:04:48 +02:00
4edf299780 Handle more gracefully when video/clip not found 2020-08-09 11:55:40 +02:00
78295a492c Improve parsing inputs for download command
* Fix overzealous regex which caused video patterns to be identified as
  clips by testing for videos before clips.
* Allow URLs with or without `www.`.
* Add tests which verify this actually works.

fixes #28
2020-08-09 11:45:39 +02:00
f456d04de6 Bump version, update changelog and readme 2020-08-07 16:38:55 +02:00
8222df3670 Allow numbers in clip slugs
issue #24
2020-08-07 16:35:56 +02:00
65663d3505 Fix url to video shown when listing videos 2020-08-07 16:24:23 +02:00
706e42d197 Make quality selectable for clip download 2020-08-07 16:23:08 +02:00
3cfa05a3ee Make quality selectable for video download 2020-08-07 16:23:00 +02:00
4f62a26c30 Bump version, changelog 2020-06-10 12:07:59 +02:00
2171a9e08e Allow unicode values in slugs
Otherwise non-ascii characters get stripped which is not good for
e.g. titles in cyrillic script.
2020-06-10 10:54:28 +02:00
15ca684286 Don't unpack options
This makes it more readable as option count increases.
2020-05-30 10:21:19 +02:00
fd56a16c41 Fix option to use kebab case like the rest 2020-05-30 09:48:31 +02:00
4885c6a3b7 Add requirements to readme 2020-05-29 13:57:02 +02:00
2cf66c022c Don't break if game is None 2020-05-29 13:55:54 +02:00
717f634dda Remove unused code 2020-05-29 13:51:51 +02:00
169f15ca30 Add --game example to README 2020-05-17 14:46:08 +02:00
58458553bc Bump version 2020-05-17 14:42:55 +02:00
cabc8ff327 Improve paging 2020-05-17 14:41:11 +02:00
d22fd74357 Add filtering videos by game 2020-05-17 14:35:33 +02:00
4241ab5d67 Make less important messages dim 2020-05-17 14:32:37 +02:00
94e9f6aa80 Extract graphql query function 2020-05-17 13:48:48 +02:00
b014d94366 Blue is nicer than cyan 2020-05-17 13:48:16 +02:00
ea01ef3d99 Add paging to videos command 2020-05-17 13:41:34 +02:00
2118cd8825 Use graphql to fetch channel videos
The old helix endpoint returns HTTP 401

fixes #18
2020-05-17 11:57:16 +02:00
6c28dd2f5e Bump version 2020-04-25 20:06:02 +02:00
e3dde90870 Specify broadcast type when listing videos
issue #13
2020-04-25 20:04:21 +02:00
c628757ac0 Fix error message 2020-04-12 11:44:01 +02:00
5e97b439a7 Bump version, changelog 2020-04-11 20:57:43 +02:00
07f3a2fa48 Implement downloading clips
issue #15
2020-04-11 16:07:17 +02:00
96f13e9cf7 Bump version, changelog 2020-04-11 14:07:14 +02:00
c9547435df Nicer otput while dowloading VODs, bright colors 2020-04-11 14:05:23 +02:00
042d35ba1e Override local file names for downloaded vods
Sometimes the playlists contain more than just file names which can
break the ffmpeg join, so just name downloaded vods sequentially.

fixes #12
2020-04-11 13:20:59 +02:00
ebc754072d Reorganise code 2020-04-11 13:08:42 +02:00
cb00accd6a Better long description 2020-04-10 16:42:35 +02:00
64157c1ef6 Bump version 2020-04-10 16:34:37 +02:00
6a8da3b01b Don't print errors messages when retrying
Only die if all retries fail.
2020-04-10 16:22:15 +02:00
e29d42e9ef Use Twitch's client ID
Fetching access token with own client ID no longer works.

Everybody else in the world seems to be doing it:
https://github.com/search?p=2&q=kimne78kx3ncx6brgo4mv6wki5h1ko&type=Code
2020-04-10 16:21:10 +02:00
100aa53b84 Bump version 2019-08-23 13:08:57 +02:00
e384f26444 Save playlists to temp dir for debugging 2019-08-23 13:08:35 +02:00
000754af8c Use m3u8 lib to parse playlists 2019-08-23 12:36:05 +02:00
6813bb51b4 Add option not to delete downloaded VODs 2019-08-23 10:16:49 +02:00
34b0592cf3 Fix usage of deprecated v3 API
related #8
2019-08-23 09:03:33 +02:00
e72f8e24ea Bump version 2019-08-13 12:40:00 +02:00
3c99e9975b Bump version, changelog 2019-08-13 12:31:44 +02:00
f807d4324b Style the url in video list 2019-08-13 12:29:42 +02:00
68a8b70948 Add offset and sort options to videos command
fixes #7
2019-08-13 12:25:25 +02:00
932b7750b9 Print video URL 2019-08-12 15:21:57 +02:00
aa5f17cbdb Show errors returned via HTTP 400 2019-08-12 15:14:13 +02:00
0fff0a4de1 Exit with nonzero code on error 2019-08-12 13:47:48 +02:00
9dc67a7ff1 Bump version, add changelog 2019-07-05 13:15:59 +02:00
3e7f310e36 Add --version option to print program version 2019-07-05 13:14:22 +02:00
cbb0d6cfbd Allow specifying the output format
i.e. the output file extension passed to ffmpeg
2019-07-05 13:04:09 +02:00
46d2654cfa Bump to stable 2019-06-06 11:48:28 +02:00
0f54c527be Remove redundant bdist_wheel setting
py3 is the default
2019-06-06 11:47:04 +02:00
8133d93436 Don't make universal wheels (py2 not supported) 2019-06-06 11:44:56 +02:00
9345dd966f Bump version, add changelog 2019-06-06 11:10:21 +02:00
e9bd706194 Allow limiting download by start and end time 2019-06-06 11:06:33 +02:00
357379a6a1 Update Makefile 2019-06-06 09:28:23 +02:00
0c88de3862 Bump version 2019-04-30 13:47:16 +02:00
29 changed files with 2233 additions and 346 deletions

3
.gitignore vendored
View File

@ -11,3 +11,6 @@ tmp/
/htmlcov
/twitch-dl-*.tar.gz
/twitch-dl.1.man
/bundle
/*.pyz
/pyrightconfig.json

165
CHANGELOG.md Normal file
View File

@ -0,0 +1,165 @@
twitch-dl changelog
===================
<!-- Do not edit. This file is automatically generated from changelog.yaml.-->
**1.20.0 (2022-02-25)**
* Add `--json` option to `videos` command (#92, thanks @miff2000)
* Add `--all` option to `videos` and `clips` commands to list all clips or
videos in one go.
* Modify how `--pager` works, will make multiple requests if needed to show all
available items, ignoring `--limit`.
**1.19.0 (2022-02-05)**
* Add support for downloading audio only (#10)
**1.18.1 (2022-02-05)**
* Fix issues with output formats (#87, #89)
* Fix issues when downloading clip with no game set (#78)
* Add option to use clip slug in `--output` format
**1.18.0 (2022-01-25)**
* Add `--output` option to `download` command which allows setting output file
template (#70)
* Ask to overwrite before downloading to avoid later prompt
**1.17.1 (2022-01-19)**
* Upgrade m3u8 lib to 1.0.0+
**1.17.0 (2021-12-03)**
* Fix speed calculation when resuming download (#75, thanks CroquetteTheThe)
* Add artist and title metadata to resulting video (#80)
**1.16.1 (2021-07-31)**
* Fix compat with older versions of python (#71)
**1.16.0 (2021-06-09)**
* Fix clips download caused by Twitch changes (#64, thanks to all participants)
**1.15.0 (2021-02-15)**
* Add support for new format of clip slug (thanks @Loveangel1337)
**1.14.1 (2021-01-14)**
* Handle videos which don't exist more gracefully
**1.14.0 (2021-01-14)**
* Added `info` command for displaying video or clip info (#51)
* Don't show there are more videos when there aren't (#52, thanks @scottyallen)
* Fixed Twitch regression for getting the access token (#53)
**1.13.1 (2020-11-23)**
* Fixed clip download issue (#45)
**1.13.0 (2020-11-10)**
* Added `clips` command for listing and batch downloading clips (#26)
**1.12.1 (2020-09-29)**
* Fix bug introduced in previous version which broke joining
**1.12.0 (2020-09-29)**
* Added `source` as alias for best available quality (#33)
* Added `--no-join` option to `download` to skip ffmpeg join (#36)
* Added `--overwrite` option to `download` to overwrite target without prompting
for confirmation (#37)
* Added `--pager` option to `videos`, don't page by default (#30)
**1.11.0 (2020-09-03)**
* Make downloading more robust, fixes issues with some VODs (#35)
* Bundle twitch-dl to a standalone archive, simplifying installation, see
installation instructions in README
**1.10.2 (2020-08-11)**
* Fix version number displayed by `twitch-dl --version` (#29)
**1.10.1 (2020-08-09)**
* Fix videos incorrectly identified as clips (#28)
* Make download command work with video URLs lacking 'www' before 'twitch.tv'
* Print an error when video or clip is not found instead of an exception trace
**1.10.0 (2020-08-07)**
* Add `--quality` option to `download` command, allows specifying the video
quality to download. In this case, twitch-dl will require no user input. (#22)
* Fix download of clips which contain numbers in their slug (#24)
* Fix URL to video displayed by `videos` command (it was missing /videos/)
**1.9.0 (2020-06-10)**
* **Breaking**: wrongly named `--max_workers` option changed to `--max-workers`.
The shorthand option `-w` remains the same.
* Fix bug where `videos` command would crash if there was no game info (#21)
* Allow unicode characters in filenames, no longer strips e.g. cyrillic script
**1.8.0 (2020-05-17)**
* Fix videos command (#18)
* **Breaking**: `videos` command no longer takes the `--offset` parameter due to
API changes
* Add paging to `videos` command to replace offset
* Add `--game` option to `videos` command to filter by game
**1.7.0 (2020-04-25)**
* Support for specifying broadcast type when listing videos (#13)
**1.6.0 (2020-04-11)**
* Support for downloading clips (#15)
**1.5.1 (2020-04-11)**
* Fix VOD naming issue (#12)
* Nice console output while downloading
**1.5.0 (2020-04-10)**
* Fix video downloads after Twitch deprecated access token access
* Don't print errors when retrying download, only if all fails
**1.4.0 (2019-08-23)**
* Fix usage of deprecated v3 API
* Use m3u8 lib for parsing playlists
* Add `--keep` option not preserve downloaded VODs
**1.3.1 (2019-08-13)**
* No changes, bumped to fix issue with pypi
**1.3.0 (2019-08-13)**
* Add `--sort` and `--offset` options to `videos` command, allows paging (#7)
* Show video URL in `videos` command output
**1.2.0 (2019-07-05)**
* Add `--format` option to `download` command for specifying the output format
(#6)
* Add `--version` option for printing program version
**1.1.0 (2019-06-06)**
* Allow limiting download by start and end time
**1.0.0 (2019-04-30)**
* Initial release

View File

@ -1,28 +1,35 @@
default : clean dist
dist :
@echo "\nMaking source"
@echo "-------------"
@python setup.py sdist
@echo "\nMaking wheel"
@echo "-------------"
@python setup.py bdist_wheel --universal
@echo "\nDone."
python setup.py sdist --formats=gztar,zip
python setup.py bdist_wheel --python-tag=py3
clean :
find . -name "*pyc" | xargs rm -rf $1
rm -rf build dist MANIFEST htmlcov deb_dist twitch-dl*.tar.gz twitch-dl.1.man
rm -rf build dist bundle MANIFEST htmlcov deb_dist twitch-dl.*.pyz twitch-dl.1.man
bundle:
mkdir bundle
cp twitchdl/__main__.py bundle
pip install . --target=bundle
rm -rf bundle/*.dist-info
find bundle/ -type d -name "__pycache__" -exec rm -rf {} +
python -m zipapp \
--python "/usr/bin/env python3" \
--output twitch-dl.`git describe`.pyz bundle \
--compress
publish :
twine upload dist/*
twine upload dist/*.tar.gz dist/*.whl
coverage:
py.test --cov=toot --cov-report html tests/
deb:
@python setup.py --command-packages=stdeb.command bdist_deb
man:
scdoc < twitch-dl.1.scd > twitch-dl.1.man
test:
pytest
changelog:
./scripts/generate_changelog > CHANGELOG.md

281
README.md
View File

@ -1,10 +1,10 @@
Twitch Downloader
=================
A simple CLI tool for downloading videos from Twitch.
CLI tool for downloading videos from twitch.tv
Inspired by youtube-dl but improves upon it by using multiple concurrent
connections to make the download faster.
Inspired by [youtube-dl](https://youtube-dl.org/) but improves upon it by using
multiple concurrent connections to make the download faster.
Resources
---------
@ -13,43 +13,278 @@ Resources
* Issues: https://github.com/ihabunek/twitch-dl/issues
* Python package: https://pypi.org/project/twitch-dl/
Requirements
------------
* Python 3.5+
* [ffmpeg](https://ffmpeg.org/download.html), installed and on the system path
Installation
------------
### Download standalone archive
Go to the [latest release](https://github.com/ihabunek/twitch-dl/releases/latest)
and download the `twitch-dl.<version>.pyz` archive.
Run the archive by either:
a) passing it to python:
```
python3 twitch-dl.1.13.0.pyz --help
```
b) making it executable and invoking it directly (linux specific):
```
chmod +x twitch-dl.1.13.0.pyz
./twitch-dl.1.13.0.pyz --help
```
Feel free to rename the archive to something more managable, like `twitch-dl`.
To upgrade to a newer version, repeat the process with the newer release.
### From PYPI using pipx
**pipx** is a tool which installs python apps into isolated environments, which
prevents all kinds of problems later so it's the suggested way to install
twitch-dl from PYPI.
Install pipx as described in
[pipx install docs](https://pipxproject.github.io/pipx/installation/).
Install twitch-dl:
```
pipx install twitch-dl
```
Check installation worked:
```
twitch-dl --help
```
If twitch-dl executable is not found, check that the pipx binary location (by
default `~/.local/bin`) is in your PATH.
To upgrade twitch-dl to the latest version:
```
pipx install twitch-dl
```
Usage
-----
List recent streams for a given channel:
This section does an overview of available features.
To see a list of available commands run:
```
twitch-dl --help
```
And to see description and all arguments for a given command run:
```
twitch-dl <command> --help
```
### Print clip or video info
Videos can be referenced by URL or ID:
```
twitch-dl info 863849735
twitch-dl info https://www.twitch.tv/videos/863849735
```
Clips by slug or ID:
```
twitch-dl info BusyBlushingCattleItsBoshyTime
twitch-dl info https://www.twitch.tv/bananasaurus_rex/clip/BusyBlushingCattleItsBoshyTime
```
Shows info about the video or clip as well as download URLs for clips and
playlist URLs for videos.
### Listing videos
List recent channel videos (10 by default):
```
twitch-dl videos bananasaurus_rex
```
Yields (trimmed):
Limit to videos of one or more games:
```
Found 33 videos
221837124
SUPER MARIO ODYSSSEY - Stream #2 / 600,000,000
Bananasaurus_Rex playing Super Mario Odyssey
Published 2018-01-24 @ 12:05:25 Length: 3h 40min
221418913
Dead Space and then SUPER MARIO ODYSSEY PogChamp
Bananasaurus_Rex playing Dead Space
Published 2018-01-23 @ 02:40:58 Length: 6h 2min
220783179
Dead Space | Got my new setup working! rexChamp
Bananasaurus_Rex playing Dead Space
Published 2018-01-21 @ 05:47:03 Length: 5h 7min
twitch-dl videos --game "doom eternal" --game "cave story" bananasaurus_rex
```
Download a stream by ID or URL:
List all channel videos at once:
```
twitch-dl videos bananasaurus_rex --all
```
List all channel videos in pages of 10:
```
twitch-dl videos bananasaurus_rex --pager
```
Page size can be adjusted by passing number of items per page:
```
twitch-dl videos bananasaurus_rex --pager 5
```
Returns all videos as a JSON list. Useful for scripting.
```
twitch-dl videos bananasaurus_rex --json --all
```
### Downloading videos
Download a video by ID or URL:
```
twitch-dl download 221837124
twitch-dl download https://www.twitch.tv/videos/221837124
```
Specify video quality to download:
```
twitch-dl download -q 720p 221837124
```
Setting quality to `source` will download the best available quality:
```
twitch-dl download -q source 221837124
```
Setting quality to `audio_only` will download only audio:
```
twitch-dl download -q audio_only 221837124
```
### Overriding file name
The target filename can be defined by passing the `--output` option followed by
the desired file name, e.g. `--output strim.mkv`.
The filename uses
[Python format string syntax](https://docs.python.org/3/library/string.html#format-string-syntax)
and may contain placeholders in curly braces which will be replaced with
relevant information tied to the downloaded video.
The supported placeholders are:
| Placeholder | Description | Example |
| ----------------- | ------------------------------ | ------------------------------ |
| `{id}` | Video ID | 1255522958 |
| `{title}` | Video title | Dark Souls 3 First playthrough |
| `{title_slug}` | Slugified video title | dark_souls_3_first_playthrough |
| `{datetime}` | Video date and time | 2022-01-07T04:00:27Z |
| `{date}` | Video date | 2022-01-07 |
| `{time}` | Video time | 04:00:27Z |
| `{channel}` | Channel name | KatLink |
| `{channel_login}` | Channel login | katlink |
| `{format}` | File extension, see `--format` | mkv |
| `{game}` | Game name | Dark Souls III |
| `{game_slug}` | Slugified game name | dark_souls_iii |
| `{slug}` | Clip slug (clips only) | AbrasivePlacidCatDxAbomb |
A couple of examples:
Pattern: `"{date}_{id}_{channel_login}_{title_slug}.{format}"`<br />
Expands to: `2022-01-07_1255522958_katlink_dark_souls_3_first_playthrough.mkv`<br />
*This is the default.*
Pattern: `"{channel} - {game} - {title}.{format}"`<br />
Expands to: `KatLink - Dark Souls III - Dark Souls 3 First playthrough.mkv`
### Listing clips
Listing clips works similar to listing videos. Shows 10 clips by default. Use
`--all` to list all in one go or `--pager` to show them in pages.
List clips for the given period:
```
twitch-dl clips bananasaurus_rex --period last_week
```
Supported periods are: `last_day`, `last_week`, `last_month`, `all_time`.
Also supports JSON output:
```
twitch-dl clips bananasaurus_rex --json --all
```
Note that this may make multiple requests to the server because each request is
limited to 100 clips, so it may take a little while. You can use `--debug` to
log requests.
### Downloading clips
Download a clip by slug or URL:
```
twitch-dl download VenomousTameWormHumbleLife
twitch-dl download https://www.twitch.tv/bananasaurus_rex/clip/VenomousTameWormHumbleLife
```
Specify clip quality to download:
```
twitch-dl download -q 720 VenomousTameWormHumbleLife
```
Note that twitch names for clip qualities have no trailing "p".
### Batch downloading clips
It's possible to download all clips for a given period:
```
twitch-dl clips bananasaurus_rex --period last_week --download
```
Clips are downloaded in source quality.
A note about clips
------------------
Currently it doesn't seem to be possible to get a list of clips ordered by time
of creation, only by view count. Clips with the same view count seem to be
returned in random order. This can break paging resulting in duplicate clips
listed or clips missed.
When batch downloading a large number of clips (over 100), it's possible that
some will be missed.
Temporary files
---------------
By default, twitch-dl will download VODs to your systems temp dir (e.g. `/tmp/`
on Linux). To change the location where the files are downloaded you can set
the `TMP` environment variable, e.g.
```
TMP=/my/tmp/path/ twitch-dl download 221837124
```
Man page
--------
@ -64,6 +299,6 @@ make man
License
-------
Copyright 2018 Ivan Habunek <ivan@habunek.com>
Copyright 2018-2022 Ivan Habunek <ivan@habunek.com>
Licensed under the GPLv3: http://www.gnu.org/licenses/gpl-3.0.html

181
changelog.yaml Normal file
View File

@ -0,0 +1,181 @@
1.20.0:
date: 2022-02-25
changes:
- "Add `--json` option to `videos` command (#92, thanks @miff2000)"
- "Add `--all` option to `videos` and `clips` commands to list all clips or videos in one go."
- "Modify how `--pager` works, will make multiple requests if needed to show all available items, ignoring `--limit`."
1.19.0:
date: 2022-02-05
changes:
- "Add support for downloading audio only (#10)"
1.18.1:
date: 2022-02-05
changes:
- "Fix issues with output formats (#87, #89)"
- "Fix issues when downloading clip with no game set (#78)"
- "Add option to use clip slug in `--output` format"
1.18.0:
date: 2022-01-25
changes:
- "Add `--output` option to `download` command which allows setting output file template (#70)"
- "Ask to overwrite before downloading to avoid later prompt"
1.17.1:
date: 2022-01-19
changes:
- "Upgrade m3u8 lib to 1.0.0+"
1.17.0:
date: 2021-12-03
changes:
- "Fix speed calculation when resuming download (#75, thanks CroquetteTheThe)"
- "Add artist and title metadata to resulting video (#80)"
1.16.1:
date: 2021-07-31
changes:
- "Fix compat with older versions of python (#71)"
1.16.0:
date: 2021-06-09
changes:
- "Fix clips download caused by Twitch changes (#64, thanks to all participants)"
1.15.0:
date: 2021-02-15
changes:
- "Add support for new format of clip slug (thanks @Loveangel1337)"
1.14.1:
date: 2021-01-14
changes:
- "Handle videos which don't exist more gracefully"
1.14.0:
date: 2021-01-14
changes:
- "Added `info` command for displaying video or clip info (#51)"
- "Don't show there are more videos when there aren't (#52, thanks @scottyallen)"
- "Fixed Twitch regression for getting the access token (#53)"
1.13.1:
date: 2020-11-23
changes:
- "Fixed clip download issue (#45)"
1.13.0:
date: 2020-11-10
changes:
- "Added `clips` command for listing and batch downloading clips (#26)"
1.12.1:
date: 2020-09-29
changes:
- "Fix bug introduced in previous version which broke joining"
1.12.0:
date: 2020-09-29
changes:
- "Added `source` as alias for best available quality (#33)"
- "Added `--no-join` option to `download` to skip ffmpeg join (#36)"
- "Added `--overwrite` option to `download` to overwrite target without prompting for confirmation (#37)"
- "Added `--pager` option to `videos`, don't page by default (#30)"
1.11.0:
date: 2020-09-03
changes:
- "Make downloading more robust, fixes issues with some VODs (#35)"
- "Bundle twitch-dl to a standalone archive, simplifying installation, see installation instructions in README"
1.10.2:
date: 2020-08-11
changes:
- "Fix version number displayed by `twitch-dl --version` (#29)"
1.10.1:
date: 2020-08-09
changes:
- "Fix videos incorrectly identified as clips (#28)"
- "Make download command work with video URLs lacking 'www' before 'twitch.tv'"
- "Print an error when video or clip is not found instead of an exception trace"
1.10.0:
date: 2020-08-07
changes:
- "Add `--quality` option to `download` command, allows specifying the video quality to download. In this case, twitch-dl will require no user input. (#22)"
- "Fix download of clips which contain numbers in their slug (#24)"
- "Fix URL to video displayed by `videos` command (it was missing /videos/)"
1.9.0:
date: 2020-06-10
changes:
- "**Breaking**: wrongly named `--max_workers` option changed to `--max-workers`. The shorthand option `-w` remains the same."
- "Fix bug where `videos` command would crash if there was no game info (#21)"
- "Allow unicode characters in filenames, no longer strips e.g. cyrillic script"
1.8.0:
date: 2020-05-17
changes:
- "Fix videos command (#18)"
- "**Breaking**: `videos` command no longer takes the `--offset` parameter due to API changes"
- "Add paging to `videos` command to replace offset"
- "Add `--game` option to `videos` command to filter by game"
1.7.0:
date: 2020-04-25
changes:
- "Support for specifying broadcast type when listing videos (#13)"
1.6.0:
date: 2020-04-11
changes:
- "Support for downloading clips (#15)"
1.5.1:
date: 2020-04-11
changes:
- "Fix VOD naming issue (#12)"
- "Nice console output while downloading"
1.5.0:
date: 2020-04-10
changes:
- "Fix video downloads after Twitch deprecated access token access"
- "Don't print errors when retrying download, only if all fails"
1.4.0:
date: 2019-08-23
changes:
- "Fix usage of deprecated v3 API"
- "Use m3u8 lib for parsing playlists"
- "Add `--keep` option not preserve downloaded VODs"
1.3.1:
date: 2019-08-13
changes:
- "No changes, bumped to fix issue with pypi"
1.3.0:
date: 2019-08-13
changes:
- "Add `--sort` and `--offset` options to `videos` command, allows paging (#7)"
- "Show video URL in `videos` command output"
1.2.0:
date: 2019-07-05
changes:
- "Add `--format` option to `download` command for specifying the output format (#6)"
- "Add `--version` option for printing program version"
1.1.0:
date: 2019-06-06
changes:
- "Allow limiting download by start and end time"
1.0.0:
date: 2019-04-30
changes:
- "Initial release"

View File

@ -1,5 +1,4 @@
pytest-cov~=2.4.0
pytest~=3.0.0
stdeb~=0.8.5
twine~=1.8.1
wheel~=0.29.0
pytest
twine
wheel
pyyaml

33
scripts/generate_changelog Executable file
View File

@ -0,0 +1,33 @@
#!/usr/bin/env python3
"""
Generates a more user-readable changelog from changelog.yaml.
"""
import textwrap
import yaml
with open("changelog.yaml", "r") as f:
data = yaml.safe_load(f)
print("twitch-dl changelog")
print("===================")
print()
print("<!-- Do not edit. This file is automatically generated from changelog.yaml.-->")
print()
for version in data.keys():
date = data[version]["date"]
changes = data[version]["changes"]
print(f"**{version} ({date})**")
print()
for c in changes:
lines = textwrap.wrap(c, 78)
initial = True
for line in lines:
if initial:
print("* " + line)
initial = False
else:
print(" " + line)
print()

68
scripts/tag_version Executable file
View File

@ -0,0 +1,68 @@
#!/usr/bin/env python3
"""
Creates an annotated git tag for a given version number.
The tag will include the version number and changes for given version.
Usage: tag_version [version]
"""
import subprocess
import sys
import textwrap
import yaml
import twitchdl
from datetime import date
from os import path
from pkg_resources import get_distribution
path = path.join(path.dirname(path.dirname(path.abspath(__file__))), "changelog.yaml")
with open(path, "r") as f:
changelog = yaml.safe_load(f)
if len(sys.argv) != 2:
print("Wrong argument count", file=sys.stderr)
sys.exit(1)
version = sys.argv[1]
changelog_item = changelog.get(version)
if not changelog_item:
print(f"Version `{version}` not found in changelog.", file=sys.stderr)
sys.exit(1)
if twitchdl.__version__ != version:
print(f"twitchdl.__version__ is `{twitchdl.__version__}`, expected {version}.", file=sys.stderr)
sys.exit(1)
dist_version = get_distribution('twitch-dl').version
if dist_version != version:
print(f"Version in setup.py is `{dist_version}`, expected {version}.", file=sys.stderr)
sys.exit(1)
release_date = changelog_item["date"]
changes = changelog_item["changes"]
if not isinstance(release_date, date):
print(f"Release date not set for version `{version}` in the changelog.", file=sys.stderr)
sys.exit(1)
commit_message = f"twitch-dl {version}\n\n"
for c in changes:
lines = textwrap.wrap(c, 70)
initial = True
for line in lines:
lead = " *" if initial else " "
initial = False
commit_message += f"{lead} {line}\n"
proc = subprocess.run(["git", "tag", "-a", version, "-m", commit_message])
if proc.returncode != 0:
sys.exit(1)
print()
print(commit_message)
print()
print(f"Version {version} tagged \\o/")

View File

@ -1,2 +0,0 @@
[bdist_wheel]
universal=1

View File

@ -1,28 +1,36 @@
#!/usr/bin/env python
from setuptools import setup
from setuptools import setup, find_packages
long_description = """
Quickly download videos from twitch.tv.
Works simliarly to youtube-dl but downloads multiple VODs in parallel which
makes it faster.
"""
setup(
name='twitch-dl',
version='0.1.0',
version='1.20.0',
description='Twitch downloader',
long_description="A simple script for downloading videos from Twitch",
long_description=long_description.strip(),
author='Ivan Habunek',
author_email='ivan@habunek.com',
url='https://github.com/ihabunek/twitch-dl/',
keywords='twitch vod video download',
license='GPLv3',
classifiers=[
'Development Status :: 4 - Beta',
'Development Status :: 5 - Production/Stable',
'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
],
packages=['twitchdl'],
packages=find_packages(),
python_requires='>=3.5',
install_requires=[
"m3u8>=1.0.0,<2.0.0",
"requests>=2.13,<3.0",
],
entry_points={

View File

@ -1,3 +0,0 @@
[DEFAULT]
X-Python3-Version: >= 3.3
Copyright-File: LICENSE

30
tests/test_api.py Normal file
View File

@ -0,0 +1,30 @@
"""
These tests depend on the channel having some videos and clips published.
"""
from twitchdl import twitch
TEST_CHANNEL = "bananasaurus_rex"
def test_get_videos():
videos = twitch.get_channel_videos(TEST_CHANNEL, 3, "time")
assert videos["pageInfo"]
assert len(videos["edges"]) > 0
video_id = videos["edges"][0]["node"]["id"]
video = twitch.get_video(video_id)
assert video["id"] == video_id
def test_get_clips():
"""
This test depends on the channel having some videos published.
"""
clips = twitch.get_channel_clips(TEST_CHANNEL, "all_time", 3)
assert clips["pageInfo"]
assert len(clips["edges"]) > 0
clip_slug = clips["edges"][0]["node"]["slug"]
clip = twitch.get_clip(clip_slug)
assert clip["slug"] == clip_slug

35
tests/test_patterns.py Normal file
View File

@ -0,0 +1,35 @@
import pytest
from twitchdl.utils import parse_video_identifier, parse_clip_identifier
TEST_VIDEO_PATTERNS = [
("702689313", "702689313"),
("702689313", "https://twitch.tv/videos/702689313"),
("702689313", "https://www.twitch.tv/videos/702689313"),
]
TEST_CLIP_PATTERNS = {
("AbrasivePlayfulMangoMau5", "AbrasivePlayfulMangoMau5"),
("AbrasivePlayfulMangoMau5", "https://clips.twitch.tv/AbrasivePlayfulMangoMau5"),
("AbrasivePlayfulMangoMau5", "https://www.twitch.tv/dracul1nx/clip/AbrasivePlayfulMangoMau5"),
("AbrasivePlayfulMangoMau5", "https://twitch.tv/dracul1nx/clip/AbrasivePlayfulMangoMau5"),
("HungryProudRadicchioDoggo", "HungryProudRadicchioDoggo"),
("HungryProudRadicchioDoggo", "https://clips.twitch.tv/HungryProudRadicchioDoggo"),
("HungryProudRadicchioDoggo", "https://www.twitch.tv/bananasaurus_rex/clip/HungryProudRadicchioDoggo?filter=clips&range=7d&sort=time"),
("HungryProudRadicchioDoggo", "https://twitch.tv/bananasaurus_rex/clip/HungryProudRadicchioDoggo?filter=clips&range=7d&sort=time"),
("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ"),
("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ"),
("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ?filter=clips&range=7d&sort=time"),
("GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ", "https://www.twitch.tv/dracul1nx/clip/GloriousColdbloodedTortoiseRuleFive-E017utJ4DZmHVpfQ?filter=clips&range=7d&sort=time"),
}
@pytest.mark.parametrize("expected,input", TEST_VIDEO_PATTERNS)
def test_video_patterns(expected, input):
assert parse_video_identifier(input) == expected
@pytest.mark.parametrize("expected,input", TEST_CLIP_PATTERNS)
def test_clip_patterns(expected, input):
assert parse_clip_identifier(input) == expected

13
tests/test_utils.py Normal file
View File

@ -0,0 +1,13 @@
from twitchdl.utils import titlify, slugify
def test_titlify():
assert titlify("Foo Bar Baz.") == "Foo Bar Baz."
assert titlify("Foo (Bar) [Baz]") == "Foo (Bar) [Baz]"
assert titlify("Foo@{} Bar Baz!\"#$%&/=?*+'🔪") == "Foo Bar Baz"
def test_slugify():
assert slugify("Foo Bar Baz") == "foo_bar_baz"
assert slugify(" Foo Bar Baz ") == "foo_bar_baz"
assert slugify("Foo@{}[] Bar Baz!\"#$%&/()=?*+'🔪") == "foo_bar_baz"

View File

@ -24,18 +24,45 @@ List recent videos from bananasaurus\_rex's channel:
twitch-dl videos bananasaurus_rex
```
Download by URL:
Download video by URL:
```
twitch-dl download https://www.twitch.tv/videos/377220226
```
Download by ID:
Download video by ID:
```
twitch-dl download 377220226
```
Specify output format:
```
twitch-dl download --format=avi 377220226
```
Partial download by setting start and end time (hh:mm or hh:mm:ss):
```
twitch-dl download --start=00:10 --end=02:15 377220226
```
Download clip by URL:
```
twitch-dl download https://www.twitch.tv/bananasaurus_rex/clip/VenomousTameWormHumbleLife
```
Download clip by slug:
```
twitch-dl download VenomousTameWormHumbleLife
```
Note that clips are a single download, and don't benefit from the paralelism
used when downloading videos.
# SEE ALSO
youtube-dl(1)

View File

@ -1,3 +1,3 @@
__version__ = "0.1.0"
__version__ = "1.20.0"
CLIENT_ID = "miwy5zk23vh2he94san0bzj5ks1r0p"
CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko"

3
twitchdl/__main__.py Normal file
View File

@ -0,0 +1,3 @@
from twitchdl.console import main
main()

View File

@ -1,208 +0,0 @@
import os
import pathlib
import re
import subprocess
import tempfile
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor, as_completed
from functools import partial
from twitchdl import twitch
from twitchdl.download import download_file
from twitchdl.exceptions import ConsoleError
from twitchdl.output import print_out
from twitchdl.utils import slugify
def read_int(msg, min, max, default):
msg = msg + " [default {}]: ".format(default)
while True:
try:
val = input(msg)
if not val:
return default
if min <= int(val) <= max:
return int(val)
except ValueError:
pass
def format_size(bytes_):
if bytes_ < 1024:
return str(bytes_)
kilo = bytes_ / 1024
if kilo < 1024:
return "{:.1f}K".format(kilo)
mega = kilo / 1024
if mega < 1024:
return "{:.1f}M".format(mega)
return "{:.1f}G".format(mega / 1024)
def format_duration(total_seconds):
total_seconds = int(total_seconds)
hours = total_seconds // 3600
remainder = total_seconds % 3600
minutes = remainder // 60
seconds = total_seconds % 60
if hours:
return "{} h {} min".format(hours, minutes)
if minutes:
return "{} min {} sec".format(minutes, seconds)
return "{} sec".format(seconds)
def _print_video(video):
published_at = video['published_at'].replace('T', ' @ ').replace('Z', '')
length = format_duration(video['length'])
name = video['channel']['display_name']
print_out("\n<bold>{}</bold>".format(video['_id'][1:]))
print_out("<green>{}</green>".format(video["title"]))
print_out("<cyan>{}</cyan> playing <cyan>{}</cyan>".format(name, video['game']))
print_out("Published <cyan>{}</cyan> Length: <cyan>{}</cyan> ".format(published_at, length))
def videos(channel_name, **kwargs):
videos = twitch.get_channel_videos(channel_name)
print("Found {} videos".format(videos["_total"]))
for video in videos['videos']:
_print_video(video)
def _select_quality(playlists):
print_out("\nAvailable qualities:")
for no, v in playlists.items():
print_out("{}) {}".format(no, v[0]))
keys = list(playlists.keys())
no = read_int("Choose quality", min=min(keys), max=max(keys), default=keys[0])
return playlists[no]
def _print_progress(futures):
counter = 1
total = len(futures)
total_size = 0
start_time = datetime.now()
for future in as_completed(futures):
size = future.result()
percentage = 100 * counter // total
total_size += size
duration = (datetime.now() - start_time).seconds
speed = total_size // duration if duration else 0
remaining = (total - counter) * duration / counter
msg = "Downloaded VOD {}/{} ({}%) total <cyan>{}B</cyan> at <cyan>{}B/s</cyan> remaining <cyan>{}</cyan>".format(
counter, total, percentage, format_size(total_size), format_size(speed), format_duration(remaining))
print_out("\r" + msg.ljust(80), end='')
counter += 1
def _download_files(base_url, directory, filenames, max_workers):
urls = [base_url.format(f) for f in filenames]
paths = ["/".join([directory, f]) for f in filenames]
partials = (partial(download_file, url, path) for url, path in zip(urls, paths))
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = [executor.submit(fn) for fn in partials]
_print_progress(futures)
return paths
def _join_vods(directory, paths, target):
input_path = "{}/files.txt".format(directory)
with open(input_path, 'w') as f:
for path in paths:
f.write('file {}\n'.format(os.path.basename(path)))
result = subprocess.run([
"ffmpeg",
"-f", "concat",
"-i", input_path,
"-c", "copy",
target,
"-stats",
"-loglevel", "warning",
])
result.check_returncode()
def _video_target_filename(video, format):
match = re.search(r"^(\d{4})-(\d{2})-(\d{2})T", video['published_at'])
date = "".join(match.groups())
name = "_".join([
date,
video['_id'][1:],
video['channel']['name'],
slugify(video['title']),
])
return name + "." + format
def parse_video_id(video_id):
"""This can be either a integer ID or an URL to the video on twitch."""
if re.search(r"^\d+$", video_id):
return int(video_id)
match = re.search(r"^https://www.twitch.tv/videos/(\d+)(\?.+)?$", video_id)
if match:
return int(match.group(1))
raise ConsoleError("Invalid video ID given, expected integer ID or Twitch URL")
def download(video_id, max_workers, format='mkv', **kwargs):
video_id = parse_video_id(video_id)
print_out("Looking up video...")
video = twitch.get_video(video_id)
print_out("Found: <blue>{}</blue> by <yellow>{}</yellow>".format(
video['title'], video['channel']['display_name']))
print_out("Fetching access token...")
access_token = twitch.get_access_token(video_id)
print_out("Fetching playlists...")
playlists = twitch.get_playlists(video_id, access_token)
quality, playlist_url = _select_quality(playlists)
print_out("\nFetching playlist...")
base_url, filenames = twitch.get_playlist_urls(playlist_url)
# Create a temp dir to store downloads if it doesn't exist
directory = '{}/twitch-dl/{}/{}'.format(tempfile.gettempdir(), video_id, quality)
pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
print_out("Download dir: {}".format(directory))
print_out("Downloading VODs with {} workers...".format(max_workers))
paths = _download_files(base_url, directory, filenames, max_workers)
print_out("\n\nJoining files...")
target = _video_target_filename(video, format)
_join_vods(directory, paths, target)
print_out("\nDeleting vods...")
for path in paths:
os.unlink(path)
print_out("\nDownloaded: {}".format(target))

View File

@ -0,0 +1,11 @@
from .clips import clips
from .download import download
from .info import info
from .videos import videos
__all__ = [
clips,
download,
info,
videos,
]

112
twitchdl/commands/clips.py Normal file
View File

@ -0,0 +1,112 @@
import re
import sys
from itertools import islice
from os import path
from twitchdl import twitch, utils
from twitchdl.commands.download import get_clip_authenticated_url
from twitchdl.download import download_file
from twitchdl.exceptions import ConsoleError
from twitchdl.output import print_out, print_clip, print_json
def clips(args):
# Ignore --limit if --pager or --all are given
limit = sys.maxsize if args.all or args.pager else args.limit
generator = twitch.channel_clips_generator(args.channel_name, args.period, limit)
if args.json:
return print_json(list(generator))
if args.download:
return _download_clips(generator)
if args.pager:
print(args)
return _print_paged(generator, args.pager)
return _print_all(generator, args)
def _continue():
print_out("Press <green><b>Enter</green> to continue, <yellow><b>Ctrl+C</yellow> to break.")
try:
input()
except KeyboardInterrupt:
return False
return True
def _target_filename(clip):
url = clip["videoQualities"][0]["sourceURL"]
_, ext = path.splitext(url)
ext = ext.lstrip(".")
match = re.search(r"^(\d{4})-(\d{2})-(\d{2})T", clip["createdAt"])
date = "".join(match.groups())
name = "_".join([
date,
clip["id"],
clip["broadcaster"]["login"],
utils.slugify(clip["title"]),
])
return "{}.{}".format(name, ext)
def _download_clips(generator):
for clip in generator:
target = _target_filename(clip)
if path.exists(target):
print_out("Already downloaded: <green>{}</green>".format(target))
else:
url = get_clip_authenticated_url(clip["slug"], "source")
print_out("Downloading: <yellow>{}</yellow>".format(target))
download_file(url, target)
def _print_all(generator, args):
for clip in generator:
print_out()
print_clip(clip)
if not args.all:
print_out(
"\n<dim>There may be more clips. " +
"Increase the --limit, use --all or --pager to see the rest.</dim>"
)
def _print_paged(generator, page_size):
iterator = iter(generator)
page = list(islice(iterator, page_size))
first = 1
last = first + len(page) - 1
while True:
print_out("-" * 80)
print_out()
for clip in page:
print_clip(clip)
print_out()
last = first + len(page) - 1
print_out("-" * 80)
print_out("<yellow>Clips {}-{}</yellow>".format(first, last))
first = first + len(page)
last = first + 1
page = list(islice(iterator, page_size))
if not page or not _continue():
break

View File

@ -0,0 +1,330 @@
import m3u8
import re
import requests
import shutil
import subprocess
import tempfile
from os import path
from pathlib import Path
from urllib.parse import urlparse, urlencode
from twitchdl import twitch, utils
from twitchdl.download import download_file, download_files
from twitchdl.exceptions import ConsoleError
from twitchdl.output import print_out
def _parse_playlists(playlists_m3u8):
playlists = m3u8.loads(playlists_m3u8)
for p in sorted(playlists.playlists, key=lambda p: p.stream_info.resolution is None):
if p.stream_info.resolution:
name = p.media[0].name
description = "x".join(str(r) for r in p.stream_info.resolution)
else:
name = p.media[0].group_id
description = None
yield name, description, p.uri
def _get_playlist_by_name(playlists, quality):
if quality == "source":
_, _, uri = playlists[0]
return uri
for name, _, uri in playlists:
if name == quality:
return uri
available = ", ".join([name for (name, _, _) in playlists])
msg = "Quality '{}' not found. Available qualities are: {}".format(quality, available)
raise ConsoleError(msg)
def _select_playlist_interactive(playlists):
print_out("\nAvailable qualities:")
for n, (name, resolution, uri) in enumerate(playlists):
if resolution:
print_out("{}) {} [{}]".format(n + 1, name, resolution))
else:
print_out("{}) {}".format(n + 1, name))
no = utils.read_int("Choose quality", min=1, max=len(playlists) + 1, default=1)
_, _, uri = playlists[no - 1]
return uri
def _join_vods(playlist_path, target, overwrite, video):
command = [
"ffmpeg",
"-i", playlist_path,
"-c", "copy",
"-metadata", "artist={}".format(video["creator"]["displayName"]),
"-metadata", "title={}".format(video["title"]),
"-metadata", "encoded_by=twitch-dl",
"-stats",
"-loglevel", "warning",
"file:{}".format(target),
]
if overwrite:
command.append("-y")
print_out("<dim>{}</dim>".format(" ".join(command)))
result = subprocess.run(command)
if result.returncode != 0:
raise ConsoleError("Joining files failed")
def _video_target_filename(video, args):
date, time = video['publishedAt'].split("T")
game = video["game"]["name"] if video["game"] else "Unknown"
subs = {
"channel": video["creator"]["displayName"],
"channel_login": video["creator"]["login"],
"date": date,
"datetime": video["publishedAt"],
"format": args.format,
"game": game,
"game_slug": utils.slugify(game),
"id": video["id"],
"time": time,
"title": utils.titlify(video["title"]),
"title_slug": utils.slugify(video["title"]),
}
try:
return args.output.format(**subs)
except KeyError as e:
supported = ", ".join(subs.keys())
raise ConsoleError("Invalid key {} used in --output. Supported keys are: {}".format(e, supported))
def _clip_target_filename(clip, args):
date, time = clip["createdAt"].split("T")
game = clip["game"]["name"] if clip["game"] else "Unknown"
url = clip["videoQualities"][0]["sourceURL"]
_, ext = path.splitext(url)
ext = ext.lstrip(".")
subs = {
"channel": clip["broadcaster"]["displayName"],
"channel_login": clip["broadcaster"]["login"],
"date": date,
"datetime": clip["createdAt"],
"format": ext,
"game": game,
"game_slug": utils.slugify(game),
"id": clip["id"],
"slug": clip["slug"],
"time": time,
"title": utils.titlify(clip["title"]),
"title_slug": utils.slugify(clip["title"]),
}
try:
return args.output.format(**subs)
except KeyError as e:
supported = ", ".join(subs.keys())
raise ConsoleError("Invalid key {} used in --output. Supported keys are: {}".format(e, supported))
def _get_vod_paths(playlist, start, end):
"""Extract unique VOD paths for download from playlist."""
files = []
vod_start = 0
for segment in playlist.segments:
vod_end = vod_start + segment.duration
# `vod_end > start` is used here becuase it's better to download a bit
# more than a bit less, similar for the end condition
start_condition = not start or vod_end > start
end_condition = not end or vod_start < end
if start_condition and end_condition and segment.uri not in files:
files.append(segment.uri)
vod_start = vod_end
return files
def _crete_temp_dir(base_uri):
"""Create a temp dir to store downloads if it doesn't exist."""
path = urlparse(base_uri).path.lstrip("/")
temp_dir = Path(tempfile.gettempdir(), "twitch-dl", path)
temp_dir.mkdir(parents=True, exist_ok=True)
return str(temp_dir)
def download(args):
video_id = utils.parse_video_identifier(args.video)
if video_id:
return _download_video(video_id, args)
clip_slug = utils.parse_clip_identifier(args.video)
if clip_slug:
return _download_clip(clip_slug, args)
raise ConsoleError("Invalid input: {}".format(args.video))
def _get_clip_url(clip, quality):
qualities = clip["videoQualities"]
# Quality given as an argument
if quality:
if quality == "source":
return qualities[0]["sourceURL"]
selected_quality = quality.rstrip("p") # allow 720p as well as 720
for q in qualities:
if q["quality"] == selected_quality:
return q["sourceURL"]
available = ", ".join([str(q["quality"]) for q in qualities])
msg = "Quality '{}' not found. Available qualities are: {}".format(quality, available)
raise ConsoleError(msg)
# Ask user to select quality
print_out("\nAvailable qualities:")
for n, q in enumerate(qualities):
print_out("{}) {} [{} fps]".format(n + 1, q["quality"], q["frameRate"]))
print_out()
no = utils.read_int("Choose quality", min=1, max=len(qualities), default=1)
selected_quality = qualities[no - 1]
return selected_quality["sourceURL"]
def get_clip_authenticated_url(slug, quality):
print_out("<dim>Fetching access token...</dim>")
access_token = twitch.get_clip_access_token(slug)
if not access_token:
raise ConsoleError("Access token not found for slug '{}'".format(slug))
url = _get_clip_url(access_token, quality)
query = urlencode({
"sig": access_token["playbackAccessToken"]["signature"],
"token": access_token["playbackAccessToken"]["value"],
})
return "{}?{}".format(url, query)
def _download_clip(slug, args):
print_out("<dim>Looking up clip...</dim>")
clip = twitch.get_clip(slug)
game = clip["game"]["name"] if clip["game"] else "Unknown"
if not clip:
raise ConsoleError("Clip '{}' not found".format(slug))
print_out("Found: <green>{}</green> by <yellow>{}</yellow>, playing <blue>{}</blue> ({})".format(
clip["title"],
clip["broadcaster"]["displayName"],
game,
utils.format_duration(clip["durationSeconds"])
))
target = _clip_target_filename(clip, args)
print_out("Target: <blue>{}</blue>".format(target))
if not args.overwrite and path.exists(target):
response = input("File exists. Overwrite? [Y/n]: ")
if response.lower().strip() not in ["", "y"]:
raise ConsoleError("Aborted")
args.overwrite = True
url = get_clip_authenticated_url(slug, args.quality)
print_out("<dim>Selected URL: {}</dim>".format(url))
print_out("<dim>Downloading clip...</dim>")
download_file(url, target)
print_out("Downloaded: <blue>{}</blue>".format(target))
def _download_video(video_id, args):
if args.start and args.end and args.end <= args.start:
raise ConsoleError("End time must be greater than start time")
print_out("<dim>Looking up video...</dim>")
video = twitch.get_video(video_id)
if not video:
raise ConsoleError("Video {} not found".format(video_id))
print_out("Found: <blue>{}</blue> by <yellow>{}</yellow>".format(
video['title'], video['creator']['displayName']))
target = _video_target_filename(video, args)
print_out("Output: <blue>{}</blue>".format(target))
if not args.overwrite and path.exists(target):
response = input("File exists. Overwrite? [Y/n]: ")
if response.lower().strip() not in ["", "y"]:
raise ConsoleError("Aborted")
args.overwrite = True
print_out("<dim>Fetching access token...</dim>")
access_token = twitch.get_access_token(video_id)
print_out("<dim>Fetching playlists...</dim>")
playlists_m3u8 = twitch.get_playlists(video_id, access_token)
playlists = list(_parse_playlists(playlists_m3u8))
playlist_uri = (_get_playlist_by_name(playlists, args.quality) if args.quality
else _select_playlist_interactive(playlists))
print_out("<dim>Fetching playlist...</dim>")
response = requests.get(playlist_uri)
response.raise_for_status()
playlist = m3u8.loads(response.text)
base_uri = re.sub("/[^/]+$", "/", playlist_uri)
target_dir = _crete_temp_dir(base_uri)
vod_paths = _get_vod_paths(playlist, args.start, args.end)
# Save playlists for debugging purposes
with open(path.join(target_dir, "playlists.m3u8"), "w") as f:
f.write(playlists_m3u8)
with open(path.join(target_dir, "playlist.m3u8"), "w") as f:
f.write(response.text)
print_out("\nDownloading {} VODs using {} workers to {}".format(
len(vod_paths), args.max_workers, target_dir))
path_map = download_files(base_uri, target_dir, vod_paths, args.max_workers)
# Make a modified playlist which references downloaded VODs
# Keep only the downloaded segments and skip the rest
org_segments = playlist.segments.copy()
playlist.segments.clear()
for segment in org_segments:
if segment.uri in path_map:
segment.uri = path_map[segment.uri]
playlist.segments.append(segment)
playlist_path = path.join(target_dir, "playlist_downloaded.m3u8")
playlist.dump(playlist_path)
if args.no_join:
print_out("\n\n<dim>Skipping joining files...</dim>")
print_out("VODs downloaded to:\n<blue>{}</blue>".format(target_dir))
return
print_out("\n\nJoining files...")
_join_vods(playlist_path, target, args.overwrite, video)
if args.keep:
print_out("\n<dim>Temporary files not deleted: {}</dim>".format(target_dir))
else:
print_out("\n<dim>Deleting temporary files...</dim>")
shutil.rmtree(target_dir)
print_out("\nDownloaded: <green>{}</green>".format(target))

79
twitchdl/commands/info.py Normal file
View File

@ -0,0 +1,79 @@
import m3u8
from twitchdl import utils, twitch
from twitchdl.exceptions import ConsoleError
from twitchdl.output import print_video, print_clip, print_json, print_out, print_log
def info(args):
video_id = utils.parse_video_identifier(args.identifier)
if video_id:
print_log("Fetching video...")
video = twitch.get_video(video_id)
if not video:
raise ConsoleError("Video {} not found".format(video_id))
print_log("Fetching access token...")
access_token = twitch.get_access_token(video_id)
print_log("Fetching playlists...")
playlists = twitch.get_playlists(video_id, access_token)
if video:
if args.json:
video_json(video, playlists)
else:
video_info(video, playlists)
return
clip_slug = utils.parse_clip_identifier(args.identifier)
if clip_slug:
print_log("Fetching clip...")
clip = twitch.get_clip(clip_slug)
if not clip:
raise ConsoleError("Clip {} not found".format(clip_slug))
if args.json:
print_json(clip)
else:
clip_info(clip)
return
raise ConsoleError("Invalid input: {}".format(args.identifier))
def video_info(video, playlists):
print_out()
print_video(video)
print_out()
print_out("Playlists:")
for p in m3u8.loads(playlists).playlists:
print_out("<b>{}</b> {}".format(p.stream_info.video, p.uri))
def video_json(video, playlists):
playlists = m3u8.loads(playlists).playlists
video["playlists"] = [
{
"bandwidth": p.stream_info.bandwidth,
"resolution": p.stream_info.resolution,
"codecs": p.stream_info.codecs,
"video": p.stream_info.video,
"uri": p.uri
} for p in playlists
]
print_json(video)
def clip_info(clip):
print_out()
print_clip(clip)
print_out()
print_out("Download links:")
for q in clip["videoQualities"]:
print_out("<b>{quality}p{frameRate}</b> {sourceURL}".format(**q))

View File

@ -0,0 +1,62 @@
import sys
from twitchdl import twitch
from twitchdl.exceptions import ConsoleError
from twitchdl.output import print_out, print_paged_videos, print_video, print_json
def videos(args):
game_ids = _get_game_ids(args.game)
# Ignore --limit if --pager or --all are given
max_videos = sys.maxsize if args.all or args.pager else args.limit
total_count, generator = twitch.channel_videos_generator(
args.channel_name, max_videos, args.sort, args.type, game_ids=game_ids)
if args.json:
videos = list(generator)
print_json({
"count": len(videos),
"totalCount": total_count,
"videos": videos
})
return
if total_count == 0:
print_out("<yellow>No videos found</yellow>")
return
if args.pager:
print_paged_videos(generator, args.pager, total_count)
return
count = 0
for video in generator:
print_out()
print_video(video)
count += 1
print_out()
print_out("-" * 80)
print_out("<yellow>Videos {}-{} of {}</yellow>".format(1, count, total_count))
if total_count > count:
print_out()
print_out(
"<dim>There are more videos. Increase the --limit, use --all or --pager to see the rest.</dim>"
)
def _get_game_ids(names):
if not names:
return []
game_ids = []
for name in names:
print_out("<dim>Looking up game '{}'...</dim>".format(name))
game_id = twitch.get_game_id(name)
if not game_id:
raise ConsoleError("Game '{}' not found".format(name))
game_ids.append(int(game_id))
return game_ids

View File

@ -1,17 +1,51 @@
# -*- coding: utf-8 -*-
from argparse import ArgumentParser
import logging
import sys
from argparse import ArgumentParser, ArgumentTypeError
from collections import namedtuple
from twitchdl.exceptions import ConsoleError
from twitchdl.output import print_err
from . import commands
from twitchdl.twitch import GQLError
from . import commands, __version__
Command = namedtuple("Command", ["name", "description", "arguments"])
CLIENT_WEBSITE = 'https://github.com/ihabunek/twitch-dl'
def time(value):
"""Parse a time string (hh:mm or hh:mm:ss) to number of seconds."""
parts = [int(p) for p in value.split(":")]
if not 2 <= len(parts) <= 3:
raise ArgumentTypeError()
hours = parts[0]
minutes = parts[1]
seconds = parts[2] if len(parts) > 2 else 0
if hours < 0 or not (0 <= minutes <= 59) or not (0 <= seconds <= 59):
raise ArgumentTypeError()
return hours * 3600 + minutes * 60 + seconds
def pos_integer(value):
try:
value = int(value)
except ValueError:
raise ArgumentTypeError("must be an integer")
if value < 1:
raise ArgumentTypeError("must be positive")
return value
COMMANDS = [
Command(
name="videos",
@ -21,23 +55,159 @@ COMMANDS = [
"help": "channel name",
"type": str,
}),
(["-g", "--game"], {
"help": "Show videos of given game (can be given multiple times)",
"action": "append",
"type": str,
}),
(["-l", "--limit"], {
"help": "Number of videos to fetch (default 10)",
"type": pos_integer,
"default": 10,
}),
(["-a", "--all"], {
"help": "Fetch all videos, overrides --limit",
"action": "store_true",
"default": False,
}),
(["-s", "--sort"], {
"help": "Sorting order of videos. (default: time)",
"type": str,
"choices": ["views", "time"],
"default": "time",
}),
(["-t", "--type"], {
"help": "Broadcast type. (default: archive)",
"type": str,
"choices": ["archive", "highlight", "upload"],
"default": "archive",
}),
(["-j", "--json"], {
"help": "Show results as JSON. Ignores --pager.",
"action": "store_true",
"default": False,
}),
(["-p", "--pager"], {
"help": "Number of videos to show per page. Disabled by default.",
"type": pos_integer,
"nargs": "?",
"const": 10,
}),
],
),
Command(
name="clips",
description="List clips",
arguments=[
(["channel_name"], {
"help": "channel name",
"type": str,
}),
(["-l", "--limit"], {
"help": "Number of videos to fetch (default 10, max 100)",
"type": pos_integer,
"default": 10,
}),
(["-a", "--all"], {
"help": "Fetch all videos, overrides --limit",
"action": "store_true",
"default": False,
}),
(["-P", "--period"], {
"help": "Period from which to return clips. (default: 'all_time')",
"type": str,
"choices": ["last_day", "last_week", "last_month", "all_time"],
"default": "all_time",
}),
(["-j", "--json"], {
"help": "Show results as JSON",
"action": "store_true",
"default": False,
}),
(["-p", "--pager"], {
"help": "Number of clips to show per page. Disabled by default.",
"type": pos_integer,
"nargs": "?",
"const": 10,
}),
(["-d", "--download"], {
"help": "Download all videos in given period (in source quality)",
"action": "store_true",
"default": False,
}),
],
),
Command(
name="download",
description="Download a video",
arguments=[
(["video_id"], {
"help": "video ID",
(["video"], {
"help": "video ID, clip slug, or URL",
"type": str,
}),
(["-w", "--max_workers"], {
"help": "maximal number of threads for downloading vods concurrently (default 5)",
(["-w", "--max-workers"], {
"help": "maximal number of threads for downloading vods "
"concurrently (default 20)",
"type": int,
"default": 20,
}),
(["-s", "--start"], {
"help": "Download video from this time (hh:mm or hh:mm:ss)",
"type": time,
"default": None,
}),
(["-e", "--end"], {
"help": "Download video up to this time (hh:mm or hh:mm:ss)",
"type": time,
"default": None,
}),
(["-f", "--format"], {
"help": "Video format to convert into, passed to ffmpeg as the "
"target file extension (default: mkv)",
"type": str,
"default": "mkv",
}),
(["-k", "--keep"], {
"help": "Don't delete downloaded VODs and playlists after merging.",
"action": "store_true",
"default": False,
}),
(["-q", "--quality"], {
"help": "Video quality, e.g. 720p. Set to 'source' to get best quality.",
"type": str,
}),
(["--no-join"], {
"help": "Don't run ffmpeg to join the downloaded vods, implies --keep.",
"action": "store_true",
"default": False,
}),
(["--overwrite"], {
"help": "Overwrite the target file if it already exists without prompting.",
"action": "store_true",
"default": False,
}),
(["-o", "--output"], {
"help": "Output file name template. See docs for details.",
"type": str,
"default": "{date}_{id}_{channel_login}_{title_slug}.{format}"
})
],
),
Command(
name="info",
description="Print information for a given Twitch URL, video ID or clip slug",
arguments=[
(["identifier"], {
"help": "identifier",
"type": str,
}),
(["-j", "--json"], {
"help": "Show results as JSON",
"action": "store_true",
"default": False,
}),
],
)
]
COMMON_ARGUMENTS = [
@ -58,6 +228,8 @@ def get_parser():
description = "A script for downloading videos from Twitch"
parser = ArgumentParser(prog='twitch-dl', description=description, epilog=CLIENT_WEBSITE)
parser.add_argument("--version", help="show version number", action='store_true')
subparsers = parser.add_subparsers(title="commands")
for command in COMMANDS:
@ -76,11 +248,27 @@ def main():
parser = get_parser()
args = parser.parse_args()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
if args.version:
print("twitch-dl v{}".format(__version__))
return
if "func" not in args:
parser.print_help()
return
try:
args.func(**args.__dict__)
args.func(args)
except ConsoleError as e:
print_err(e)
sys.exit(1)
except KeyboardInterrupt:
print_err("Operation canceled")
sys.exit(1)
except GQLError as e:
print_err(e)
for err in e.errors:
print_err("*", err["message"])
sys.exit(1)

View File

@ -1,11 +1,18 @@
import os
import requests
from collections import OrderedDict
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
from functools import partial
from requests.exceptions import RequestException
from twitchdl.output import print_out
from twitchdl.utils import format_size, format_duration
CHUNK_SIZE = 1024
CONNECT_TIMEOUT = 5
RETRY_COUNT = 5
class DownloadFailed(Exception):
@ -25,14 +32,72 @@ def _download(url, path):
return size
def download_file(url, path, retries=3):
def download_file(url, path, retries=RETRY_COUNT):
if os.path.exists(path):
return 0
from_disk = True
return (os.path.getsize(path), from_disk)
from_disk = False
for _ in range(retries):
try:
return _download(url, path)
except RequestException as e:
print("Download failed: {}".format(e))
return (_download(url, path), from_disk)
except RequestException:
pass
raise DownloadFailed(":(")
def _print_progress(futures):
downloaded_count = 0
downloaded_size = 0
max_msg_size = 0
start_time = datetime.now()
total_count = len(futures)
current_download_size = 0
current_downloaded_count = 0
for future in as_completed(futures):
size, from_disk = future.result()
downloaded_count += 1
downloaded_size += size
# If we find something on disk, we don't want to take it in account in
# the speed calculation
if not from_disk:
current_download_size += size
current_downloaded_count += 1
percentage = 100 * downloaded_count // total_count
est_total_size = int(total_count * downloaded_size / downloaded_count)
duration = (datetime.now() - start_time).seconds
speed = current_download_size // duration if duration else 0
remaining = (total_count - downloaded_count) * duration / current_downloaded_count \
if current_downloaded_count else 0
msg = " ".join([
"Downloaded VOD {}/{}".format(downloaded_count, total_count),
"({}%)".format(percentage),
"<cyan>{}</cyan>".format(format_size(downloaded_size)),
"of <cyan>~{}</cyan>".format(format_size(est_total_size)),
"at <cyan>{}/s</cyan>".format(format_size(speed)) if speed > 0 else "",
"remaining <cyan>~{}</cyan>".format(format_duration(remaining)) if remaining > 0 else "",
])
max_msg_size = max(len(msg), max_msg_size)
print_out("\r" + msg.ljust(max_msg_size), end="")
def download_files(base_url, target_dir, vod_paths, max_workers):
"""
Downloads a list of VODs defined by a common `base_url` and a list of
`vod_paths`, returning a dict which maps the paths to the downloaded files.
"""
urls = [base_url + path for path in vod_paths]
targets = [os.path.join(target_dir, "{:05d}.ts".format(k)) for k, _ in enumerate(vod_paths)]
partials = (partial(download_file, url, path) for url, path in zip(urls, targets))
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = [executor.submit(fn) for fn in partials]
_print_progress(futures)
return OrderedDict(zip(vod_paths, targets))

View File

@ -1,16 +1,24 @@
# -*- coding: utf-8 -*-
import json
import sys
import re
from itertools import islice
from twitchdl import utils
START_CODES = {
'bold': '\033[1m',
'red': '\033[31m',
'green': '\033[32m',
'yellow': '\033[33m',
'blue': '\033[34m',
'magenta': '\033[35m',
'cyan': '\033[36m',
'b': '\033[1m',
'dim': '\033[2m',
'i': '\033[3m',
'u': '\033[4m',
'red': '\033[91m',
'green': '\033[92m',
'yellow': '\033[93m',
'blue': '\033[94m',
'magenta': '\033[95m',
'cyan': '\033[96m',
}
END_CODE = '\033[0m'
@ -45,7 +53,95 @@ def print_out(*args, **kwargs):
print(*args, **kwargs)
def print_json(data):
print(json.dumps(data))
def print_err(*args, **kwargs):
args = ["<red>{}</red>".format(a) for a in args]
args = [colorize(a) if USE_ANSI_COLOR else strip_tags(a) for a in args]
print(*args, file=sys.stderr, **kwargs)
def print_log(*args, **kwargs):
args = ["<dim>{}</dim>".format(a) for a in args]
args = [colorize(a) if USE_ANSI_COLOR else strip_tags(a) for a in args]
print(*args, file=sys.stderr, **kwargs)
def print_video(video):
published_at = video["publishedAt"].replace("T", " @ ").replace("Z", "")
length = utils.format_duration(video["lengthSeconds"])
channel = "<blue>{}</blue>".format(video["creator"]["displayName"]) if video["creator"] else ""
playing = "playing <blue>{}</blue>".format(video["game"]["name"]) if video["game"] else ""
# Can't find URL in video object, strange
url = "https://www.twitch.tv/videos/{}".format(video["id"])
print_out("<b>Video {}</b>".format(video["id"]))
print_out("<green>{}</green>".format(video["title"]))
if channel or playing:
print_out(" ".join([channel, playing]))
print_out("Published <blue>{}</blue> Length: <blue>{}</blue> ".format(published_at, length))
print_out("<i>{}</i>".format(url))
def print_paged_videos(generator, page_size, total_count):
iterator = iter(generator)
page = list(islice(iterator, page_size))
first = 1
last = first + len(page) - 1
while True:
print_out("-" * 80)
print_out()
for video in page:
print_video(video)
print_out()
last = first + len(page) - 1
print_out("-" * 80)
print_out("<yellow>Videos {}-{} of {}</yellow>".format(first, last, total_count))
first = first + len(page)
last = first + 1
page = list(islice(iterator, page_size))
if not page or not _continue():
break
def print_clip(clip):
published_at = clip["createdAt"].replace("T", " @ ").replace("Z", "")
length = utils.format_duration(clip["durationSeconds"])
channel = clip["broadcaster"]["displayName"]
playing = (
"playing <blue>{}</blue>".format(clip["game"]["name"])
if clip["game"] else ""
)
print_out("Clip <b>{}</b>".format(clip["slug"]))
print_out("<green>{}</green>".format(clip["title"]))
print_out("<blue>{}</blue> {}".format(channel, playing))
print_out(
"Published <blue>{}</blue>"
" Length: <blue>{}</blue>"
" Views: <blue>{}</blue>".format(published_at, length, clip["viewCount"]))
print_out("<i>{}</i>".format(clip["url"]))
def _continue():
print_out("Press <green><b>Enter</green> to continue, <yellow><b>Ctrl+C</yellow> to break.")
try:
input()
except KeyboardInterrupt:
return False
return True

View File

@ -1,28 +0,0 @@
import re
from collections import OrderedDict
def parse_playlists(data):
media_pattern = re.compile(r'^#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="(?P<group>\w+)",NAME="(?P<name>\w+)"')
playlists = OrderedDict()
n = 1
name = None
for line in data.split():
match = re.match(media_pattern, line)
if match:
name = match.group('name')
elif line.startswith('http'):
playlists[n] = (name, line)
n += 1
return playlists
def parse_playlist(url, data):
base_url = re.sub("/[^/]+$", "/{}", url)
filenames = [line for line in data.split() if re.match(r"\d+\.ts", line)]
return base_url, filenames

View File

@ -1,65 +1,358 @@
"""
Twitch API access.
"""
import requests
from twitchdl import CLIENT_ID
from twitchdl.parse import parse_playlists, parse_playlist
from twitchdl.exceptions import ConsoleError
def authenticated_get(url, params={}):
headers = {'Client-ID': CLIENT_ID}
class GQLError(Exception):
def __init__(self, errors):
super().__init__("GraphQL query failed")
self.errors = errors
def authenticated_get(url, params={}, headers={}):
headers['Client-ID'] = CLIENT_ID
response = requests.get(url, params, headers=headers)
if 400 <= response.status_code < 500:
data = response.json()
# TODO: this does not look nice in the console since data["message"]
# can contain a JSON encoded object.
raise ConsoleError(data["message"])
response.raise_for_status()
return response
def authenticated_post(url, data=None, json=None, headers={}):
headers['Client-ID'] = CLIENT_ID
response = requests.post(url, data=data, json=json, headers=headers)
if response.status_code == 400:
data = response.json()
raise ConsoleError(data["message"])
response.raise_for_status()
return response
def gql_post(query):
url = "https://gql.twitch.tv/gql"
response = authenticated_post(url, data=query).json()
if "errors" in response:
raise GQLError(response["errors"])
return response
def gql_query(query):
url = "https://gql.twitch.tv/gql"
response = authenticated_post(url, json={"query": query}).json()
if "errors" in response:
raise GQLError(response["errors"])
return response
VIDEO_FIELDS = """
id
title
publishedAt
broadcastType
lengthSeconds
game {
name
}
creator {
login
displayName
}
"""
CLIP_FIELDS = """
id
slug
title
createdAt
viewCount
durationSeconds
url
videoQualities {
frameRate
quality
sourceURL
}
game {
id
name
}
broadcaster {
displayName
login
}
"""
def get_video(video_id):
query = """
{{
video(id: "{video_id}") {{
{fields}
}}
}}
"""
https://dev.twitch.tv/docs/v5/reference/videos#get-video
"""
url = "https://api.twitch.tv/kraken/videos/%d" % video_id
return authenticated_get(url).json()
query = query.format(video_id=video_id, fields=VIDEO_FIELDS)
response = gql_query(query)
return response["data"]["video"]
def get_channel_videos(channel_name, limit=20):
def get_clip(slug):
query = """
{{
clip(slug: "{}") {{
{fields}
}}
}}
"""
https://dev.twitch.tv/docs/v5/reference/channels#get-channel-videos
"""
url = "https://api.twitch.tv/kraken/channels/%s/videos" % channel_name
return authenticated_get(url, {
"broadcast_type": "archive",
"limit": limit,
}).json()
response = gql_query(query.format(slug, fields=CLIP_FIELDS))
return response["data"]["clip"]
def get_clip_access_token(slug):
query = """
{{
"operationName": "VideoAccessToken_Clip",
"variables": {{
"slug": "{slug}"
}},
"extensions": {{
"persistedQuery": {{
"version": 1,
"sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11"
}}
}}
}}
"""
response = gql_post(query.format(slug=slug).strip())
return response["data"]["clip"]
def get_channel_clips(channel_id, period, limit, after=None):
"""
List channel clips.
At the time of writing this:
* filtering by game name returns an error
* sorting by anything but VIEWS_DESC or TRENDING returns an error
* sorting by VIEWS_DESC and TRENDING returns the same results
* there is no totalCount
"""
query = """
{{
user(login: "{channel_id}") {{
clips(first: {limit}, after: "{after}", criteria: {{ period: {period}, sort: VIEWS_DESC }}) {{
pageInfo {{
hasNextPage
hasPreviousPage
}}
edges {{
cursor
node {{
{fields}
}}
}}
}}
}}
}}
"""
query = query.format(
channel_id=channel_id,
after=after if after else "",
limit=limit,
period=period.upper(),
fields=CLIP_FIELDS
)
response = gql_query(query)
user = response["data"]["user"]
if not user:
raise ConsoleError("Channel {} not found".format(channel_id))
return response["data"]["user"]["clips"]
def channel_clips_generator(channel_id, period, limit):
def _generator(clips, limit):
for clip in clips["edges"]:
if limit < 1:
return
yield clip["node"]
limit -= 1
has_next = clips["pageInfo"]["hasNextPage"]
if limit < 1 or not has_next:
return
req_limit = min(limit, 100)
cursor = clips["edges"][-1]["cursor"]
clips = get_channel_clips(channel_id, period, req_limit, cursor)
yield from _generator(clips, limit)
req_limit = min(limit, 100)
clips = get_channel_clips(channel_id, period, req_limit)
return _generator(clips, limit)
def channel_clips_generator_old(channel_id, period, limit):
cursor = ""
while True:
clips = get_channel_clips(
channel_id, period, limit, after=cursor)
if not clips["edges"]:
break
has_next = clips["pageInfo"]["hasNextPage"]
cursor = clips["edges"][-1]["cursor"] if has_next else None
yield clips, has_next
if not cursor:
break
def get_channel_videos(channel_id, limit, sort, type="archive", game_ids=[], after=None):
query = """
{{
user(login: "{channel_id}") {{
videos(
first: {limit},
type: {type},
sort: {sort},
after: "{after}",
options: {{
gameIDs: {game_ids}
}}
) {{
totalCount
pageInfo {{
hasNextPage
}}
edges {{
cursor
node {{
{fields}
}}
}}
}}
}}
}}
"""
query = query.format(
channel_id=channel_id,
game_ids=game_ids,
after=after if after else "",
limit=limit,
sort=sort.upper(),
type=type.upper(),
fields=VIDEO_FIELDS
)
response = gql_query(query)
if not response["data"]["user"]:
raise ConsoleError("Channel {} not found".format(channel_id))
return response["data"]["user"]["videos"]
def channel_videos_generator(channel_id, max_videos, sort, type, game_ids=None):
def _generator(videos, max_videos):
for video in videos["edges"]:
if max_videos < 1:
return
yield video["node"]
max_videos -= 1
has_next = videos["pageInfo"]["hasNextPage"]
if max_videos < 1 or not has_next:
return
limit = min(max_videos, 100)
cursor = videos["edges"][-1]["cursor"]
videos = get_channel_videos(channel_id, limit, sort, type, game_ids, cursor)
yield from _generator(videos, max_videos)
limit = min(max_videos, 100)
videos = get_channel_videos(channel_id, limit, sort, type, game_ids)
return videos["totalCount"], _generator(videos, max_videos)
def get_access_token(video_id):
url = "https://api.twitch.tv/api/vods/%d/access_token" % video_id
query = """
{{
videoPlaybackAccessToken(
id: {video_id},
params: {{
platform: "web",
playerBackend: "mediaplayer",
playerType: "site"
}}
) {{
signature
value
}}
}}
"""
return authenticated_get(url).json()
query = query.format(video_id=video_id)
response = gql_query(query)
return response["data"]["videoPlaybackAccessToken"]
def get_playlists(video_id, access_token):
"""
For a given video return a playlist which contains possible video qualities.
"""
url = "http://usher.twitch.tv/vod/{}".format(video_id)
response = requests.get(url, params={
"nauth": access_token['token'],
"nauthsig": access_token['sig'],
"nauth": access_token['value'],
"nauthsig": access_token['signature'],
"allow_audio_only": "true",
"allow_source": "true",
"player": "twitchweb",
})
response.raise_for_status()
data = response.content.decode('utf-8')
return parse_playlists(data)
return response.content.decode('utf-8')
def get_playlist_urls(url):
response = requests.get(url)
response.raise_for_status()
def get_game_id(name):
query = """
{{
game(name: "{}") {{
id
}}
}}
"""
data = response.content.decode('utf-8')
return parse_playlist(url, data)
response = gql_query(query.format(name.strip()))
game = response["data"]["game"]
if game:
return game["id"]

View File

@ -2,10 +2,95 @@ import re
import unicodedata
def _format_size(value, digits, unit):
if digits > 0:
return "{{:.{}f}}{}".format(digits, unit).format(value)
else:
return "{{:d}}{}".format(unit).format(value)
def format_size(bytes_, digits=1):
if bytes_ < 1024:
return _format_size(bytes_, digits, "B")
kilo = bytes_ / 1024
if kilo < 1024:
return _format_size(kilo, digits, "kB")
mega = kilo / 1024
if mega < 1024:
return _format_size(mega, digits, "MB")
return _format_size(mega / 1024, digits, "GB")
def format_duration(total_seconds):
total_seconds = int(total_seconds)
hours = total_seconds // 3600
remainder = total_seconds % 3600
minutes = remainder // 60
seconds = total_seconds % 60
if hours:
return "{} h {} min".format(hours, minutes)
if minutes:
return "{} min {} sec".format(minutes, seconds)
return "{} sec".format(seconds)
def read_int(msg, min, max, default):
msg = msg + " [default {}]: ".format(default)
while True:
try:
val = input(msg)
if not val:
return default
if min <= int(val) <= max:
return int(val)
except ValueError:
pass
def slugify(value):
re_pattern = re.compile(r'[^\w\s-]', flags=re.U)
re_spaces = re.compile(r'[-\s]+', flags=re.U)
value = str(value)
value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
value = re_pattern.sub('', value).strip().lower()
return re_spaces.sub('-', value)
value = unicodedata.normalize('NFKC', str(value))
value = re.sub(r'[^\w\s_-]', '', value)
value = re.sub(r'[\s_-]+', '_', value)
return value.strip("_").lower()
def titlify(value):
value = unicodedata.normalize('NFKC', str(value))
value = re.sub(r'[^\w\s\[\]().-]', '', value)
value = re.sub(r'\s+', ' ', value)
return value.strip()
VIDEO_PATTERNS = [
r"^(?P<id>\d+)?$",
r"^https://(www.)?twitch.tv/videos/(?P<id>\d+)(\?.+)?$",
]
CLIP_PATTERNS = [
r"^(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)$",
r"^https://(www.)?twitch.tv/\w+/clip/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
r"^https://clips.twitch.tv/(?P<slug>[A-Za-z0-9]+(?:-[A-Za-z0-9_-]{16})?)(\?.+)?$",
]
def parse_video_identifier(identifier):
"""Given a video ID or URL returns the video ID, or null if not matched"""
for pattern in VIDEO_PATTERNS:
match = re.match(pattern, identifier)
if match:
return match.group("id")
def parse_clip_identifier(identifier):
"""Given a clip slug or URL returns the clip slug, or null if not matched"""
for pattern in CLIP_PATTERNS:
match = re.match(pattern, identifier)
if match:
return match.group("slug")