Merge remote-tracking branch 'repo-a/master'

* repo-a/master: (289 commits)
update callback documentation
gitter chat link
update pycharm dictionary
update Pipfile.lock
version bump
expiration in UTC
stream test
stream expiration property
better defaults, avoid unnecessary head call
version bump
added test_stream_to_buffer
docstrings and linting
multi range file streaming
update makefile
Update install.rst
update README
added tests
added test
fixes for logging setup
print cli message if file already downloaded
...

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+source = pytube

.deepsource.toml

@@ -0,0 +1,16 @@
+version = 1
+
+test_patterns = [
+  "tests/**"
+]
+
+exclude_patterns = [
+  "setup.py"
+]
+
+[[analyzers]]
+name = "python"
+enabled = true
+
+  [analyzers.meta]
+  runtime_version = "3.x.x"

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+ignore = E231,E203,W503,Q000,WPS111,WPS305,WPS348,WPS602,D400,DAR201,S101,DAR101,C812,D104,I001,WPS306,WPS214,D401,WPS229,WPS420,WPS230,WPS414,WPS114,WPS226,WPS442,C819,WPS601,T001,RST304,WPS410,WPS428,A003,A002,I003,WPS221,WPS326,WPS201,S405,DAR301,WPS210,WPS202,WPS213,WPS301,P103,WPS407,WPS432,WPS211,S314,S310,S001,IF100,PT001
+max-line-length = 95
+
+[isort]

.gitattributes

@@ -1 +1,4 @@
 tests/mock_data/* linguist-vendored
+/docs export-ignore
+.travis.yml export-ignore
+*.sh text eol=lf

.gitignore

@@ -1,4 +1,10 @@
-*.py[oc]
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
 
 # Temp files
 *~
@@ -7,18 +13,52 @@
 \#*
 .#*
 *#
-dist
-.DS_Store
 
-# Build files
-build
-dist
-pkg
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
 *.egg
-*.egg-info
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
 
-# Test files
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
 .pytest_cache/
+*.mp4
+
+# Performance profiling
+prof/
+*.cprof
 
 # Debian Files
 debian/files
@@ -30,8 +70,6 @@ doc/_build
 # Generated man page
 doc/aws_hostname.1
 
-.coverage
-.cache
 _run.py
 _devfiles/*
 
@@ -41,9 +79,54 @@ _templates
 _autosummary
 .pytest_cache*
 
-# IDE Files
-.idea/
-#Pycharm stuff
-.idea/*
-
 .vscode/
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# Mac
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon?
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+.dropbox
+
+# Generated
+test/**/*.xml
+/*.gv
+/*.dot
+/*.xml
+
+# PyCharm
+.idea/workspace.xml
+.idea/usage.statistics.xml
+.idea/tasks.xml
+.idea/modules.xml
+.idea/*.iml

.idea/dictionaries/haroldmartin.xml

@@ -0,0 +1,49 @@
+<component name="ProjectDictionaryState">
+  <dictionary name="haroldmartin">
+    <words>
+      <w>acodec</w>
+      <w>akamaized</w>
+      <w>bitrate</w>
+      <w>capsys</w>
+      <w>descr</w>
+      <w>descramble</w>
+      <w>descrambler</w>
+      <w>descrambles</w>
+      <w>descrambling</w>
+      <w>eurl</w>
+      <w>ffmpeg</w>
+      <w>ficano</w>
+      <w>filenames</w>
+      <w>filesize</w>
+      <w>filetype</w>
+      <w>gangnam</w>
+      <w>itag</w>
+      <w>itags</w>
+      <w>lsig</w>
+      <w>maxresdefault</w>
+      <w>meth</w>
+      <w>monostate</w>
+      <w>mypy</w>
+      <w>nficano</w>
+      <w>noqa</w>
+      <w>noreorder</w>
+      <w>nosec</w>
+      <w>ntfs</w>
+      <w>prog</w>
+      <w>pylint</w>
+      <w>pytube</w>
+      <w>recompiles</w>
+      <w>samp</w>
+      <w>scodecs</w>
+      <w>streamability</w>
+      <w>stty</w>
+      <w>tracklist</w>
+      <w>uniqueify</w>
+      <w>vcodec</w>
+      <w>vorbis</w>
+      <w>webm</w>
+      <w>youtu</w>
+      <w>ytplayer</w>
+    </words>
+  </dictionary>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7" project-jdk-type="Python SDK" />
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

.readthedocs.yml

@@ -0,0 +1,19 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats: all
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  version: 3.7
+  install:
+    - requirements: docs/requirements.txt

.travis.yml

@@ -3,20 +3,17 @@ cache:
   - apt
   - pip
 python:
-  - "2.7"
-  - "3.4"
-  - "3.5"
-  - "3.5-dev"  # 3.5 development branch
   - "3.6"
-  - "3.6-dev"  # 3.6 development branch
-  - "3.7-dev"  # 3.7 development branch
+  - "3.7"
+  - "3.8"
 install: "make"
 script:
   - make ci
 before_install:
-  - pip install pipenv flake8 --upgrade
+  - pip install pipenv --upgrade
 sudo: false
 after_success:
-  coveralls
+  - codecov
+  - coveralls
 notifications:
-  slack: watchcloud:rNoT5kJJakPqwLSKuev6oa4C
+#  slack: watchcloud:rNoT5kJJakPqwLSKuev6oa4C

CODE_OF_CONDUCT.md

@@ -1,46 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at [email protected]. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
-
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/

MANIFEST.in

@@ -1,2 +1,2 @@
-include README.rst LICENSE NOTICE HISTORY.rst pytest.ini
+include README.md LICENSE
 recursive-include tests *.py

Makefile

@@ -4,11 +4,19 @@ help:
 	@echo "clean-pyc - remove Python file artifacts"
 	@echo "install - install the package to the active Python's site-packages"
 
-ci:
+pipenv:
 	pip install pipenv
 	pipenv install --dev
-	pipenv run flake8
-	pipenv run pytest --cov-report term-missing --cov=pytube --ignore=W605
+
+test:
+	pipenv run flake8 pytube/
+	pipenv run flake8 tests/
+	pipenv run black pytube --check
+	pipenv run black tests --check
+	pipenv run mypy pytube
+	pipenv run pytest --cov-report term-missing --cov=pytube
+
+ci: pipenv test
 
 clean: clean-build clean-pyc
 
@@ -26,6 +34,26 @@ clean-pyc:
 	find . -name '*~' -exec rm -f {} +
 	find . -name '__pycache__' -exec rm -fr {} +
 	find . -name '.pytest_cache' -exec rm -fr {} +
+	find . -name '.mypy_cache' -exec rm -fr {} +
 
 install: clean
 	python setup.py install
+
+package: clean
+	pipenv run python setup.py sdist bdist_wheel
+
+upload:
+	twine upload dist/*
+
+tag:
+	git diff-index --quiet HEAD --  # checks for unstaged/uncomitted files
+	git tag "v`pipenv run python pytube/version.py`"
+	git push --tags
+
+check-master:
+	if [[ `git rev-parse --abbrev-ref HEAD` != "master" ]]; then exit 1; fi
+
+pull:
+	git pull
+
+release: check-master pull clean test tag package upload

Pipfile

@@ -4,21 +4,37 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
+typing_extensions = "*"
 
 [dev-packages]
-"flake8" = "*"
-pytest = "*"
-mock = "*"
-pytest-mock = "*"
+black = "==19.10b0"
+codecov = "*"
+coveralls = "*"
+flake8 = "*"
+flake8-breakpoint = "*"
+flake8-broken-line = "*"
+flake8-bugbear ="*"
+flake8-builtins = "*"
+flake8-comprehensions ="*"
+flake8-eradicate = "*"
+flake8-executable = "*"
+flake8-if-expr = "*"
+flake8-isort = "*"
+flake8-logging-format = "*"
+flake8-mock = "*"
+flake8-mutable = "*"
+flake8-print = "*"
+flake8-pytest = "*"
+flake8-pytest-style = "*"
+flake8-quotes = "*"
+flake8-return = "*"
+flake8-strict = "*"
+flake8-string-format = "*"
+mypy = "*"
+pep8-naming = "*"
 pre-commit = "*"
-"enum34" = "*"
+pytest = "*"
 pytest-cov = "*"
-"pathlib2" = "*"
-"scandir" = "*"
-bumpversion = "*"
-coveralls = "*"
-twine = "*"
-more-itertools = "==5.0.0"
-
-[requires]
-python_version = "3.6"
+pytest-mock = "*"
+pytest-profiling = "*"
+sphinx_rtd_theme = "*"

Pipfile.lock

@@ -1,12 +1,10 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "5a2d404725db87789c428cc6fb3f2945c4232b4838e18c4ad95d5f07d002315a"
+            "sha256": "67eed8580be32eb9e1c105500479ae3464231ce063bb0b404cc29a43e20262da"
         },
         "pipfile-spec": 6,
-        "requires": {
-            "python_version": "3.6"
-        },
+        "requires": {},
         "sources": [
             {
                 "name": "pypi",
@@ -15,57 +13,74 @@
             }
         ]
     },
-    "default": {},
+    "default": {
+        "typing-extensions": {
+            "hashes": [
+                "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2",
+                "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d",
+                "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"
+            ],
+            "index": "pypi",
+            "version": "==3.7.4.1"
+        }
+    },
     "develop": {
-        "aspy.yaml": {
+        "alabaster": {
             "hashes": [
-                "sha256:ae249074803e8b957c83fdd82a99160d0d6d26dff9ba81ba608b42eebd7d8cd3",
-                "sha256:c7390d79f58eb9157406966201abf26da0d56c07e0ff0deadc39c8f4dbc13482"
+                "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359",
+                "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"
             ],
-            "version": "==1.2.0"
+            "version": "==0.7.12"
+        },
+        "appdirs": {
+            "hashes": [
+                "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92",
+                "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e"
+            ],
+            "version": "==1.4.3"
         },
-        "atomicwrites": {
+        "aspy.yaml": {
             "hashes": [
-                "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
-                "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
+                "sha256:463372c043f70160a9ec950c3f1e4c3a82db5fca01d334b6bc89c7164d744bdc",
+                "sha256:e7c742382eff2caed61f87a39d13f99109088e5e93f04d76eb8d4b28aa143f45"
             ],
             "version": "==1.3.0"
         },
         "attrs": {
             "hashes": [
-                "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
-                "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
+                "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
+                "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
             ],
-            "version": "==19.1.0"
+            "version": "==19.3.0"
         },
-        "bleach": {
+        "babel": {
             "hashes": [
-                "sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16",
-                "sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa"
+                "sha256:1aac2ae2d0d8ea368fa90906567f5c08463d98ade155c0c4bfedd6a0f7160e38",
+                "sha256:d670ea0b10f8b723672d3a6abeb87b565b244da220d76b4dba1b66269ec152d4"
             ],
-            "version": "==3.1.0"
+            "version": "==2.8.0"
         },
-        "bumpversion": {
+        "black": {
             "hashes": [
-                "sha256:6744c873dd7aafc24453d8b6a1a0d6d109faf63cd0cd19cb78fd46e74932c77e",
-                "sha256:6753d9ff3552013e2130f7bc03c1007e24473b4835952679653fb132367bdd57"
+                "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b",
+                "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"
             ],
             "index": "pypi",
-            "version": "==0.5.3"
+            "version": "==19.10b0"
         },
         "certifi": {
             "hashes": [
-                "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5",
-                "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae"
+                "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
+                "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
             ],
-            "version": "==2019.3.9"
+            "version": "==2019.11.28"
         },
         "cfgv": {
             "hashes": [
-                "sha256:6e9f2feea5e84bc71e56abd703140d7a2c250fc5ba38b8702fd6a68ed4e3b2ef",
-                "sha256:e7f186d4a36c099a9e20b04ac3108bd8bb9b9257e692ce18c8c3764d5cb12172"
+                "sha256:04b093b14ddf9fd4d17c53ebfd55582d27b76ed30050193c14e560770c5360eb",
+                "sha256:f22b426ed59cd2ab2b54ff96608d846c33dfb8766a67f0b4a6ce130ce244414f"
             ],
-            "version": "==1.6.0"
+            "version": "==3.0.0"
         },
         "chardet": {
             "hashes": [
@@ -74,49 +89,70 @@
             ],
             "version": "==3.0.4"
         },
+        "click": {
+            "hashes": [
+                "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
+                "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
+            ],
+            "version": "==7.0"
+        },
+        "codecov": {
+            "hashes": [
+                "sha256:8ed8b7c6791010d359baed66f84f061bba5bd41174bf324c31311e8737602788",
+                "sha256:ae00d68e18d8a20e9c3288ba3875ae03db3a8e892115bf9b83ef20507732bed4"
+            ],
+            "index": "pypi",
+            "version": "==2.0.15"
+        },
         "coverage": {
             "hashes": [
-                "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9",
-                "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74",
-                "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390",
-                "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8",
-                "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe",
-                "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf",
-                "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e",
-                "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741",
-                "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09",
-                "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd",
-                "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034",
-                "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420",
-                "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c",
-                "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab",
-                "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba",
-                "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e",
-                "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609",
-                "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2",
-                "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49",
-                "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b",
-                "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d",
-                "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce",
-                "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9",
-                "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4",
-                "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773",
-                "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723",
-                "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c",
-                "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f",
-                "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1",
-                "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260",
-                "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a"
-            ],
-            "version": "==4.5.3"
+                "sha256:15cf13a6896048d6d947bf7d222f36e4809ab926894beb748fc9caa14605d9c3",
+                "sha256:1daa3eceed220f9fdb80d5ff950dd95112cd27f70d004c7918ca6dfc6c47054c",
+                "sha256:1e44a022500d944d42f94df76727ba3fc0a5c0b672c358b61067abb88caee7a0",
+                "sha256:25dbf1110d70bab68a74b4b9d74f30e99b177cde3388e07cc7272f2168bd1477",
+                "sha256:3230d1003eec018ad4a472d254991e34241e0bbd513e97a29727c7c2f637bd2a",
+                "sha256:3dbb72eaeea5763676a1a1efd9b427a048c97c39ed92e13336e726117d0b72bf",
+                "sha256:5012d3b8d5a500834783689a5d2292fe06ec75dc86ee1ccdad04b6f5bf231691",
+                "sha256:51bc7710b13a2ae0c726f69756cf7ffd4362f4ac36546e243136187cfcc8aa73",
+                "sha256:527b4f316e6bf7755082a783726da20671a0cc388b786a64417780b90565b987",
+                "sha256:722e4557c8039aad9592c6a4213db75da08c2cd9945320220634f637251c3894",
+                "sha256:76e2057e8ffba5472fd28a3a010431fd9e928885ff480cb278877c6e9943cc2e",
+                "sha256:77afca04240c40450c331fa796b3eab6f1e15c5ecf8bf2b8bee9706cd5452fef",
+                "sha256:7afad9835e7a651d3551eab18cbc0fdb888f0a6136169fbef0662d9cdc9987cf",
+                "sha256:9bea19ac2f08672636350f203db89382121c9c2ade85d945953ef3c8cf9d2a68",
+                "sha256:a8b8ac7876bc3598e43e2603f772d2353d9931709345ad6c1149009fd1bc81b8",
+                "sha256:b0840b45187699affd4c6588286d429cd79a99d509fe3de0f209594669bb0954",
+                "sha256:b26aaf69713e5674efbde4d728fb7124e429c9466aeaf5f4a7e9e699b12c9fe2",
+                "sha256:b63dd43f455ba878e5e9f80ba4f748c0a2156dde6e0e6e690310e24d6e8caf40",
+                "sha256:be18f4ae5a9e46edae3f329de2191747966a34a3d93046dbdf897319923923bc",
+                "sha256:c312e57847db2526bc92b9bfa78266bfbaabac3fdcd751df4d062cd4c23e46dc",
+                "sha256:c60097190fe9dc2b329a0eb03393e2e0829156a589bd732e70794c0dd804258e",
+                "sha256:c62a2143e1313944bf4a5ab34fd3b4be15367a02e9478b0ce800cb510e3bbb9d",
+                "sha256:cc1109f54a14d940b8512ee9f1c3975c181bbb200306c6d8b87d93376538782f",
+                "sha256:cd60f507c125ac0ad83f05803063bed27e50fa903b9c2cfee3f8a6867ca600fc",
+                "sha256:d513cc3db248e566e07a0da99c230aca3556d9b09ed02f420664e2da97eac301",
+                "sha256:d649dc0bcace6fcdb446ae02b98798a856593b19b637c1b9af8edadf2b150bea",
+                "sha256:d7008a6796095a79544f4da1ee49418901961c97ca9e9d44904205ff7d6aa8cb",
+                "sha256:da93027835164b8223e8e5af2cf902a4c80ed93cb0909417234f4a9df3bcd9af",
+                "sha256:e69215621707119c6baf99bda014a45b999d37602cb7043d943c76a59b05bf52",
+                "sha256:ea9525e0fef2de9208250d6c5aeeee0138921057cd67fcef90fbed49c4d62d37",
+                "sha256:fca1669d464f0c9831fd10be2eef6b86f5ebd76c724d1e0706ebdff86bb4adf0"
+            ],
+            "version": "==5.0.3"
         },
         "coveralls": {
             "hashes": [
-                "sha256:baa26648430d5c2225ab12d7e2067f75597a4b967034bba7e3d5ab7501d207a1",
-                "sha256:ff9b7823b15070f26f654837bb02a201d006baaf2083e0514ffd3b34a3ffed81"
+                "sha256:4b6bfc2a2a77b890f556bc631e35ba1ac21193c356393b66c84465c06218e135",
+                "sha256:67188c7ec630c5f708c31552f2bcdac4580e172219897c4136504f14b823132f"
             ],
             "index": "pypi",
-            "version": "==1.7.0"
+            "version": "==1.11.1"
+        },
+        "distlib": {
+            "hashes": [
+                "sha256:2e166e231a26b36d6dfe35a48c4464346620f8645ed0ace01ee31822b288de21"
+            ],
+            "version": "==0.3.0"
         },
         "docopt": {
             "hashes": [
@@ -126,11 +162,10 @@
         },
         "docutils": {
             "hashes": [
-                "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6",
-                "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274",
-                "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6"
+                "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af",
+                "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"
             ],
-            "version": "==0.14"
+            "version": "==0.16"
         },
         "entrypoints": {
             "hashes": [
@@ -139,31 +174,209 @@
             ],
             "version": "==0.3"
         },
-        "enum34": {
+        "enum-compat": {
             "hashes": [
-                "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850",
-                "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a",
-                "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79",
-                "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1"
+                "sha256:3677daabed56a6f724451d585662253d8fb4e5569845aafa8bb0da36b1a8751e",
+                "sha256:88091b617c7fc3bbbceae50db5958023c48dc40b50520005aa3bf27f8f7ea157"
             ],
-            "index": "pypi",
-            "version": "==1.1.6"
+            "version": "==0.0.3"
+        },
+        "eradicate": {
+            "hashes": [
+                "sha256:4ffda82aae6fd49dfffa777a857cb758d77502a1f2e0f54c9ac5155a39d2d01a"
+            ],
+            "version": "==1.0"
+        },
+        "filelock": {
+            "hashes": [
+                "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59",
+                "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"
+            ],
+            "version": "==3.0.12"
         },
         "flake8": {
             "hashes": [
-                "sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661",
-                "sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8"
+                "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb",
+                "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca"
             ],
             "index": "pypi",
-            "version": "==3.7.7"
+            "version": "==3.7.9"
         },
-        "identify": {
+        "flake8-breakpoint": {
+            "hashes": [
+                "sha256:27e0cb132647f9ef348b4a3c3126e7350bedbb22e8e221cd11712a223855ea0b",
+                "sha256:5bc70d478f0437a3655d094e1d2fca81ddacabaa84d99db45ad3630bf2004064"
+            ],
+            "index": "pypi",
+            "version": "==1.1.0"
+        },
+        "flake8-broken-line": {
+            "hashes": [
+                "sha256:30378a3749911e453d0a9e03204156cbbd35bcc03fb89f12e6a5206e5baf3537",
+                "sha256:7721725dce3aeee1df371a252822f1fcecfaf2766dcf5bac54ee1b3f779ee9d1"
+            ],
+            "index": "pypi",
+            "version": "==0.1.1"
+        },
+        "flake8-bugbear": {
             "hashes": [
-                "sha256:443f419ca6160773cbaf22dbb302b1e436a386f23129dbb5482b68a147c2eca9",
-                "sha256:bd7f15fe07112b713fb68fbdde3a34dd774d9062128f2c398104889f783f989d"
+                "sha256:a3ddc03ec28ba2296fc6f89444d1c946a6b76460f859795b35b77d4920a51b63",
+                "sha256:bd02e4b009fb153fe6072c31c52aeab5b133d508095befb2ffcf3b41c4823162"
             ],
+            "index": "pypi",
+            "version": "==20.1.4"
+        },
+        "flake8-builtins": {
+            "hashes": [
+                "sha256:29bc0f7e68af481d088f5c96f8aeb02520abdfc900500484e3af969f42a38a5f",
+                "sha256:c44415fb19162ef3737056e700d5b99d48c3612a533943b4e16419a5d3de3a64"
+            ],
+            "index": "pypi",
             "version": "==1.4.2"
         },
+        "flake8-comprehensions": {
+            "hashes": [
+                "sha256:d08323aa801aef33477cd33f2f5ce3acb1aafd26803ab0d171d85d514c1273a2",
+                "sha256:e7db586bb6eb95afdfd87ed244c90e57ae1352db8ef0ad3012fca0200421e5df"
+            ],
+            "index": "pypi",
+            "version": "==3.2.2"
+        },
+        "flake8-eradicate": {
+            "hashes": [
+                "sha256:b0bcdbb70a489fb799f9ee11fefc57bd0d3251e1ea9bdc5bf454443cccfd620c",
+                "sha256:b693e9dfe6da42dbc7fb75af8486495b9414d1ab0372d15efcf85a2ac85fd368"
+            ],
+            "index": "pypi",
+            "version": "==0.2.4"
+        },
+        "flake8-executable": {
+            "hashes": [
+                "sha256:968618c475a23a538ced9b957a741b818d37610838f99f6abcea249e4de7c9ec",
+                "sha256:a636ff78b14b63b1245d1c4d509db2f6ea0f2e27a86ee7eb848f3827bef7e16d"
+            ],
+            "index": "pypi",
+            "version": "==2.0.3"
+        },
+        "flake8-if-expr": {
+            "hashes": [
+                "sha256:173f6ceefdecbff532180aafe0360f6d1dd4da8b4a9b10193ddc1781291d580e",
+                "sha256:890c5bd0103c864492e7088bfaf4f9f5a987c336b03b2b285178456d08db3025"
+            ],
+            "index": "pypi",
+            "version": "==1.0.0"
+        },
+        "flake8-isort": {
+            "hashes": [
+                "sha256:64454d1f154a303cfe23ee715aca37271d4f1d299b2f2663f45b73bff14e36a9",
+                "sha256:aa0c4d004e6be47e74f122f5b7f36554d0d78ad8bf99b497a460dedccaa7cce9"
+            ],
+            "index": "pypi",
+            "version": "==2.8.0"
+        },
+        "flake8-logging-format": {
+            "hashes": [
+                "sha256:ca5f2b7fc31c3474a0aa77d227e022890f641a025f0ba664418797d979a779f8"
+            ],
+            "index": "pypi",
+            "version": "==0.6.0"
+        },
+        "flake8-mock": {
+            "hashes": [
+                "sha256:2fa775e7589f4e1ad74f35d60953eb20937f5d7355235e54bf852c6837f2bede"
+            ],
+            "index": "pypi",
+            "version": "==0.3"
+        },
+        "flake8-mutable": {
+            "hashes": [
+                "sha256:38fd9dadcbcda6550a916197bc40ed76908119dabb37fbcca30873666c31d2d5",
+                "sha256:ee9b77111b867d845177bbc289d87d541445ffcc6029a0c5c65865b42b18c6a6"
+            ],
+            "index": "pypi",
+            "version": "==1.2.0"
+        },
+        "flake8-plugin-utils": {
+            "hashes": [
+                "sha256:1ac5eb19773d5c7fdde60b0d901ae86be9c751bf697c61fdb6609b86872f3c6e",
+                "sha256:24b4a3b216ad588951d3d7adef4645dcb3b32a33b878e03baa790b5a66bf3a73"
+            ],
+            "version": "==1.0.0"
+        },
+        "flake8-polyfill": {
+            "hashes": [
+                "sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9",
+                "sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda"
+            ],
+            "version": "==1.0.2"
+        },
+        "flake8-print": {
+            "hashes": [
+                "sha256:324f9e59a522518daa2461bacd7f82da3c34eb26a4314c2a54bd493f8b394a68"
+            ],
+            "index": "pypi",
+            "version": "==3.1.4"
+        },
+        "flake8-pytest": {
+            "hashes": [
+                "sha256:61686128a79e1513db575b2bcac351081d5a293811ddce2d5dfc25e8c762d33e",
+                "sha256:b4d6703f7d7b646af1e2660809e795886dd349df11843613dbe6515efa82c0f3"
+            ],
+            "index": "pypi",
+            "version": "==1.3"
+        },
+        "flake8-pytest-style": {
+            "hashes": [
+                "sha256:1c2303998c509cd65c3fb047cd536787ddf953e8113bc7f086c0cd7468db4b1f",
+                "sha256:820503cb50b7f6aa13a9889f4c47ba35bbd666877a72ed138ae5682a9bccaf9d"
+            ],
+            "index": "pypi",
+            "version": "==0.1.3"
+        },
+        "flake8-quotes": {
+            "hashes": [
+                "sha256:11a15d30c92ca5f04c2791bd7019cf62b6f9d3053eb050d02a135557eb118bfc"
+            ],
+            "index": "pypi",
+            "version": "==2.1.1"
+        },
+        "flake8-return": {
+            "hashes": [
+                "sha256:03b920cf2784370af4447a754fb7133ce165a6ecf6d4f506a95c4032ece48d8a",
+                "sha256:a219b619cdca3cd07dae150772f21083a11ce5280e2198acbac82bd9be0f574f"
+            ],
+            "index": "pypi",
+            "version": "==1.1.1"
+        },
+        "flake8-strict": {
+            "hashes": [
+                "sha256:2ef66f75f9215c2084ae7d1b18e158a3c392141a5621ecab28858256ea75d41e",
+                "sha256:75d5c11babe3f3b2bc5349e645112571a1d80d6183bda99afe5ffdfc70192d10"
+            ],
+            "index": "pypi",
+            "version": "==0.2.1"
+        },
+        "flake8-string-format": {
+            "hashes": [
+                "sha256:65f3da786a1461ef77fca3780b314edb2853c377f2e35069723348c8917deaa2",
+                "sha256:812ff431f10576a74c89be4e85b8e075a705be39bc40c4b4278b5b13e2afa9af"
+            ],
+            "index": "pypi",
+            "version": "==0.3.0"
+        },
+        "gprof2dot": {
+            "hashes": [
+                "sha256:b43fe04ebb3dfe181a612bbfc69e90555b8957022ad6a466f0308ed9c7f22e99"
+            ],
+            "version": "==2019.11.30"
+        },
+        "identify": {
+            "hashes": [
+                "sha256:1222b648251bdcb8deb240b294f450fbf704c7984e08baa92507e4ea10b436d5",
+                "sha256:d824ebe21f38325c771c41b08a95a761db1982f1fc0eee37c6c97df3f1636b96"
+            ],
+            "version": "==1.4.11"
+        },
         "idna": {
             "hashes": [
                 "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
@@ -171,20 +384,75 @@
             ],
             "version": "==2.8"
         },
-        "importlib-metadata": {
+        "imagesize": {
             "hashes": [
-                "sha256:46fc60c34b6ed7547e2a723fc8de6dc2e3a1173f8423246b3ce497f064e9c3de",
-                "sha256:bc136180e961875af88b1ab85b4009f4f1278f8396a60526c0009f503a1a96ca"
+                "sha256:6965f19a6a2039c7d48bca7dba2473069ff854c36ae6f19d2cde309d998228a1",
+                "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1"
             ],
-            "version": "==0.9"
+            "version": "==1.2.0"
         },
-        "importlib-resources": {
+        "importlib-metadata": {
             "hashes": [
-                "sha256:6e2783b2538bd5a14678284a3962b0660c715e5a0f10243fd5e00a4b5974f50b",
-                "sha256:d3279fd0f6f847cced9f7acc19bd3e5df54d34f93a2e7bb5f238f81545787078"
-            ],
-            "markers": "python_version < '3.7'",
-            "version": "==1.0.2"
+                "sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302",
+                "sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==1.5.0"
+        },
+        "isort": {
+            "extras": [
+                "pyproject"
+            ],
+            "hashes": [
+                "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1",
+                "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"
+            ],
+            "version": "==4.3.21"
+        },
+        "jinja2": {
+            "hashes": [
+                "sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250",
+                "sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49"
+            ],
+            "version": "==2.11.1"
+        },
+        "markupsafe": {
+            "hashes": [
+                "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473",
+                "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161",
+                "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235",
+                "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5",
+                "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42",
+                "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff",
+                "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b",
+                "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1",
+                "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e",
+                "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183",
+                "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66",
+                "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b",
+                "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1",
+                "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15",
+                "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1",
+                "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e",
+                "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b",
+                "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905",
+                "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735",
+                "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d",
+                "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e",
+                "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d",
+                "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c",
+                "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21",
+                "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2",
+                "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5",
+                "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b",
+                "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6",
+                "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f",
+                "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f",
+                "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2",
+                "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7",
+                "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"
+            ],
+            "version": "==1.1.1"
         },
         "mccabe": {
             "hashes": [
@@ -193,72 +461,89 @@
             ],
             "version": "==0.6.1"
         },
-        "mock": {
+        "more-itertools": {
             "hashes": [
-                "sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1",
-                "sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba"
+                "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c",
+                "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507"
             ],
-            "index": "pypi",
-            "version": "==2.0.0"
+            "version": "==8.2.0"
         },
-        "more-itertools": {
+        "mypy": {
             "hashes": [
-                "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4",
-                "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc",
-                "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9"
+                "sha256:0a9a45157e532da06fe56adcfef8a74629566b607fa2c1ac0122d1ff995c748a",
+                "sha256:2c35cae79ceb20d47facfad51f952df16c2ae9f45db6cb38405a3da1cf8fc0a7",
+                "sha256:4b9365ade157794cef9685791032521233729cb00ce76b0ddc78749abea463d2",
+                "sha256:53ea810ae3f83f9c9b452582261ea859828a9ed666f2e1ca840300b69322c474",
+                "sha256:634aef60b4ff0f650d3e59d4374626ca6153fcaff96ec075b215b568e6ee3cb0",
+                "sha256:7e396ce53cacd5596ff6d191b47ab0ea18f8e0ec04e15d69728d530e86d4c217",
+                "sha256:7eadc91af8270455e0d73565b8964da1642fe226665dd5c9560067cd64d56749",
+                "sha256:7f672d02fffcbace4db2b05369142e0506cdcde20cea0e07c7c2171c4fd11dd6",
+                "sha256:85baab8d74ec601e86134afe2bcccd87820f79d2f8d5798c889507d1088287bf",
+                "sha256:87c556fb85d709dacd4b4cb6167eecc5bbb4f0a9864b69136a0d4640fdc76a36",
+                "sha256:a6bd44efee4dc8c3324c13785a9dc3519b3ee3a92cada42d2b57762b7053b49b",
+                "sha256:c6d27bd20c3ba60d5b02f20bd28e20091d6286a699174dfad515636cb09b5a72",
+                "sha256:e2bb577d10d09a2d8822a042a23b8d62bc3b269667c9eb8e60a6edfa000211b1",
+                "sha256:f97a605d7c8bc2c6d1172c2f0d5a65b24142e11a58de689046e62c2d632ca8c1"
             ],
             "index": "pypi",
-            "version": "==5.0.0"
+            "version": "==0.761"
+        },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
+                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
+            ],
+            "version": "==0.4.3"
         },
         "nodeenv": {
             "hashes": [
-                "sha256:ad8259494cf1c9034539f6cced78a1da4840a4b157e23640bc4a0c0546b0cb7a"
+                "sha256:5b2438f2e42af54ca968dd1b374d14a1194848955187b0e5e4be1f73813a5212"
             ],
-            "version": "==1.3.3"
+            "version": "==1.3.5"
         },
-        "pathlib2": {
+        "packaging": {
             "hashes": [
-                "sha256:25199318e8cc3c25dcb45cbe084cc061051336d5a9ea2a12448d3d8cb748f742",
-                "sha256:5887121d7f7df3603bca2f710e7219f3eca0eb69e0b7cc6e0a022e155ac931a7"
+                "sha256:170748228214b70b672c581a3dd610ee51f733018650740e98c7df862a583f73",
+                "sha256:e665345f9eef0c621aa0bf2f8d78cf6d21904eef16a93f020240b704a57f1334"
             ],
-            "index": "pypi",
-            "version": "==2.3.3"
+            "version": "==20.1"
         },
-        "pbr": {
+        "pathspec": {
             "hashes": [
-                "sha256:6901995b9b686cb90cceba67a0f6d4d14ae003cd59bc12beb61549bdfbe3bc89",
-                "sha256:d950c64aeea5456bbd147468382a5bb77fe692c13c9f00f0219814ce5b642755"
+                "sha256:163b0632d4e31cef212976cf57b43d9fd6b0bac6e67c26015d611a647d5e7424",
+                "sha256:562aa70af2e0d434367d9790ad37aed893de47f1693e4201fd1d3dca15d19b96"
             ],
-            "version": "==5.2.0"
+            "version": "==0.7.0"
         },
-        "pkginfo": {
+        "pep8-naming": {
             "hashes": [
-                "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb",
-                "sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32"
+                "sha256:45f330db8fcfb0fba57458c77385e288e7a3be1d01e8ea4268263ef677ceea5f",
+                "sha256:a33d38177056321a167decd6ba70b890856ba5025f0a8eca6a3eda607da93caf"
             ],
-            "version": "==1.5.0.1"
+            "index": "pypi",
+            "version": "==0.9.1"
         },
         "pluggy": {
             "hashes": [
-                "sha256:19ecf9ce9db2fce065a7a0586e07cfb4ac8614fe96edf628a264b1c70116cf8f",
-                "sha256:84d306a647cc805219916e62aab89caa97a33a1dd8c342e87a37f91073cd4746"
+                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
+                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
             ],
-            "version": "==0.9.0"
+            "version": "==0.13.1"
         },
         "pre-commit": {
             "hashes": [
-                "sha256:2576a2776098f3902ef9540a84696e8e06bf18a337ce43a6a889e7fa5d26c4c5",
-                "sha256:82f2f2d657d7f9280de9f927ae56886d60b9ef7f3714eae92d12713cd9cb9e11"
+                "sha256:0385479a0fe0765b1d32241f6b5358668cb4b6496a09aaf9c79acc6530489dbb",
+                "sha256:bf80d9dd58bea4f45d5d71845456fdcb78c1027eda9ed562db6fa2bd7a680c3a"
             ],
             "index": "pypi",
-            "version": "==1.15.2"
+            "version": "==2.0.1"
         },
         "py": {
             "hashes": [
-                "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa",
-                "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"
+                "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa",
+                "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0"
             ],
-            "version": "==1.8.0"
+            "version": "==1.8.1"
         },
         "pycodestyle": {
             "hashes": [
@@ -276,145 +561,253 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a",
-                "sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d"
+                "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b",
+                "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe"
+            ],
+            "version": "==2.5.2"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f",
+                "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"
             ],
-            "version": "==2.3.1"
+            "version": "==2.4.6"
         },
         "pytest": {
             "hashes": [
-                "sha256:3773f4c235918987d51daf1db66d51c99fac654c81d6f2f709a046ab446d5e5d",
-                "sha256:b7802283b70ca24d7119b32915efa7c409982f59913c1a6c0640aacf118b95f5"
+                "sha256:0d5fe9189a148acc3c3eb2ac8e1ac0742cb7618c084f3d228baaec0c254b318d",
+                "sha256:ff615c761e25eb25df19edddc0b970302d2a9091fbce0e7213298d85fb61fef6"
             ],
             "index": "pypi",
-            "version": "==4.4.1"
+            "version": "==5.3.5"
         },
         "pytest-cov": {
             "hashes": [
-                "sha256:0ab664b25c6aa9716cbf203b17ddb301932383046082c081b9848a0edf5add33",
-                "sha256:230ef817450ab0699c6cc3c9c8f7a829c34674456f2ed8df1fe1d39780f7c87f"
+                "sha256:cc6742d8bac45070217169f5f72ceee1e0e55b0221f54bcf24845972d3a47f2b",
+                "sha256:cdbdef4f870408ebdbfeb44e63e07eb18bb4619fae852f6e760645fa36172626"
             ],
             "index": "pypi",
-            "version": "==2.6.1"
+            "version": "==2.8.1"
         },
         "pytest-mock": {
             "hashes": [
-                "sha256:43ce4e9dd5074993e7c021bb1c22cbb5363e612a2b5a76bc6d956775b10758b7",
-                "sha256:5bf5771b1db93beac965a7347dc81c675ec4090cb841e49d9d34637a25c30568"
+                "sha256:b35eb281e93aafed138db25c8772b95d3756108b601947f89af503f8c629413f",
+                "sha256:cb67402d87d5f53c579263d37971a164743dc33c159dfb4fb4a86f37c5552307"
             ],
             "index": "pypi",
-            "version": "==1.10.4"
+            "version": "==2.0.0"
         },
-        "pyyaml": {
+        "pytest-profiling": {
             "hashes": [
-                "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c",
-                "sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95",
-                "sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2",
-                "sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4",
-                "sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad",
-                "sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba",
-                "sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1",
-                "sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e",
-                "sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673",
-                "sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13",
-                "sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19"
+                "sha256:93938f147662225d2b8bd5af89587b979652426a8a6ffd7e73ec4a23e24b7f29",
+                "sha256:999cc9ac94f2e528e3f5d43465da277429984a1c237ae9818f8cfd0b06acb019"
             ],
-            "version": "==5.1"
+            "index": "pypi",
+            "version": "==1.7.0"
         },
-        "readme-renderer": {
+        "pytz": {
             "hashes": [
-                "sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f",
-                "sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d"
+                "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
+                "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
             ],
-            "version": "==24.0"
+            "version": "==2019.3"
+        },
+        "pyyaml": {
+            "hashes": [
+                "sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6",
+                "sha256:1cf708e2ac57f3aabc87405f04b86354f66799c8e62c28c5fc5f88b5521b2dbf",
+                "sha256:24521fa2890642614558b492b473bee0ac1f8057a7263156b02e8b14c88ce6f5",
+                "sha256:4fee71aa5bc6ed9d5f116327c04273e25ae31a3020386916905767ec4fc5317e",
+                "sha256:70024e02197337533eef7b85b068212420f950319cc8c580261963aefc75f811",
+                "sha256:74782fbd4d4f87ff04159e986886931456a1894c61229be9eaf4de6f6e44b99e",
+                "sha256:940532b111b1952befd7db542c370887a8611660d2b9becff75d39355303d82d",
+                "sha256:cb1f2f5e426dc9f07a7681419fe39cee823bb74f723f36f70399123f439e9b20",
+                "sha256:dbbb2379c19ed6042e8f11f2a2c66d39cceb8aeace421bfc29d085d93eda3689",
+                "sha256:e3a057b7a64f1222b56e47bcff5e4b94c4f61faac04c7c4ecb1985e18caa3994",
+                "sha256:e9f45bd5b92c7974e59bcd2dcc8631a6b6cc380a904725fce7bc08872e691615"
+            ],
+            "version": "==5.3"
+        },
+        "regex": {
+            "hashes": [
+                "sha256:07b39bf943d3d2fe63d46281d8504f8df0ff3fe4c57e13d1656737950e53e525",
+                "sha256:0932941cdfb3afcbc26cc3bcf7c3f3d73d5a9b9c56955d432dbf8bbc147d4c5b",
+                "sha256:0e182d2f097ea8549a249040922fa2b92ae28be4be4895933e369a525ba36576",
+                "sha256:10671601ee06cf4dc1bc0b4805309040bb34c9af423c12c379c83d7895622bb5",
+                "sha256:23e2c2c0ff50f44877f64780b815b8fd2e003cda9ce817a7fd00dea5600c84a0",
+                "sha256:26ff99c980f53b3191d8931b199b29d6787c059f2e029b2b0c694343b1708c35",
+                "sha256:27429b8d74ba683484a06b260b7bb00f312e7c757792628ea251afdbf1434003",
+                "sha256:3e77409b678b21a056415da3a56abfd7c3ad03da71f3051bbcdb68cf44d3c34d",
+                "sha256:4e8f02d3d72ca94efc8396f8036c0d3bcc812aefc28ec70f35bb888c74a25161",
+                "sha256:4eae742636aec40cf7ab98171ab9400393360b97e8f9da67b1867a9ee0889b26",
+                "sha256:6a6ae17bf8f2d82d1e8858a47757ce389b880083c4ff2498dba17c56e6c103b9",
+                "sha256:6a6ba91b94427cd49cd27764679024b14a96874e0dc638ae6bdd4b1a3ce97be1",
+                "sha256:7bcd322935377abcc79bfe5b63c44abd0b29387f267791d566bbb566edfdd146",
+                "sha256:98b8ed7bb2155e2cbb8b76f627b2fd12cf4b22ab6e14873e8641f266e0fb6d8f",
+                "sha256:bd25bb7980917e4e70ccccd7e3b5740614f1c408a642c245019cff9d7d1b6149",
+                "sha256:d0f424328f9822b0323b3b6f2e4b9c90960b24743d220763c7f07071e0778351",
+                "sha256:d58e4606da2a41659c84baeb3cfa2e4c87a74cec89a1e7c56bee4b956f9d7461",
+                "sha256:e3cd21cc2840ca67de0bbe4071f79f031c81418deb544ceda93ad75ca1ee9f7b",
+                "sha256:e6c02171d62ed6972ca8631f6f34fa3281d51db8b326ee397b9c83093a6b7242",
+                "sha256:e7c7661f7276507bce416eaae22040fd91ca471b5b33c13f8ff21137ed6f248c",
+                "sha256:ecc6de77df3ef68fee966bb8cb4e067e84d4d1f397d0ef6fce46913663540d77"
+            ],
+            "version": "==2020.1.8"
         },
         "requests": {
             "hashes": [
-                "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
-                "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
+                "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
+                "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
             ],
-            "version": "==2.21.0"
+            "version": "==2.22.0"
         },
-        "requests-toolbelt": {
+        "six": {
             "hashes": [
-                "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f",
-                "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"
+                "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a",
+                "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"
             ],
-            "version": "==0.9.1"
+            "version": "==1.14.0"
+        },
+        "snowballstemmer": {
+            "hashes": [
+                "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0",
+                "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52"
+            ],
+            "version": "==2.0.0"
         },
-        "scandir": {
+        "sphinx": {
             "hashes": [
-                "sha256:2586c94e907d99617887daed6c1d102b5ca28f1085f90446554abf1faf73123e",
-                "sha256:2ae41f43797ca0c11591c0c35f2f5875fa99f8797cb1a1fd440497ec0ae4b022",
-                "sha256:2b8e3888b11abb2217a32af0766bc06b65cc4a928d8727828ee68af5a967fa6f",
-                "sha256:2c712840c2e2ee8dfaf36034080108d30060d759c7b73a01a52251cc8989f11f",
-                "sha256:4d4631f6062e658e9007ab3149a9b914f3548cb38bfb021c64f39a025ce578ae",
-                "sha256:67f15b6f83e6507fdc6fca22fedf6ef8b334b399ca27c6b568cbfaa82a364173",
-                "sha256:7d2d7a06a252764061a020407b997dd036f7bd6a175a5ba2b345f0a357f0b3f4",
-                "sha256:8c5922863e44ffc00c5c693190648daa6d15e7c1207ed02d6f46a8dcc2869d32",
-                "sha256:92c85ac42f41ffdc35b6da57ed991575bdbe69db895507af88b9f499b701c188",
-                "sha256:b24086f2375c4a094a6b51e78b4cf7ca16c721dcee2eddd7aa6494b42d6d519d",
-                "sha256:cb925555f43060a1745d0a321cca94bcea927c50114b623d73179189a4e100ac"
+                "sha256:5024a67f065fe60d9db2005580074d81f22a02dd8f00a5b1ec3d5f4d42bc88d8",
+                "sha256:f929b72e0cfe45fa581b8964d54457117863a6a6c9369ecc1a65b8827abd3bf2"
+            ],
+            "version": "==2.4.1"
+        },
+        "sphinx-rtd-theme": {
+            "hashes": [
+                "sha256:00cf895504a7895ee433807c62094cf1e95f065843bf3acd17037c3e9a2becd4",
+                "sha256:728607e34d60456d736cc7991fd236afb828b21b82f956c5ea75f94c8414040a"
             ],
             "index": "pypi",
-            "version": "==1.10.0"
+            "version": "==0.4.3"
         },
-        "six": {
+        "sphinxcontrib-applehelp": {
             "hashes": [
-                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
-                "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
+                "sha256:edaa0ab2b2bc74403149cb0209d6775c96de797dfd5b5e2a71981309efab3897",
+                "sha256:fb8dee85af95e5c30c91f10e7eb3c8967308518e0f7488a2828ef7bc191d0d5d"
             ],
-            "version": "==1.12.0"
+            "version": "==1.0.1"
         },
-        "toml": {
+        "sphinxcontrib-devhelp": {
             "hashes": [
-                "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c",
-                "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e"
+                "sha256:6c64b077937330a9128a4da74586e8c2130262f014689b4b89e2d08ee7294a34",
+                "sha256:9512ecb00a2b0821a146736b39f7aeb90759834b07e81e8cc23a9c70bacb9981"
             ],
-            "version": "==0.10.0"
+            "version": "==1.0.1"
+        },
+        "sphinxcontrib-htmlhelp": {
+            "hashes": [
+                "sha256:4670f99f8951bd78cd4ad2ab962f798f5618b17675c35c5ac3b2132a14ea8422",
+                "sha256:d4fd39a65a625c9df86d7fa8a2d9f3cd8299a3a4b15db63b50aac9e161d8eff7"
+            ],
+            "version": "==1.0.2"
+        },
+        "sphinxcontrib-jsmath": {
+            "hashes": [
+                "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178",
+                "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"
+            ],
+            "version": "==1.0.1"
+        },
+        "sphinxcontrib-qthelp": {
+            "hashes": [
+                "sha256:513049b93031beb1f57d4daea74068a4feb77aa5630f856fcff2e50de14e9a20",
+                "sha256:79465ce11ae5694ff165becda529a600c754f4bc459778778c7017374d4d406f"
+            ],
+            "version": "==1.0.2"
+        },
+        "sphinxcontrib-serializinghtml": {
+            "hashes": [
+                "sha256:c0efb33f8052c04fd7a26c0a07f1678e8512e0faec19f4aa8f2473a8b81d5227",
+                "sha256:db6615af393650bf1151a6cd39120c29abaf93cc60db8c48eb2dddbfdc3a9768"
+            ],
+            "version": "==1.1.3"
         },
-        "tqdm": {
+        "testfixtures": {
             "hashes": [
-                "sha256:d385c95361699e5cf7622485d9b9eae2d4864b21cd5a2374a9c381ffed701021",
-                "sha256:e22977e3ebe961f72362f6ddfb9197cc531c9737aaf5f607ef09740c849ecd05"
+                "sha256:0a8a369dba5e01fe6b8da9300379d60fe62094536c8d971b559ec8167ab1fce3",
+                "sha256:fb42846633b159e38f2c7ef2056818e9f15ee9689f5b0a8a88b4775957853048"
             ],
-            "version": "==4.31.1"
+            "version": "==6.12.1"
         },
-        "twine": {
+        "toml": {
             "hashes": [
-                "sha256:0fb0bfa3df4f62076cab5def36b1a71a2e4acb4d1fa5c97475b048117b1a6446",
-                "sha256:d6c29c933ecfc74e9b1d9fa13aa1f87c5d5770e119f5a4ce032092f0ff5b14dc"
+                "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c",
+                "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e"
+            ],
+            "version": "==0.10.0"
+        },
+        "typed-ast": {
+            "hashes": [
+                "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355",
+                "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919",
+                "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa",
+                "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652",
+                "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75",
+                "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01",
+                "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d",
+                "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1",
+                "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907",
+                "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c",
+                "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3",
+                "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b",
+                "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614",
+                "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb",
+                "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b",
+                "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41",
+                "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6",
+                "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34",
+                "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe",
+                "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4",
+                "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"
+            ],
+            "version": "==1.4.1"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2",
+                "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d",
+                "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"
             ],
             "index": "pypi",
-            "version": "==1.13.0"
+            "version": "==3.7.4.1"
         },
         "urllib3": {
             "hashes": [
-                "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0",
-                "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3"
+                "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc",
+                "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"
             ],
-            "version": "==1.24.2"
+            "version": "==1.25.8"
         },
         "virtualenv": {
             "hashes": [
-                "sha256:15ee248d13e4001a691d9583948ad3947bcb8a289775102e4c4aa98a8b7a6d73",
-                "sha256:bfc98bb9b42a3029ee41b96dc00a34c2f254cbf7716bec824477b2c82741a5c4"
+                "sha256:08f3623597ce73b85d6854fb26608a6f39ee9d055c81178dc6583803797f8994",
+                "sha256:de2cbdd5926c48d7b84e0300dea9e8f276f61d186e8e49223d71d91250fbaebd"
             ],
-            "version": "==16.5.0"
+            "version": "==20.0.4"
         },
-        "webencodings": {
+        "wcwidth": {
             "hashes": [
-                "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
-                "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"
+                "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603",
+                "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8"
             ],
-            "version": "==0.5.1"
+            "version": "==0.1.8"
         },
         "zipp": {
             "hashes": [
-                "sha256:139391b239594fd8b91d856bc530fbd2df0892b17dd8d98a91f018715954185f",
-                "sha256:8047e4575ce8d700370a3301bbfc972896a5845eb62dd535da395b86be95dfad"
+                "sha256:5c56e330306215cd3553342cfafc73dda2c60792384117893f3a83f8a1209f50",
+                "sha256:d65287feb793213ffe11c0f31b81602be31448f38aeb8ffc2eb286c4f6f6657e"
             ],
-            "version": "==0.4.0"
+            "version": "==2.2.0"
         }
     }
 }

README.md

@@ -1,69 +1,68 @@
 
 <div align="center">
-  <p>
-  <img src="https://github.com/nficano/pytube/blob/master/images/pytube.png?raw=true" width="350" height="328" alt="pytube logo" />
-  </p>
   <p align="center">
-	  <img src="https://img.shields.io/pypi/v/pytube.svg" alt="pypi">
-	  <a href="https://travis-ci.org/nficano/pytube"><img src="https://travis-ci.org/nficano/pytube.svg?branch=master" /></a>
-	  <a href="http://python-pytube.readthedocs.io/en/latest/?badge=latest"><img src="https://readthedocs.org/projects/python-pytube/badge/?version=latest" /></a>
-	  <a href="https://coveralls.io/github/nficano/pytube?branch=master"><img src="https://coveralls.io/repos/github/nficano/pytube/badge.svg?branch=master#23e6f7ac56dd3bde" /></a>
-      <a href="https://pypi.org/project/pytube/"><img src="https://img.shields.io/pypi/dm/pytube.svg" alt="pypi"></a>
-	  <a href="https://pypi.python.org/pypi/pytube/"><img src="https://img.shields.io/pypi/pyversions/pytube.svg" /></a>
+	  <a href="https://pypi.org/project/pytube3/"><img src="https://img.shields.io/pypi/v/pytube3.svg" alt="pypi"></a>
+	  <a href="https://pypi.python.org/pypi/pytube3/"><img src="https://img.shields.io/pypi/pyversions/pytube3.svg" /></a>
+	  <a href="https://travis-ci.com/hbmartin/pytube3/"><img src="https://travis-ci.org/hbmartin/pytube3.svg?branch=master" /></a>
+	  <a href='https://pytube3.readthedocs.io/en/latest/?badge=latest'><img src='https://readthedocs.org/projects/pytube3/badge/?version=latest' alt='Documentation Status' /></a>
+	  <a href="https://codecov.io/gh/hbmartin/pytube3"><img src="https://codecov.io/gh/hbmartin/pytube3/branch/master/graph/badge.svg" /></a>
+	  <a href="https://www.codefactor.io/repository/github/hbmartin/pytube3/overview/master"><img src="https://www.codefactor.io/repository/github/hbmartin/pytube3/badge/master" alt="CodeFactor" /></a>
+	  <a href="https://gitter.im/pytube3/community"><img src="https://img.shields.io/badge/chat-gitter-lightgrey" /></a>
   </p>
 </div>
 
-# pytube
-*pytube* is a very serious, lightweight, dependency-free Python library (and command-line utility) for downloading YouTube Videos.
+# pytube3
 
-## Description
-YouTube is the most popular video-sharing platform in the world and as a hacker you may encounter a situation where you want to script something to download videos.  For this I present to you *pytube*.
+## Table of Contents
+  * [Installation](#installation)
+  * [Quick start](#quick-start)
+  * [Features](#features)
+  * [Usage](#usage)
+  * [Command-line interface](#command-line-interface)
+  * [Development](#development)
+  * [GUIs and other libraries](#guis-and-other-libraries)
 
-*pytube* is a lightweight library written in Python. It has no third party dependencies and aims to be highly reliable.
+## Installation
 
-*pytube* also makes pipelining easy, allowing you to specify callback functions for different download events, such as  ``on progress`` or ``on complete``.
+Download using pip via pypi.
 
-Finally *pytube* also includes a command-line utility, allowing you to quickly download videos right from terminal.
+```bash
+$ pip install pytube3 --upgrade
+```
+(Mac/homebrew users may need to use ``pip3``)
 
-### Behold, a perfect balance of simplicity versus flexibility:
 
+## Quick start
 ```python
- >>> YouTube('https://youtu.be/9bZkp7q19f0').streams.first().download()
+ >>> from pytube import YouTube
+ >>> YouTube('https://youtu.be/9bZkp7q19f0').streams.get_highest_resolution().download()
+ >>>
  >>> yt = YouTube('http://youtube.com/watch?v=9bZkp7q19f0')
  >>> yt.streams
   ... .filter(progressive=True, file_extension='mp4')
-  ... .order_by('resolution')
-  ... .desc()
-  ... .first()
+  ... .order_by('resolution')[-1]
   ... .download()
 ```
+A GUI frontend for pytube3 is available at [YouTubeDownload](https://github.com/YouTubeDownload/YouTubeDownload)
 
 ## Features
-- Support for Both Progressive & DASH Streams
-- Support for downloading complete playlist
-- Easily Register ``on_download_progress`` & ``on_download_complete`` callbacks
-- Command-line Interfaced Included
-- Caption Track Support
-- Outputs Caption Tracks to .srt format (SubRip Subtitle)
-- Ability to Capture Thumbnail URL.
-- Extensively Documented Source Code
-- No Third-Party Dependencies
-
-## Installation
-
-Download using pip via pypi.
-
-```bash
-$ pip install pytube
-```
-
-## Getting started
+  * Support for Both Progressive & DASH Streams
+  * Support for downloading complete playlist
+  * Easily Register ``on_download_progress`` & ``on_download_complete`` callbacks
+  * Command-line Interfaced Included
+  * Caption Track Support
+  * Outputs Caption Tracks to .srt format (SubRip Subtitle)
+  * Ability to Capture Thumbnail URL.
+  * Extensively Documented Source Code
+  * No Third-Party Dependencies
+
+## Usage
 
 Let's begin with showing how easy it is to download a video with pytube:
 
 ```python
 >>> from pytube import YouTube
->>> YouTube('http://youtube.com/watch?v=9bZkp7q19f0').streams.first().download()
+>>> YouTube('http://youtube.com/watch?v=9bZkp7q19f0').streams[0].download()
 ```
 This example will download the highest quality progressive download stream available.
 
@@ -71,7 +70,7 @@ Next, let's explore how we would view what video streams are available:
 
 ```python
 >>> yt = YouTube('http://youtube.com/watch?v=9bZkp7q19f0')
->>> yt.streams.all()
+>>> print(yt.streams)
  [<Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2">,
  <Stream: itag="43" mime_type="video/webm" res="360p" fps="30fps" vcodec="vp8.0" acodec="vorbis">,
  <Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2">,
@@ -95,6 +94,9 @@ Next, let's explore how we would view what video streams are available:
  <Stream: itag="250" mime_type="audio/webm" abr="70kbps" acodec="opus">,
  <Stream: itag="251" mime_type="audio/webm" abr="160kbps" acodec="opus">]
 ```
+
+### Selecting an itag
+
 You may notice that some streams listed have both a video codec and audio codec, while others have just video or just audio, this is a result of YouTube supporting a streaming technique called Dynamic Adaptive Streaming over HTTP (DASH).
 
 In the context of pytube, the implications are for the highest quality streams; you now need to download both the audio and video tracks and then post-process them with software like FFmpeg to merge them.
@@ -104,7 +106,7 @@ The legacy streams that contain the audio and video in a single file (referred t
 To only view these progressive download streams:
 
 ```python
- >>> yt.streams.filter(progressive=True).all()
+ >>> yt.streams.filter(progressive=True)
   [<Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2">,
   <Stream: itag="43" mime_type="video/webm" res="360p" fps="30fps" vcodec="vp8.0" acodec="vorbis">,
   <Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2">,
@@ -115,7 +117,7 @@ To only view these progressive download streams:
 Conversely, if you only want to see the DASH streams (also referred to as "adaptive") you can do:
 
 ```python
->>> yt.streams.filter(adaptive=True).all()
+>>> yt.streams.filter(adaptive=True)
  [<Stream: itag="137" mime_type="video/mp4" res="1080p" fps="30fps" vcodec="avc1.640028">,
   <Stream: itag="248" mime_type="video/webm" res="1080p" fps="30fps" vcodec="vp9">,
   <Stream: itag="136" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.4d401f">,
@@ -135,24 +137,26 @@ Conversely, if you only want to see the DASH streams (also referred to as "adapt
   <Stream: itag="251" mime_type="audio/webm" abr="160kbps" acodec="opus">]
 ```
 
+### Playlists
+
 You can also download a complete Youtube playlist:
 
 ```python
 >>> from pytube import Playlist
->>> pl = Playlist("https://www.youtube.com/watch?v=Edpy1szoG80&list=PL153hDY-y1E00uQtCVCVC8xJ25TYX8yPU")
->>> pl.download_all()
->>> # or if you want to download in a specific directory
->>> pl.download_all('/path/to/directory/')
+>>> playlist = Playlist("https://www.youtube.com/playlist?list=PLynhp4cZEpTbRs_PYISQ8v_uwO0_mDg_X")
+>>> for video in playlist:
+>>> 	video.streams.get_highest_resolution().download()
 ```
-This will download the highest progressive stream available (generally 720p) from the given playlist. Later more options would be given for user's flexibility
-to choose video resolution.
+This will download the highest progressive stream available (generally 720p) from the given playlist.
+
+### Filtering
 
 Pytube allows you to filter on every property available (see the documentation for the complete list), let's take a look at some of the most useful ones.
 
 To list the audio only streams:
 
 ```python
->>> yt.streams.filter(only_audio=True).all()
+>>> yt.streams.filter(only_audio=True)
   [<Stream: itag="140" mime_type="audio/mp4" abr="128kbps" acodec="mp4a.40.2">,
   <Stream: itag="171" mime_type="audio/webm" abr="128kbps" acodec="vorbis">,
   <Stream: itag="249" mime_type="audio/webm" abr="50kbps" acodec="opus">,
@@ -163,7 +167,7 @@ To list the audio only streams:
 To list only ``mp4`` streams:
 
 ```python
->>> yt.streams.filter(subtype='mp4').all()
+>>> yt.streams.filter(subtype='mp4')
  [<Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2">,
   <Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2">,
   <Stream: itag="137" mime_type="video/mp4" res="1080p" fps="30fps" vcodec="avc1.640028">,
@@ -178,9 +182,9 @@ To list only ``mp4`` streams:
 Multiple filters can also be specified:
 
 ```python
->>> yt.streams.filter(subtype='mp4', progressive=True).all()
+>>> yt.streams.filter(subtype='mp4', progressive=True)
 >>> # this can also be expressed as:
->>> yt.streams.filter(subtype='mp4').filter(progressive=True).all()
+>>> yt.streams.filter(subtype='mp4').filter(progressive=True)
   [<Stream: itag="22" mime_type="video/mp4" res="720p" fps="30fps" vcodec="avc1.64001F" acodec="mp4a.40.2">,
   <Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2">]
 ```
@@ -194,14 +198,16 @@ You also have an interface to select streams by their itag, without needing to f
 If you need to optimize for a specific feature, such as the "highest resolution" or "lowest average bitrate":
 
 ```python
->>> yt.streams.filter(progressive=True).order_by('resolution').desc().all()
+>>> yt.streams.filter(progressive=True).order_by('resolution').desc()
 ```
-Note that ``order_by`` cannot be used if your attribute is undefined in any of the Stream instances, so be sure to apply a filter to remove those before calling it.
+Note: Using ``order_by`` on a given attribute will filter out all streams missing that attribute.
+
+### Callbacks
 
 If your application requires post-processing logic, pytube allows you to specify an "on download complete" callback function:
 
 ```python
- >>> def convert_to_aac(stream, file_handle):
+ >>> def convert_to_aac(stream: Stream, file_path: str):
          return  # do work
 
  >>> yt.register_on_complete_callback(convert_to_aac)
@@ -210,7 +216,7 @@ If your application requires post-processing logic, pytube allows you to specify
 Similarly, if your application requires on-download progress logic, pytube exposes a callback for this as well:
 
 ```python
- >>> def show_progress_bar(stream, chunk, file_handle, bytes_remaining):
+ >>> def show_progress_bar(stream: Stream, chunk: bytes, bytes_remaining: int):
          return  # do work
 
  >>> yt.register_on_progress_callback(show_progress_bar)
@@ -218,17 +224,86 @@ Similarly, if your application requires on-download progress logic, pytube expos
 
 ## Command-line interface
 
-pytube also ships with a tiny cli interface for downloading and probing videos.
+pytube3 ships with a simple CLI interface for downloading videos, playlists, and captions.
 
 Let's start with downloading:
 
 ```bash
-$ pytube http://youtube.com/watch?v=9bZkp7q19f0 --itag=22
+$ pytube3 http://youtube.com/watch?v=9bZkp7q19f0 --itag=18
 ```
 To view available streams:
 
 ```bash
-$ pytube http://youtube.com/watch?v=9bZkp7q19f0 --list
+$ pytube3 http://youtube.com/watch?v=9bZkp7q19f0 --list
+```
+
+The complete set of flags are:
+
+```
+usage: pytube3 [-h] [--version] [--itag ITAG] [-r RESOLUTION] [-l] [-v]
+               [--build-playback-report] [-c [CAPTION_CODE]] [-t TARGET]
+               [-a [AUDIO]] [-f [FFMPEG]]
+               [url]
+
+Command line application to download youtube videos.
+
+positional arguments:
+  url                   The YouTube /watch or /playlist url
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --version             show program's version number and exit
+  --itag ITAG           The itag for the desired stream
+  -r RESOLUTION, --resolution RESOLUTION
+                        The resolution for the desired stream
+  -l, --list            The list option causes pytube cli to return a list of
+                        streams available to download
+  -v, --verbose         Verbosity level, use up to 4 to increase logging -vvvv
+  --build-playback-report
+                        Save the html and js to disk
+  -c [CAPTION_CODE], --caption-code [CAPTION_CODE]
+                        Download srt captions for given language code. Prints
+                        available language codes if no argument given
+  -t TARGET, --target TARGET
+                        The output directory for the downloaded stream.
+                        Default is current working directory
+  -a [AUDIO], --audio [AUDIO]
+                        Download the audio for a given URL at the highest
+                        bitrate availableDefaults to mp4 format if none is
+                        specified
+  -f [FFMPEG], --ffmpeg [FFMPEG]
+                        Downloads the audio and video stream for resolution
+                        providedIf no resolution is provided, downloads the
+                        best resolutionRuns the command line program ffmpeg to
+                        combine the audio and video
 ```
 
-Finally, if you're filing a bug report, the cli contains a switch called ``--build-playback-report``, which bundles up the state, allowing others to easily replay your issue.
+
+## Development
+
+<a href="https://deepsource.io/gh/hbmartin/pytube3/?ref=repository-badge" target="_blank"><img alt="DeepSource" title="DeepSource" src="https://static.deepsource.io/deepsource-badge-light-mini.svg"></a>
+<a href="https://www.codacy.com/manual/hbmartin/pytube3?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=hbmartin/pytube3&amp;utm_campaign=Badge_Grade"><img src="https://api.codacy.com/project/badge/Grade/53794f06983a46829620b3284c6a5596"/></a>
+<a href="https://github.com/ambv/black"><img src="https://img.shields.io/badge/code%20style-black-000000.svg" /></a>
+
+Pull requests are welcome. For major changes, please open an issue first to discuss what you would like to change.
+
+To run code checking before a PR use ``make test``
+
+#### Virtual environment
+
+Virtual environment is setup with [pipenv](https://pipenv-fork.readthedocs.io/en/latest/) and can be automatically activated with [direnv](https://direnv.net/docs/installation.html)
+
+#### Code Formatting
+
+This project is linted with [pyflakes](https://github.com/PyCQA/pyflakes), formatted with [black](https://github.com/ambv/black), and typed with [mypy](https://mypy.readthedocs.io/en/latest/introduction.html)
+
+
+#### Code of Conduct
+
+Treat other people with helpfulness, gratitude, and consideration! See the [Python Community Code of Conduct](https://www.python.org/psf/codeofconduct/).
+
+## GUIs and other libraries
+* [YouTubeDownload](https://github.com/YouTubeDownload/YouTubeDownload) - Featured GUI frontend for pytube3
+* [Pytube-GUI](https://github.com/GAO23/Pytube-GUI) - Simple GUI frontend for pytube3
+* [StackOverflow questions](https://stackoverflow.com/questions/tagged/pytube)
+* [PySlackers](https://pyslackers.com/web) - Python Slack group

docs/api.rst

@@ -65,12 +65,6 @@ Mixins
 .. automodule:: pytube.mixins
     :members:
 
-Compat
-------
-
-.. automodule:: pytube.compat
-    :members:
-
 
 Helpers
 -------

docs/conf.py

@@ -34,9 +34,9 @@ source_suffix = '.rst'
 master_doc = 'index'
 
 # General information about the project.
-project = 'pytube'
+project = 'pytube3'
 copyright = '2019, Nick Ficano'
-author = 'Nick Ficano'
+author = 'Nick Ficano, Harold Martin'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -108,7 +108,7 @@ html_sidebars = {
 # -- Options for HTMLHelp output ------------------------------------------
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'pytubedoc'
+htmlhelp_basename = 'pytube3doc'
 
 
 # -- Options for LaTeX output ---------------------------------------------
@@ -120,7 +120,7 @@ latex_elements = {}
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
     (
-        master_doc, 'pytube.tex', 'pytube Documentation',
+        master_doc, 'pytube3.tex', 'pytube3 Documentation',
         'Nick Ficano', 'manual',
     ),
 ]
@@ -132,7 +132,7 @@ latex_documents = [
 # (source start file, name, description, authors, manual section).
 man_pages = [
     (
-        master_doc, 'pytube', 'pytube Documentation',
+        master_doc, 'pytube3', 'pytube3 Documentation',
         [author], 1,
     ),
 ]
@@ -145,8 +145,8 @@ man_pages = [
 #  dir menu entry, description, category)
 texinfo_documents = [
     (
-        master_doc, 'pytube', 'pytube Documentation',
-        author, 'pytube', 'One line description of project.',
+        master_doc, 'pytube3', 'pytube3 Documentation',
+        author, 'pytube3', 'One line description of project.',
         'Miscellaneous',
     ),
 ]

docs/index.rst

@@ -1,31 +1,24 @@
-.. pytube documentation master file, created by
-   sphinx-quickstart on Mon Oct  9 02:11:41 2017.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
+.. pytube3 documentation master file, created by sphinx-quickstart on Mon Oct  9 02:11:41 2017.
 
-pytube
+pytube3
 ======
 Release v\ |version|. (:ref:`Installation <install>`)
 
-.. image:: https://img.shields.io/pypi/v/pytube.svg
+.. image:: https://img.shields.io/pypi/v/pytube3.svg
   :alt: Pypi
-  :target: https://pypi.python.org/pypi/pytube/
+  :target: https://pypi.python.org/pypi/pytube3/
 
-.. image:: https://travis-ci.org/nficano/pytube.svg?branch=master
+.. image:: https://travis-ci.org/hbmartin/pytube3.svg?branch=master
    :alt: Build status
-   :target: https://travis-ci.org/nficano/pytube
-
-.. image:: https://readthedocs.org/projects/python-pytube/badge/?version=latest
-  :target: http://python-pytube.readthedocs.io/en/latest/?badge=latest
-  :alt: Documentation Status
+   :target: https://travis-ci.org/hbmartin/pytube3
 
 .. image:: https://coveralls.io/repos/github/nficano/pytube/badge.svg?branch=master
   :alt: Coverage
   :target: https://coveralls.io/github/nficano/pytube?branch=master
 
-.. image:: https://img.shields.io/pypi/pyversions/pytube.svg
+.. image:: https://img.shields.io/pypi/pyversions/pytube3.svg
   :alt: Python Versions
-  :target: https://pypi.python.org/pypi/pytube/
+  :target: https://pypi.python.org/pypi/pytube3/
 
 **pytube** is a lightweight, Pythonic, dependency-free, library (and command-line utility) for downloading YouTube Videos.
 
@@ -33,6 +26,7 @@ Release v\ |version|. (:ref:`Installation <install>`)
 
 **Behold, a perfect balance of simplicity versus flexibility**::
 
+    >>> from pytube import YouTube
     >>> YouTube('https://youtu.be/9bZkp7q19f0').streams.first().download()
     >>> yt = YouTube('http://youtube.com/watch?v=9bZkp7q19f0')
     >>> yt.streams

docs/requirements.txt

@@ -0,0 +1 @@
+typing_extensions

docs/user/install.rst

@@ -7,20 +7,20 @@ This part of the documentation covers the installation of pytube.
 
 To install pytube, run the following command in your terminal::
 
-    $ pip install pytube
+    $ pip install pytube3
 
 Get the Source Code
 -------------------
 
-pytube is actively developed on GitHub, where the source is `available <https://github.com/nficano/pytube>`_.
+pytube is actively developed on GitHub, where the source is `available <https://github.com/hbmartin/pytube3>`_.
 
 You can either clone the public repository::
 
     $ git clone git://github.com/nficano/pytube.git
 
-Or, download the `tarball <https://github.com/nficano/pytube/tarball/master>`_::
+Or, download the `tarball <https://github.com/hbmartin/pytube3/tarball/master>`_::
 
-    $ curl -OL https://github.com/nficano/pytube/tarball/master
+    $ curl -OL https://github.com/hbmartin/pytube3/tarball/master
     # optionally, zipball is also available (for Windows users).
 
 Once you have a copy of the source, you can embed it in your Python package, or install it into your site-packages by running::

pytube/__init__.py

@@ -1,22 +1,18 @@
 # -*- coding: utf-8 -*-
-# flake8: noqa
+# flake8: noqa: F401
 # noreorder
 """
 Pytube: a very serious Python library for downloading YouTube Videos.
 """
-__title__ = 'pytube'
-__version__ = '9.5.3'
-__author__ = 'Nick Ficano'
-__license__ = 'MIT License'
-__copyright__ = 'Copyright 2019 Nick Ficano'
+__title__ = "pytube3"
+__author__ = "Nick Ficano, Harold Martin"
+__license__ = "MIT License"
+__copyright__ = "Copyright 2019 Nick Ficano"
 
-from pytube.logging import create_logger
-from pytube.query import CaptionQuery
-from pytube.query import StreamQuery
+from pytube.version import __version__
 from pytube.streams import Stream
 from pytube.captions import Caption
-from pytube.contrib.playlist import Playlist
+from pytube.query import CaptionQuery
+from pytube.query import StreamQuery
 from pytube.__main__ import YouTube
-
-logger = create_logger()
-logger.info('%s v%s', __title__, __version__)
+from pytube.contrib.playlist import Playlist

pytube/__main__.py

@@ -7,38 +7,43 @@ exclusively on the developer interface. Pytube offloads the heavy lifting to
 smaller peripheral modules and functions.
 
 """
-from __future__ import absolute_import
 
 import json
 import logging
+from typing import Optional, Dict, List
+from urllib.parse import parse_qsl
+from html import unescape
 
 from pytube import Caption
 from pytube import CaptionQuery
 from pytube import extract
-from pytube import mixins
 from pytube import request
 from pytube import Stream
 from pytube import StreamQuery
-from pytube.compat import install_proxy
-from pytube.compat import parse_qsl
+from pytube.extract import apply_descrambler, apply_signature, get_ytplayer_config
+from pytube.helpers import install_proxy
 from pytube.exceptions import VideoUnavailable
-from pytube.helpers import apply_mixin
+from pytube.monostate import OnProgress, OnComplete, Monostate
 
 logger = logging.getLogger(__name__)
 
 
-class YouTube(object):
+class YouTube:
     """Core developer interface for pytube."""
 
     def __init__(
-        self, url=None, defer_prefetch_init=False, on_progress_callback=None,
-        on_complete_callback=None, proxies=None,
+        self,
+        url: str,
+        defer_prefetch_init: bool = False,
+        on_progress_callback: Optional[OnProgress] = None,
+        on_complete_callback: Optional[OnComplete] = None,
+        proxies: Dict[str, str] = None,
     ):
         """Construct a :class:`YouTube <YouTube>`.
 
         :param str url:
             A valid YouTube watch URL.
-        :param bool defer_init:
+        :param bool defer_prefetch_init:
             Defers executing any network requests.
         :param func on_progress_callback:
             (Optional) User defined callback function for stream download
@@ -48,55 +53,45 @@ class YouTube(object):
             complete events.
 
         """
-        self.js = None      # js fetched by js_url
-        self.js_url = None  # the url to the js, parsed from watch html
+        self.js: Optional[str] = None  # js fetched by js_url
+        self.js_url: Optional[str] = None  # the url to the js, parsed from watch html
 
         # note: vid_info may eventually be removed. It sounds like it once had
         # additional formats, but that doesn't appear to still be the case.
 
-        self.vid_info = None      # content fetched by vid_info_url
-        self.vid_info_url = None  # the url to vid info, parsed from watch html
+        # the url to vid info, parsed from watch html
+        self.vid_info_url: Optional[str] = None
+        self.vid_info_raw: Optional[str] = None  # content fetched by vid_info_url
+        self.vid_info: Optional[Dict] = None  # parsed content of vid_info_raw
 
-        self.watch_html = None     # the html of /watch?v=<video_id>
-        self.embed_html = None
-        self.player_config_args = None  # inline js in the html containing
+        self.watch_html: Optional[str] = None  # the html of /watch?v=<video_id>
+        self.embed_html: Optional[str] = None
+        self.player_config_args: Dict = {}  # inline js in the html containing
+        self.player_response: Dict = {}
         # streams
-        self.age_restricted = None
+        self.age_restricted: Optional[bool] = None
 
-        self.fmt_streams = []  # list of :class:`Stream <Stream>` instances
-        self.caption_tracks = []
+        self.fmt_streams: List[Stream] = []
 
         # video_id part of /watch?v=<video_id>
         self.video_id = extract.video_id(url)
 
-        # https://www.youtube.com/watch?v=<video_id>
-        self.watch_url = extract.watch_url(self.video_id)
+        self.watch_url = f"https://youtube.com/watch?v={self.video_id}"
+        self.embed_url = f"https://www.youtube.com/embed/{self.video_id}"
 
-        self.embed_url = extract.embed_url(self.video_id)
-        # A dictionary shared between all instances of :class:`Stream <Stream>`
-        # (Borg pattern).
-        self.stream_monostate = {
-            # user defined callback functions.
-            'on_progress': on_progress_callback,
-            'on_complete': on_complete_callback,
-        }
+        # Shared between all instances of `Stream` (Borg pattern).
+        self.stream_monostate = Monostate(
+            on_progress=on_progress_callback, on_complete=on_complete_callback
+        )
 
         if proxies:
             install_proxy(proxies)
 
         if not defer_prefetch_init:
-            self.prefetch_init()
-
-    def prefetch_init(self):
-        """Download data, descramble it, and build Stream instances.
-
-        :rtype: None
+            self.prefetch()
+            self.descramble()
 
-        """
-        self.prefetch()
-        self.init()
-
-    def init(self):
+    def descramble(self) -> None:
         """Descramble the stream data and build Stream instances.
 
         The initialization process takes advantage of Python's
@@ -107,60 +102,55 @@ class YouTube(object):
         :rtype: None
 
         """
-        logger.info('init started')
+        logger.info("init started")
 
-        self.vid_info = {k: v for k, v in parse_qsl(self.vid_info)}
+        self.vid_info = dict(parse_qsl(self.vid_info_raw))
         if self.age_restricted:
             self.player_config_args = self.vid_info
         else:
-            self.player_config_args = extract.get_ytplayer_config(
-                self.watch_html,
-            )['args']
+            assert self.watch_html is not None
+            self.player_config_args = get_ytplayer_config(self.watch_html)["args"]
 
             # Fix for KeyError: 'title' issue #434
-            if 'title' not in self.player_config_args:
-                i_start = (
-                    self.watch_html
-                    .lower()
-                    .index('<title>') + len('<title>')
-                )
-                i_end = self.watch_html.lower().index('</title>')
+            if "title" not in self.player_config_args:  # type: ignore
+                i_start = self.watch_html.lower().index("<title>") + len("<title>")
+                i_end = self.watch_html.lower().index("</title>")
                 title = self.watch_html[i_start:i_end].strip()
-                index = title.lower().rfind(' - youtube')
+                index = title.lower().rfind(" - youtube")
                 title = title[:index] if index > 0 else title
-                self.player_config_args['title'] = title
+                self.player_config_args["title"] = unescape(title)
 
-        self.vid_descr = extract.get_vid_descr(self.watch_html)
         # https://github.com/nficano/pytube/issues/165
-        stream_maps = ['url_encoded_fmt_stream_map']
-        if 'adaptive_fmts' in self.player_config_args:
-            stream_maps.append('adaptive_fmts')
+        stream_maps = ["url_encoded_fmt_stream_map"]
+        if "adaptive_fmts" in self.player_config_args:
+            stream_maps.append("adaptive_fmts")
 
         # unscramble the progressive and adaptive stream manifests.
         for fmt in stream_maps:
             if not self.age_restricted and fmt in self.vid_info:
-                mixins.apply_descrambler(self.vid_info, fmt)
-            mixins.apply_descrambler(self.player_config_args, fmt)
-
-            try:
-                mixins.apply_signature(self.player_config_args, fmt, self.js)
-            except TypeError:
-                self.js_url = extract.js_url(
-                    self.embed_html, self.age_restricted,
-                )
+                apply_descrambler(self.vid_info, fmt)
+            apply_descrambler(self.player_config_args, fmt)
+
+            if not self.js:
+                if not self.embed_html:
+                    self.embed_html = request.get(url=self.embed_url)
+                self.js_url = extract.js_url(self.embed_html)
                 self.js = request.get(self.js_url)
-                mixins.apply_signature(self.player_config_args, fmt, self.js)
+
+            apply_signature(self.player_config_args, fmt, self.js)
 
             # build instances of :class:`Stream <Stream>`
             self.initialize_stream_objects(fmt)
 
         # load the player_response object (contains subtitle information)
-        apply_mixin(self.player_config_args, 'player_response', json.loads)
+        self.player_response = json.loads(self.player_config_args["player_response"])
+        del self.player_config_args["player_response"]
+        self.stream_monostate.title = self.title
+        self.stream_monostate.duration = self.length
 
-        self.initialize_caption_objects()
-        logger.info('init finished successfully')
+        logger.info("init finished successfully")
 
-    def prefetch(self):
+    def prefetch(self) -> None:
         """Eagerly download all necessary data.
 
         Eagerly executes all necessary network requests so all other
@@ -168,26 +158,32 @@ class YouTube(object):
         which blocks for long periods of time.
 
         :rtype: None
-
         """
         self.watch_html = request.get(url=self.watch_url)
-        if '<img class="icon meh" src="/yts/img' not in self.watch_html:
-            raise VideoUnavailable('This video is unavailable.')
-        self.embed_html = request.get(url=self.embed_url)
+        if self.watch_html is None:
+            raise VideoUnavailable(video_id=self.video_id)
         self.age_restricted = extract.is_age_restricted(self.watch_html)
-        self.vid_info_url = extract.video_info_url(
-            video_id=self.video_id,
-            watch_url=self.watch_url,
-            watch_html=self.watch_html,
-            embed_html=self.embed_html,
-            age_restricted=self.age_restricted,
-        )
-        self.vid_info = request.get(self.vid_info_url)
+
+        if not self.age_restricted and "This video is private" in self.watch_html:
+            raise VideoUnavailable(video_id=self.video_id)
+
+        if self.age_restricted:
+            if not self.embed_html:
+                self.embed_html = request.get(url=self.embed_url)
+            self.vid_info_url = extract.video_info_url_age_restricted(
+                self.video_id, self.watch_url
+            )
+        else:
+            self.vid_info_url = extract.video_info_url(
+                video_id=self.video_id, watch_url=self.watch_url
+            )
+
+        self.vid_info_raw = request.get(self.vid_info_url)
         if not self.age_restricted:
-            self.js_url = extract.js_url(self.watch_html, self.age_restricted)
+            self.js_url = extract.js_url(self.watch_html)
             self.js = request.get(self.js_url)
 
-    def initialize_stream_objects(self, fmt):
+    def initialize_stream_objects(self, fmt: str) -> None:
         """Convert manifest data to instances of :class:`Stream <Stream>`.
 
         Take the unscrambled stream data and uses it to initialize
@@ -210,127 +206,131 @@ class YouTube(object):
             )
             self.fmt_streams.append(video)
 
-    def initialize_caption_objects(self):
-        """Populate instances of :class:`Caption <Caption>`.
-
-        Take the unscrambled player response data, and use it to initialize
-        instances of :class:`Caption <Caption>`.
-
-        :rtype: None
+    @property
+    def caption_tracks(self) -> List[Caption]:
+        """Get a list of :class:`Caption <Caption>`.
 
+        :rtype: List[Caption]
         """
-        if 'captions' not in self.player_config_args['player_response']:
-            return
-        # https://github.com/nficano/pytube/issues/167
-        caption_tracks = (
-            self.player_config_args
-            .get('player_response', {})
-            .get('captions', {})
-            .get('playerCaptionsTracklistRenderer', {})
-            .get('captionTracks', [])
+        raw_tracks = (
+            self.player_response.get("captions", {})
+            .get("playerCaptionsTracklistRenderer", {})
+            .get("captionTracks", [])
         )
-        for caption_track in caption_tracks:
-            self.caption_tracks.append(Caption(caption_track))
+        return [Caption(track) for track in raw_tracks]
 
     @property
-    def captions(self):
+    def captions(self) -> CaptionQuery:
         """Interface to query caption tracks.
 
         :rtype: :class:`CaptionQuery <CaptionQuery>`.
         """
-        return CaptionQuery([c for c in self.caption_tracks])
+        return CaptionQuery(self.caption_tracks)
 
     @property
-    def streams(self):
+    def streams(self) -> StreamQuery:
         """Interface to query both adaptive (DASH) and progressive streams.
 
         :rtype: :class:`StreamQuery <StreamQuery>`.
         """
-        return StreamQuery([s for s in self.fmt_streams])
+        return StreamQuery(self.fmt_streams)
 
     @property
-    def thumbnail_url(self):
+    def thumbnail_url(self) -> str:
         """Get the thumbnail url image.
 
         :rtype: str
 
         """
-        return self.player_config_args['thumbnail_url']
+        thumbnail_details = (
+            self.player_response.get("videoDetails", {})
+            .get("thumbnail", {})
+            .get("thumbnails")
+        )
+        if thumbnail_details:
+            thumbnail_details = thumbnail_details[-1]  # last item has max size
+            return thumbnail_details["url"]
+
+        return f"https://img.youtube.com/vi/{self.video_id}/maxresdefault.jpg"
 
     @property
-    def title(self):
+    def title(self) -> str:
         """Get the video title.
 
         :rtype: str
 
         """
-        return self.player_config_args['title']
+        return self.player_config_args.get("title") or (
+            self.player_response.get("videoDetails", {}).get("title")
+        )
 
     @property
-    def description(self):
+    def description(self) -> str:
         """Get the video description.
 
         :rtype: str
 
         """
-        return self.vid_descr
+        return self.player_response.get("videoDetails", {}).get(
+            "shortDescription"
+        ) or extract._get_vid_descr(self.watch_html)
 
     @property
-    def rating(self):
+    def rating(self) -> float:
         """Get the video average rating.
 
-        :rtype: str
+        :rtype: float
 
         """
-        return (
-            self.player_config_args
-            .get('player_response', {})
-            .get('videoDetails', {})
-            .get('averageRating')
-        )
+        return self.player_response.get("videoDetails", {}).get("averageRating")
 
     @property
-    def length(self):
+    def length(self) -> int:
         """Get the video length in seconds.
 
         :rtype: str
 
         """
-        return self.player_config_args['length_seconds']
+        return int(
+            self.player_config_args.get("length_seconds")
+            or (self.player_response.get("videoDetails", {}).get("lengthSeconds"))
+        )
 
     @property
-    def views(self):
+    def views(self) -> int:
         """Get the number of the times the video has been viewed.
 
         :rtype: str
 
         """
-        return (
-            self.player_config_args
-            .get('player_response', {})
-            .get('videoDetails', {})
-            .get('viewCount')
-        )
+        return int(self.player_response.get("videoDetails", {}).get("viewCount"))
+
+    @property
+    def author(self) -> str:
+        """Get the video author.
+        :rtype: str
+        """
+        return self.player_response.get("videoDetails", {}).get("author", "unknown")
 
-    def register_on_progress_callback(self, func):
+    def register_on_progress_callback(self, func: OnProgress):
         """Register a download progress callback function post initialization.
 
         :param callable func:
             A callback function that takes ``stream``, ``chunk``,
-            ``file_handle``, ``bytes_remaining`` as parameters.
+             and ``bytes_remaining`` as parameters.
 
         :rtype: None
 
         """
-        self.stream_monostate['on_progress'] = func
+        self.stream_monostate.on_progress = func
 
-    def register_on_complete_callback(self, func):
+    def register_on_complete_callback(self, func: OnComplete):
         """Register a download complete callback function post initialization.
 
         :param callable func:
-            A callback function that takes ``stream`` and  ``file_handle``.
+            A callback function that takes ``stream`` and  ``file_path``.
 
         :rtype: None
 
         """
-        self.stream_monostate['on_complete'] = func
+        self.stream_monostate.on_complete = func

pytube/captions.py

@@ -1,32 +1,33 @@
 # -*- coding: utf-8 -*-
-"""This module contrains a container for caption tracks."""
 import math
+import os
 import time
 import xml.etree.ElementTree as ElementTree
-
+from typing import Dict, Optional
 from pytube import request
-from pytube.compat import unescape
+from html import unescape
+from pytube.helpers import safe_filename, target_directory
 
 
 class Caption:
     """Container for caption tracks."""
 
-    def __init__(self, caption_track):
+    def __init__(self, caption_track: Dict):
         """Construct a :class:`Caption <Caption>`.
 
         :param dict caption_track:
             Caption track data extracted from ``watch_html``.
         """
-        self.url = caption_track.get('baseUrl')
-        self.name = caption_track['name']['simpleText']
-        self.code = caption_track['languageCode']
+        self.url = caption_track.get("baseUrl")
+        self.name = caption_track["name"]["simpleText"]
+        self.code = caption_track["languageCode"]
 
     @property
-    def xml_captions(self):
+    def xml_captions(self) -> str:
         """Download the xml caption tracks."""
         return request.get(self.url)
 
-    def generate_srt_captions(self):
+    def generate_srt_captions(self) -> str:
         """Generate "SubRip Subtitle" captions.
 
         Takes the xml captions from :meth:`~pytube.Caption.xml_captions` and
@@ -34,22 +35,22 @@ class Caption:
         """
         return self.xml_caption_to_srt(self.xml_captions)
 
-    def float_to_srt_time_format(self, d):
+    @staticmethod
+    def float_to_srt_time_format(d: float) -> str:
         """Convert decimal durations into proper srt format.
 
         :rtype: str
         :returns:
             SubRip Subtitle (str) formatted time duration.
 
-        >>> float_to_srt_time_format(3.89)
-        '00:00:03,890'
+        float_to_srt_time_format(3.89) -> '00:00:03,890'
         """
-        frac, whole = math.modf(d)
-        time_fmt = time.strftime('%H:%M:%S,', time.gmtime(whole))
-        ms = '{:.3f}'.format(frac).replace('0.', '')
+        fraction, whole = math.modf(d)
+        time_fmt = time.strftime("%H:%M:%S,", time.gmtime(whole))
+        ms = f"{fraction:.3f}".replace("0.", "")
         return time_fmt + ms
 
-    def xml_caption_to_srt(self, xml_captions):
+    def xml_caption_to_srt(self, xml_captions: str) -> str:
         """Convert xml caption tracks to "SubRip Subtitle (srt)".
 
         :param str xml_captions:
@@ -57,28 +58,79 @@ class Caption:
         """
         segments = []
         root = ElementTree.fromstring(xml_captions)
-        for i, child in enumerate(root.getchildren()):
-            text = child.text or ''
-            caption = unescape(
-                text
-                .replace('\n', ' ')
-                .replace('  ', ' '),
-            )
-            duration = float(child.attrib['dur'])
-            start = float(child.attrib['start'])
+        for i, child in enumerate(list(root)):
+            text = child.text or ""
+            caption = unescape(text.replace("\n", " ").replace("  ", " "),)
+            duration = float(child.attrib["dur"])
+            start = float(child.attrib["start"])
             end = start + duration
             sequence_number = i + 1  # convert from 0-indexed to 1.
-            line = (
-                '{seq}\n{start} --> {end}\n{text}\n'.format(
-                    seq=sequence_number,
-                    start=self.float_to_srt_time_format(start),
-                    end=self.float_to_srt_time_format(end),
-                    text=caption,
-                )
+            line = "{seq}\n{start} --> {end}\n{text}\n".format(
+                seq=sequence_number,
+                start=self.float_to_srt_time_format(start),
+                end=self.float_to_srt_time_format(end),
+                text=caption,
             )
             segments.append(line)
-        return '\n'.join(segments).strip()
+        return "\n".join(segments).strip()
+
+    def download(
+        self,
+        title: str,
+        srt: bool = True,
+        output_path: Optional[str] = None,
+        filename_prefix: Optional[str] = None,
+    ) -> str:
+        """Write the media stream to disk.
+
+        :param title:
+            Output filename (stem only) for writing media file.
+            If one is not specified, the default filename is used.
+        :type title: str
+        :param srt:
+            Set to True to download srt, false to download xml. Defaults to True.
+        :type srt bool
+        :param output_path:
+            (optional) Output path for writing media file. If one is not
+            specified, defaults to the current working directory.
+        :type output_path: str or None
+        :param filename_prefix:
+            (optional) A string that will be prepended to the filename.
+            For example a number in a playlist or the name of a series.
+            If one is not specified, nothing will be prepended
+            This is separate from filename so you can use the default
+            filename but still add a prefix.
+        :type filename_prefix: str or None
+
+        :rtype: str
+        """
+        if title.endswith(".srt") or title.endswith(".xml"):
+            filename = ".".join(title.split(".")[:-1])
+        else:
+            filename = title
+
+        if filename_prefix:
+            filename = f"{safe_filename(filename_prefix)}{filename}"
+
+        filename = safe_filename(filename)
+
+        filename += f" ({self.code})"
+
+        if srt:
+            filename += ".srt"
+        else:
+            filename += ".xml"
+
+        file_path = os.path.join(target_directory(output_path), filename)
+
+        with open(file_path, "w", encoding="utf-8") as file_handle:
+            if srt:
+                file_handle.write(self.generate_srt_captions())
+            else:
+                file_handle.write(self.xml_captions)
+
+        return file_path
 
     def __repr__(self):
         """Printable object representation."""
-        return'<Caption lang="{s.name}" code="{s.code}">'.format(s=self)
+        return '<Caption lang="{s.name}" code="{s.code}">'.format(s=self)

pytube/cipher.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
+
 """
-This module countains all logic necessary to decipher the signature.
+This module contains all logic necessary to decipher the signature.
 
 YouTube's strategy to restrict downloading videos is to send a ciphered version
 of the signature to the client, along with the decryption algorithm obfuscated
@@ -13,48 +14,116 @@ functions" (2) maps them to Python equivalents and (3) taking the ciphered
 signature and decoding it.
 
 """
-from __future__ import absolute_import
-
 import logging
-import pprint
 import re
 from itertools import chain
+from typing import List, Tuple, Dict, Callable, Any, Optional
 
 from pytube.exceptions import RegexMatchError
-from pytube.helpers import regex_search
-
+from pytube.helpers import regex_search, cache
 
 logger = logging.getLogger(__name__)
 
 
-def get_initial_function_name(js):
+class Cipher:
+    def __init__(self, js: str):
+        self.transform_plan: List[str] = get_transform_plan(js)
+        var, _ = self.transform_plan[0].split(".")
+        self.transform_map = get_transform_map(js, var)
+        self.js_func_regex = re.compile(r"\w+\.(\w+)\(\w,(\d+)\)")
+
+    def get_signature(self, ciphered_signature: str) -> str:
+        """Decipher the signature.
+
+        Taking the ciphered signature, applies the transform functions.
+
+        :param str ciphered_signature:
+            The ciphered signature sent in the ``player_config``.
+        :rtype: str
+        :returns:
+           Decrypted signature required to download the media content.
+        """
+        signature = list(ciphered_signature)
+
+        for js_func in self.transform_plan:
+            name, argument = self.parse_function(js_func)  # type: ignore
+            signature = self.transform_map[name](signature, argument)
+            logger.debug(
+                "applied transform function\n"
+                "output: %s\n"
+                "js_function: %s\n"
+                "argument: %d\n"
+                "function: %s",
+                "".join(signature),
+                name,
+                argument,
+                self.transform_map[name],
+            )
+
+        return "".join(signature)
+
+    @cache
+    def parse_function(self, js_func: str) -> Tuple[str, int]:
+        """Parse the Javascript transform function.
+
+        Break a JavaScript transform function down into a two element ``tuple``
+        containing the function name and some integer-based argument.
+
+        :param str js_func:
+            The JavaScript version of the transform function.
+        :rtype: tuple
+        :returns:
+            two element tuple containing the function name and an argument.
+
+        **Example**:
+
+        parse_function('DE.AJ(a,15)')
+        ('AJ', 15)
+
+        """
+        logger.debug("parsing transform function")
+        parse_match = self.js_func_regex.search(js_func)
+        if not parse_match:
+            raise RegexMatchError(caller="parse_function", pattern="js_func_regex")
+        fn_name, fn_arg = parse_match.groups()
+        return fn_name, int(fn_arg)
+
+
+def get_initial_function_name(js: str) -> str:
     """Extract the name of the function responsible for computing the signature.
-
     :param str js:
         The contents of the base.js asset file.
-
+    :rtype: str
+    :returns:
+       Function name from regex match
     """
-    # c&&d.set("signature", EE(c));
 
-    pattern = [
-        r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',  # noqa: E501
-        r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',  # noqa: E501
+    function_patterns = [
+        r"\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(",  # noqa: E501
+        r"\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(",  # noqa: E501
+        r'\b(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',  # noqa: E501
         r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',  # noqa: E501
         r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-        r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
-        r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<si$',  # noqa: E501
-        r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',  # noqa: E501
-        r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',  # noqa: E501
-        r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',  # noqa: E501
-        r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',  # noqa: E501
-        r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',  # noqa: E501
+        r"\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(",
+        r"yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(",  # noqa: E501
+        r"\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(",  # noqa: E501
+        r"\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(",  # noqa: E501
+        r"\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(",  # noqa: E501
+        r"\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(",  # noqa: E501
+        r"\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(",  # noqa: E501
     ]
+    logger.debug("finding initial function name")
+    for pattern in function_patterns:
+        regex = re.compile(pattern)
+        function_match = regex.search(js)
+        if function_match:
+            logger.debug("finished regex search, matched: %s", pattern)
+            return function_match.group(1)
 
-    logger.debug('finding initial function name')
-    return regex_search(pattern, js, group=1)
+    raise RegexMatchError(caller="get_initial_function_name", pattern="multiple")
 
 
-def get_transform_plan(js):
+def get_transform_plan(js: str) -> List[str]:
     """Extract the "transform plan".
 
     The "transform plan" is the functions that the ciphered signature is
@@ -65,7 +134,6 @@ def get_transform_plan(js):
 
     **Example**:
 
-    >>> get_transform_plan(js)
     ['DE.AJ(a,15)',
     'DE.VR(a,3)',
     'DE.AJ(a,51)',
@@ -76,12 +144,12 @@ def get_transform_plan(js):
     'DE.kT(a,21)']
     """
     name = re.escape(get_initial_function_name(js))
-    pattern = r'%s=function\(\w\){[a-z=\.\(\"\)]*;(.*);(?:.+)}' % name
-    logger.debug('getting transform plan')
-    return regex_search(pattern, js, group=1).split(';')
+    pattern = r"%s=function\(\w\){[a-z=\.\(\"\)]*;(.*);(?:.+)}" % name
+    logger.debug("getting transform plan")
+    return regex_search(pattern, js, group=1).split(";")
 
 
-def get_transform_object(js, var):
+def get_transform_object(js: str, var: str) -> List[str]:
     """Extract the "transform object".
 
     The "transform object" contains the function definitions referenced in the
@@ -103,16 +171,17 @@ def get_transform_object(js, var):
     'kT:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b]=c}']
 
     """
-    pattern = r'var %s={(.*?)};' % re.escape(var)
-    logger.debug('getting transform object')
-    return (
-        regex_search(pattern, js, group=1, flags=re.DOTALL)
-        .replace('\n', ' ')
-        .split(', ')
-    )
+    pattern = r"var %s={(.*?)};" % re.escape(var)
+    logger.debug("getting transform object")
+    regex = re.compile(pattern, flags=re.DOTALL)
+    transform_match = regex.search(js)
+    if not transform_match:
+        raise RegexMatchError(caller="get_transform_object", pattern=pattern)
 
+    return transform_match.group(1).replace("\n", " ").split(", ")
 
-def get_transform_map(js, var):
+
+def get_transform_map(js: str, var: str) -> Dict:
     """Build a transform function lookup.
 
     Build a lookup table of obfuscated JavaScript function names to the
@@ -129,13 +198,13 @@ def get_transform_map(js, var):
     mapper = {}
     for obj in transform_object:
         # AJ:function(a){a.reverse()} => AJ, function(a){a.reverse()}
-        name, function = obj.split(':', 1)
+        name, function = obj.split(":", 1)
         fn = map_functions(function)
         mapper[name] = fn
     return mapper
 
 
-def reverse(arr, b):
+def reverse(arr: List, _: Optional[Any]):
     """Reverse elements in a list.
 
     This function is equivalent to:
@@ -155,7 +224,7 @@ def reverse(arr, b):
     return arr[::-1]
 
 
-def splice(arr, b):
+def splice(arr: List, b: int):
     """Add/remove items to/from a list.
 
     This function is equivalent to:
@@ -169,10 +238,10 @@ def splice(arr, b):
     >>> splice([1, 2, 3, 4], 2)
     [1, 2]
     """
-    return arr[:b] + arr[b * 2:]
+    return arr[:b] + arr[b * 2 :]
 
 
-def swap(arr, b):
+def swap(arr: List, b: int):
     """Swap positions at b modulus the list length.
 
     This function is equivalent to:
@@ -187,10 +256,10 @@ def swap(arr, b):
     [3, 2, 1, 4]
     """
     r = b % len(arr)
-    return list(chain([arr[r]], arr[1:r], [arr[0]], arr[r + 1:]))
+    return list(chain([arr[r]], arr[1:r], [arr[0]], arr[r + 1 :]))
 
 
-def map_functions(js_func):
+def map_functions(js_func: str) -> Callable:
     """For a given JavaScript transform function, return the Python equivalent.
 
     :param str js_func:
@@ -199,80 +268,19 @@ def map_functions(js_func):
     """
     mapper = (
         # function(a){a.reverse()}
-        ('{\w\.reverse\(\)}', reverse),
+        (r"{\w\.reverse\(\)}", reverse),
         # function(a,b){a.splice(0,b)}
-        ('{\w\.splice\(0,\w\)}', splice),
+        (r"{\w\.splice\(0,\w\)}", splice),
         # function(a,b){var c=a[0];a[0]=a[b%a.length];a[b]=c}
-        ('{var\s\w=\w\[0\];\w\[0\]=\w\[\w\%\w.length\];\w\[\w\]=\w}', swap),
+        (r"{var\s\w=\w\[0\];\w\[0\]=\w\[\w\%\w.length\];\w\[\w\]=\w}", swap),
         # function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}
         (
-            '{var\s\w=\w\[0\];\w\[0\]=\w\[\w\%\w.length\];'
-            '\w\[\w\%\w.length\]=\w}', swap,
+            r"{var\s\w=\w\[0\];\w\[0\]=\w\[\w\%\w.length\];\w\[\w\%\w.length\]=\w}",
+            swap,
         ),
     )
 
     for pattern, fn in mapper:
         if re.search(pattern, js_func):
             return fn
-    raise RegexMatchError(
-        'could not find python equivalent function for: ',
-        js_func,
-    )
-
-
-def parse_function(js_func):
-    """Parse the Javascript transform function.
-
-    Break a JavaScript transform function down into a two element ``tuple``
-    containing the function name and some integer-based argument.
-
-    :param str js_func:
-        The JavaScript version of the transform function.
-    :rtype: tuple
-    :returns:
-        two element tuple containing the function name and an argument.
-
-    **Example**:
-
-    >>> parse_function('DE.AJ(a,15)')
-    ('AJ', 15)
-
-    """
-    logger.debug('parsing transform function')
-    return regex_search(r'\w+\.(\w+)\(\w,(\d+)\)', js_func, groups=True)
-
-
-def get_signature(js, ciphered_signature):
-    """Decipher the signature.
-
-    Taking the ciphered signature, applies the transform functions.
-
-    :param str js:
-        The contents of the base.js asset file.
-    :param str ciphered_signature:
-        The ciphered signature sent in the ``player_config``.
-    :rtype: str
-    :returns:
-       Decrypted signature required to download the media content.
-
-    """
-    tplan = get_transform_plan(js)
-    # DE.AJ(a,15) => DE, AJ(a,15)
-    var, _ = tplan[0].split('.')
-    tmap = get_transform_map(js, var)
-    signature = [s for s in ciphered_signature]
-
-    for js_func in tplan:
-        name, argument = parse_function(js_func)
-        signature = tmap[name](signature, int(argument))
-        logger.debug(
-            'applied transform function\n%s', pprint.pformat(
-                {
-                    'output': ''.join(signature),
-                    'js_function': name,
-                    'argument': int(argument),
-                    'function': tmap[name],
-                }, indent=2,
-            ),
-        )
-    return ''.join(signature)
+    raise RegexMatchError(caller="map_functions", pattern="multiple")

pytube/cli.py

@@ -1,7 +1,6 @@
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """A simple command line application to download youtube videos."""
-from __future__ import absolute_import
-from __future__ import print_function
 
 import argparse
 import datetime as dt
@@ -9,97 +8,176 @@ import gzip
 import json
 import logging
 import os
+import shutil
 import sys
+import subprocess  # nosec
+from typing import List, Optional
 
-from pytube import __version__
+from pytube import __version__, CaptionQuery, Stream, Playlist
 from pytube import YouTube
-
-
-logger = logging.getLogger(__name__)
+from pytube.exceptions import PytubeError
+from pytube.helpers import safe_filename, setup_logger
 
 
 def main():
     """Command line application to download youtube videos."""
+    # noinspection PyTypeChecker
     parser = argparse.ArgumentParser(description=main.__doc__)
-    parser.add_argument('url', help='The YouTube /watch url', nargs='?')
+    args = _parse_args(parser)
+    if args.verbosity:
+        log_level = min(args.verbosity, 4) * 10
+        setup_logger(logging.FATAL - log_level)
+
+    if not args.url or "youtu" not in args.url:
+        parser.print_help()
+        sys.exit(1)
+
+    if "/playlist" in args.url:
+        print("Loading playlist...")
+        playlist = Playlist(args.url)
+        if not args.target:
+            args.target = safe_filename(playlist.title())
+        for youtube_video in playlist.videos:
+            try:
+                _perform_args_on_youtube(youtube_video, args)
+            except PytubeError as e:
+                print(f"There was an error with video: {youtube_video}")
+                print(e)
+    else:
+        print("Loading video...")
+        youtube = YouTube(args.url)
+        _perform_args_on_youtube(youtube, args)
+
+
+def _perform_args_on_youtube(youtube: YouTube, args: argparse.Namespace) -> None:
+    if args.list:
+        display_streams(youtube)
+    if args.build_playback_report:
+        build_playback_report(youtube)
+    if args.itag:
+        download_by_itag(youtube=youtube, itag=args.itag, target=args.target)
+    if hasattr(args, "caption_code"):
+        download_caption(
+            youtube=youtube, lang_code=args.caption_code, target=args.target
+        )
+    if args.resolution:
+        download_by_resolution(
+            youtube=youtube, resolution=args.resolution, target=args.target
+        )
+    if args.audio:
+        download_audio(youtube=youtube, filetype=args.audio, target=args.target)
+    if args.ffmpeg:
+        ffmpeg_process(youtube=youtube, resolution=args.ffmpeg, target=args.target)
+
+
+def _parse_args(
+    parser: argparse.ArgumentParser, args: Optional[List] = None
+) -> argparse.Namespace:
+    parser.add_argument("url", help="The YouTube /watch or /playlist url", nargs="?")
+    parser.add_argument(
+        "--version", action="version", version="%(prog)s " + __version__,
+    )
+    parser.add_argument(
+        "--itag", type=int, help="The itag for the desired stream",
+    )
     parser.add_argument(
-        '--version', action='version',
-        version='%(prog)s ' + __version__,
+        "-r", "--resolution", type=str, help="The resolution for the desired stream",
     )
     parser.add_argument(
-        '--itag', type=int, help=(
-            'The itag for the desired stream'
+        "-l",
+        "--list",
+        action="store_true",
+        help=(
+            "The list option causes pytube cli to return a list of streams "
+            "available to download"
         ),
     )
     parser.add_argument(
-        '-l', '--list', action='store_true', help=(
-            'The list option causes pytube cli to return a list of streams '
-            'available to download'
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        dest="verbosity",
+        help="Verbosity level, use up to 4 to increase logging -vvvv",
+    )
+    parser.add_argument(
+        "--build-playback-report",
+        action="store_true",
+        help="Save the html and js to disk",
+    )
+    parser.add_argument(
+        "-c",
+        "--caption-code",
+        type=str,
+        default=argparse.SUPPRESS,
+        nargs="?",
+        help=(
+            "Download srt captions for given language code. "
+            "Prints available language codes if no argument given"
         ),
     )
     parser.add_argument(
-        '-v', '--verbose', action='count', default=0, dest='verbosity',
-        help='Verbosity level',
+        "-t",
+        "--target",
+        help=(
+            "The output directory for the downloaded stream. "
+            "Default is current working directory"
+        ),
     )
     parser.add_argument(
-        '--build-playback-report', action='store_true', help=(
-            'Save the html and js to disk'
+        "-a",
+        "--audio",
+        const="mp4",
+        nargs="?",
+        help=(
+            "Download the audio for a given URL at the highest bitrate available"
+            "Defaults to mp4 format if none is specified"
+        ),
+    )
+    parser.add_argument(
+        "-f",
+        "--ffmpeg",
+        const="best",
+        nargs="?",
+        help=(
+            "Downloads the audio and video stream for resolution provided"
+            "If no resolution is provided, downloads the best resolution"
+            "Runs the command line program ffmpeg to combine the audio and video"
         ),
     )
 
-    args = parser.parse_args()
-    logging.getLogger().setLevel(max(3 - args.verbosity, 0) * 10)
-
-    if not args.url:
-        parser.print_help()
-        sys.exit(1)
-
-    if args.list:
-        display_streams(args.url)
-
-    elif args.build_playback_report:
-        build_playback_report(args.url)
-
-    elif args.itag:
-        download(args.url, args.itag)
+    return parser.parse_args(args)
 
 
-def build_playback_report(url):
+def build_playback_report(youtube: YouTube) -> None:
     """Serialize the request data to json for offline debugging.
 
-    :param str url:
-        A valid YouTube watch URL.
+    :param YouTube youtube:
+        A YouTube object.
     """
-    yt = YouTube(url)
     ts = int(dt.datetime.utcnow().timestamp())
-    fp = os.path.join(
-        os.getcwd(),
-        'yt-video-{yt.video_id}-{ts}.json.gz'.format(yt=yt, ts=ts),
-    )
+    fp = os.path.join(os.getcwd(), f"yt-video-{youtube.video_id}-{ts}.json.gz")
 
-    js = yt.js
-    watch_html = yt.watch_html
-    vid_info = yt.vid_info
+    js = youtube.js
+    watch_html = youtube.watch_html
+    vid_info = youtube.vid_info
 
-    with gzip.open(fp, 'wb') as fh:
+    with gzip.open(fp, "wb") as fh:
         fh.write(
-            json.dumps({
-                'url': url,
-                'js': js,
-                'watch_html': watch_html,
-                'video_info': vid_info,
-            })
-            .encode('utf8'),
+            json.dumps(
+                {
+                    "url": youtube.watch_url,
+                    "js": js,
+                    "watch_html": watch_html,
+                    "video_info": vid_info,
+                }
+            ).encode("utf8"),
         )
 
 
-def get_terminal_size():
-    """Return the terminal size in rows and columns."""
-    rows, columns = os.popen('stty size', 'r').read().split()
-    return int(rows), int(columns)
-
-
-def display_progress_bar(bytes_received, filesize, ch='█', scale=0.55):
+def display_progress_bar(
+    bytes_received: int, filesize: int, ch: str = "█", scale: float = 0.55
+) -> None:
     """Display a simple, pretty progress bar.
 
     Example:
@@ -112,77 +190,285 @@ def display_progress_bar(bytes_received, filesize, ch='█', scale=0.55):
         written to disk.
     :param int filesize:
         File size of the media stream in bytes.
-    :param ch str:
+    :param str ch:
         Character to use for presenting progress segment.
     :param float scale:
-        Scale multipler to reduce progress bar size.
+        Scale multiplier to reduce progress bar size.
 
     """
-    _, columns = get_terminal_size()
+    columns = shutil.get_terminal_size().columns
     max_width = int(columns * scale)
 
     filled = int(round(max_width * bytes_received / float(filesize)))
     remaining = max_width - filled
-    bar = ch * filled + ' ' * remaining
+    progress_bar = ch * filled + " " * remaining
     percent = round(100.0 * bytes_received / float(filesize), 1)
-    text = ' ↳ |{bar}| {percent}%\r'.format(bar=bar, percent=percent)
+    text = f" ↳ |{progress_bar}| {percent}%\r"
     sys.stdout.write(text)
     sys.stdout.flush()
 
 
-def on_progress(stream, chunk, file_handle, bytes_remaining):
-    """On download progress callback function.
-
-    :param object stream:
-        An instance of :class:`Stream <Stream>` being downloaded.
-    :param file_handle:
-        The file handle where the media is being written to.
-    :type file_handle:
-        :py:class:`io.BufferedWriter`
-    :param int bytes_remaining:
-        How many bytes have been downloaded.
-
-    """
+# noinspection PyUnusedLocal
+def on_progress(
+    stream: Stream, chunk: bytes, bytes_remaining: int
+) -> None:  # pylint: disable=W0613
     filesize = stream.filesize
     bytes_received = filesize - bytes_remaining
     display_progress_bar(bytes_received, filesize)
 
 
-def download(url, itag):
+def _download(
+    stream: Stream, target: Optional[str] = None, filename: Optional[str] = None
+) -> None:
+    filesize_megabytes = stream.filesize // 1048576
+    print(f"{filename or stream.default_filename} | {filesize_megabytes} MB")
+    file_path = stream.get_file_path(filename=filename, output_path=target)
+    if stream.exists_at_path(file_path):
+        print(f"Already downloaded at:\n{file_path}")
+        return
+
+    stream.download(output_path=target, filename=filename)
+    sys.stdout.write("\n")
+
+
+def _unique_name(base: str, subtype: str, media_type: str, target: str) -> str:
+    """
+    Given a base name, the file format, and the target directory, will generate
+    a filename unique for that directory and file format.
+    :param str base:
+        The given base-name.
+    :param str subtype:
+        The filetype of the video which will be downloaded.
+    :param str media_type:
+        The media_type of the file, ie. "audio" or "video"
+    :param Path target:
+        Target directory for download.
+    """
+    counter = 0
+    while True:
+        file_name = f"{base}_{media_type}_{counter}"
+        file_path = os.path.join(target, f"{file_name}.{subtype}")
+        if not os.path.exists(file_path):
+            return file_name
+        counter += 1
+
+
+def ffmpeg_process(
+    youtube: YouTube, resolution: str, target: Optional[str] = None
+) -> None:
+    """
+    Decides the correct video stream to download, then calls _ffmpeg_downloader.
+
+    :param YouTube youtube:
+        A valid YouTube object.
+    :param str resolution:
+        YouTube video resolution.
+    :param str target:
+        Target directory for download
+    """
+    youtube.register_on_progress_callback(on_progress)
+    target = target or os.getcwd()
+
+    if resolution == "best":
+        highest_quality_stream = (
+            youtube.streams.filter(progressive=False).order_by("resolution").last()
+        )
+        mp4_stream = (
+            youtube.streams.filter(progressive=False, subtype="mp4")
+            .order_by("resolution")
+            .last()
+        )
+        if highest_quality_stream.resolution == mp4_stream.resolution:
+            video_stream = mp4_stream
+        else:
+            video_stream = highest_quality_stream
+    else:
+        video_stream = youtube.streams.filter(
+            progressive=False, resolution=resolution, subtype="mp4"
+        ).first()
+        if not video_stream:
+            video_stream = youtube.streams.filter(
+                progressive=False, resolution=resolution
+            ).first()
+    if video_stream is None:
+        print(f"Could not find a stream with resolution: {resolution}")
+        print("Try one of these:")
+        display_streams(youtube)
+        sys.exit()
+
+    audio_stream = youtube.streams.get_audio_only(video_stream.subtype)
+    if not audio_stream:
+        audio_stream = youtube.streams.filter(only_audio=True).order_by("abr").last()
+    if not audio_stream:
+        print("Could not find an audio only stream")
+        sys.exit()
+    _ffmpeg_downloader(
+        audio_stream=audio_stream, video_stream=video_stream, target=target
+    )
+
+
+def _ffmpeg_downloader(audio_stream: Stream, video_stream: Stream, target: str) -> None:
+    """
+    Given a YouTube Stream object, finds the correct audio stream, downloads them both
+    giving them a unique name, them uses ffmpeg to create a new file with the audio
+    and video from the previously downloaded files. Then deletes the original adaptive
+    streams, leaving the combination.
+
+    :param Stream audio_stream:
+        A valid Stream object representing the audio to download
+    :param Stream video_stream:
+        A valid Stream object representing the video to download
+    :param Path target:
+        A valid Path object
+    """
+    video_unique_name = _unique_name(
+        safe_filename(video_stream.title), video_stream.subtype, "video", target=target
+    )
+    audio_unique_name = _unique_name(
+        safe_filename(video_stream.title), audio_stream.subtype, "audio", target=target
+    )
+    _download(stream=video_stream, target=target, filename=video_unique_name)
+    print("Loading audio...")
+    _download(stream=audio_stream, target=target, filename=audio_unique_name)
+
+    video_path = os.path.join(target, f"{video_unique_name}.{video_stream.subtype}")
+    audio_path = os.path.join(target, f"{audio_unique_name}.{audio_stream.subtype}")
+    final_path = os.path.join(
+        target, f"{safe_filename(video_stream.title)}.{video_stream.subtype}"
+    )
+
+    subprocess.run(  # nosec
+        ["ffmpeg", "-i", video_path, "-i", audio_path, "-codec", "copy", final_path,]
+    )
+    os.unlink(video_path)
+    os.unlink(audio_path)
+
+
+def download_by_itag(youtube: YouTube, itag: int, target: Optional[str] = None) -> None:
     """Start downloading a YouTube video.
 
-    :param str url:
-        A valid YouTube watch URL.
-    :param str itag:
+    :param YouTube youtube:
+        A valid YouTube object.
+    :param int itag:
         YouTube format identifier code.
+    :param str target:
+        Target directory for download
+    """
+    stream = youtube.streams.get_by_itag(itag)
+    if stream is None:
+        print(f"Could not find a stream with itag: {itag}")
+        print("Try one of these:")
+        display_streams(youtube)
+        sys.exit()
+
+    youtube.register_on_progress_callback(on_progress)
+
+    try:
+        _download(stream, target=target)
+    except KeyboardInterrupt:
+        sys.exit()
+
+
+def download_by_resolution(
+    youtube: YouTube, resolution: str, target: Optional[str] = None
+) -> None:
+    """Start downloading a YouTube video.
 
+    :param YouTube youtube:
+        A valid YouTube object.
+    :param str resolution:
+        YouTube video resolution.
+    :param str target:
+        Target directory for download
     """
-    # TODO(nficano): allow download target to be specified
     # TODO(nficano): allow dash itags to be selected
-    yt = YouTube(url, on_progress_callback=on_progress)
-    stream = yt.streams.get_by_itag(itag)
-    print('\n{fn} | {fs} bytes'.format(
-        fn=stream.default_filename,
-        fs=stream.filesize,
-    ))
+    stream = youtube.streams.get_by_resolution(resolution)
+    if stream is None:
+        print(f"Could not find a stream with resolution: {resolution}")
+        print("Try one of these:")
+        display_streams(youtube)
+        sys.exit()
+
+    youtube.register_on_progress_callback(on_progress)
+
     try:
-        stream.download()
-        sys.stdout.write('\n')
+        _download(stream, target=target)
     except KeyboardInterrupt:
         sys.exit()
 
 
-def display_streams(url):
+def display_streams(youtube: YouTube) -> None:
     """Probe YouTube video and lists its available formats.
 
-    :param str url:
+    :param YouTube youtube:
         A valid YouTube watch URL.
 
     """
-    yt = YouTube(url)
-    for stream in yt.streams.all():
+    for stream in youtube.streams:
         print(stream)
 
 
-if __name__ == '__main__':
+def _print_available_captions(captions: CaptionQuery) -> None:
+    print(f"Available caption codes are: {', '.join(c.code for c in captions)}")
+
+
+def download_caption(
+    youtube: YouTube, lang_code: Optional[str], target: Optional[str] = None
+) -> None:
+    """Download a caption for the YouTube video.
+
+    :param YouTube youtube:
+        A valid YouTube object.
+    :param str lang_code:
+        Language code desired for caption file.
+        Prints available codes if the value is None
+        or the desired code is not available.
+    :param str target:
+        Target directory for download
+    """
+    if lang_code is None:
+        _print_available_captions(youtube.captions)
+        return
+
+    try:
+        caption = youtube.captions[lang_code]
+        downloaded_path = caption.download(title=youtube.title, output_path=target)
+        print(f"Saved caption file to: {downloaded_path}")
+    except KeyError:
+        print(f"Unable to find caption with code: {lang_code}")
+        _print_available_captions(youtube.captions)
+
+
+def download_audio(
+    youtube: YouTube, filetype: str, target: Optional[str] = None
+) -> None:
+    """
+    Given a filetype, downloads the highest quality available audio stream for a
+    YouTube video.
+
+    :param YouTube youtube:
+        A valid YouTube object.
+    :param str filetype:
+        Desired file format to download.
+    :param str target:
+        Target directory for download
+    """
+    audio = (
+        youtube.streams.filter(only_audio=True, subtype=filetype).order_by("abr").last()
+    )
+
+    if audio is None:
+        print("No audio only stream found. Try one of these:")
+        display_streams(youtube)
+        sys.exit()
+
+    youtube.register_on_progress_callback(on_progress)
+
+    try:
+        _download(audio, target=target)
+    except KeyboardInterrupt:
+        sys.exit()
+
+
+if __name__ == "__main__":
     main()

pytube/compat.py

@@ -1,70 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# flake8: noqa
-"""Python 2/3 compatibility support."""
-import sys
-
-
-PY2 = sys.version_info[0] == 2
-PY3 = sys.version_info[0] == 3
-PY33 = sys.version_info[0:2] >= (3, 3)
-
-if PY2:
-    reload(sys)
-    sys.setdefaultencoding('utf8')
-    import urllib2
-    from urllib import urlencode
-    from urllib2 import URLError
-    from urllib2 import quote
-    from urllib2 import unquote
-    from urllib2 import urlopen
-    from urlparse import parse_qsl
-    from HTMLParser import HTMLParser
-
-    def install_proxy(proxy_handler):
-        """
-        install global proxy.
-        :param proxy_handler:
-            :samp:`{"http":"http://my.proxy.com:1234", "https":"https://my.proxy.com:1234"}`
-        :return:
-        """
-        proxy_support = urllib2.ProxyHandler(proxy_handler)
-        opener = urllib2.build_opener(proxy_support)
-        urllib2.install_opener(opener)
-
-    def unescape(s):
-        """Strip HTML entries from a string."""
-        html_parser = HTMLParser()
-        return html_parser.unescape(s)
-
-    def unicode(s):
-        """Encode a string to utf-8."""
-        return s.encode('utf-8')
-
-elif PY3:
-    from urllib.error import URLError
-    from urllib.parse import parse_qsl
-    from urllib.parse import quote
-    from urllib.parse import unquote
-    from urllib.parse import urlencode
-    from urllib.request import urlopen
-    from urllib import request
-
-    def install_proxy(proxy_handler):
-        proxy_support = request.ProxyHandler(proxy_handler)
-        opener = request.build_opener(proxy_support)
-        request.install_opener(opener)
-
-    def unicode(s):
-        """No-op."""
-        return s
-
-    if PY33:
-        from html.parser import HTMLParser
-
-        def unescape(s):
-            """Strip HTML entries from a string."""
-            html_parser = HTMLParser()
-            return html_parser.unescape(s)
-    else:
-        from html import unescape

pytube/contrib/playlist.py

@@ -1,107 +1,172 @@
 # -*- coding: utf-8 -*-
-"""
-Module to download a complete playlist from a youtube channel
-"""
+
+"""Module to download a complete playlist from a youtube channel."""
+
 import json
 import logging
 import re
-from collections import OrderedDict
+from datetime import date, datetime
+from typing import List, Optional, Iterable, Dict, Union
+from urllib.parse import parse_qs
+from collections.abc import Sequence
 
-from pytube import request
-from pytube.__main__ import YouTube
+from pytube import request, YouTube
+from pytube.helpers import cache, deprecated, install_proxy, uniqueify
 
 logger = logging.getLogger(__name__)
 
 
-class Playlist(object):
-    """Handles all the task of manipulating and downloading a whole YouTube
-    playlist
-    """
-
-    def __init__(self, url, suppress_exception=False):
-        self.playlist_url = url
-        self.video_urls = []
-        self.suppress_exception = suppress_exception
+class Playlist(Sequence):
+    """Load a YouTube playlist with URL or ID"""
 
-    def construct_playlist_url(self):
-        """There are two kinds of playlist urls in YouTube. One that contains
-        watch?v= in URL, another one contains the "playlist?list=" portion. It
-        is preferable to work with the later one.
-
-        :return: playlist url
-        """
+    def __init__(self, url: str, proxies: Optional[Dict[str, str]] = None):
+        if proxies:
+            install_proxy(proxies)
 
-        if 'watch?v=' in self.playlist_url:
-            base_url = 'https://www.youtube.com/playlist?list='
-            playlist_code = self.playlist_url.split('&list=')[1]
-            return base_url + playlist_code
-
-        # url is already in the desired format, so just return it
-        return self.playlist_url
-
-    def _load_more_url(self, req):
-        """Given an html page or a fragment thereof, looks for
-        and returns the "load more" url if found.
-        """
         try:
-            load_more_url = 'https://www.youtube.com' + re.search(
-                r'data-uix-load-more-href=\"(/browse_ajax\?'
-                'action_continuation=.*?)\"', req,
-            ).group(1)
-        except AttributeError:
-            load_more_url = ''
-        return load_more_url
-
-    def parse_links(self):
-        """Parse the video links from the page source, extracts and
-        returns the /watch?v= part from video link href
-        It's an alternative for BeautifulSoup
+            self.playlist_id: str = parse_qs(url.split("?")[1])["list"][0]
+        except IndexError:  # assume that url is just the id
+            self.playlist_id = url
+
+        self.playlist_url = f"https://www.youtube.com/playlist?list={self.playlist_id}"
+        self.html = request.get(self.playlist_url)
+
+        # Needs testing with non-English
+        self.last_update: Optional[date] = None
+        date_match = re.search(
+            r"<li>Last updated on (\w{3}) (\d{1,2}), (\d{4})</li>", self.html
+        )
+        if date_match:
+            month, day, year = date_match.groups()
+            self.last_update = datetime.strptime(
+                f"{month} {day:0>2} {year}", "%b %d %Y"
+            ).date()
+
+        self._video_regex = re.compile(r"href=\"(/watch\?v=[\w-]*)")
+
+    @staticmethod
+    def _find_load_more_url(req: str) -> Optional[str]:
+        """Given an html page or fragment, returns the "load more" url if found."""
+        match = re.search(
+            r"data-uix-load-more-href=\"(/browse_ajax\?" 'action_continuation=.*?)"',
+            req,
+        )
+        if match:
+            return f"https://www.youtube.com{match.group(1)}"
+
+        return None
+
+    @deprecated("This function will be removed in the future, please use .video_urls")
+    def parse_links(self) -> List[str]:  # pragma: no cover
+        """ Deprecated function for returning list of URLs
+
+        :return: List[str]
         """
+        return self.video_urls
 
-        url = self.construct_playlist_url()
-        req = request.get(url)
-
-        # split the page source by line and process each line
-        content = [x for x in req.split('\n') if 'pl-video-title-link' in x]
-        link_list = [x.split('href="', 1)[1].split('&', 1)[0] for x in content]
+    def _paginate(self, until_watch_id: Optional[str] = None) -> Iterable[List[str]]:
+        """Parse the video links from the page source, yields the /watch?v= part from video link
+        """
+        req = self.html
+        videos_urls = self._extract_videos(req)
+        if until_watch_id:
+            try:
+                trim_index = videos_urls.index(f"/watch?v={until_watch_id}")
+                yield videos_urls[:trim_index]
+                return
+            except ValueError:
+                pass
+        yield videos_urls
 
         # The above only returns 100 or fewer links
         # Simulating a browser request for the load more link
-        load_more_url = self._load_more_url(req)
-        while len(load_more_url):   # there is an url found
-            logger.debug('load more url: %s' % load_more_url)
+        load_more_url = self._find_load_more_url(req)
+
+        while load_more_url:  # there is an url found
+            logger.debug("load more url: %s", load_more_url)
             req = request.get(load_more_url)
             load_more = json.loads(req)
-            videos = re.findall(
-                r'href=\"(/watch\?v=[\w-]*)',
-                load_more['content_html'],
-            )
-            # remove duplicates
-            link_list.extend(list(OrderedDict.fromkeys(videos)))
-            load_more_url = self._load_more_url(
-                load_more['load_more_widget_html'],
+            try:
+                html = load_more["content_html"]
+            except KeyError:
+                logger.debug("Could not find content_html")
+                return
+            videos_urls = self._extract_videos(html)
+            if until_watch_id:
+                try:
+                    trim_index = videos_urls.index(f"/watch?v={until_watch_id}")
+                    yield videos_urls[:trim_index]
+                    return
+                except ValueError:
+                    pass
+            yield videos_urls
+
+            load_more_url = self._find_load_more_url(
+                load_more["load_more_widget_html"],
             )
 
-        return link_list
+        return
+
+    def _extract_videos(self, html: str) -> List[str]:
+        return uniqueify(self._video_regex.findall(html))
+
+    def trimmed(self, video_id: str) -> Iterable[str]:
+        """Retrieve a list of YouTube video URLs trimmed at the given video ID
+
+        i.e. if the playlist has video IDs 1,2,3,4 calling trimmed(3) returns [1,2]
+        :type video_id: str
+            video ID to trim the returned list of playlist URLs at
+        :rtype: List[str]
+        :returns:
+            List of video URLs from the playlist trimmed at the given ID
+        """
+        for page in self._paginate(until_watch_id=video_id):
+            yield from (self._video_url(watch_path) for watch_path in page)
+
+    @property  # type: ignore
+    @cache
+    def video_urls(self) -> List[str]:
+        """Complete links of all the videos in playlist
+
+        :rtype: List[str]
+        :returns: List of video URLs
+        """
+        return [
+            self._video_url(video) for page in list(self._paginate()) for video in page
+        ]
 
-    def populate_video_urls(self):
-        """Construct complete links of all the videos in playlist and
-        populate video_urls list
+    @property
+    def videos(self) -> Iterable[YouTube]:
+        """Yields YouTube objects of videos in this playlist
 
-        :return: urls -> string
+        :Yields: YouTube
         """
+        yield from (YouTube(url) for url in self.video_urls)
+
+    def __getitem__(self, i: Union[slice, int]) -> Union[str, List[str]]:
+        return self.video_urls[i]
+
+    def __len__(self) -> int:
+        return len(self.video_urls)
 
-        base_url = 'https://www.youtube.com'
-        link_list = self.parse_links()
+    def __repr__(self) -> str:
+        return f"{self.video_urls}"
 
-        for video_id in link_list:
-            complete_url = base_url + video_id
-            self.video_urls.append(complete_url)
+    @deprecated(
+        "This call is unnecessary, you can directly access .video_urls or .videos"
+    )
+    def populate_video_urls(self) -> List[str]:  # pragma: no cover
+        """Complete links of all the videos in playlist
 
-    def _path_num_prefix_generator(self, reverse=False):
+        :rtype: List[str]
+        :returns: List of video URLs
         """
-        This generator function generates number prefixes, for the items
-        in the playlist.
+        return self.video_urls
+
+    @deprecated("This function will be removed in the future.")
+    def _path_num_prefix_generator(self, reverse=False):  # pragma: no cover
+        """Generate number prefixes for the items in the playlist.
+
         If the number of digits required to name a file,is less than is
         required to name the last file,it prepends 0s.
         So if you have a playlist of 100 videos it will number them like:
@@ -116,17 +181,17 @@ class Playlist(object):
             start, stop, step = (1, len(self.video_urls) + 1, 1)
         return (str(i).zfill(digits) for i in range(start, stop, step))
 
+    @deprecated(
+        "This function will be removed in the future. Please iterate through .videos"
+    )
     def download_all(
         self,
-        download_path=None,
-        prefix_number=True,
-        reverse_numbering=False,
-    ):
-        """Download all the videos in the the playlist. Initially, download
-        resolution is 720p (or highest available), later more option
-        should be added to download resolution of choice
-
-        TODO(nficano): Add option to download resolution of user's choice
+        download_path: Optional[str] = None,
+        prefix_number: bool = True,
+        reverse_numbering: bool = False,
+        resolution: str = "720p",
+    ) -> None:  # pragma: no cover
+        """Download all the videos in the the playlist.
 
         :param download_path:
             (optional) Output path for the playlist If one is not
@@ -139,57 +204,49 @@ class Playlist(object):
         :type prefix_number: bool
         :param reverse_numbering:
             (optional) Lets you number playlists in reverse, since some
-            playlists are ordered newest -> oldests.
+            playlists are ordered newest -> oldest.
         :type reverse_numbering: bool
+        :param resolution:
+            Video resolution i.e. "720p", "480p", "360p", "240p", "144p"
+        :type resolution: str
         """
-
-        self.populate_video_urls()
-        logger.debug('total videos found: %d', len(self.video_urls))
-        logger.debug('starting download')
+        logger.debug("total videos found: %d", len(self.video_urls))
+        logger.debug("starting download")
 
         prefix_gen = self._path_num_prefix_generator(reverse_numbering)
 
         for link in self.video_urls:
-            try:
-                yt = YouTube(link)
-            except Exception as e:
-                logger.debug(e)
-                if not self.suppress_exception:
-                    raise e
-                else:
-                    logger.debug('Exception suppressed')
+            youtube = YouTube(link)
+            dl_stream = (
+                youtube.streams.get_by_resolution(resolution=resolution)
+                or youtube.streams.get_lowest_resolution()
+            )
+            assert dl_stream is not None
+
+            logger.debug("download path: %s", download_path)
+            if prefix_number:
+                prefix = next(prefix_gen)
+                logger.debug("file prefix is: %s", prefix)
+                dl_stream.download(download_path, filename_prefix=prefix)
             else:
-                # TODO: this should not be hardcoded to a single user's
-                # preference
-                dl_stream = yt.streams.filter(
-                    progressive=True, subtype='mp4',
-                ).order_by('resolution').desc().first()
-
-                logger.debug('download path: %s', download_path)
-                if prefix_number:
-                    prefix = next(prefix_gen)
-                    logger.debug('file prefix is: %s', prefix)
-                    dl_stream.download(download_path, filename_prefix=prefix)
-                else:
-                    dl_stream.download(download_path)
-                logger.debug('download complete')
-
-    def title(self):
-        """return playlist title (name)
+                dl_stream.download(download_path)
+            logger.debug("download complete")
+
+    @cache
+    def title(self) -> Optional[str]:
+        """Extract playlist title
+
+        :return: playlist title (name)
+        :rtype: Optional[str]
         """
-        try:
-            url = self.construct_playlist_url()
-            req = request.get(url)
-            open_tag = '<title>'
-            end_tag = '</title>'
-            matchresult = re.compile(open_tag + '(.+?)' + end_tag)
-            matchresult = matchresult.search(req).group()
-            matchresult = matchresult.replace(open_tag, '')
-            matchresult = matchresult.replace(end_tag, '')
-            matchresult = matchresult.replace('- YouTube', '')
-            matchresult = matchresult.strip()
-
-            return matchresult
-        except Exception as e:
-            logger.debug(e)
+        pattern = re.compile("<title>(.+?)</title>")
+        match = pattern.search(self.html)
+
+        if match is None:
             return None
+
+        return match.group(1).replace("- YouTube", "").strip()
+
+    @staticmethod
+    def _video_url(watch_path: str):
+        return f"https://www.youtube.com{watch_path}"

pytube/exceptions.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
+
 """Library specific exception definitions."""
-import sys
+from typing import Union, Pattern
 
 
 class PytubeError(Exception):
@@ -15,30 +16,47 @@ class PytubeError(Exception):
 class ExtractError(PytubeError):
     """Data extraction based exception."""
 
-    def __init__(self, msg, video_id=None):
-        """Construct an instance of a :class:`ExtractError <ExtractError>`.
 
-        :param str msg:
-            User defined error message.
+class RegexMatchError(ExtractError):
+    """Regex pattern did not return any matches."""
+
+    def __init__(self, caller: str, pattern: Union[str, Pattern]):
+        """
+        :param str caller:
+            Calling function
+        :param str pattern:
+            Pattern that failed to match
+        """
+        super().__init__(f"{caller}: could not find match for {pattern}")
+        self.caller = caller
+        self.pattern = pattern
+
+
+class LiveStreamError(ExtractError):
+    """Video is a live stream."""
+
+    def __init__(self, video_id: str):
+        """
         :param str video_id:
             A YouTube video identifier.
         """
-        if video_id is not None:
-            msg = '{video_id}: {msg}'.format(video_id=video_id, msg=msg)
-
-        super(ExtractError, self).__init__(msg)
+        super().__init__(f"{video_id} is streaming live and cannot be loaded")
 
-        self.exc_info = sys.exc_info()
         self.video_id = video_id
 
 
-class RegexMatchError(ExtractError):
-    """Regex pattern did not return any matches."""
+class VideoUnavailable(PytubeError):
+    """Video is unavailable."""
 
+    def __init__(self, video_id: str):
+        """
+        :param str video_id:
+            A YouTube video identifier.
+        """
+        super().__init__(f"{video_id} is unavailable")
 
-class LiveStreamError(ExtractError):
-    """Video is a live stream."""
+        self.video_id = video_id
 
 
-class VideoUnavailable(PytubeError):
-    """Video is unavailable."""
+class HTMLParseError(PytubeError):
+    """HTML could not be parsed"""

pytube/extract.py

@@ -1,43 +1,52 @@
 # -*- coding: utf-8 -*-
 """This module contains all non-cipher related data extraction logic."""
 import json
+import logging
+import re
 from collections import OrderedDict
+from html.parser import HTMLParser
+from typing import Any, Optional, Tuple, List, Dict
+from urllib.parse import quote, parse_qs, unquote, parse_qsl
+from urllib.parse import urlencode
 
-from pytube.compat import HTMLParser
-from pytube.compat import quote
-from pytube.compat import urlencode
-from pytube.exceptions import RegexMatchError
+from pytube.cipher import Cipher
+from pytube.exceptions import RegexMatchError, HTMLParseError, LiveStreamError
 from pytube.helpers import regex_search
 
+logger = logging.getLogger(__name__)
+
 
 class PytubeHTMLParser(HTMLParser):
     in_vid_descr = False
     in_vid_descr_br = False
-    vid_descr = ''
+    vid_descr = ""
 
     def handle_starttag(self, tag, attrs):
-        if tag == 'p':
+        if tag == "p":
             for attr in attrs:
-                if attr[0] == 'id' and attr[1] == 'eow-description':
+                if attr[0] == "id" and attr[1] == "eow-description":
                     self.in_vid_descr = True
 
     def handle_endtag(self, tag):
-        if self.in_vid_descr and tag == 'p':
+        if self.in_vid_descr and tag == "p":
             self.in_vid_descr = False
 
     def handle_startendtag(self, tag, attrs):
-        if self.in_vid_descr and tag == 'br':
+        if self.in_vid_descr and tag == "br":
             self.in_vid_descr_br = True
 
     def handle_data(self, data):
         if self.in_vid_descr_br:
-            self.vid_descr += '\n{}'.format(data)
+            self.vid_descr += f"\n{data}"
             self.in_vid_descr_br = False
         elif self.in_vid_descr:
             self.vid_descr += data
 
+    def error(self, message):
+        raise HTMLParseError(message)
+
 
-def is_age_restricted(watch_html):
+def is_age_restricted(watch_html: str) -> bool:
     """Check if content is age restricted.
 
     :param str watch_html:
@@ -47,13 +56,13 @@ def is_age_restricted(watch_html):
         Whether or not the content is age restricted.
     """
     try:
-        regex_search(r'og:restrictions:age', watch_html, group=0)
+        regex_search(r"og:restrictions:age", watch_html, group=0)
     except RegexMatchError:
         return False
     return True
 
 
-def video_id(url):
+def video_id(url: str) -> str:
     """Extract the ``video_id`` from a YouTube url.
 
     This function supports the following patterns:
@@ -68,88 +77,74 @@ def video_id(url):
     :returns:
         YouTube video id.
     """
-    return regex_search(r'(?:v=|\/)([0-9A-Za-z_-]{11}).*', url, group=1)
+    return regex_search(r"(?:v=|\/)([0-9A-Za-z_-]{11}).*", url, group=1)
 
 
-def watch_url(video_id):
-    """Construct a sanitized YouTube watch url, given a video id.
+def video_info_url(video_id: str, watch_url: str) -> str:
+    """Construct the video_info url.
 
     :param str video_id:
         A YouTube video identifier.
+    :param str watch_url:
+        A YouTube watch url.
     :rtype: str
     :returns:
-        Sanitized YouTube watch url.
+        :samp:`https://youtube.com/get_video_info` with necessary GET
+        parameters.
     """
-    return 'https://youtube.com/watch?v=' + video_id
-
-
-def embed_url(video_id):
-    return 'https://www.youtube.com/embed/{}'.format(video_id)
-
-
-def eurl(video_id):
-    return 'https://youtube.googleapis.com/v/{}'.format(video_id)
-
-
-def video_info_url(
-    video_id, watch_url, watch_html, embed_html,
-    age_restricted,
-):
+    params = OrderedDict(
+        [
+            ("video_id", video_id),
+            ("el", "$el"),
+            ("ps", "default"),
+            ("eurl", quote(watch_url)),
+            ("hl", "en_US"),
+        ]
+    )
+    return _video_info_url(params)
+
+
+def video_info_url_age_restricted(video_id: str, embed_html: str) -> str:
     """Construct the video_info url.
 
     :param str video_id:
         A YouTube video identifier.
-    :param str watch_url:
-        A YouTube watch url.
-    :param str watch_html:
-        The html contents of the watch page.
     :param str embed_html:
         The html contents of the embed page (for age restricted videos).
-    :param bool age_restricted:
-        Is video age restricted.
     :rtype: str
     :returns:
         :samp:`https://youtube.com/get_video_info` with necessary GET
         parameters.
     """
-    if age_restricted:
+    try:
         sts = regex_search(r'"sts"\s*:\s*(\d+)', embed_html, group=1)
-        # Here we use ``OrderedDict`` so that the output is consistent between
-        # Python 2.7+.
-        params = OrderedDict([
-            ('video_id', video_id),
-            ('eurl', eurl(video_id)),
-            ('sts', sts),
-        ])
-    else:
-        params = OrderedDict([
-            ('video_id', video_id),
-            ('el', '$el'),
-            ('ps', 'default'),
-            ('eurl', quote(watch_url)),
-            ('hl', 'en_US'),
-        ])
-    return 'https://youtube.com/get_video_info?' + urlencode(params)
+    except RegexMatchError:
+        sts = ""
+    # Here we use ``OrderedDict`` so that the output is consistent between
+    # Python 2.7+.
+    eurl = f"https://youtube.googleapis.com/v/{video_id}"
+    params = OrderedDict([("video_id", video_id), ("eurl", eurl), ("sts", sts),])
+    return _video_info_url(params)
 
 
-def js_url(html, age_restricted=False):
+def _video_info_url(params: OrderedDict) -> str:
+    return "https://youtube.com/get_video_info?" + urlencode(params)
+
+
+def js_url(html: str) -> str:
     """Get the base JavaScript url.
 
     Construct the base JavaScript url, which contains the decipher
     "transforms".
 
-    :param str watch_html:
+    :param str html:
         The html contents of the watch page.
-    :param bool age_restricted:
-        Is video age restricted.
-
     """
-    ytplayer_config = get_ytplayer_config(html, age_restricted)
-    base_js = ytplayer_config['assets']['js']
-    return 'https://youtube.com' + base_js
+    base_js = get_ytplayer_config(html)["assets"]["js"]
+    return "https://youtube.com" + base_js
 
 
-def mime_type_codec(mime_type_codec):
+def mime_type_codec(mime_type_codec: str) -> Tuple[str, List[str]]:
     """Parse the type data.
 
     Breaks up the data in the ``type`` key of the manifest, which contains the
@@ -158,8 +153,7 @@ def mime_type_codec(mime_type_codec):
 
     **Example**:
 
-    >>> mime_type_codec('audio/webm; codecs="opus"')
-    ('audio/webm', ['opus'])
+    mime_type_codec('audio/webm; codecs="opus"') -> ('audio/webm', ['opus'])
 
     :param str mime_type_codec:
         String containing mime type and codecs.
@@ -168,35 +162,160 @@ def mime_type_codec(mime_type_codec):
         The mime type and a list of codecs.
 
     """
-    pattern = r'(\w+\/\w+)\;\scodecs=\"([a-zA-Z-0-9.,\s]*)\"'
-    mime_type, codecs = regex_search(pattern, mime_type_codec, groups=True)
-    return mime_type, [c.strip() for c in codecs.split(',')]
+    pattern = r"(\w+\/\w+)\;\scodecs=\"([a-zA-Z-0-9.,\s]*)\""
+    regex = re.compile(pattern)
+    results = regex.search(mime_type_codec)
+    if not results:
+        raise RegexMatchError(caller="mime_type_codec", pattern=pattern)
+    mime_type, codecs = results.groups()
+    return mime_type, [c.strip() for c in codecs.split(",")]
 
 
-def get_ytplayer_config(html, age_restricted=False):
+def get_ytplayer_config(html: str) -> Any:
     """Get the YouTube player configuration data from the watch html.
 
     Extract the ``ytplayer_config``, which is json data embedded within the
     watch html and serves as the primary source of obtaining the stream
     manifest data.
 
-    :param str watch_html:
+    :param str html:
         The html contents of the watch page.
-    :param bool age_restricted:
-        Is video age restricted.
     :rtype: str
     :returns:
         Substring of the html containing the encoded manifest data.
     """
-    if age_restricted:
-        pattern = r";yt\.setConfig\(\{'PLAYER_CONFIG':\s*({.*})(,'EXPERIMENT_FLAGS'|;)"  # noqa: E501
-    else:
-        pattern = r';ytplayer\.config\s*=\s*({.*?});'
-    yt_player_config = regex_search(pattern, html, group=1)
-    return json.loads(yt_player_config)
-
-
-def get_vid_descr(html):
+    config_patterns = [
+        r";ytplayer\.config\s*=\s*({.*?});",
+        r";ytplayer\.config\s*=\s*({.+?});ytplayer",
+        r";yt\.setConfig\(\{'PLAYER_CONFIG':\s*({.*})}\);",
+        r";yt\.setConfig\(\{'PLAYER_CONFIG':\s*({.*})(,'EXPERIMENT_FLAGS'|;)",  # noqa: E501
+    ]
+    logger.debug("finding initial function name")
+    for pattern in config_patterns:
+        regex = re.compile(pattern)
+        function_match = regex.search(html)
+        if function_match:
+            logger.debug("finished regex search, matched: %s", pattern)
+            yt_player_config = function_match.group(1)
+            return json.loads(yt_player_config)
+
+    raise RegexMatchError(caller="get_ytplayer_config", pattern="config_patterns")
+
+
+def _get_vid_descr(html: Optional[str]) -> str:
     html_parser = PytubeHTMLParser()
-    html_parser.feed(html)
+    if html:
+        html_parser.feed(html)
     return html_parser.vid_descr
+
+
+def apply_signature(config_args: Dict, fmt: str, js: str) -> None:
+    """Apply the decrypted signature to the stream manifest.
+
+    :param dict config_args:
+        Details of the media streams available.
+    :param str fmt:
+        Key in stream manifests (``ytplayer_config``) containing progressive
+        download or adaptive streams (e.g.: ``url_encoded_fmt_stream_map`` or
+        ``adaptive_fmts``).
+    :param str js:
+        The contents of the base.js asset file.
+
+    """
+    cipher = Cipher(js=js)
+    stream_manifest = config_args[fmt]
+
+    for i, stream in enumerate(stream_manifest):
+        try:
+            url: str = stream["url"]
+        except KeyError:
+            live_stream = (
+                json.loads(config_args["player_response"])
+                .get("playabilityStatus", {},)
+                .get("liveStreamability")
+            )
+            if live_stream:
+                raise LiveStreamError("UNKNOWN")
+        # 403 Forbidden fix.
+        if "signature" in url or (
+            "s" not in stream and ("&sig=" in url or "&lsig=" in url)
+        ):
+            # For certain videos, YouTube will just provide them pre-signed, in
+            # which case there's no real magic to download them and we can skip
+            # the whole signature descrambling entirely.
+            logger.debug("signature found, skip decipher")
+            continue
+
+        signature = cipher.get_signature(ciphered_signature=stream["s"])
+
+        logger.debug("finished descrambling signature for itag=%s", stream["itag"])
+        # 403 forbidden fix
+        stream_manifest[i]["url"] = url + "&sig=" + signature
+
+
+def apply_descrambler(stream_data: Dict, key: str) -> None:
+    """Apply various in-place transforms to YouTube's media stream data.
+
+    Creates a ``list`` of dictionaries by string splitting on commas, then
+    taking each list item, parsing it as a query string, converting it to a
+    ``dict`` and unquoting the value.
+
+    :param dict stream_data:
+        Dictionary containing query string encoded values.
+    :param str key:
+        Name of the key in dictionary.
+
+    **Example**:
+
+    >>> d = {'foo': 'bar=1&var=test,em=5&t=url%20encoded'}
+    >>> apply_descrambler(d, 'foo')
+    >>> print(d)
+    {'foo': [{'bar': '1', 'var': 'test'}, {'em': '5', 't': 'url encoded'}]}
+
+    """
+    otf_type = "FORMAT_STREAM_TYPE_OTF"
+
+    if key == "url_encoded_fmt_stream_map" and not stream_data.get(
+        "url_encoded_fmt_stream_map"
+    ):
+        formats = json.loads(stream_data["player_response"])["streamingData"]["formats"]
+        formats.extend(
+            json.loads(stream_data["player_response"])["streamingData"][
+                "adaptiveFormats"
+            ]
+        )
+        try:
+            stream_data[key] = [
+                {
+                    "url": format_item["url"],
+                    "type": format_item["mimeType"],
+                    "quality": format_item["quality"],
+                    "itag": format_item["itag"],
+                    "bitrate": format_item.get("bitrate"),
+                    "is_otf": (format_item.get("type") == otf_type),
+                }
+                for format_item in formats
+            ]
+        except KeyError:
+            cipher_url = [
+                parse_qs(formats[i]["cipher"]) for i, data in enumerate(formats)
+            ]
+            stream_data[key] = [
+                {
+                    "url": cipher_url[i]["url"][0],
+                    "s": cipher_url[i]["s"][0],
+                    "type": format_item["mimeType"],
+                    "quality": format_item["quality"],
+                    "itag": format_item["itag"],
+                    "bitrate": format_item.get("bitrate"),
+                    "is_otf": (format_item.get("type") == otf_type),
+                }
+                for i, format_item in enumerate(formats)
+            ]
+    else:
+        stream_data[key] = [
+            {k: unquote(v) for k, v in parse_qsl(i)}
+            for i in stream_data[key].split(",")
+        ]
+
+    logger.debug("applying descrambler")

pytube/helpers.py

@@ -1,107 +1,44 @@
 # -*- coding: utf-8 -*-
-"""Various helper functions implemented by pytube."""
-from __future__ import absolute_import
 
+"""Various helper functions implemented by pytube."""
+import functools
 import logging
-import pprint
+import os
 import re
+import warnings
+from typing import TypeVar, Callable, Optional, Dict, List, Any
+from urllib import request
 
-from pytube.compat import unicode
 from pytube.exceptions import RegexMatchError
 
-
 logger = logging.getLogger(__name__)
 
 
-def regex_search(pattern, string, groups=False, group=None, flags=0):
+def regex_search(pattern: str, string: str, group: int) -> str:
     """Shortcut method to search a string for a given pattern.
 
     :param str pattern:
         A regular expression pattern.
     :param str string:
         A target string to search.
-    :param bool groups:
-        Should the return value be ``.groups()``.
     :param int group:
         Index of group to return.
-    :param int flags:
-        Expression behavior modifiers.
     :rtype:
         str or tuple
     :returns:
         Substring pattern matches.
     """
-    if type(pattern) == list:
-        for p in pattern:
-            regex = re.compile(p, flags)
-            results = regex.search(string)
-            if not results:
-                raise RegexMatchError(
-                    'regex pattern ({pattern}) had zero matches'
-                    .format(pattern=p),
-                )
-            else:
-                logger.debug(
-                    'finished regex search: %s',
-                    pprint.pformat(
-                        {
-                            'pattern': p,
-                            'results': results.group(0),
-                        }, indent=2,
-                    ),
-                )
-                if groups:
-                    return results.groups()
-                elif group is not None:
-                    return results.group(group)
-                else:
-                    return results
-    else:
-        regex = re.compile(pattern, flags)
-        results = regex.search(string)
-        if not results:
-            raise RegexMatchError(
-                'regex pattern ({pattern}) had zero matches'
-                .format(pattern=pattern),
-            )
-        else:
-            logger.debug(
-                'finished regex search: %s',
-                pprint.pformat(
-                    {
-                        'pattern': pattern,
-                        'results': results.group(0),
-                    }, indent=2,
-                ),
-            )
-            if groups:
-                return results.groups()
-            elif group is not None:
-                return results.group(group)
-            else:
-                return results
-
-
-def apply_mixin(dct, key, func, *args, **kwargs):
-    r"""Apply in-place data mutation to a dictionary.
-
-    :param dict dct:
-        Dictionary to apply mixin function to.
-    :param str key:
-        Key within dictionary to apply mixin function to.
-    :param callable func:
-        Transform function to apply to ``dct[key]``.
-    :param \*args:
-        (optional) positional arguments that ``func`` takes.
-    :param \*\*kwargs:
-        (optional) keyword arguments that ``func`` takes.
-    :rtype:
-        None
-    """
-    dct[key] = func(dct[key], *args, **kwargs)
+    regex = re.compile(pattern)
+    results = regex.search(string)
+    if not results:
+        raise RegexMatchError(caller="regex_search", pattern=pattern)
+
+    logger.debug("matched regex search: %s", pattern)
+
+    return results.group(group)
 
 
-def safe_filename(s, max_length=255):
+def safe_filename(s: str, max_length: int = 255) -> str:
     """Sanitize a string making it safe to use as a filename.
 
     This function was based off the limitations outlined here:
@@ -116,12 +53,120 @@ def safe_filename(s, max_length=255):
         A sanitized string.
     """
     # Characters in range 0-31 (0x00-0x1F) are not allowed in ntfs filenames.
-    ntfs_chrs = [chr(i) for i in range(0, 31)]
-    chrs = [
-        '\"', '\#', '\$', '\%', '\'', '\*', '\,', '\.', '\/', '\:', '"',
-        '\;', '\<', '\>', '\?', '\\', '\^', '\|', '\~', '\\\\',
+    ntfs_characters = [chr(i) for i in range(0, 31)]
+    characters = [
+        r'"',
+        r"\#",
+        r"\$",
+        r"\%",
+        r"'",
+        r"\*",
+        r"\,",
+        r"\.",
+        r"\/",
+        r"\:",
+        r'"',
+        r"\;",
+        r"\<",
+        r"\>",
+        r"\?",
+        r"\\",
+        r"\^",
+        r"\|",
+        r"\~",
+        r"\\\\",
     ]
-    pattern = '|'.join(ntfs_chrs + chrs)
+    pattern = "|".join(ntfs_characters + characters)
     regex = re.compile(pattern, re.UNICODE)
-    filename = regex.sub('', s)
-    return unicode(filename[:max_length].rsplit(' ', 0)[0])
+    filename = regex.sub("", s)
+    return filename[:max_length].rsplit(" ", 0)[0]
+
+
+def setup_logger(level: int = logging.ERROR):
+    """Create a configured instance of logger.
+
+    :param int level:
+        Describe the severity level of the logs to handle.
+    """
+    fmt = "[%(asctime)s] %(levelname)s in %(module)s: %(message)s"
+    date_fmt = "%H:%M:%S"
+    formatter = logging.Formatter(fmt, datefmt=date_fmt)
+
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+
+    # https://github.com/nficano/pytube/issues/163
+    logger = logging.getLogger("pytube")
+    logger.addHandler(handler)
+    logger.setLevel(level)
+
+
+GenericType = TypeVar("GenericType")
+
+
+def cache(func: Callable[..., GenericType]) -> GenericType:
+    """ mypy compatible annotation wrapper for lru_cache"""
+    return functools.lru_cache()(func)  # type: ignore
+
+
+def deprecated(reason: str) -> Callable:
+    """
+    This is a decorator which can be used to mark functions
+    as deprecated. It will result in a warning being emitted
+    when the function is used.
+    """
+
+    def decorator(func1):
+        message = "Call to deprecated function {name} ({reason})."
+
+        @functools.wraps(func1)
+        def new_func1(*args, **kwargs):
+            warnings.simplefilter("always", DeprecationWarning)
+            warnings.warn(
+                message.format(name=func1.__name__, reason=reason),
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+            warnings.simplefilter("default", DeprecationWarning)
+            return func1(*args, **kwargs)
+
+        return new_func1
+
+    return decorator
+
+
+def target_directory(output_path: Optional[str] = None) -> str:
+    """
+    Function for determining target directory of a download.
+    Returns an absolute path (if relative one given) or the current
+    path (if none given). Makes directory if it does not exist.
+
+    :type output_path: str
+        :rtype: str
+    :returns:
+        An absolute directory path as a string.
+    """
+    if output_path:
+        if not os.path.isabs(output_path):
+            output_path = os.path.join(os.getcwd(), output_path)
+    else:
+        output_path = os.getcwd()
+    os.makedirs(output_path, exist_ok=True)
+    return output_path
+
+
+def install_proxy(proxy_handler: Dict[str, str]) -> None:
+    proxy_support = request.ProxyHandler(proxy_handler)
+    opener = request.build_opener(proxy_support)
+    request.install_opener(opener)
+
+
+def uniqueify(duped_list: List) -> List:
+    seen: Dict[Any, bool] = {}
+    result = []
+    for item in duped_list:
+        if item in seen:
+            continue
+        seen[item] = True
+        result.append(item)
+    return result

pytube/itags.py

@@ -1,92 +1,91 @@
 # -*- coding: utf-8 -*-
 """This module contains a lookup table of YouTube's itag values."""
+from typing import Dict
 
 ITAGS = {
-    5: ('240p', '64kbps'),
-    6: ('270p', '64kbps'),
-    13: ('144p', None),
-    17: ('144p', '24kbps'),
-    18: ('360p', '96kbps'),
-    22: ('720p', '192kbps'),
-    34: ('360p', '128kbps'),
-    35: ('480p', '128kbps'),
-    36: ('240p', None),
-    37: ('1080p', '192kbps'),
-    38: ('3072p', '192kbps'),
-    43: ('360p', '128kbps'),
-    44: ('480p', '128kbps'),
-    45: ('720p', '192kbps'),
-    46: ('1080p', '192kbps'),
-    59: ('480p', '128kbps'),
-    78: ('480p', '128kbps'),
-    82: ('360p', '128kbps'),
-    83: ('480p', '128kbps'),
-    84: ('720p', '192kbps'),
-    85: ('1080p', '192kbps'),
-    91: ('144p', '48kbps'),
-    92: ('240p', '48kbps'),
-    93: ('360p', '128kbps'),
-    94: ('480p', '128kbps'),
-    95: ('720p', '256kbps'),
-    96: ('1080p', '256kbps'),
-    100: ('360p', '128kbps'),
-    101: ('480p', '192kbps'),
-    102: ('720p', '192kbps'),
-    132: ('240p', '48kbps'),
-    151: ('720p', '24kbps'),
-
+    5: ("240p", "64kbps"),
+    6: ("270p", "64kbps"),
+    13: ("144p", None),
+    17: ("144p", "24kbps"),
+    18: ("360p", "96kbps"),
+    22: ("720p", "192kbps"),
+    34: ("360p", "128kbps"),
+    35: ("480p", "128kbps"),
+    36: ("240p", None),
+    37: ("1080p", "192kbps"),
+    38: ("3072p", "192kbps"),
+    43: ("360p", "128kbps"),
+    44: ("480p", "128kbps"),
+    45: ("720p", "192kbps"),
+    46: ("1080p", "192kbps"),
+    59: ("480p", "128kbps"),
+    78: ("480p", "128kbps"),
+    82: ("360p", "128kbps"),
+    83: ("480p", "128kbps"),
+    84: ("720p", "192kbps"),
+    85: ("1080p", "192kbps"),
+    91: ("144p", "48kbps"),
+    92: ("240p", "48kbps"),
+    93: ("360p", "128kbps"),
+    94: ("480p", "128kbps"),
+    95: ("720p", "256kbps"),
+    96: ("1080p", "256kbps"),
+    100: ("360p", "128kbps"),
+    101: ("480p", "192kbps"),
+    102: ("720p", "192kbps"),
+    132: ("240p", "48kbps"),
+    151: ("720p", "24kbps"),
     # DASH Video
-    133: ('240p', None),
-    134: ('360p', None),
-    135: ('480p', None),
-    136: ('720p', None),
-    137: ('1080p', None),
-    138: ('2160p', None),
-    160: ('144p', None),
-    167: ('360p', None),
-    168: ('480p', None),
-    169: ('720p', None),
-    170: ('1080p', None),
-    212: ('480p', None),
-    218: ('480p', None),
-    219: ('480p', None),
-    242: ('240p', None),
-    243: ('360p', None),
-    244: ('480p', None),
-    245: ('480p', None),
-    246: ('480p', None),
-    247: ('720p', None),
-    248: ('1080p', None),
-    264: ('1440p', None),
-    266: ('2160p', None),
-    271: ('1440p', None),
-    272: ('2160p', None),
-    278: ('144p', None),
-    298: ('720p', None),
-    299: ('1080p', None),
-    302: ('720p', None),
-    303: ('1080p', None),
-    308: ('1440p', None),
-    313: ('2160p', None),
-    315: ('2160p', None),
-    330: ('144p', None),
-    331: ('240p', None),
-    332: ('360p', None),
-    333: ('480p', None),
-    334: ('720p', None),
-    335: ('1080p', None),
-    336: ('1440p', None),
-    337: ('2160p', None),
-
+    133: ("240p", None),
+    134: ("360p", None),
+    135: ("480p", None),
+    136: ("720p", None),
+    137: ("1080p", None),
+    138: ("2160p", None),
+    160: ("144p", None),
+    167: ("360p", None),
+    168: ("480p", None),
+    169: ("720p", None),
+    170: ("1080p", None),
+    212: ("480p", None),
+    218: ("480p", None),
+    219: ("480p", None),
+    242: ("240p", None),
+    243: ("360p", None),
+    244: ("480p", None),
+    245: ("480p", None),
+    246: ("480p", None),
+    247: ("720p", None),
+    248: ("1080p", None),
+    264: ("1440p", None),
+    266: ("2160p", None),
+    271: ("1440p", None),
+    272: ("2160p", None),
+    278: ("144p", None),
+    298: ("720p", None),
+    299: ("1080p", None),
+    302: ("720p", None),
+    303: ("1080p", None),
+    308: ("1440p", None),
+    313: ("2160p", None),
+    315: ("2160p", None),
+    330: ("144p", None),
+    331: ("240p", None),
+    332: ("360p", None),
+    333: ("480p", None),
+    334: ("720p", None),
+    335: ("1080p", None),
+    336: ("1440p", None),
+    337: ("2160p", None),
     # DASH Audio
-    139: (None, '48kbps'),
-    140: (None, '128kbps'),
-    141: (None, '256kbps'),
-    171: (None, '128kbps'),
-    172: (None, '256kbps'),
-    249: (None, '50kbps'),
-    250: (None, '70kbps'),
-    251: (None, '160kbps'),
+    139: (None, "48kbps"),
+    140: (None, "128kbps"),
+    141: (None, "256kbps"),
+    171: (None, "128kbps"),
+    172: (None, "256kbps"),
+    249: (None, "50kbps"),
+    250: (None, "70kbps"),
+    251: (None, "160kbps"),
     256: (None, None),
     258: (None, None),
     325: (None, None),
@@ -97,10 +96,36 @@ HDR = [330, 331, 332, 333, 334, 335, 336, 337]
 _60FPS = [298, 299, 302, 303, 308, 315] + HDR
 _3D = [82, 83, 84, 85, 100, 101, 102]
 LIVE = [91, 92, 93, 94, 95, 96, 132, 151]
+DASH_MP4_VIDEO = [133, 134, 135, 136, 137, 138, 160, 212, 264, 266, 298, 299]
+DASH_MP4_AUDIO = [139, 140, 141, 256, 258, 325, 328]
+DASH_WEBM_VIDEO = [
+    167,
+    168,
+    169,
+    170,
+    218,
+    219,
+    278,
+    242,
+    243,
+    244,
+    245,
+    246,
+    247,
+    248,
+    271,
+    272,
+    302,
+    303,
+    308,
+    313,
+    315,
+]
+DASH_WEBM_AUDIO = [171, 172, 249, 250, 251]
 
 
-def get_format_profile(itag):
-    """Get dditional format information for a given itag.
+def get_format_profile(itag: int) -> Dict:
+    """Get additional format information for a given itag.
 
     :param str itag:
         YouTube format identifier code.
@@ -111,10 +136,14 @@ def get_format_profile(itag):
     else:
         res, bitrate = None, None
     return {
-        'resolution': res,
-        'abr': bitrate,
-        'is_live': itag in LIVE,
-        'is_3d': itag in _3D,
-        'is_hdr': itag in HDR,
-        'fps': 60 if itag in _60FPS else 30,
+        "resolution": res,
+        "abr": bitrate,
+        "is_live": itag in LIVE,
+        "is_3d": itag in _3D,
+        "is_hdr": itag in HDR,
+        "fps": 60 if itag in _60FPS else 30,
+        "is_dash": itag in DASH_MP4_VIDEO
+        or itag in DASH_MP4_AUDIO
+        or itag in DASH_WEBM_VIDEO
+        or itag in DASH_WEBM_AUDIO,
     }

pytube/logging.py

@@ -1,25 +0,0 @@
-# -*- coding: utf-8 -*-
-"""This module implements a log factory."""
-from __future__ import absolute_import
-
-import logging
-
-
-def create_logger(level=logging.ERROR):
-    """Create a configured instance of logger.
-
-    :param int level:
-        Describe the severity level of the logs to handle.
-    """
-    fmt = '[%(asctime)s] %(levelname)s in %(module)s: %(message)s'
-    date_fmt = '%H:%M:%S'
-    formatter = logging.Formatter(fmt, datefmt=date_fmt)
-
-    handler = logging.StreamHandler()
-    handler.setFormatter(formatter)
-
-    # https://github.com/nficano/pytube/issues/163
-    logger = logging.getLogger('pytube')
-    logger.addHandler(handler)
-    logger.setLevel(level)
-    return logger

pytube/mixins.py

@@ -1,101 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Applies in-place data mutations."""
-from __future__ import absolute_import
-
-import json
-import logging
-import pprint
-
-from pytube import cipher
-from pytube.compat import parse_qsl
-from pytube.compat import unquote
-from pytube.exceptions import LiveStreamError
-
-
-logger = logging.getLogger(__name__)
-
-
-def apply_signature(config_args, fmt, js):
-    """Apply the decrypted signature to the stream manifest.
-
-    :param dict config_args:
-        Details of the media streams available.
-    :param str fmt:
-        Key in stream manifests (``ytplayer_config``) containing progressive
-        download or adaptive streams (e.g.: ``url_encoded_fmt_stream_map`` or
-        ``adaptive_fmts``).
-    :param str js:
-        The contents of the base.js asset file.
-
-    """
-    stream_manifest = config_args[fmt]
-    live_stream = json.loads(config_args['player_response']).get(
-        'playabilityStatus', {},
-    ).get('liveStreamability')
-    for i, stream in enumerate(stream_manifest):
-        if 'url' in stream:
-            url = stream['url']
-        elif live_stream:
-            raise LiveStreamError('Video is currently being streamed live')
-        # 403 Forbidden fix.
-        if (
-            'signature' in url or (
-                's' not in stream and (
-                    '&sig=' in url or '&lsig=' in url
-                )
-            )
-        ):
-            # For certain videos, YouTube will just provide them pre-signed, in
-            # which case there's no real magic to download them and we can skip
-            # the whole signature descrambling entirely.
-            logger.debug('signature found, skip decipher')
-            continue
-
-        if js is not None:
-            signature = cipher.get_signature(js, stream['s'])
-        else:
-            # signature not present in url (line 33), need js to descramble
-            # TypeError caught in __main__
-            raise TypeError('JS is None')
-
-        logger.debug(
-            'finished descrambling signature for itag=%s\n%s',
-            stream['itag'], pprint.pformat(
-                {
-                    's': stream['s'],
-                    'signature': signature,
-                }, indent=2,
-            ),
-        )
-        # 403 forbidden fix
-        stream_manifest[i]['url'] = url + '&sig=' + signature
-
-
-def apply_descrambler(stream_data, key):
-    """Apply various in-place transforms to YouTube's media stream data.
-
-    Creates a ``list`` of dictionaries by string splitting on commas, then
-    taking each list item, parsing it as a query string, converting it to a
-    ``dict`` and unquoting the value.
-
-    :param dict dct:
-        Dictionary containing query string encoded values.
-    :param str key:
-        Name of the key in dictionary.
-
-    **Example**:
-
-    >>> d = {'foo': 'bar=1&var=test,em=5&t=url%20encoded'}
-    >>> apply_descrambler(d, 'foo')
-    >>> print(d)
-    {'foo': [{'bar': '1', 'var': 'test'}, {'em': '5', 't': 'url encoded'}]}
-
-    """
-    stream_data[key] = [
-        {k: unquote(v) for k, v in parse_qsl(i)}
-        for i in stream_data[key].split(',')
-    ]
-    logger.debug(
-        'applying descrambler\n%s',
-        pprint.pformat(stream_data[key], indent=2),
-    )

pytube/monostate.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+
+from typing import Any, Optional
+from typing_extensions import Protocol
+
+
+class OnProgress(Protocol):
+    def __call__(self, stream: Any, chunk: bytes, bytes_remaining: int) -> None:
+        """On download progress callback function.
+
+        :param stream:
+            An instance of :class:`Stream <Stream>` being downloaded.
+        :type stream:
+            :py:class:`pytube.Stream`
+        :param bytes chunk:
+            Segment of media file binary data, not yet written to disk.
+        :param int bytes_remaining:
+            How many bytes have been downloaded.
+
+        """
+        ...
+
+
+class OnComplete(Protocol):
+    def __call__(self, stream: Any, file_path: Optional[str]) -> None:
+        """On download complete handler function.
+
+        :param stream:
+            An instance of :class:`Stream <Stream>` being downloaded.
+        :type stream:
+            :py:class:`pytube.Stream`
+        :param file_path:
+            The file handle where the media is being written to.
+        :type file_path: str
+
+        :rtype: None
+        """
+        ...
+
+
+class Monostate:
+    def __init__(
+        self,
+        on_progress: Optional[OnProgress],
+        on_complete: Optional[OnComplete],
+        title: Optional[str] = None,
+        duration: Optional[int] = None,
+    ):
+        self.on_progress = on_progress
+        self.on_complete = on_complete
+        self.title = title
+        self.duration = duration

pytube/query.py

@@ -1,8 +1,14 @@
 # -*- coding: utf-8 -*-
+
 """This module provides a query interface for media streams and captions."""
+from typing import Callable, List, Optional, Union
+from collections.abc import Mapping, Sequence
+
+from pytube import Stream, Caption
+from pytube.helpers import deprecated
 
 
-class StreamQuery:
+class StreamQuery(Sequence):
     """Interface for querying the available media streams."""
 
     def __init__(self, fmt_streams):
@@ -15,12 +21,24 @@ class StreamQuery:
         self.itag_index = {int(s.itag): s for s in fmt_streams}
 
     def filter(
-            self, fps=None, res=None, resolution=None, mime_type=None,
-            type=None, subtype=None, file_extension=None, abr=None,
-            bitrate=None, video_codec=None, audio_codec=None,
-            only_audio=None, only_video=None,
-            progressive=None, adaptive=None,
-            custom_filter_functions=None,
+        self,
+        fps=None,
+        res=None,
+        resolution=None,
+        mime_type=None,
+        type=None,
+        subtype=None,
+        file_extension=None,
+        abr=None,
+        bitrate=None,
+        video_codec=None,
+        audio_codec=None,
+        only_audio=None,
+        only_video=None,
+        progressive=None,
+        adaptive=None,
+        is_dash=None,
+        custom_filter_functions=None,
     ):
         """Apply the given filtering criterion.
 
@@ -89,6 +107,9 @@ class StreamQuery:
             Excludes progressive streams (audio and video are on separate
             tracks).
 
+        :param bool is_dash:
+            Include/exclude dash streams.
+
         :param bool only_audio:
             Excludes streams with video tracks.
 
@@ -129,16 +150,12 @@ class StreamQuery:
 
         if only_audio:
             filters.append(
-                lambda s: (
-                    s.includes_audio_track and not s.includes_video_track
-                ),
+                lambda s: (s.includes_audio_track and not s.includes_video_track),
             )
 
         if only_video:
             filters.append(
-                lambda s: (
-                    s.includes_video_track and not s.includes_audio_track
-                ),
+                lambda s: (s.includes_video_track and not s.includes_audio_track),
             )
 
         if progressive:
@@ -148,43 +165,49 @@ class StreamQuery:
             filters.append(lambda s: s.is_adaptive)
 
         if custom_filter_functions:
-            for fn in custom_filter_functions:
-                filters.append(fn)
+            filters.extend(custom_filter_functions)
+
+        if is_dash is not None:
+            filters.append(lambda s: s.is_dash == is_dash)
 
+        return self._filter(filters)
+
+    def _filter(self, filters: List[Callable]) -> "StreamQuery":
         fmt_streams = self.fmt_streams
-        for fn in filters:
-            fmt_streams = list(filter(fn, fmt_streams))
-        return StreamQuery(fmt_streams)
+        for filter_lambda in filters:
+            fmt_streams = filter(filter_lambda, fmt_streams)
+        return StreamQuery(list(fmt_streams))
 
-    def order_by(self, attribute_name):
-        """Apply a sort order to a resultset.
+    def order_by(self, attribute_name: str) -> "StreamQuery":
+        """Apply a sort order. Filters out stream the do not have the attribute.
 
         :param str attribute_name:
             The name of the attribute to sort by.
         """
-        integer_attr_repr = {}
-        for stream in self.fmt_streams:
-            attr = getattr(stream, attribute_name)
-            if attr is None:
-                break
-            num = ''.join(x for x in attr if x.isdigit())
-            integer_attr_repr[attr] = int(''.join(num)) if num else None
-
-        # if every attribute has an integer representation
-        if integer_attr_repr and all(integer_attr_repr.values()):
-            def key(s):
-                return integer_attr_repr[getattr(s, attribute_name)]
-        else:
-            def key(s):
-                return getattr(s, attribute_name)
-
-        fmt_streams = sorted(
-            self.fmt_streams,
-            key=key,
+        has_attribute = [
+            s for s in self.fmt_streams if getattr(s, attribute_name) is not None
+        ]
+        # Check that the attributes have string values.
+        if has_attribute and isinstance(getattr(has_attribute[0], attribute_name), str):
+            # Try to return a StreamQuery sorted by the integer representations
+            # of the values.
+            try:
+                return StreamQuery(
+                    sorted(
+                        has_attribute,
+                        key=lambda s: int(
+                            "".join(filter(str.isdigit, getattr(s, attribute_name)))
+                        ),  # type: ignore  # noqa: E501
+                    )
+                )
+            except ValueError:
+                pass
+
+        return StreamQuery(
+            sorted(has_attribute, key=lambda s: getattr(s, attribute_name))
         )
-        return StreamQuery(fmt_streams)
 
-    def desc(self):
+    def desc(self) -> "StreamQuery":
         """Sort streams in descending order.
 
         :rtype: :class:`StreamQuery <StreamQuery>`
@@ -192,7 +215,7 @@ class StreamQuery:
         """
         return StreamQuery(self.fmt_streams[::-1])
 
-    def asc(self):
+    def asc(self) -> "StreamQuery":
         """Sort streams in ascending order.
 
         :rtype: :class:`StreamQuery <StreamQuery>`
@@ -200,10 +223,10 @@ class StreamQuery:
         """
         return self
 
-    def get_by_itag(self, itag):
+    def get_by_itag(self, itag: int) -> Optional[Stream]:
         """Get the corresponding :class:`Stream <Stream>` for a given itag.
 
-        :param str int itag:
+        :param int itag:
             YouTube format identifier code.
         :rtype: :class:`Stream <Stream>` or None
         :returns:
@@ -211,12 +234,71 @@ class StreamQuery:
             not found.
 
         """
-        try:
-            return self.itag_index[int(itag)]
-        except KeyError:
-            pass
+        return self.itag_index.get(int(itag))
+
+    def get_by_resolution(self, resolution: str) -> Optional[Stream]:
+        """Get the corresponding :class:`Stream <Stream>` for a given resolution.
+
+        Stream must be a progressive mp4.
+
+        :param str resolution:
+            Video resolution i.e. "720p", "480p", "360p", "240p", "144p"
+        :rtype: :class:`Stream <Stream>` or None
+        :returns:
+            The :class:`Stream <Stream>` matching the given itag or None if
+            not found.
+
+        """
+        return self.filter(
+            progressive=True, subtype="mp4", resolution=resolution
+        ).first()
+
+    def get_lowest_resolution(self) -> Optional[Stream]:
+        """Get lowest resolution stream that is a progressive mp4.
+
+        :rtype: :class:`Stream <Stream>` or None
+        :returns:
+            The :class:`Stream <Stream>` matching the given itag or None if
+            not found.
+
+        """
+        return (
+            self.filter(progressive=True, subtype="mp4").order_by("resolution").first()
+        )
+
+    def get_highest_resolution(self) -> Optional[Stream]:
+        """Get highest resolution stream that is a progressive video.
+
+        :rtype: :class:`Stream <Stream>` or None
+        :returns:
+            The :class:`Stream <Stream>` matching the given itag or None if
+            not found.
+
+        """
+        return self.filter(progressive=True).order_by("resolution").last()
+
+    def get_audio_only(self, subtype: str = "mp4") -> Optional[Stream]:
+        """Get highest bitrate audio stream for given codec (defaults to mp4)
+
+        :param str subtype:
+            Audio subtype, defaults to mp4
+        :rtype: :class:`Stream <Stream>` or None
+        :returns:
+            The :class:`Stream <Stream>` matching the given itag or None if
+            not found.
+        """
+        return self.filter(only_audio=True, subtype=subtype).order_by("abr").last()
+
+    def otf(self, is_otf: bool = False) -> "StreamQuery":
+        """Filter stream by OTF, useful if some streams have 404 URLs
+
+        :param bool is_otf: Set to False to retrieve only non-OTF streams
+        :rtype: :class:`StreamQuery <StreamQuery>`
+        :returns: A StreamQuery object with otf filtered streams
+        """
+        return self._filter([lambda s: s.is_otf == is_otf])
 
-    def first(self):
+    def first(self) -> Optional[Stream]:
         """Get the first :class:`Stream <Stream>` in the results.
 
         :rtype: :class:`Stream <Stream>` or None
@@ -228,7 +310,7 @@ class StreamQuery:
         try:
             return self.fmt_streams[0]
         except IndexError:
-            pass
+            return None
 
     def last(self):
         """Get the last :class:`Stream <Stream>` in the results.
@@ -244,15 +326,19 @@ class StreamQuery:
         except IndexError:
             pass
 
-    def count(self):
-        """Get the count the query would return.
+    @deprecated("Get the size of this list directly using len()")
+    def count(self, value: Optional[str] = None) -> int:  # pragma: no cover
+        """Get the count of items in the list.
 
         :rtype: int
-
         """
-        return len(self.fmt_streams)
+        if value:
+            return self.fmt_streams.count(value)
+
+        return len(self)
 
-    def all(self):
+    @deprecated("This object can be treated as a list, all() is useless")
+    def all(self) -> List[Stream]:  # pragma: no cover
         """Get all the results represented by this query as a list.
 
         :rtype: list
@@ -260,21 +346,32 @@ class StreamQuery:
         """
         return self.fmt_streams
 
+    def __getitem__(self, i: Union[slice, int]):
+        return self.fmt_streams[i]
+
+    def __len__(self) -> int:
+        return len(self.fmt_streams)
+
+    def __repr__(self) -> str:
+        return f"{self.fmt_streams}"
 
-class CaptionQuery:
+
+class CaptionQuery(Mapping):
     """Interface for querying the available captions."""
 
-    def __init__(self, captions):
+    def __init__(self, captions: List[Caption]):
         """Construct a :class:`Caption <Caption>`.
 
         param list captions:
             list of :class:`Caption <Caption>` instances.
 
         """
-        self.captions = captions
         self.lang_code_index = {c.code: c for c in captions}
 
-    def get_by_language_code(self, lang_code):
+    @deprecated("This object can be treated as a dictionary, i.e. captions['en']")
+    def get_by_language_code(
+        self, lang_code: str
+    ) -> Optional[Caption]:  # pragma: no cover
         """Get the :class:`Caption <Caption>` for a given ``lang_code``.
 
         :param str lang_code:
@@ -286,10 +383,23 @@ class CaptionQuery:
         """
         return self.lang_code_index.get(lang_code)
 
-    def all(self):
+    @deprecated("This object can be treated as a dictionary")
+    def all(self) -> List[Caption]:  # pragma: no cover
         """Get all the results represented by this query as a list.
 
         :rtype: list
 
         """
-        return self.captions
+        return list(self.lang_code_index.values())
+
+    def __getitem__(self, i: str):
+        return self.lang_code_index[i]
+
+    def __len__(self) -> int:
+        return len(self.lang_code_index)
+
+    def __iter__(self):
+        return iter(self.lang_code_index.values())
+
+    def __repr__(self) -> str:
+        return f"{self.lang_code_index}"

pytube/request.py

@@ -1,47 +1,89 @@
 # -*- coding: utf-8 -*-
+
 """Implements a simple wrapper around urlopen."""
-import urllib.request
+import logging
+from functools import lru_cache
+from http.client import HTTPResponse
+from typing import Iterable, Dict, Optional
+from urllib.request import Request
+from urllib.request import urlopen
+
+logger = logging.getLogger(__name__)
 
-from pytube.compat import urlopen
-# 403 forbidden fix
 
+def _execute_request(
+    url: str, method: Optional[str] = None, headers: Optional[Dict[str, str]] = None
+) -> HTTPResponse:
+    base_headers = {"User-Agent": "Mozilla/5.0"}
+    if headers:
+        base_headers.update(headers)
+    if url.lower().startswith("http"):
+        request = Request(url, headers=base_headers, method=method)
+    else:
+        raise ValueError("Invalid URL")
+    return urlopen(request)  # nosec
 
-def get(
-    url=None, headers=False,
-    streaming=False, chunk_size=8 * 1024,
-):
+
+def get(url) -> str:
     """Send an http GET request.
 
     :param str url:
         The URL to perform the GET request for.
-    :param bool headers:
-        Only return the http headers.
-    :param bool streaming:
-        Returns the response body in chunks via a generator.
-    :param int chunk_size:
-        The size in bytes of each chunk.
+    :rtype: str
+    :returns:
+        UTF-8 encoded string of response
+    """
+    return _execute_request(url).read().decode("utf-8")
+
+
+def stream(
+    url: str, chunk_size: int = 4096, range_size: int = 9437184
+) -> Iterable[bytes]:
+    """Read the response in chunks.
+    :param str url: The URL to perform the GET request for.
+    :param int chunk_size: The size in bytes of each chunk. Defaults to 4KB
+    :param int range_size: The size in bytes of each range request. Defaults to 9MB
+    :rtype: Iterable[bytes]
     """
+    file_size: int = range_size  # fake filesize to start
+    downloaded = 0
+    while downloaded < file_size:
+        stop_pos = min(downloaded + range_size, file_size) - 1
+        range_header = f"bytes={downloaded}-{stop_pos}"
+        response = _execute_request(url, method="GET", headers={"Range": range_header})
+        if file_size == range_size:
+            try:
+                content_range = response.info()["Content-Range"]
+                file_size = int(content_range.split("/")[1])
+            except (KeyError, IndexError, ValueError) as e:
+                logger.error(e)
+        while True:
+            chunk = response.read(chunk_size)
+            if not chunk:
+                break
+            downloaded += len(chunk)
+            yield chunk
+    return  # pylint: disable=R1711
+
+
+@lru_cache(maxsize=None)
+def filesize(url: str) -> int:
+    """Fetch size in bytes of file at given URL
 
-    # https://github.com/nficano/pytube/pull/465
-    req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
-    response = urlopen(req)
-
-    if streaming:
-        return stream_response(response, chunk_size)
-    elif headers:
-        # https://github.com/nficano/pytube/issues/160
-        return {k.lower(): v for k, v in response.info().items()}
-    return (
-        response
-        .read()
-        .decode('utf-8')
-    )
-
-
-def stream_response(response, chunk_size=8 * 1024):
-    """Read the response in chunks."""
-    while True:
-        buf = response.read(chunk_size)
-        if not buf:
-            break
-        yield buf
+    :param str url: The URL to get the size of
+    :returns: int: size in bytes of remote file
+    """
+    return int(head(url)["content-length"])
+
+
+def head(url: str) -> Dict:
+    """Fetch headers returned http GET request.
+
+    :param str url:
+        The URL to perform the GET request for.
+    :rtype: dict
+    :returns:
+        dictionary of lowercase headers
+    """
+    response_headers = _execute_request(url, method="HEAD").info()
+    return {k.lower(): v for k, v in response_headers.items()}

pytube/streams.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+
 """
 This module contains a container for stream manifest data.
 
@@ -7,26 +8,26 @@ combined). This was referred to as ``Video`` in the legacy pytube version, but
 has been renamed to accommodate DASH (which serves the audio and video
 separately).
 """
-from __future__ import absolute_import
 
-import io
+from datetime import datetime
 import logging
 import os
-import pprint
+from typing import Dict, Tuple, Optional, BinaryIO
+from urllib.parse import parse_qs
 
 from pytube import extract
 from pytube import request
-from pytube.helpers import safe_filename
+from pytube.helpers import safe_filename, target_directory
 from pytube.itags import get_format_profile
-
+from pytube.monostate import Monostate
 
 logger = logging.getLogger(__name__)
 
 
-class Stream(object):
+class Stream:
     """Container for stream manifest data."""
 
-    def __init__(self, stream, player_config_args, monostate):
+    def __init__(self, stream: Dict, player_config_args: Dict, monostate: Monostate):
         """Construct a :class:`Stream <Stream>`.
 
         :param dict stream:
@@ -42,67 +43,52 @@ class Stream(object):
         # (Borg pattern).
         self._monostate = monostate
 
-        self.abr = None   # average bitrate (audio streams only)
-        self.fps = None   # frames per second (video streams only)
-        self.itag = None  # stream format id (youtube nomenclature)
-        self.res = None   # resolution (e.g.: 480p, 720p, 1080p)
-        self.url = None   # signed download url
-
-        self._filesize = None  # filesize in bytes
-        self.mime_type = None  # content identifier (e.g.: video/mp4)
-        self.type = None       # the part of the mime before the slash
-        self.subtype = None    # the part of the mime after the slash
-
-        self.codecs = []         # audio/video encoders (e.g.: vp8, mp4a)
-        self.audio_codec = None  # audio codec of the stream (e.g.: vorbis)
-        self.video_codec = None  # video codec of the stream (e.g.: vp8)
-
-        # Iterates over the key/values of stream and sets them as class
-        # attributes. This is an anti-pattern and should be removed.
-        self.set_attributes_from_dict(stream)
+        self.url = stream["url"]  # signed download url
+        self.itag = int(stream["itag"])  # stream format id (youtube nomenclature)
 
-        # Additional information about the stream format, such as resolution,
-        # frame rate, and whether the stream is live (HLS) or 3D.
-        self.fmt_profile = get_format_profile(self.itag)
-
-        # Same as above, except for the format profile attributes.
-        self.set_attributes_from_dict(self.fmt_profile)
-
-        # The player configuration which contains information like the video
-        # title.
-        # TODO(nficano): this should be moved to the monostate.
-        self.player_config_args = player_config_args
+        # set type and codec info
 
         # 'video/webm; codecs="vp8, vorbis"' -> 'video/webm', ['vp8', 'vorbis']
-        self.mime_type, self.codecs = extract.mime_type_codec(self.type)
+        self.mime_type, self.codecs = extract.mime_type_codec(stream["type"])
 
         # 'video/webm' -> 'video', 'webm'
-        self.type, self.subtype = self.mime_type.split('/')
+        self.type, self.subtype = self.mime_type.split("/")
 
         # ['vp8', 'vorbis'] -> video_codec: vp8, audio_codec: vorbis. DASH
         # streams return NoneType for audio/video depending.
         self.video_codec, self.audio_codec = self.parse_codecs()
 
-    def set_attributes_from_dict(self, dct):
-        """Set class attributes from dictionary items.
+        self.is_otf: bool = stream["is_otf"]
+        self.bitrate: Optional[int] = stream["bitrate"]
 
-        :rtype: None
-        """
-        for key, val in dct.items():
-            setattr(self, key, val)
+        self._filesize: Optional[int] = None  # filesize in bytes
+
+        # Additional information about the stream format, such as resolution,
+        # frame rate, and whether the stream is live (HLS) or 3D.
+        itag_profile = get_format_profile(self.itag)
+        self.is_dash = itag_profile["is_dash"]
+        self.abr = itag_profile["abr"]  # average bitrate (audio streams only)
+        self.fps = itag_profile["fps"]  # frames per second (video streams only)
+        self.resolution = itag_profile["resolution"]  # resolution (e.g.: "480p")
+        self.is_3d = itag_profile["is_3d"]
+        self.is_hdr = itag_profile["is_hdr"]
+        self.is_live = itag_profile["is_live"]
+
+        # The player configuration, contains info like the video title.
+        self.player_config_args = player_config_args
 
     @property
-    def is_adaptive(self):
+    def is_adaptive(self) -> bool:
         """Whether the stream is DASH.
 
         :rtype: bool
         """
         # if codecs has two elements (e.g.: ['vp8', 'vorbis']): 2 % 2 = 0
         # if codecs has one element (e.g.: ['vp8']) 1 % 2 = 1
-        return len(self.codecs) % 2
+        return bool(len(self.codecs) % 2)
 
     @property
-    def is_progressive(self):
+    def is_progressive(self) -> bool:
         """Whether the stream is progressive.
 
         :rtype: bool
@@ -110,26 +96,22 @@ class Stream(object):
         return not self.is_adaptive
 
     @property
-    def includes_audio_track(self):
+    def includes_audio_track(self) -> bool:
         """Whether the stream only contains audio.
 
         :rtype: bool
         """
-        if self.is_progressive:
-            return True
-        return self.type == 'audio'
+        return self.is_progressive or self.type == "audio"
 
     @property
-    def includes_video_track(self):
+    def includes_video_track(self) -> bool:
         """Whether the stream only contains video.
 
         :rtype: bool
         """
-        if self.is_progressive:
-            return True
-        return self.type == 'video'
+        return self.is_progressive or self.type == "video"
 
-    def parse_codecs(self):
+    def parse_codecs(self) -> Tuple[Optional[str], Optional[str]]:
         """Get the video/audio codecs from list of codecs.
 
         Parse a variable length sized list of codecs and returns a
@@ -153,7 +135,7 @@ class Stream(object):
         return video, audio
 
     @property
-    def filesize(self):
+    def filesize(self) -> int:
         """File size of the media stream in bytes.
 
         :rtype: int
@@ -161,45 +143,57 @@ class Stream(object):
             Filesize (in bytes) of the stream.
         """
         if self._filesize is None:
-            headers = request.get(self.url, headers=True)
-            self._filesize = int(headers['content-length'])
+            self._filesize = request.filesize(self.url)
         return self._filesize
 
     @property
-    def title(self):
+    def title(self) -> str:
         """Get title of video
 
         :rtype: str
         :returns:
             Youtube video title
         """
-        player_config_args = self.player_config_args or {}
+        return self._monostate.title or "Unknown YouTube Video Title"
+
+    @property
+    def filesize_approx(self) -> int:
+        """Get approximate filesize of the video
 
-        if 'title' in player_config_args:
-            return player_config_args['title']
+        Falls back to HTTP call if there is not sufficient information to approximate
 
-        details = self.player_config_args.get(
-            'player_response', {},
-        ).get('videoDetails', {})
+        :rtype: int
+        :returns: size of video in bytes
+        """
+        if self._monostate.duration and self.bitrate:
+            bits_in_byte = 8
+            return int((self._monostate.duration * self.bitrate) / bits_in_byte)
 
-        if 'title' in details:
-            return details['title']
+        return self.filesize
 
-        return 'Unknown YouTube Video Title'
+    @property
+    def expiration(self) -> datetime:
+        expire = parse_qs(self.url.split("?")[1])["expire"][0]
+        return datetime.utcfromtimestamp(int(expire))
 
     @property
-    def default_filename(self):
+    def default_filename(self) -> str:
         """Generate filename based on the video title.
 
         :rtype: str
         :returns:
             An os file system compatible filename.
         """
-
         filename = safe_filename(self.title)
-        return '{filename}.{s.subtype}'.format(filename=filename, s=self)
-
-    def download(self, output_path=None, filename=None, filename_prefix=None):
+        return f"{filename}.{self.subtype}"
+
+    def download(
+        self,
+        output_path: Optional[str] = None,
+        filename: Optional[str] = None,
+        filename_prefix: Optional[str] = None,
+        skip_existing: bool = True,
+    ) -> str:
         """Write the media stream to disk.
 
         :param output_path:
@@ -214,71 +208,82 @@ class Stream(object):
             (optional) A string that will be prepended to the filename.
             For example a number in a playlist or the name of a series.
             If one is not specified, nothing will be prepended
-            This is seperate from filename so you can use the default
+            This is separate from filename so you can use the default
             filename but still add a prefix.
         :type filename_prefix: str or None
-
+        :param skip_existing:
+            (optional) skip existing files, defaults to True
+        :type skip_existing: bool
+        :returns:
+            Path to the saved video
         :rtype: str
 
         """
-        output_path = output_path or os.getcwd()
-        if filename:
-            safe = safe_filename(filename)
-            filename = '{filename}.{s.subtype}'.format(filename=safe, s=self)
-        filename = filename or self.default_filename
+        file_path = self.get_file_path(
+            filename=filename, output_path=output_path, filename_prefix=filename_prefix
+        )
 
-        if filename_prefix:
-            filename = '{prefix}{filename}'\
-                .format(
-                    prefix=safe_filename(filename_prefix),
-                    filename=filename,
-                )
+        if skip_existing and self.exists_at_path(file_path):
+            logger.debug("file %s already exists, skipping", file_path)
+            self.on_complete(file_path)
+            return file_path
 
-        # file path
-        fp = os.path.join(output_path, filename)
         bytes_remaining = self.filesize
         logger.debug(
-            'downloading (%s total bytes) file to %s',
-            self.filesize, fp,
+            "downloading (%s total bytes) file to %s", self.filesize, file_path,
         )
 
-        with open(fp, 'wb') as fh:
-            for chunk in request.get(self.url, streaming=True):
+        with open(file_path, "wb") as fh:
+            for chunk in request.stream(self.url):
                 # reduce the (bytes) remainder by the length of the chunk.
                 bytes_remaining -= len(chunk)
                 # send to the on_progress callback.
                 self.on_progress(chunk, fh, bytes_remaining)
-        self.on_complete(fh)
-        return fp
+        self.on_complete(file_path)
+        return file_path
+
+    def get_file_path(
+        self,
+        filename: Optional[str],
+        output_path: Optional[str],
+        filename_prefix: Optional[str] = None,
+    ) -> str:
+        if filename:
+            filename = f"{safe_filename(filename)}.{self.subtype}"
+        else:
+            filename = self.default_filename
+        if filename_prefix:
+            filename = f"{safe_filename(filename_prefix)}{filename}"
+        return os.path.join(target_directory(output_path), filename)
 
-    def stream_to_buffer(self):
+    def exists_at_path(self, file_path: str) -> bool:
+        return os.path.isfile(file_path) and os.path.getsize(file_path) == self.filesize
+
+    def stream_to_buffer(self, buffer: BinaryIO) -> None:
         """Write the media stream to buffer
 
         :rtype: io.BytesIO buffer
         """
-        buffer = io.BytesIO()
         bytes_remaining = self.filesize
-        logger.debug(
-            'downloading (%s total bytes) file to BytesIO buffer',
-            self.filesize,
+        logger.info(
+            "downloading (%s total bytes) file to buffer", self.filesize,
         )
 
-        for chunk in request.get(self.url, streaming=True):
+        for chunk in request.stream(self.url):
             # reduce the (bytes) remainder by the length of the chunk.
             bytes_remaining -= len(chunk)
             # send to the on_progress callback.
             self.on_progress(chunk, buffer, bytes_remaining)
-        self.on_complete(buffer)
-        return buffer
+        self.on_complete(None)
 
-    def on_progress(self, chunk, file_handler, bytes_remaining):
+    def on_progress(self, chunk: bytes, file_handler: BinaryIO, bytes_remaining: int):
         """On progress callback function.
 
         This function writes the binary data to the file, then checks if an
         additional callback is defined in the monostate. This is exposed to
         allow things like displaying a progress bar.
 
-        :param str chunk:
+        :param bytes chunk:
             Segment of media file binary data, not yet written to disk.
         :param file_handler:
             The file handle where the media is being written to.
@@ -292,56 +297,43 @@ class Stream(object):
 
         """
         file_handler.write(chunk)
-        logger.debug(
-            'download progress\n%s',
-            pprint.pformat(
-                {
-                    'chunk_size': len(chunk),
-                    'bytes_remaining': bytes_remaining,
-                }, indent=2,
-            ),
-        )
-        on_progress = self._monostate['on_progress']
-        if on_progress:
-            logger.debug('calling on_progress callback %s', on_progress)
-            on_progress(self, chunk, file_handler, bytes_remaining)
+        logger.debug("download remaining: %s", bytes_remaining)
+        if self._monostate.on_progress:
+            self._monostate.on_progress(self, chunk, bytes_remaining)
 
-    def on_complete(self, file_handle):
+    def on_complete(self, file_path: Optional[str]):
         """On download complete handler function.
 
-        :param file_handle:
+        :param file_path:
             The file handle where the media is being written to.
-        :type file_handle:
-            :py:class:`io.BufferedWriter`
+        :type file_path: str
 
         :rtype: None
 
         """
-        logger.debug('download finished')
-        on_complete = self._monostate['on_complete']
+        logger.debug("download finished")
+        on_complete = self._monostate.on_complete
         if on_complete:
-            logger.debug('calling on_complete callback %s', on_complete)
-            on_complete(self, file_handle)
+            logger.debug("calling on_complete callback %s", on_complete)
+            on_complete(self, file_path)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         """Printable object representation.
 
         :rtype: str
         :returns:
             A string representation of a :class:`Stream <Stream>` object.
         """
-        # TODO(nficano): this can probably be written better.
         parts = ['itag="{s.itag}"', 'mime_type="{s.mime_type}"']
         if self.includes_video_track:
             parts.extend(['res="{s.resolution}"', 'fps="{s.fps}fps"'])
             if not self.is_adaptive:
-                parts.extend([
-                    'vcodec="{s.video_codec}"',
-                    'acodec="{s.audio_codec}"',
-                ])
+                parts.extend(
+                    ['vcodec="{s.video_codec}"', 'acodec="{s.audio_codec}"',]
+                )
             else:
                 parts.extend(['vcodec="{s.video_codec}"'])
         else:
             parts.extend(['abr="{s.abr}"', 'acodec="{s.audio_codec}"'])
-        parts = ' '.join(parts).format(s=self)
-        return '<Stream: {parts}>'.format(parts=parts)
+        parts.extend(['progressive="{s.is_progressive}"', 'type="{s.type}"'])
+        return f"<Stream: {' '.join(parts).format(s=self)}>"

pytube/version.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+__version__ = "9.6.4"
+
+if __name__ == "__main__":
+    print(__version__)

setup.cfg

@@ -1,23 +0,0 @@
-[bumpversion]
-commit = True
-tag = True
-current_version = 9.5.3
-parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+))?
-serialize = 
-	{major}.{minor}.{patch}
-
-[metadata]
-description-file = README.md
-
-[bumpversion:file:setup.py]
-
-[bumpversion:file:pytube/__init__.py]
-
-[coverage:run]
-source = pytube
-omit = 
-	pytube/compat.py
-
-[flake8]
-ignore = W605
-

setup.py

@@ -1,92 +1,58 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-"""This module contains setup instructions for pytube."""
+"""This module contains setup instructions for pytube3."""
 import codecs
 import os
-import sys
-from shutil import rmtree
-
-from setuptools import Command
 from setuptools import setup
 
 here = os.path.abspath(os.path.dirname(__file__))
 
-with codecs.open(os.path.join(here, 'README.md'), encoding='utf-8') as fh:
-    long_description = '\n' + fh.read()
-
-
-class UploadCommand(Command):
-    """Support setup.py publish."""
-
-    description = 'Build and publish the package.'
-    user_options = []
-
-    @staticmethod
-    def status(s):
-        """Prints things in bold."""
-        print('\033[1m{0}\033[0m'.format(s))
-
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        try:
-            self.status('Removing previous builds ...')
-            rmtree(os.path.join(here, 'dist'))
-        except Exception:
-            pass
-        self.status('Building Source distribution ...')
-        os.system('{0} setup.py sdist bdist_wheel'.format(sys.executable))
-        self.status('Uploading the package to PyPI via Twine ...')
-        os.system('twine upload dist/*')
-        sys.exit()
+with codecs.open(os.path.join(here, "README.md"), encoding="utf-8") as fh:
+    long_description = "\n" + fh.read()
 
+with open(os.path.join(here, "pytube", "version.py")) as fp:
+    exec(fp.read())
 
 setup(
-    name='pytube',
-    version='9.5.3',
-    author='Nick Ficano',
-    author_email='[email protected]',
-    packages=['pytube', 'pytube.contrib'],
-    package_data={
-        '': ['LICENSE'],
-    },
-    url='https://github.com/nficano/pytube',
-    license='MIT',
-    entry_points={
-        'console_scripts': [
-            'pytube = pytube.cli:main',
-        ],
-    },
+    name="pytube3",
+    version=__version__,  # noqa: F821
+    author="Nick Ficano, Harold Martin",
+    author_email="[email protected], [email protected]",
+    packages=["pytube", "pytube.contrib"],
+    package_data={"": ["LICENSE"],},
+    url="https://github.com/hbmartin/pytube3",
+    license="MIT",
+    entry_points={"console_scripts": ["pytube3 = pytube.cli:main",],},
+    install_requires=["typing_extensions"],
     classifiers=[
-        'Development Status :: 5 - Production/Stable',
-        'Environment :: Console',
-        'Intended Audience :: Developers',
-        'License :: OSI Approved :: MIT License',
-        'Natural Language :: English',
-        'Operating System :: MacOS',
-        'Operating System :: Microsoft',
-        'Operating System :: POSIX',
-        'Operating System :: Unix',
-        'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python',
-        'Topic :: Internet',
-        'Topic :: Multimedia :: Video',
-        'Topic :: Software Development :: Libraries :: Python Modules',
-        'Topic :: Terminals',
-        'Topic :: Utilities',
+        "Development Status :: 5 - Production/Stable",
+        "Environment :: Console",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: MIT License",
+        "Natural Language :: English",
+        "Operating System :: MacOS",
+        "Operating System :: Microsoft",
+        "Operating System :: POSIX",
+        "Operating System :: Unix",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python",
+        "Topic :: Internet",
+        "Topic :: Multimedia :: Video",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        "Topic :: Terminals",
+        "Topic :: Utilities",
     ],
-    description=('A pythonic library for downloading YouTube Videos.'),
+    description=("Python 3 library for downloading YouTube Videos."),
     include_package_data=True,
-    long_description_content_type='text/markdown',
+    long_description_content_type="text/markdown",
     long_description=long_description,
     zip_safe=True,
-    cmdclass={'upload': UploadCommand},
+    python_requires=">=3.6",
+    project_urls={
+        "Bug Reports": "https://github.com/hbmartin/pytube3/issues",
+        "Read the Docs": "https://pytube3.readthedocs.io/en/latest/?badge=latest",
+    },
+    keywords=["youtube", "download", "video", "stream",],
 )

tests/conftest.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 """Reusable dependency injected testing components."""
-from __future__ import unicode_literals
 
 import gzip
 import json
@@ -15,39 +14,61 @@ def load_playback_file(filename):
     """Load a gzip json playback file."""
     cur_fp = os.path.realpath(__file__)
     cur_dir = os.path.dirname(cur_fp)
-    fp = os.path.join(cur_dir, 'mocks', filename)
-    with gzip.open(fp, 'rb') as fh:
-        content = fh.read().decode('utf-8')
+    fp = os.path.join(cur_dir, "mocks", filename)
+    with gzip.open(fp, "rb") as fh:
+        content = fh.read().decode("utf-8")
         return json.loads(content)
 
 
 def load_and_init_from_playback_file(filename):
     """Load a gzip json playback file and create YouTube instance."""
     pb = load_playback_file(filename)
-    yt = YouTube(pb['url'], defer_prefetch_init=True)
-    yt.watch_html = pb['watch_html']
-    yt.js = pb['js']
-    yt.vid_info = pb['video_info']
-    yt.init()
+    yt = YouTube(pb["url"], defer_prefetch_init=True)
+    yt.watch_html = pb["watch_html"]
+    yt.js = pb["js"]
+    yt.vid_info = pb["video_info"]
+    yt.descramble()
     return yt
 
 
 @pytest.fixture
 def cipher_signature():
     """Youtube instance initialized with video id 9bZkp7q19f0."""
-    filename = 'yt-video-9bZkp7q19f0-1507588332.json.tar.gz'
+    filename = "yt-video-9bZkp7q19f0.json.gz"
     return load_and_init_from_playback_file(filename)
 
 
 @pytest.fixture
 def presigned_video():
     """Youtube instance initialized with video id QRS8MkLhQmM."""
-    filename = 'yt-video-QRS8MkLhQmM-1507588031.json.tar.gz'
+    filename = "yt-video-QRS8MkLhQmM.json.gz"
     return load_and_init_from_playback_file(filename)
 
 
 @pytest.fixture
 def age_restricted():
     """Youtube instance initialized with video id zRbsm3e2ltw."""
-    filename = 'yt-video-zRbsm3e2ltw-1507777044.json.tar.gz'
+    filename = "yt-video-irauhITDrsE.json.gz"
     return load_playback_file(filename)
+
+
+@pytest.fixture
+def playlist_html():
+    """Youtube playlist HTML loaded on 2020-01-25 from
+    https://www.youtube.com/playlist?list=PLzMcBGfZo4-mP7qA9cagf68V06sko5otr"""
+    file_path = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "mocks", "playlist.html.gz"
+    )
+    with gzip.open(file_path, "rb") as f:
+        return f.read().decode("utf-8")
+
+
+@pytest.fixture
+def playlist_long_html():
+    """Youtube playlist HTML loaded on 2020-01-25 from
+    https://www.youtube.com/playlist?list=PLzMcBGfZo4-mP7qA9cagf68V06sko5otr"""
+    file_path = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "mocks", "playlist_long.html.gz"
+    )
+    with gzip.open(file_path, "rb") as f:
+        return f.read().decode("utf-8")

tests/contrib/test_playlist.py

@@ -1,10 +1,220 @@
 # -*- coding: utf-8 -*-
+import datetime
+from unittest import mock
+from unittest.mock import MagicMock
+
 from pytube import Playlist
 
 
-def test_title():
-    list_key = 'PLsyeobzWxl7poL9JTVyndKe62ieoN-MZ3'
-    url = 'https://www.youtube.com/playlist?list=' + list_key
+@mock.patch("pytube.contrib.playlist.request.get")
+def test_title(request_get):
+    request_get.return_value = (
+        "<title>(149) Python Tutorial for Beginners "
+        "(For Absolute Beginners) - YouTube</title>"
+    )
+    url = "https://www.fakeurl.com/playlist?list=PLS1QulWo1RIaJECMeUT4LFwJ-ghgoSH6n"
     pl = Playlist(url)
     pl_title = pl.title()
-    assert pl_title == 'Python Tutorial for Beginners'
+    assert pl_title == "(149) Python Tutorial for Beginners (For Absolute Beginners)"
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+def test_init_with_playlist_url(request_get):
+    request_get.return_value = ""
+    url = "https://www.youtube.com/playlist?list=PLS1QulWo1RIaJECMeUT4LFwJ-ghgoSH6n"
+    playlist = Playlist(url)
+    assert playlist.playlist_url == url
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+def test_init_with_watch_url(request_get):
+    request_get.return_value = ""
+    url = (
+        "https://www.youtube.com/watch?v=1KeYzjILqDo&"
+        "list=PLS1QulWo1RIaJECMeUT4LFwJ-ghgoSH6n&index=2&t=661s"
+    )
+    playlist = Playlist(url)
+    assert (
+        playlist.playlist_url
+        == "https://www.youtube.com/playlist?list=PLS1QulWo1RIaJECMeUT4LFwJ-ghgoSH6n"
+    )
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+def test_last_update(request_get, playlist_html):
+    expected = datetime.date(2019, 3, 7)
+    request_get.return_value = playlist_html
+    playlist = Playlist("url")
+    assert playlist.last_update == expected
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+def test_init_with_watch_id(request_get):
+    request_get.return_value = ""
+    playlist = Playlist("PLS1QulWo1RIaJECMeUT4LFwJ-ghgoSH6n")
+    assert (
+        playlist.playlist_url
+        == "https://www.youtube.com/playlist?list=PLS1QulWo1RIaJECMeUT4LFwJ-ghgoSH6n"
+    )
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+def test_video_urls(request_get, playlist_html):
+    url = "https://www.fakeurl.com/playlist?list=whatever"
+    request_get.return_value = playlist_html
+    playlist = Playlist(url)
+    playlist._find_load_more_url = MagicMock(return_value=None)
+    request_get.assert_called()
+    assert playlist.video_urls == [
+        "https://www.youtube.com/watch?v=ujTCoH21GlA",
+        "https://www.youtube.com/watch?v=45ryDIPHdGg",
+        "https://www.youtube.com/watch?v=1BYu65vLKdA",
+        "https://www.youtube.com/watch?v=3AQ_74xrch8",
+        "https://www.youtube.com/watch?v=ddqQUz9mZaM",
+        "https://www.youtube.com/watch?v=vwLT6bZrHEE",
+        "https://www.youtube.com/watch?v=TQKI0KE-JYY",
+        "https://www.youtube.com/watch?v=dNBvQ38MlT8",
+        "https://www.youtube.com/watch?v=JHxyrMgOUWI",
+        "https://www.youtube.com/watch?v=l2I8NycJMCY",
+        "https://www.youtube.com/watch?v=g1Zbuk1gAfk",
+        "https://www.youtube.com/watch?v=zixd-si9Q-o",
+    ]
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+def test_repr(request_get, playlist_html):
+    url = "https://www.fakeurl.com/playlist?list=whatever"
+    request_get.return_value = playlist_html
+    playlist = Playlist(url)
+    playlist._find_load_more_url = MagicMock(return_value=None)
+    request_get.assert_called()
+    assert (
+        repr(playlist) == "['https://www.youtube.com/watch?v=ujTCoH21GlA', "
+        "'https://www.youtube.com/watch?v=45ryDIPHdGg', "
+        "'https://www.youtube.com/watch?v=1BYu65vLKdA', "
+        "'https://www.youtube.com/watch?v=3AQ_74xrch8', "
+        "'https://www.youtube.com/watch?v=ddqQUz9mZaM', "
+        "'https://www.youtube.com/watch?v=vwLT6bZrHEE', "
+        "'https://www.youtube.com/watch?v=TQKI0KE-JYY', "
+        "'https://www.youtube.com/watch?v=dNBvQ38MlT8', "
+        "'https://www.youtube.com/watch?v=JHxyrMgOUWI', "
+        "'https://www.youtube.com/watch?v=l2I8NycJMCY', "
+        "'https://www.youtube.com/watch?v=g1Zbuk1gAfk', "
+        "'https://www.youtube.com/watch?v=zixd-si9Q-o']"
+    )
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+def test_sequence(request_get, playlist_html):
+    url = "https://www.fakeurl.com/playlist?list=whatever"
+    request_get.return_value = playlist_html
+    playlist = Playlist(url)
+    playlist._find_load_more_url = MagicMock(return_value=None)
+    assert playlist[0] == "https://www.youtube.com/watch?v=ujTCoH21GlA"
+    assert len(playlist) == 12
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.cli.YouTube.__init__", return_value=None)
+def test_videos(youtube, request_get, playlist_html):
+    url = "https://www.fakeurl.com/playlist?list=whatever"
+    request_get.return_value = playlist_html
+    playlist = Playlist(url)
+    playlist._find_load_more_url = MagicMock(return_value=None)
+    request_get.assert_called()
+    assert len(list(playlist.videos)) == 12
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.cli.YouTube.__init__", return_value=None)
+def test_load_more(youtube, request_get, playlist_html):
+    url = "https://www.fakeurl.com/playlist?list=whatever"
+    request_get.side_effect = [
+        playlist_html,
+        '{"content_html":"", "load_more_widget_html":""}',
+    ]
+    playlist = Playlist(url)
+    playlist._find_load_more_url = MagicMock(side_effect=["dummy", None])
+    request_get.assert_called()
+    assert len(list(playlist.videos)) == 12
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.contrib.playlist.install_proxy", return_value=None)
+def test_proxy(install_proxy, request_get):
+    url = "https://www.fakeurl.com/playlist?list=whatever"
+    request_get.return_value = ""
+    Playlist(url, proxies={"http": "things"})
+    install_proxy.assert_called_with({"http": "things"})
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+def test_trimmed(request_get, playlist_html):
+    url = "https://www.fakeurl.com/playlist?list=whatever"
+    request_get.return_value = playlist_html
+    playlist = Playlist(url)
+    playlist._find_load_more_url = MagicMock(return_value=None)
+    assert request_get.call_count == 1
+    trimmed = list(playlist.trimmed("1BYu65vLKdA"))
+    assert trimmed == [
+        "https://www.youtube.com/watch?v=ujTCoH21GlA",
+        "https://www.youtube.com/watch?v=45ryDIPHdGg",
+    ]
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+def test_playlist_failed_pagination(request_get, playlist_long_html):
+    url = "https://www.fakeurl.com/playlist?list=whatever"
+    request_get.side_effect = [
+        playlist_long_html,
+        "{}",
+    ]
+    playlist = Playlist(url)
+    video_urls = playlist.video_urls
+    assert len(video_urls) == 100
+    assert request_get.call_count == 2
+    request_get.assert_called_with(
+        "https://www.youtube.com/browse_ajax?action_continuation=1&amp;continuation"
+        "=4qmFsgIsEhpWTFVVYS12aW9HaGUyYnRCY1puZWFQb25LQRoOZWdaUVZEcERSMUUlM0Q%253D"
+    )
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+def test_playlist_pagination(request_get, playlist_html, playlist_long_html):
+    url = "https://www.fakeurl.com/playlist?list=whatever"
+    request_get.side_effect = [
+        playlist_long_html,
+        '{"content_html":"<a href=\\"/watch?v=BcWz41-4cDk&amp;feature=plpp_video&amp;ved'
+        '=CCYQxjQYACITCO33n5-pn-cCFUG3xAodLogN2yj6LA\\">}", "load_more_widget_html":""}',
+        "{}",
+    ]
+    playlist = Playlist(url)
+    assert len(playlist.video_urls) == 101
+    assert request_get.call_count == 2
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+def test_trimmed_pagination(request_get, playlist_html, playlist_long_html):
+    url = "https://www.fakeurl.com/playlist?list=whatever"
+    request_get.side_effect = [
+        playlist_long_html,
+        '{"content_html":"<a href=\\"/watch?v=BcWz41-4cDk&amp;feature=plpp_video&amp;ved'
+        '=CCYQxjQYACITCO33n5-pn-cCFUG3xAodLogN2yj6LA\\">}", "load_more_widget_html":""}',
+        "{}",
+    ]
+    playlist = Playlist(url)
+    assert len(list(playlist.trimmed("FN9vC8aR7Yk"))) == 3
+    assert request_get.call_count == 1
+
+
+@mock.patch("pytube.contrib.playlist.request.get")
+def test_trimmed_pagination_not_found(request_get, playlist_html, playlist_long_html):
+    url = "https://www.fakeurl.com/playlist?list=whatever"
+    request_get.side_effect = [
+        playlist_long_html,
+        '{"content_html":"<a href=\\"/watch?v=BcWz41-4cDk&amp;feature=plpp_video&amp;ved'
+        '=CCYQxjQYACITCO33n5-pn-cCFUG3xAodLogN2yj6LA\\">}", "load_more_widget_html":""}',
+        "{}",
+    ]
+    playlist = Playlist(url)
+    assert len(list(playlist.trimmed("wont-be-found"))) == 101

tests/generate_fixture.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+# flake8: noqa: E402
+
+from os import path
+import sys
+import json
+
+currentdir = path.dirname(path.realpath(__file__))
+parentdir = path.dirname(currentdir)
+sys.path.append(parentdir)
+
+from pytube import YouTube
+
+yt = YouTube(sys.argv[1], defer_prefetch_init=True)
+yt.prefetch()
+output = {
+    "url": sys.argv[1],
+    "watch_html": yt.watch_html,
+    "video_info": yt.vid_info,
+    "js": yt.js,
+    "embed_html": yt.embed_html,
+}
+
+outpath = path.join(currentdir, "mocks", "yt-video-" + yt.video_id + ".json")
+print("Writing to: " + outpath)
+with open(outpath, "w") as f:
+    json.dump(output, f)