Fix for the new youtube playlist requests (#950)

* Fix for the new youtube playlist requests

* Implements pytube.request.post in order to fetch continuations

* Implements ytcfg extraction

* Slight refactor of object parser

* Extends _execute_request to take a data argument for post requests

* Adjusted tests to reflect changes

pytube/contrib/playlist.py

@@ -30,12 +30,33 @@ class Playlist(Sequence):
         if proxies:
             install_proxy(proxies)
 
+        # These need to be initialized as None for the properties.
+        self._html = None
+        self._ytcfg = None
+
         self.playlist_id = extract.playlist_id(url)
 
         self.playlist_url = (
             f"https://www.youtube.com/playlist?list={self.playlist_id}"
         )
-        self.html = request.get(self.playlist_url)
+
+    @property
+    def html(self):
+        if self._html:
+            return self._html
+        self._html = request.get(self.playlist_url)
+        return self._html
+
+    @property
+    def ytcfg(self):
+        if self._ytcfg:
+            return self._ytcfg
+        self._ytcfg = extract.get_ytcfg(self.html)
+        return self._ytcfg
+
+    @property
+    def yt_api_key(self):
+        return self.ytcfg['INNERTUBE_API_KEY']
 
     def _paginate(
         self, until_watch_id: Optional[str] = None
@@ -49,7 +70,6 @@ class Playlist(Sequence):
         :rtype: Iterable[List[str]]
         :returns: Iterable of lists of YouTube watch ids
         """
-        req = self.html
         videos_urls, continuation = self._extract_videos(
             json.dumps(extract.initial_data(self.html))
         )
@@ -67,15 +87,15 @@ class Playlist(Sequence):
         # than 100 songs inside a playlist, so we need to add further requests
         # to gather all of them
         if continuation:
-            load_more_url, headers = self._build_continuation_url(continuation)
+            load_more_url, headers, data = self._build_continuation_url(continuation)
         else:
-            load_more_url, headers = None, None
+            load_more_url, headers, data = None, None, None
 
-        while load_more_url and headers:  # there is an url found
+        while load_more_url and headers and data:  # there is an url found
             logger.debug("load more url: %s", load_more_url)
             # requesting the next page of videos with the url generated from the
-            # previous page
-            req = request.get(load_more_url, extra_headers=headers)
+            # previous page, needs to be a post
+            req = request.post(load_more_url, extra_headers=headers, data=data)
             # extract up to 100 songs from the page loaded
             # returns another continuation if more videos are available
             videos_urls, continuation = self._extract_videos(req)
@@ -89,32 +109,43 @@ class Playlist(Sequence):
             yield videos_urls
 
             if continuation:
-                load_more_url, headers = self._build_continuation_url(
+                load_more_url, headers, data = self._build_continuation_url(
                     continuation
                 )
             else:
-                load_more_url, headers = None, None
+                load_more_url, headers, data = None, None, None
 
-    @staticmethod
-    def _build_continuation_url(continuation: str) -> Tuple[str, dict]:
+    def _build_continuation_url(self, continuation: str) -> Tuple[str, dict, dict]:
         """Helper method to build the url and headers required to request
         the next page of videos
 
         :param str continuation: Continuation extracted from the json response
             of the last page
-        :rtype: Tuple[str, dict]
+        :rtype: Tuple[str, dict, dict]
         :returns: Tuple of an url and required headers for the next http
             request
         """
         return (
             (
-                f"https://www.youtube.com/browse_ajax?ctoken="
-                f"{continuation}&continuation={continuation}"
+                # was changed to this format (and post requests)
+                # between 2021.03.02 and 2021.03.03
+                "https://www.youtube.com/youtubei/v1/browse?key="
+                f"{self.yt_api_key}"
             ),
             {
                 "X-YouTube-Client-Name": "1",
                 "X-YouTube-Client-Version": "2.20200720.00.02",
             },
+            # extra data required for post request
+            {
+                "continuation": continuation,
+                "context": {
+                    "client": {
+                        "clientName": "WEB",
+                        "clientVersion": "2.20200720.00.02"
+                    }
+                }
+            }
         )
 
     @staticmethod
@@ -150,9 +181,9 @@ class Playlist(Sequence):
             try:
                 # this is the json tree structure, if the json was directly sent
                 # by the server in a continuation response
-                important_content = initial_data[1]['response']['onResponseReceivedActions'][
-                    0
-                ]['appendContinuationItemsAction']['continuationItems']
+                # no longer a list and no longer has the "response" key
+                important_content = initial_data['onResponseReceivedActions'][0][
+                    'appendContinuationItemsAction']['continuationItems']
                 videos = important_content
             except (KeyError, IndexError, TypeError) as p:
                 print(p)

pytube/extract.py

@@ -24,6 +24,7 @@ from pytube.exceptions import RegexMatchError
 from pytube.helpers import regex_search
 from pytube.metadata import YouTubeMetadata
 from pytube.parser import parse_for_object
+from pytube.parser import parse_for_all_objects
 
 logger = logging.getLogger(__name__)
 
@@ -354,6 +355,40 @@ def get_ytplayer_config(html: str) -> Any:
     )
 
 
+def get_ytcfg(html: str) -> str:
+    """Get the entirety of the ytcfg object.
+
+    This is built over multiple pieces, so we have to find all matches and
+    combine the dicts together.
+
+    :param str html:
+        The html contents of the watch page.
+    :rtype: str
+    :returns:
+        Substring of the html containing the encoded manifest data.
+    """
+    ytcfg = {}
+    ytcfg_patterns = [
+        r"ytcfg\s=\s",
+        r"ytcfg\.set\("
+    ]
+    for pattern in ytcfg_patterns:
+        # Try each pattern consecutively and try to build a cohesive object
+        try:
+            found_objects = parse_for_all_objects(html, pattern)
+            for obj in found_objects:
+                ytcfg.update(obj)
+        except HTMLParseError:
+            continue
+
+    if len(ytcfg) > 0:
+        return ytcfg
+
+    raise RegexMatchError(
+        caller="get_ytcfg", pattern="ytcfg_pattenrs"
+    )
+
+
 def apply_signature(config_args: Dict, fmt: str, js: str) -> None:
     """Apply the decrypted signature to the stream manifest.
 

pytube/parser.py

@@ -4,6 +4,39 @@ import re
 from pytube.exceptions import HTMLParseError
 
 
+def parse_for_all_objects(html, preceding_regex):
+    """Parses input html to find all matches for the input starting point.
+
+    :param str html:
+        HTML to be parsed for an object.
+    :param str preceding_regex:
+        Regex to find the string preceding the object.
+    :rtype list:
+    :returns:
+        A list of dicts created from parsing the objects.
+    """
+    result = []
+    regex = re.compile(preceding_regex)
+    match_iter = regex.finditer(html)
+    for match in match_iter:
+        if match:
+            start_index = match.end()
+            try:
+                obj = parse_for_object_from_startpoint(html, start_index)
+            except HTMLParseError:
+                # Some of the instances might fail because set is technically
+                # a method of the ytcfg object. We'll skip these since they
+                # don't seem relevant at the moment.
+                continue
+            else:
+                result.append(obj)
+
+    if len(result) == 0:
+        raise HTMLParseError(f'No matches for regex {preceding_regex}')
+
+    return result
+
+
 def parse_for_object(html, preceding_regex):
     """Parses input html to find the end of a JavaScript object.
 
@@ -20,11 +53,11 @@ def parse_for_object(html, preceding_regex):
     if not result:
         raise HTMLParseError(f'No matches for regex {preceding_regex}')
 
-    start_index = result.span()[1]
+    start_index = result.end()
     return parse_for_object_from_startpoint(html, start_index)
 
 
-def parse_for_object_from_startpoint(html, start_point):
+def find_object_from_startpoint(html, start_point):
     """Parses input html to find the end of a JavaScript object.
 
     :param str html:
@@ -77,10 +110,25 @@ def parse_for_object_from_startpoint(html, start_point):
         i += 1
 
     full_obj = html[:i]
+    return full_obj  # noqa: R504
+
+
+def parse_for_object_from_startpoint(html, start_point):
+    """JSONifies an object parsed from HTML.
+
+    :param str html:
+        HTML to be parsed for an object.
+    :param int start_point:
+        Index of where the object starts.
+    :rtype dict:
+    :returns:
+        A dict created from parsing the object.
+    """
+    full_obj = find_object_from_startpoint(html, start_point)
     try:
         return json.loads(full_obj)
     except json.decoder.JSONDecodeError:
         try:
             return ast.literal_eval(full_obj)
-        except ValueError:
+        except (ValueError, SyntaxError):
             raise HTMLParseError('Could not parse object.')

pytube/request.py

@@ -3,6 +3,7 @@
 import logging
 from functools import lru_cache
 import re
+import json
 from urllib import parse
 from urllib.request import Request
 from urllib.request import urlopen
@@ -15,12 +16,16 @@ default_chunk_size = 4096  # 4kb
 default_range_size = 9437184  # 9MB
 
 
-def _execute_request(url, method=None, headers=None):
+def _execute_request(url, method=None, headers=None, data=None):
     base_headers = {"User-Agent": "Mozilla/5.0", "accept-language": "en-US,en"}
     if headers:
         base_headers.update(headers)
+    if data:
+        # encode data for request
+        if not isinstance(data, bytes):
+            data = bytes(json.dumps(data), encoding="utf-8")
     if url.lower().startswith("http"):
-        request = Request(url, headers=base_headers, method=method)
+        request = Request(url, headers=base_headers, method=method, data=data)
     else:
         raise ValueError("Invalid URL")
     return urlopen(request)  # nosec
@@ -42,6 +47,31 @@ def get(url, extra_headers=None):
     return _execute_request(url, headers=extra_headers).read().decode("utf-8")
 
 
+def post(url, extra_headers=None, data=None):
+    """Send an http POST request.
+
+    :param str url:
+        The URL to perform the POST request for.
+    :param dict extra_headers:
+        Extra headers to add to the request
+    :param dict data:
+        The data to send on the POST request
+    :rtype: str
+    :returns:
+        UTF-8 encoded string of response
+    """
+    # could technically be implemented in get,
+    # but to avoid confusion implemented like this
+    if extra_headers is None:
+        extra_headers = {}
+    if data is None:
+        data = {}
+    # required because the youtube servers are strict on content type
+    # raises HTTPError [400]: Bad Request otherwise
+    extra_headers.update({"Content-Type": "application/json"})
+    return _execute_request(url, headers=extra_headers, data=data).read().decode("utf-8")
+
+
 def seq_stream(url, chunk_size=default_chunk_size, range_size=default_range_size):
     """Read the response in sequence.
     :param str url: The URL to perform the GET request for.

tests/contrib/test_playlist.py

@@ -1,12 +1,11 @@
 # -*- coding: utf-8 -*-
 import datetime
 from unittest import mock
-from unittest.mock import MagicMock
 
 from pytube import Playlist
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_title(request_get):
     request_get.return_value = (
         "<title>(149) Python Tutorial for Beginners "
@@ -24,7 +23,7 @@ def test_title(request_get):
     )
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_init_with_playlist_url(request_get):
     request_get.return_value = ""
     url = (
@@ -35,7 +34,7 @@ def test_init_with_playlist_url(request_get):
     assert playlist.playlist_url == url
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_init_with_watch_url(request_get):
     request_get.return_value = ""
     url = (
@@ -49,7 +48,7 @@ def test_init_with_watch_url(request_get):
     )
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_last_updated(request_get, playlist_html):
     expected = datetime.date(2020, 3, 11)
     request_get.return_value = playlist_html
@@ -60,13 +59,11 @@ def test_last_updated(request_get, playlist_html):
     assert playlist.last_updated == expected
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_video_urls(request_get, playlist_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.return_value = playlist_html
     playlist = Playlist(url)
-    playlist._find_load_more_url = MagicMock(return_value=None)
-    request_get.assert_called()
     assert playlist.video_urls == [
         "https://www.youtube.com/watch?v=ujTCoH21GlA",
         "https://www.youtube.com/watch?v=45ryDIPHdGg",
@@ -81,15 +78,23 @@ def test_video_urls(request_get, playlist_html):
         "https://www.youtube.com/watch?v=g1Zbuk1gAfk",
         "https://www.youtube.com/watch?v=zixd-si9Q-o",
     ]
+    request_get.assert_called()
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
-def test_repr(request_get, playlist_html):
+@mock.patch("pytube.request.get")
+def test_html(request_get, playlist_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.return_value = playlist_html
     playlist = Playlist(url)
-    playlist._find_load_more_url = MagicMock(return_value=None)
+    assert playlist.html == playlist_html
     request_get.assert_called()
+
+
+@mock.patch("pytube.request.get")
+def test_repr(request_get, playlist_html):
+    url = "https://www.fakeurl.com/playlist?list=whatever"
+    request_get.return_value = playlist_html
+    playlist = Playlist(url)
     assert (
         repr(playlist) == "["
         "'https://www.youtube.com/watch?v=ujTCoH21GlA', "
@@ -106,30 +111,29 @@ def test_repr(request_get, playlist_html):
         "'https://www.youtube.com/watch?v=zixd-si9Q-o'"
         "]"
     )
+    request_get.assert_called()
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_sequence(request_get, playlist_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.return_value = playlist_html
     playlist = Playlist(url)
-    playlist._find_load_more_url = MagicMock(return_value=None)
     assert playlist[0] == "https://www.youtube.com/watch?v=ujTCoH21GlA"
     assert len(playlist) == 12
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 @mock.patch("pytube.cli.YouTube.__init__", return_value=None)
 def test_videos(youtube, request_get, playlist_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.return_value = playlist_html
     playlist = Playlist(url)
-    playlist._find_load_more_url = MagicMock(return_value=None)
-    request_get.assert_called()
     assert len(list(playlist.videos)) == 12
+    request_get.assert_called()
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 @mock.patch("pytube.cli.YouTube.__init__", return_value=None)
 def test_load_more(youtube, request_get, playlist_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
@@ -138,12 +142,11 @@ def test_load_more(youtube, request_get, playlist_html):
         '{"content_html":"", "load_more_widget_html":""}',
     ]
     playlist = Playlist(url)
-    playlist._find_load_more_url = MagicMock(side_effect=["dummy", None])
-    request_get.assert_called()
     assert len(list(playlist.videos)) == 12
+    request_get.assert_called()
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 @mock.patch("pytube.contrib.playlist.install_proxy", return_value=None)
 def test_proxy(install_proxy, request_get):
     url = "https://www.fakeurl.com/playlist?list=whatever"
@@ -152,31 +155,34 @@ def test_proxy(install_proxy, request_get):
     install_proxy.assert_called_with({"http": "things"})
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_trimmed(request_get, playlist_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.return_value = playlist_html
     playlist = Playlist(url)
-    playlist._find_load_more_url = MagicMock(return_value=None)
-    assert request_get.call_count == 1
     trimmed = list(playlist.trimmed("1BYu65vLKdA"))
     assert trimmed == [
         "https://www.youtube.com/watch?v=ujTCoH21GlA",
         "https://www.youtube.com/watch?v=45ryDIPHdGg",
     ]
+    assert request_get.call_count == 1
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
-def test_playlist_failed_pagination(request_get, playlist_long_html):
+@mock.patch("pytube.request.get")
+@mock.patch("pytube.request.post")
+def test_playlist_failed_pagination(request_post, request_get, playlist_long_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.side_effect = [
         playlist_long_html,
-        "{}",
+    ]
+    request_post.side_effect = [
+        "{}"
     ]
     playlist = Playlist(url)
     video_urls = playlist.video_urls
     assert len(video_urls) == 100
-    assert request_get.call_count == 2
+    assert request_get.call_count == 1
+    assert request_post.call_count == 1
     # TODO: Cannot get this test to work probably
     # request_get.assert_called_with(
     #    "https://www.youtube.com/browse_ajax?ctoken" # noqa
@@ -189,11 +195,14 @@ def test_playlist_failed_pagination(request_get, playlist_long_html):
     # ) # noqa
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
-def test_playlist_pagination(request_get, playlist_html, playlist_long_html):
+@mock.patch("pytube.request.get")
+@mock.patch("pytube.request.post")
+def test_playlist_pagination(request_post, request_get, playlist_html, playlist_long_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.side_effect = [
-        playlist_long_html,
+        playlist_long_html
+    ]
+    request_post.side_effect = [
         '{"content_html":"<a '
         'href=\\"/watch?v=BcWz41-4cDk&amp;feature=plpp_video&amp;ved'
         '=CCYQxjQYACITCO33n5-pn-cCFUG3xAodLogN2yj6LA\\">}", '
@@ -202,10 +211,11 @@ def test_playlist_pagination(request_get, playlist_html, playlist_long_html):
     ]
     playlist = Playlist(url)
     assert len(playlist.video_urls) == 100
-    assert request_get.call_count == 2
+    assert request_get.call_count == 1
+    assert request_post.call_count == 1
 
 
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_trimmed_pagination(request_get, playlist_html, playlist_long_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"
     request_get.side_effect = [
@@ -222,7 +232,7 @@ def test_trimmed_pagination(request_get, playlist_html, playlist_long_html):
 
 
 # TODO: Test case not clear to me
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_trimmed_pagination_not_found(
     request_get, playlist_html, playlist_long_html
 ):
@@ -241,7 +251,7 @@ def test_trimmed_pagination_not_found(
 
 
 # test case for playlist with submenus
-@mock.patch("pytube.contrib.playlist.request.get")
+@mock.patch("pytube.request.get")
 def test_playlist_submenu(
         request_get, playlist_submenu_html):
     url = "https://www.fakeurl.com/playlist?list=whatever"