Defer web requests until absolutely necessary (#953)

* Defers web requests until they're actually necessary to fetch information

* Adjusts tests to reflect changes to how web requests are made

* Removes "defer_prefetch_init" arg from YouTube object initialization

* Converted most attributes into caching properties to facilitate request deferring.

* Added some documentation to uniqueify to explain its purpose.

* Prefetching added to conftest to improve test speed

* Added some setters for YouTube properties.

pytube/__main__.py

@@ -44,7 +44,6 @@ class YouTube:
     def __init__(
         self,
         url: str,
-        defer_prefetch_init: bool = False,
         on_progress_callback: Optional[Callable[[Any, bytes, int], None]] = None,
         on_complete_callback: Optional[Callable[[Any, Optional[str]], None]] = None,
         proxies: Dict[str, str] = None,
@@ -63,27 +62,27 @@ class YouTube:
             complete events.
 
         """
-        self.js: Optional[str] = None  # js fetched by js_url
-        self.js_url: Optional[str] = None  # the url to the js, parsed from watch html
+        self._js: Optional[str] = None  # js fetched by js_url
+        self._js_url: Optional[str] = None  # the url to the js, parsed from watch html
 
         # note: vid_info may eventually be removed. It sounds like it once had
         # additional formats, but that doesn't appear to still be the case.
 
         # the url to vid info, parsed from watch html
-        self.vid_info_url: Optional[str] = None
-        self.vid_info_raw: Optional[str] = None  # content fetched by vid_info_url
-        self.vid_info: Optional[Dict] = None  # parsed content of vid_info_raw
-
-        self.watch_html: Optional[str] = None  # the html of /watch?v=<video_id>
-        self.embed_html: Optional[str] = None
-        self.player_config_args: Dict = {}  # inline js in the html containing
-        self.player_response: Dict = {}
+        self._vid_info_url: Optional[str] = None
+        self._vid_info_raw: Optional[str] = None  # content fetched by vid_info_url
+        self._vid_info: Optional[Dict] = None  # parsed content of vid_info_raw
+
+        self._watch_html: Optional[str] = None  # the html of /watch?v=<video_id>
+        self._embed_html: Optional[str] = None
+        self._player_config_args: Optional[Dict] = None  # inline js in the html containing
+        self._player_response: Optional[Dict] = None
         # streams
-        self.age_restricted: Optional[bool] = None
+        self._age_restricted: Optional[bool] = None
 
-        self.fmt_streams: List[Stream] = []
+        self._fmt_streams: Optional[List[Stream]] = None
 
-        self.initial_data = {}
+        self._initial_data = None
         self._metadata: Optional[YouTubeMetadata] = None
 
         # video_id part of /watch?v=<video_id>
@@ -100,60 +99,108 @@ class YouTube:
         if proxies:
             install_proxy(proxies)
 
-        if not defer_prefetch_init:
-            self.prefetch()
-            self.descramble()
+        self._author = None
+        self._title = None
+        self._publish_date = None
 
-    def check_availability(self):
-        """Check whether the video is available.
-        Raises different exceptions based on why the video is unavailable,
-        otherwise does nothing.
+    @property
+    def watch_html(self):
+        if self._watch_html:
+            return self._watch_html
+        self._watch_html = request.get(url=self.watch_url)
+        return self._watch_html
 
-        """
-        if self.watch_html is None:
-            raise VideoUnavailable(video_id=self.video_id)
+    @property
+    def embed_html(self):
+        if self._embed_html:
+            return self._embed_html
+        self._embed_html = request.get(url=self.embed_url)
+        return self._embed_html
 
-        status, messages = extract.playability_status(self.watch_html)
+    @property
+    def vid_info_raw(self):
+        if self._vid_info_raw:
+            return self._vid_info_raw
+        self._vid_info_raw = request.get(self.vid_info_url)
+        return self._vid_info_raw
 
-        for reason in messages:
-            if status == 'UNPLAYABLE':
-                if reason == (
-                    'Join this channel to get access to members-only content '
-                    'like this video, and other exclusive perks.'
-                ):
-                    raise MembersOnly(video_id=self.video_id)
-                elif reason == 'This live stream recording is not available.':
-                    raise RecordingUnavailable(video_id=self.video_id)
-                else:
-                    if reason == 'Video unavailable':
-                        if extract.is_region_blocked(self.watch_html):
-                            raise VideoRegionBlocked(video_id=self.video_id)
-                    raise VideoUnavailable(video_id=self.video_id)
-            elif status == 'LOGIN_REQUIRED':
-                if reason == (
-                    'This is a private video. '
-                    'Please sign in to verify that you may see it.'
-                ):
-                    raise VideoPrivate(video_id=self.video_id)
-            elif status == 'ERROR':
-                if reason == 'Video unavailable':
-                    raise VideoUnavailable(video_id=self.video_id)
+    @property
+    def age_restricted(self):
+        if self._age_restricted:
+            return self._age_restricted
+        self._age_restricted = extract.is_age_restricted(self.watch_html)
+        return self._age_restricted
 
-    def descramble(self) -> None:
-        """Descramble the stream data and build Stream instances.
+    @property
+    def vid_info_url(self):
+        if self._vid_info_url:
+            return self._vid_info_url
 
-        The initialization process takes advantage of Python's
-        "call-by-reference evaluation," which allows dictionary transforms to
-        be applied in-place, instead of holding references to mutations at each
-        interstitial step.
+        if self.age_restricted:
+            self._vid_info_url = extract.video_info_url_age_restricted(
+                self.video_id, self.watch_url
+            )
+        else:
+            self._vid_info_url = extract.video_info_url(
+                video_id=self.video_id, watch_url=self.watch_url
+            )
+        return self._vid_info_url
 
-        :rtype: None
+    @property
+    def js_url(self):
+        if self._js_url:
+            return self._js_url
 
-        """
-        self.vid_info = dict(parse_qsl(self.vid_info_raw))
-        self.player_config_args = self.vid_info
-        self.player_response = json.loads(self.vid_info['player_response'])
+        if self.age_restricted:
+            self._js_url = extract.js_url(self.embed_html)
+        else:
+            self._js_url = extract.js_url(self.watch_html)
+
+        return self._js_url
+
+    @property
+    def js(self):
+        if self._js:
+            return self._js
+
+        # If the js_url doesn't match the cached url, fetch the new js and update
+        #  the cache; otherwise, load the cache.
+        if pytube.__js_url__ != self.js_url:
+            self._js = request.get(self.js_url)
+            pytube.__js__ = self._js
+            pytube.__js_url__ = self.js_url
+        else:
+            self._js = pytube.__js__
+
+        return self._js
+
+    @property
+    def player_response(self):
+        """The player response contains subtitle information and video details."""
+        if self._player_response:
+            return self._player_response
 
+        if isinstance(self.player_config_args["player_response"], str):
+            self._player_response = json.loads(
+                self.player_config_args["player_response"]
+            )
+        else:
+            self._player_response = self.player_config_args["player_response"]
+        return self._player_response
+
+    @property
+    def initial_data(self):
+        if self._initial_data:
+            return self._initial_data
+        self._initial_data = extract.initial_data(self.watch_html)
+        return self._initial_data
+
+    @property
+    def player_config_args(self):
+        if self._player_config_args:
+            return self._player_config_args
+
+        self._player_config_args = self.vid_info
         # On pre-signed videos, we need to use get_ytplayer_config to fix
         #  the player_response item
         if 'streamingData' not in self.player_config_args['player_response']:
@@ -163,6 +210,20 @@ class YouTube:
             else:
                 self.player_config_args['player_response'] = config_response
 
+        return self._player_config_args
+
+    @property
+    def fmt_streams(self):
+        """Returns a list of streams if they have been initialized.
+
+        If the streams have not been initialized, finds all relevant
+        streams and initializes them.
+        """
+        self.check_availability()
+        if self._fmt_streams:
+            return self._fmt_streams
+
+        self._fmt_streams = []
         # https://github.com/nficano/pytube/issues/165
         stream_maps = ["url_encoded_fmt_stream_map"]
         if "adaptive_fmts" in self.player_config_args:
@@ -177,80 +238,60 @@ class YouTube:
             apply_signature(self.player_config_args, fmt, self.js)
 
             # build instances of :class:`Stream <Stream>`
-            self.initialize_stream_objects(fmt)
+            # Initialize stream objects
+            stream_manifest = self.player_config_args[fmt]
+            for stream in stream_manifest:
+                video = Stream(
+                    stream=stream,
+                    player_config_args=self.player_config_args,
+                    monostate=self.stream_monostate,
+                )
+                self._fmt_streams.append(video)
 
-        # load the player_response object (contains subtitle information)
-        if isinstance(self.player_config_args["player_response"], str):
-            self.player_response = json.loads(
-                self.player_config_args["player_response"]
-            )
-        else:
-            self.player_response = self.player_config_args["player_response"]
-        del self.player_config_args["player_response"]
         self.stream_monostate.title = self.title
         self.stream_monostate.duration = self.length
 
-    def prefetch(self) -> None:
-        """Eagerly download all necessary data.
+        return self._fmt_streams
 
-        Eagerly executes all necessary network requests so all other
-        operations don't does need to make calls outside of the interpreter
-        which blocks for long periods of time.
+    def check_availability(self):
+        """Check whether the video is available.
 
-        :rtype: None
+        Raises different exceptions based on why the video is unavailable,
+        otherwise does nothing.
         """
-        self.watch_html = request.get(url=self.watch_url)
-        self.check_availability()
-        self.age_restricted = extract.is_age_restricted(self.watch_html)
-
-        if self.age_restricted:
-            if not self.embed_html:
-                self.embed_html = request.get(url=self.embed_url)
-            self.vid_info_url = extract.video_info_url_age_restricted(
-                self.video_id, self.watch_url
-            )
-            self.js_url = extract.js_url(self.embed_html)
-        else:
-            self.vid_info_url = extract.video_info_url(
-                video_id=self.video_id, watch_url=self.watch_url
-            )
-            self.js_url = extract.js_url(self.watch_html)
-
-        self.initial_data = extract.initial_data(self.watch_html)
-
-        self.vid_info_raw = request.get(self.vid_info_url)
-
-        # If the js_url doesn't match the cached url, fetch the new js and update
-        #  the cache; otherwise, load the cache.
-        if pytube.__js_url__ != self.js_url:
-            self.js = request.get(self.js_url)
-            pytube.__js__ = self.js
-            pytube.__js_url__ = self.js_url
-        else:
-            self.js = pytube.__js__
-
-    def initialize_stream_objects(self, fmt: str) -> None:
-        """Convert manifest data to instances of :class:`Stream <Stream>`.
-
-        Take the unscrambled stream data and uses it to initialize
-        instances of :class:`Stream <Stream>` for each media stream.
+        status, messages = extract.playability_status(self.watch_html)
 
-        :param str fmt:
-            Key in stream manifest (``ytplayer_config``) containing progressive
-            download or adaptive streams (e.g.: ``url_encoded_fmt_stream_map``
-            or ``adaptive_fmts``).
+        for reason in messages:
+            if status == 'UNPLAYABLE':
+                if reason == (
+                    'Join this channel to get access to members-only content '
+                    'like this video, and other exclusive perks.'
+                ):
+                    raise MembersOnly(video_id=self.video_id)
+                elif reason == 'This live stream recording is not available.':
+                    raise RecordingUnavailable(video_id=self.video_id)
+                else:
+                    if reason == 'Video unavailable':
+                        if extract.is_region_blocked(self.watch_html):
+                            raise VideoRegionBlocked(video_id=self.video_id)
+                    raise VideoUnavailable(video_id=self.video_id)
+            elif status == 'LOGIN_REQUIRED':
+                if reason == (
+                    'This is a private video. '
+                    'Please sign in to verify that you may see it.'
+                ):
+                    raise VideoPrivate(video_id=self.video_id)
+            elif status == 'ERROR':
+                if reason == 'Video unavailable':
+                    raise VideoUnavailable(video_id=self.video_id)
 
-        :rtype: None
+    @property
+    def vid_info(self):
+        """Parse the raw vid info and return the parsed result.
 
+        :rtype: Dict[Any, Any]
         """
-        stream_manifest = self.player_config_args[fmt]
-        for stream in stream_manifest:
-            video = Stream(
-                stream=stream,
-                player_config_args=self.player_config_args,
-                monostate=self.stream_monostate,
-            )
-            self.fmt_streams.append(video)
+        return dict(parse_qsl(self.vid_info_raw))
 
     @property
     def caption_tracks(self) -> List[Caption]:
@@ -279,6 +320,7 @@ class YouTube:
 
         :rtype: :class:`StreamQuery <StreamQuery>`.
         """
+        self.check_availability()
         return StreamQuery(self.fmt_streams)
 
     @property
@@ -286,7 +328,6 @@ class YouTube:
         """Get the thumbnail url image.
 
         :rtype: str
-
         """
         thumbnail_details = (
             self.player_response.get("videoDetails", {})
@@ -304,25 +345,38 @@ class YouTube:
         """Get the publish date.
 
         :rtype: datetime
-
         """
-        return extract.publish_date(self.watch_html)
+        if self._publish_date:
+            return self._publish_date
+        self._publish_date = extract.publish_date(self.watch_html)
+        return self._publish_date
+
+    @publish_date.setter
+    def publish_date(self, value):
+        """Sets the publish date."""
+        self._publish_date = value
 
     @property
     def title(self) -> str:
         """Get the video title.
 
         :rtype: str
-
         """
-        return self.player_response['videoDetails']['title']
+        if self._title:
+            return self._title
+        self._title = self.player_response['videoDetails']['title']
+        return self._title
+
+    @title.setter
+    def title(self, value):
+        """Sets the title value."""
+        self._title = value
 
     @property
     def description(self) -> str:
         """Get the video description.
 
         :rtype: str
-
         """
         return self.player_response.get("videoDetails", {}).get("shortDescription")
 
@@ -340,7 +394,6 @@ class YouTube:
         """Get the video length in seconds.
 
         :rtype: int
-
         """
         return int(
             self.player_config_args.get("length_seconds")
@@ -356,7 +409,6 @@ class YouTube:
         """Get the number of the times the video has been viewed.
 
         :rtype: int
-
         """
         return int(
             self.player_response.get("videoDetails", {}).get("viewCount")
@@ -367,9 +419,17 @@ class YouTube:
         """Get the video author.
         :rtype: str
         """
-        return self.player_response.get("videoDetails", {}).get(
+        if self._author:
+            return self._author
+        self._author = self.player_response.get("videoDetails", {}).get(
             "author", "unknown"
         )
+        return self._author
+
+    @author.setter
+    def author(self, value):
+        """Set the video author."""
+        self._author = value
 
     @property
     def keywords(self) -> List[str]:

pytube/helpers.py

@@ -173,6 +173,14 @@ def install_proxy(proxy_handler: Dict[str, str]) -> None:
 
 
 def uniqueify(duped_list: List) -> List:
+    """Remove duplicate items from a list, while maintaining list order.
+
+    :param List duped_list
+        List to remove duplicates from
+
+    :return List result
+        De-duplicated list
+    """
     seen: Dict[Any, bool] = {}
     result = []
     for item in duped_list:
@@ -205,11 +213,7 @@ def create_mock_html_json(vid_id) -> Dict[str, Any]:
     pytube_mocks_path = os.path.join(pytube_dir_path, 'tests', 'mocks')
     gzip_filepath = os.path.join(pytube_mocks_path, gzip_filename)
 
-    yt = YouTube(
-        'https://www.youtube.com/watch?v=%s' % vid_id,
-        defer_prefetch_init=True
-    )
-    yt.prefetch()
+    yt = YouTube(f'https://www.youtube.com/watch?v={vid_id}')
     html_data = {
         'url': yt.watch_url,
         'js': yt.js,

tests/conftest.py

@@ -34,7 +34,18 @@ def load_and_init_from_playback_file(filename, mock_urlopen):
     ]
     mock_urlopen.return_value = mock_url_open_object
 
-    return YouTube(pb["url"])
+    # Pytest caches this result, so we can speed up the tests
+    #  by causing the object to fetch all the relevant information
+    #  it needs. Previously, this was handled by prefetch_init()
+    #  and descramble(), but this functionality has since been
+    #  deferred
+    v = YouTube(pb["url"])
+    v.watch_html
+    v.vid_info_raw
+    v.js
+    v.fmt_streams
+    v.player_response
+    return v
 
 
 @pytest.fixture

tests/test_exceptions.py

@@ -79,7 +79,7 @@ def test_raises_video_private(private):
         ]
         mock_url_open.return_value = mock_url_open_object
         with pytest.raises(VideoPrivate):
-            YouTube('https://youtube.com/watch?v=m8uHb5jIGN8')
+            YouTube('https://youtube.com/watch?v=m8uHb5jIGN8').streams
 
 
 def test_raises_recording_unavailable(missing_recording):
@@ -91,7 +91,7 @@ def test_raises_recording_unavailable(missing_recording):
         ]
         mock_url_open.return_value = mock_url_open_object
         with pytest.raises(RecordingUnavailable):
-            YouTube('https://youtube.com/watch?v=5YceQ8YqYMc')
+            YouTube('https://youtube.com/watch?v=5YceQ8YqYMc').streams
 
 
 def test_raises_video_region_blocked(region_blocked):
@@ -103,4 +103,4 @@ def test_raises_video_region_blocked(region_blocked):
         ]
         mock_url_open.return_value = mock_url_open_object
         with pytest.raises(VideoRegionBlocked):
-            YouTube('https://youtube.com/watch?v=hZpzr8TbF08')
+            YouTube('https://youtube.com/watch?v=hZpzr8TbF08').streams

tests/test_helpers.py

@@ -117,14 +117,16 @@ def test_create_mock_html_json(mock_url_open, mock_open):
     mock_url_open_object = mock.Mock()
 
     # Order is:
-    # 1. watch_html -- must have js match
-    # 2. vid_info_raw
-    # 3. js
+    # 1. watch_html -- must have jsurl match
+    # 2. embed html
+    # 3. watch html
+    # 4. raw vid info
     mock_url_open_object.read.side_effect = [
         (b'yt.setConfig({"PLAYER_CONFIG":{"args":[]}});ytInitialData = {};ytInitialPlayerResponse = {};'  # noqa: E501
          b'"jsUrl":"/s/player/13371337/player_ias.vflset/en_US/base.js"'),
+        b'embed_html',
+        b'watch_html',
         b'vid_info_raw',
-        b'js_result',
     ]
     mock_url_open.return_value = mock_url_open_object
 

tests/test_main.py

@@ -5,7 +5,7 @@ import pytest
 
 import pytube
 from pytube import YouTube
-from pytube.exceptions import VideoUnavailable
+from pytube.exceptions import RegexMatchError
 
 
 @mock.patch("pytube.__main__.YouTube")
@@ -21,7 +21,6 @@ def test_install_proxy(opener):
     proxies = {"http": "http://www.example.com:3128/"}
     YouTube(
         "https://www.youtube.com/watch?v=9bZkp7q19f0",
-        defer_prefetch_init=True,
         proxies=proxies,
     )
     opener.assert_called()
@@ -29,12 +28,10 @@ def test_install_proxy(opener):
 
 @mock.patch("pytube.request.get")
 def test_video_unavailable(get):
-    get.return_value = None
-    youtube = YouTube(
-        "https://www.youtube.com/watch?v=9bZkp7q19f0", defer_prefetch_init=True
-    )
-    with pytest.raises(VideoUnavailable):
-        youtube.prefetch()
+    get.return_value = ""
+    youtube = YouTube("https://www.youtube.com/watch?v=9bZkp7q19f0")
+    with pytest.raises(RegexMatchError):
+        youtube.check_availability()
 
 
 def test_video_keywords(cipher_signature):
@@ -62,17 +59,3 @@ def test_js_caching(cipher_signature):
     assert pytube.__js_url__ is not None
     assert pytube.__js__ == cipher_signature.js
     assert pytube.__js_url__ == cipher_signature.js_url
-
-    with mock.patch('pytube.request.urlopen') as mock_urlopen:
-        mock_urlopen_object = mock.Mock()
-
-        # We should never read the js from this
-        mock_urlopen_object.read.side_effect = [
-            cipher_signature.watch_html.encode('utf-8'),
-            cipher_signature.vid_info_raw.encode('utf-8'),
-            cipher_signature.js.encode('utf-8')
-        ]
-
-        mock_urlopen.return_value = mock_urlopen_object
-        cipher_signature.prefetch()
-        assert mock_urlopen.call_count == 2

tests/test_streams.py

@@ -251,17 +251,18 @@ def test_on_complete_hook(cipher_signature):
 
 def test_author(cipher_signature):
     expected = "Test author"
-    cipher_signature.player_response = {"videoDetails": {"author": expected}}
+    cipher_signature._player_response = {"videoDetails": {"author": expected}}
     assert cipher_signature.author == expected
 
     expected = "unknown"
-    cipher_signature.player_response = {}
+    cipher_signature.author = None
+    cipher_signature._player_response = {'key': 'value'}
     assert cipher_signature.author == expected
 
 
 def test_thumbnail_when_in_details(cipher_signature):
     expected = "some url"
-    cipher_signature.player_response = {
+    cipher_signature._player_response = {
         "videoDetails": {"thumbnail": {"thumbnails": [{"url": expected}]}}
     }
     assert cipher_signature.thumbnail_url == expected
@@ -269,7 +270,7 @@ def test_thumbnail_when_in_details(cipher_signature):
 
 def test_thumbnail_when_not_in_details(cipher_signature):
     expected = "https://img.youtube.com/vi/2lAe1cqCOXo/maxresdefault.jpg"
-    cipher_signature.player_response = {}
+    cipher_signature._player_response = {'key': 'value'}
     assert cipher_signature.thumbnail_url == expected