Merge pull request #2 from MarcAbonce/bump_py_version

Support py 3.11 by mocking HTTP reqs with httpretty
MarcAbonce · Nov 4, 2024 · b4905c2 · b4905c2
2 parents 9fb7825 + 9127a94
commit b4905c2
Show file tree

Hide file tree

Showing 8 changed files with 86 additions and 123 deletions.
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.7, 3.11, 3.12]
 
     steps:
     - uses: actions/checkout@v2

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -1,3 +1,4 @@
 flake8
 mypy
 pydoc-markdown
+httpretty
diff --git a/setup.py b/setup.py
@@ -31,10 +31,6 @@ def get_version():
         "Intended Audience :: Developers",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
     ],
     install_requires=get_file_contents('requirements.txt', break_lines=True),
     packages=find_packages(),

diff --git a/test/__init__.py b/test/__init__.py
@@ -1,55 +0,0 @@
-import os
-from unittest import mock
-from urllib.parse import urlparse, quote
-
-
-TEST_DIR = os.path.dirname(os.path.realpath(__file__))
-
-
-# Generate filename for html files in test_assets
-def url_to_filename(url):
-    filename = quote(url[:url.find('&')].replace('/', '_'))
-    if not filename.endswith('.html'):
-        filename += '.html'
-    return filename
-
-
-# Patch to mock an HTTP response
-def mock_urlopen(*args, **kwargs):
-    url = args[0]
-    headers = {}
-    ext = urlparse(url).path.split('.')[-1].lower()
-    if ext in ['gif', 'jpg', 'jpeg', 'png', 'webp']:
-        return mock_empty_image_response(*args, **kwargs)
-    else:
-        headers['Content-Type'] = 'text/html; charset=utf-8'
-        content = get_video_html(*args, **kwargs).encode()
-
-    return mock.MagicMock(status=200, headers=headers, read=lambda: content)
-
-
-# Patch to open local webpage instead of downloading it
-def get_video_html(url, *args, **kwargs):
-    filename = url_to_filename(url)
-    full_path = os.path.join(TEST_DIR, 'test_assets', filename)
-    with open(full_path) as f:
-        return f.read()
-
-
-def mock_empty_html_response(*args, **kwargs):
-    headers = {'Content-Type': 'text/html; charset=utf-8'}
-    content = "<!DOCTYPE html><html><head></head><body></body></html>"
-    return mock.MagicMock(status=200, headers=headers, read=lambda: content.encode())
-
-
-def mock_empty_json_response(*args, **kwargs):
-    headers = {'Content-Type': 'application/json; charset=utf-8'}
-    content = "{}"
-    return mock.MagicMock(status=200, headers=headers, read=lambda: content.encode())
-
-
-def mock_empty_image_response(*args, **kwargs):
-    headers = {'Content-Type': 'image/webp'}
-    content = (b'RIFF$\x00\x00\x00WEBPVP8 '
-               b'\x18\x00\x00\x000\x01\x00\x9d\x01*\x01\x00\x01\x00\x0f\xc0\xfe%\xa4\x00\x03p\x00\xfe\xe6\xb5\x00\x00')
-    return mock.MagicMock(status=200, headers=headers, read=lambda: content)

diff --git a/....youtube.com_watch%3Fv%3DWhWc3b3KhnY.html → ...t_assets/www_youtube_com_WhWc3b3KhnY.html b/....youtube.com_watch%3Fv%3DWhWc3b3KhnY.html → ...t_assets/www_youtube_com_WhWc3b3KhnY.html
diff --git a/...o_id%3DWhWc3b3KhnY%26el%3Ddetailpage.html → ...et_video_info_WhWc3b3KhnY_detailpage.html b/...o_id%3DWhWc3b3KhnY%26el%3Ddetailpage.html → ...et_video_info_WhWc3b3KhnY_detailpage.html
diff --git a/test/test_youtube.py b/test/test_youtube.py
@@ -1,20 +1,63 @@
+import re
+import os
+import unittest
+
+import httpretty  # type: ignore
+
 from numbers import Number
-from unittest import mock, TestCase
 from urllib.parse import urlparse
 
 from youtube_dl.utils import ExtractorError
 from thumbframes_dl import YouTubeFrames
 
-from . import mock_urlopen, mock_empty_json_response
 
+TEST_DIR = os.path.dirname(os.path.realpath(__file__))
 
-@mock.patch('youtube_dl.YoutubeDL.urlopen', side_effect=mock_urlopen)
-class TestYouTubeFrames(TestCase):
+
+class TestYouTubeFrames(unittest.TestCase):
 
     # Spring | Blender Animation Studio | CC BY 4.0
     VIDEO_ID = 'WhWc3b3KhnY'
     VIDEO_URL = 'https://www.youtube.com/watch?v=WhWc3b3KhnY'
 
+    # Mock YoutubeDL's internal HTTP requests
+    def setUp(self):
+        httpretty.reset()
+        httpretty.enable(allow_net_connect=False)
+
+        # main video page
+        video_path = 'www_youtube_com_WhWc3b3KhnY.html'
+        with open(os.path.join(TEST_DIR, 'test_assets', video_path)) as f:
+            video_page = f.read()
+        httpretty.register_uri(
+            httpretty.GET,
+            self.VIDEO_URL,
+            body=video_page
+        )
+
+        # file with video data
+        details_path = 'www_youtube_com_get_video_info_WhWc3b3KhnY_detailpage.html'
+        with open(os.path.join(TEST_DIR, 'test_assets', details_path)) as f:
+            details_page = f.read()
+        httpretty.register_uri(
+            httpretty.POST,
+            'https://www.youtube.com/youtubei/v1/player',
+            body=details_page
+        )
+
+        # images
+        httpretty.register_uri(
+            httpretty.GET,
+            re.compile('^.*(jpg|jpeg|webp|png|gif)$'),
+            body=(b'RIFF$\x00\x00\x00WEBPVP8 '
+                  b'\x18\x00\x00\x000\x01\x00\x9d\x01*'
+                  b'\x01\x00\x01\x00\x0f\xc0\xfe%\xa4\x00\x03p\x00\xfe\xe6\xb5\x00\x00'),
+            forcing_headers={'Content-Type': 'image/webp'}
+        )
+
+    def tearDown(self):
+        httpretty.disable()
+
     # Assert that ThumbFramesImage objects look reasonably well
     def assertThumbFrames(self, tf_images):
         self.assertIsNotNone(tf_images)
@@ -42,29 +85,41 @@ def assertThumbFrames(self, tf_images):
             # check that mime type was set by image, not URL
             self.assertEqual(tf_image.mime_type, 'webp')
 
-    def test_init_with_video_id(self, mock_http_request):
+    def test_init_with_video_id(self):
         video = YouTubeFrames(self.VIDEO_ID)
         self.assertEqual(video.video_id, self.VIDEO_ID)
         self.assertEqual(video.video_url, self.VIDEO_URL)
 
-    def test_init_with_video_url(self, mock_http_request):
+    def test_init_with_video_url(self):
         video = YouTubeFrames(self.VIDEO_URL)
         self.assertEqual(video.video_id, self.VIDEO_ID)
         self.assertEqual(video.video_url, self.VIDEO_URL)
 
-    def test_fail_init_with_bad_url(self, mock_http_request):
+    def test_fail_init_with_bad_url(self):
         BAD_URL = 'BAD_URL'
         with self.assertRaises(ExtractorError):
             _ = YouTubeFrames(BAD_URL)
 
-    def test_thumbframes_not_found(self, mock_http_request):
-        mock_http_request.side_effect = mock_empty_json_response
-
-        # this represents a "valid" YouTube video that returns no thumbframes
+    def test_thumbframes_not_found(self):
+        # mock responses with no thumbframes
+        httpretty.reset()
+        httpretty.register_uri(
+            httpretty.GET,
+            re.compile('.*'),
+            body='<!DOCTYPE html><html><head></head><body></body></html>'
+        )
+        httpretty.register_uri(
+            httpretty.POST,
+            re.compile('.*'),
+            body='{}'
+        )
+
+        # this represents a valid YouTube video that returns no thumbframes
         video = YouTubeFrames('ZERO_THUMBS')
 
-        # downloaded both webpage and video_info to try to find thumbframes
-        self.assertEqual(mock_http_request.call_count, 2)
+        # number of HTTP requests that YouTubeDL tries at first
+        # it's set as a variable in case lib changes internally
+        number_of_requests = len(httpretty.latest_requests())
 
         # thumbframes info is an empty structure but NOT a None
         self.assertIsNotNone(video._thumbframes)
@@ -75,72 +130,56 @@ def test_thumbframes_not_found(self, mock_http_request):
         self.assertIsNone(video.get_thumbframe_format())
 
         # should NOT re-try download even if thumbframes info is empty
-        self.assertEqual(mock_http_request.call_count, 2)
+        self.assertEqual(len(httpretty.latest_requests()), number_of_requests)
 
-    def test_page_only_downloads_once(self, mock_http_request):
+    def test_page_only_downloads_once(self):
         video = YouTubeFrames(self.VIDEO_ID)
 
         # get thumbframes for the first time, which downloads page with frames info
         self.assertIsNotNone(video.get_thumbframes('L0'))
-        self.assertTrue(mock_http_request.called)
-        self.assertEqual(mock_http_request.call_count, 1)
-
-        mock_http_request.reset_mock()
+        self.assertEqual(len(httpretty.latest_requests()), 1)
 
         # should NOT download again if thumbframes data has already been obtained before
         self.assertIsNotNone(video.get_thumbframes('L1'))
-        self.assertFalse(mock_http_request.called)
-        self.assertEqual(mock_http_request.call_count, 0)
+        self.assertEqual(len(httpretty.latest_requests()), 1)
 
-    def test_images_only_download_once(self, mock_http_request):
+    def test_images_only_download_once(self):
         video = YouTubeFrames(self.VIDEO_ID)
-        mock_http_request.reset_mock()
 
         # get thumbframes for the first time, which downloads each image
         thumbframes = video.get_thumbframes('L2')
         for tf_image in video.get_thumbframes('L2'):
             self.assertIsNotNone(tf_image.get_image())
-        self.assertTrue(mock_http_request.called)
-        self.assertEqual(mock_http_request.call_count, len(thumbframes))
-
-        mock_http_request.reset_mock()
+        self.assertEqual(len(httpretty.latest_requests()), len(thumbframes) + 1)
 
         # should NOT download images again if they have already been downloaded before
         same_thumbframes_as_before = video.get_thumbframes('L2')
         for tf_image in same_thumbframes_as_before:
             self.assertIsNotNone(tf_image.get_image())
-        self.assertFalse(mock_http_request.called)
-        self.assertEqual(mock_http_request.call_count, 0)
+        self.assertEqual(len(httpretty.latest_requests()), len(thumbframes) + 1)
 
-    def test_lazy_thumbframes(self, mock_http_request):
+    def test_lazy_thumbframes(self):
         video = YouTubeFrames(self.VIDEO_ID)
-        mock_http_request.reset_mock()
 
         # get thumbframes for the first time, which downloads each image
         for counter, tf_image in enumerate(video.get_thumbframes('L2'), start=1):
             self.assertIsNotNone(tf_image.get_image())
-            self.assertTrue(mock_http_request.called)
-            self.assertEqual(mock_http_request.call_count, counter)
+            self.assertEqual(len(httpretty.latest_requests()), counter + 1)
 
-    def test_non_lazy_thumbframes(self, mock_http_request):
+    def test_non_lazy_thumbframes(self):
         video = YouTubeFrames(self.VIDEO_ID)
-        mock_http_request.reset_mock()
 
         # call method in non lazy mode so all images are downloaded right away
         thumbframes = video.get_thumbframes('L2', lazy=False)
-        self.assertTrue(mock_http_request.called)
-        self.assertEqual(mock_http_request.call_count, len(thumbframes))
-
-        mock_http_request.reset_mock()
+        self.assertEqual(len(httpretty.latest_requests()), len(thumbframes) + 1)
 
         # no additional download here because images are already set
         for tf_image in thumbframes:
             self.assertIsNotNone(tf_image.get_image())
-            self.assertFalse(mock_http_request.called)
-        self.assertEqual(mock_http_request.call_count, 0)
+        self.assertEqual(len(httpretty.latest_requests()), len(thumbframes) + 1)
 
     # Test that internal _thumbframes dict is set correctly.
-    def test_get_thumbframes_info(self, mock_http_request):
+    def test_get_thumbframes_info(self):
 
         video = YouTubeFrames(self.VIDEO_ID)
 
@@ -153,7 +192,7 @@ def test_get_thumbframes_info(self, mock_http_request):
             self.assertIn(tf_id, video._thumbframes)
             self.assertThumbFrames(video._thumbframes[tf_id])
 
-    def test_thumbframes_formats(self, mock_http_request):
+    def test_thumbframes_formats(self):
         video = YouTubeFrames(self.VIDEO_ID)
         self.assertEqual(len(video.thumbframe_formats), 3)
 
@@ -178,25 +217,7 @@ def test_thumbframes_formats(self, mock_http_request):
         self.assertEqual(video.thumbframe_formats[2].total_frames, 100)
         self.assertEqual(video.thumbframe_formats[2].total_images, 1)
 
-    def test_get_thumbframes(self, mock_http_request):
-        video = YouTubeFrames(self.VIDEO_ID)
-
-        # download 1 image from the L0 set
-        mock_http_request.reset_mock()
-        self.assertThumbFrames(video.get_thumbframes('L0'))
-        self.assertEqual(mock_http_request.call_count, 1)
-
-        # download 1 image from the L1 set
-        mock_http_request.reset_mock()
-        self.assertThumbFrames(video.get_thumbframes('L1'))
-        self.assertEqual(mock_http_request.call_count, 1)
-
-        # download 4 images from the L2 set
-        mock_http_request.reset_mock()
-        self.assertThumbFrames(video.get_thumbframes('L2'))
-        self.assertEqual(mock_http_request.call_count, 4)
-
-    def test_get_thumbframes_default_to_best_format(self, mock_http_request):
+    def test_get_thumbframes_default_to_best_format(self):
         video = YouTubeFrames(self.VIDEO_ID)
 
         default_thumbframes = video.get_thumbframes()

diff --git a/thumbframes_dl/version.py b/thumbframes_dl/version.py
@@ -1 +1 @@
-__version__ = "0.11.0"
+__version__ = "0.13.0"