Merge pull request #3312 from seleniumbase/cdp-mode-patch-16

CDP Mode - Patch 16

Author: mdmintz

examples/cdp_mode/ReadMe.md

@@ -22,28 +22,30 @@
 * Backwards compatibility for existing UC Mode scripts.
 * More configuration options when launching browsers.
 * More methods. (And bug-fixes for existing methods.)
+* `PyAutoGUI` integration for advanced stealth abilities.
 * Faster response time for support. (Eg. [Discord Chat](https://discord.gg/EdhQTn3EyE))
 
 --------
 
-### 🐙 <b translate="no">CDP Mode</b> usage:
+### 🐙 <b translate="no">CDP Mode</b> Usage:
 
 * **`sb.activate_cdp_mode(url)`**
 
 > (Call that from a **UC Mode** script)
 
 That disconnects WebDriver from Chrome (which prevents detection), and gives you access to `sb.cdp` methods (which don't trigger anti-bot checks).
 
-### 🐙 Here are some common `sb.cdp` methods:
+### 🐙 Here are a few common `sb.cdp` methods:
 
 * `sb.cdp.click(selector)`
 * `sb.cdp.click_if_visible(selector)`
+* `sb.cdp.gui_click_element(selector)`
 * `sb.cdp.type(selector, text)`
 * `sb.cdp.press_keys(selector, text)`
 * `sb.cdp.select_all(selector)`
 * `sb.cdp.get_text(selector)`
 
-When `type()` is too fast, use the slower `press_keys()` to avoid detection. You can also use `sb.sleep(seconds)` to slow things down.
+When `type()` is too fast, use the slower `press_keys()` to avoid detection. You can also use `sb.sleep(seconds)` to slow things down. Methods that start with `sb.cdp.gui` use `PyAutoGUI` for interaction.
 
 To use WebDriver methods again, call:
 
@@ -63,17 +65,13 @@ To find out if WebDriver is connected or disconnected, call:
 
 --------
 
-### 🐙 <b translate="no">CDP Mode</b> examples:
+### 🐙 <b translate="no">CDP Mode</b> Examples ([SeleniumBase/examples/cdp_mode](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode))
 
-> [SeleniumBase/examples/cdp_mode](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode)
-
-### 🔖 Example 1: (Pokemon site using Incapsula/Imperva protection with invisible reCAPTCHA)
-
-> [SeleniumBase/examples/cdp_mode/raw_pokemon.py](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode/raw_pokemon.py)
+<p><div /></p>
 
 <div></div>
 <details>
-<summary> ▶️ (<b>Click to expand code preview</b>)</summary>
+<summary> ▶️ 🔖 <b>Example 1: (Pokemon site using Incapsula/Imperva protection with invisible reCAPTCHA)</b></summary>
 
 ```python
 from seleniumbase import SB
@@ -127,13 +125,12 @@ with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb:
 
 </details>
 
-### 🔖 Example 2: (Hyatt site using Kasada protection)
+> [SeleniumBase/examples/cdp_mode/raw_pokemon.py](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode/raw_pokemon.py)
 
-> [SeleniumBase/examples/cdp_mode/raw_hyatt.py](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode/raw_hyatt.py)
 
 <div></div>
 <details>
-<summary> ▶️ (<b>Click to expand code preview</b>)</summary>
+<summary> ▶️ 🔖 <b>Example 2: (Hyatt site using Kasada protection)</b></summary>
 
 ```python
 from seleniumbase import SB
@@ -157,30 +154,30 @@ with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb:
     sb.sleep(1)
     sb.cdp.click('button[data-locator="find-hotels"]')
     sb.sleep(5)
-    hotel_names = sb.cdp.select_all(
-        'div[data-booking-status="BOOKABLE"] [class*="HotelCard_header"]'
-    )
-    hotel_prices = sb.cdp.select_all(
-        'div[data-booking-status="BOOKABLE"] div.rate'
-    )
-    sb.assert_true(len(hotel_names) == len(hotel_prices))
+    card_info = 'div[data-booking-status="BOOKABLE"] [class*="HotelCard_info"]'
+    hotels = sb.cdp.select_all(card_info)
     print("Hyatt Hotels in %s:" % location)
     print("(" + sb.cdp.get_text("ul.b-color_text-white") + ")")
-    if len(hotel_names) == 0:
+    if len(hotels) == 0:
         print("No availability over the selected dates!")
-    for i, hotel in enumerate(hotel_names):
-        print("* %s: %s => %s" % (i + 1, hotel.text, hotel_prices[i].text))
+    for hotel in hotels:
+        info = hotel.text.strip()
+        if "Avg/Night" in info and not info.startswith("Rates from"):
+            name = info.split("  (")[0].split(" + ")[0].split(" Award Cat")[0]
+            price = "?"
+            if "Rates from : " in info:
+                price = info.split("Rates from : ")[1].split(" Avg/Night")[0]
+            print("* %s => %s" % (name, price))
 ```
 
 </details>
 
-### 🔖 Example 3: (BestWestern site using DataDome protection)
+> [SeleniumBase/examples/cdp_mode/raw_hyatt.py](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode/raw_hyatt.py)
 
-* [SeleniumBase/examples/cdp_mode/raw_bestwestern.py](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode/raw_bestwestern.py)
 
 <div></div>
 <details>
-<summary> ▶️ (<b>Click to expand code preview</b>)</summary>
+<summary> ▶️ 🔖 <b>Example 3: (BestWestern site using DataDome protection)</b></summary>
 
 ```python
 from seleniumbase import SB
@@ -218,13 +215,12 @@ with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb:
 
 </details>
 
-### 🔖 Example 4: (Walmart site using Akamai protection with PerimeterX)
+> [SeleniumBase/examples/cdp_mode/raw_bestwestern.py](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode/raw_bestwestern.py)
 
-* [SeleniumBase/examples/cdp_mode/raw_walmart.py](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode/raw_walmart.py)
 
 <div></div>
 <details>
-<summary> ▶️ (<b>Click to expand code preview</b>)</summary>
+<summary> ▶️ 🔖 <b>Example 4: (Walmart site using Akamai protection with PerimeterX)</b></summary>
 
 ```python
 from seleniumbase import SB
@@ -264,13 +260,12 @@ with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb:
 
 </details>
 
-### 🔖 Example 5: (Nike site using Shape Security)
+> [SeleniumBase/examples/cdp_mode/raw_walmart.py](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode/raw_walmart.py)
 
-* [SeleniumBase/examples/cdp_mode/raw_nike.py](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode/raw_nike.py)
 
 <div></div>
 <details>
-<summary> ▶️ (<b>Click to expand code preview</b>)</summary>
+<summary> ▶️ 🔖 <b>Example 5: (Nike site using Shape Security)</b></summary>
 
 ```python
 from seleniumbase import SB
@@ -294,13 +289,17 @@ with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb:
 
 </details>
 
+> [SeleniumBase/examples/cdp_mode/raw_nike.py](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode/raw_nike.py)
+
+<p><div /></p>
+
 (<b>Note:</b> Extra <code translate="no">sb.sleep()</code> calls have been added to prevent bot-detection because some sites will flag you as a bot if you perform actions too quickly.)
 
 (<b>Note:</b> Some sites may IP-block you for 36 hours or more if they catch you using regular <span translate="no">Selenium WebDriver</span>. Be extra careful when creating and/or modifying automation scripts that run on them.)
 
 --------
 
-### 🐙 CDP Mode API / Methods
+### 🐙 <b translate="no">CDP Mode</b> API / Methods
 
 (Some method args have been left out for simplicity. Eg: <code translate="no">timeout</code>)
 
@@ -323,6 +322,9 @@ sb.cdp.find_visible_elements(selector)
 sb.cdp.click_nth_element(selector, number)
 sb.cdp.click_nth_visible_element(selector, number)
 sb.cdp.click_link(link_text)
+sb.cdp.go_back()
+sb.cdp.go_forward()
+sb.cdp.get_navigation_history()
 sb.cdp.tile_windows(windows=None, max_columns=0)
 sb.cdp.get_all_cookies(*args, **kwargs)
 sb.cdp.set_all_cookies(*args, **kwargs)
@@ -434,7 +436,7 @@ sb.cdp.save_screenshot(name, folder=None, selector=None)
 
 --------
 
-### 🐙 CDP Mode WebElement API / Methods
+### 🐙 <b translate="no">CDP Mode</b> WebElement API / Methods
 
 ```python
 element.clear_input()

examples/cdp_mode/raw_hyatt.py

@@ -19,16 +19,17 @@
     sb.sleep(1)
     sb.cdp.click('button[data-locator="find-hotels"]')
     sb.sleep(5)
-    hotel_names = sb.cdp.select_all(
-        'div[data-booking-status="BOOKABLE"] [class*="HotelCard_header"]'
-    )
-    hotel_prices = sb.cdp.select_all(
-        'div[data-booking-status="BOOKABLE"] div.rate'
-    )
-    sb.assert_true(len(hotel_names) == len(hotel_prices))
+    card_info = 'div[data-booking-status="BOOKABLE"] [class*="HotelCard_info"]'
+    hotels = sb.cdp.select_all(card_info)
     print("Hyatt Hotels in %s:" % location)
     print("(" + sb.cdp.get_text("ul.b-color_text-white") + ")")
-    if len(hotel_names) == 0:
+    if len(hotels) == 0:
         print("No availability over the selected dates!")
-    for i, hotel in enumerate(hotel_names):
-        print("* %s: %s => %s" % (i + 1, hotel.text, hotel_prices[i].text))
+    for hotel in hotels:
+        info = hotel.text.strip()
+        if "Avg/Night" in info and not info.startswith("Rates from"):
+            name = info.split("  (")[0].split(" + ")[0].split(" Award Cat")[0]
+            price = "?"
+            if "Rates from : " in info:
+                price = info.split("Rates from : ")[1].split(" Avg/Night")[0]
+            print("* %s => %s" % (name, price))

examples/presenter/uc_presentation_4.py

@@ -695,24 +695,24 @@ def test_presentation_4(self):
             sb.sleep(1)
             sb.cdp.click('button[data-locator="find-hotels"]')
             sb.sleep(5)
-            hotel_names = sb.cdp.select_all(
-                'div[data-booking-status="BOOKABLE"]'
-                ' [class*="HotelCard_header"]'
+            card_info = (
+                'div[data-booking-status="BOOKABLE"] [class*="HotelCard_info"]'
             )
-            hotel_prices = sb.cdp.select_all(
-                'div[data-booking-status="BOOKABLE"] div.rate'
-            )
-            sb.assert_true(len(hotel_names) == len(hotel_prices))
-            print("\n\nHyatt Hotels in %s:" % location)
+            hotels = sb.cdp.select_all(card_info)
+            print("Hyatt Hotels in %s:" % location)
             print("(" + sb.cdp.get_text("ul.b-color_text-white") + ")")
-            if len(hotel_names) == 0:
+            if len(hotels) == 0:
                 print("No availability over the selected dates!")
-            for i, hotel in enumerate(hotel_names):
-                with suppress(Exception):
-                    print(
-                        "* %s: %s => %s"
-                        % (i + 1, hotel.text, hotel_prices[i].text)
-                    )
+            for hotel in hotels:
+                info = hotel.text.strip()
+                if "Avg/Night" in info and not info.startswith("Rates from"):
+                    name = info.split("  (")[0]
+                    name = name.split(" + ")[0].split(" Award Cat")[0]
+                    price = "?"
+                    if "Rates from : " in info:
+                        price = info.split("Rates from : ")[1]
+                        price = price.split(" Avg/Night")[0]
+                    print("* %s => %s" % (name, price))
 
         self.create_presentation(theme="serif", transition="none")
         self.add_slide(

mkdocs_build/requirements.txt

@@ -20,7 +20,7 @@ lxml==5.3.0
 pyquery==2.0.1
 readtime==3.0.0
 mkdocs==1.6.1
-mkdocs-material==9.5.46
+mkdocs-material==9.5.47
 mkdocs-exclude-search==0.6.6
 mkdocs-simple-hooks==0.1.5
 mkdocs-material-extensions==1.3.1

requirements.txt

@@ -44,7 +44,7 @@ execnet==2.1.1
 iniconfig==2.0.0
 pluggy==1.5.0
 py==1.11.0
-pytest==8.3.3
+pytest==8.3.4
 pytest-html==2.0.1
 pytest-metadata==3.1.1
 pytest-ordering==0.6

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.33.2"
+__version__ = "4.33.3"

seleniumbase/core/browser_launcher.py

@@ -606,6 +606,9 @@ def uc_open_with_cdp_mode(driver, url=None):
     cdp.click_nth_element = CDPM.click_nth_element
     cdp.click_nth_visible_element = CDPM.click_nth_visible_element
     cdp.click_link = CDPM.click_link
+    cdp.go_back = CDPM.go_back
+    cdp.go_forward = CDPM.go_forward
+    cdp.get_navigation_history = CDPM.get_navigation_history
     cdp.tile_windows = CDPM.tile_windows
     cdp.get_all_cookies = CDPM.get_all_cookies
     cdp.set_all_cookies = CDPM.set_all_cookies
@@ -1419,11 +1422,8 @@ def _uc_gui_handle_captcha_(driver, frame="iframe", ctype=None):
             page_actions.switch_to_window(
                 driver, driver.current_window_handle, 2, uc_lock=False
             )
-        if (
-            IS_WINDOWS
-            and hasattr(pyautogui, "getActiveWindowTitle")
-        ):
-            py_a_g_title = pyautogui.getActiveWindowTitle()
+        if IS_WINDOWS and hasattr(pyautogui, "getActiveWindowTitle"):
+            py_a_g_title = pyautogui.getActiveWindowTitle() or ""
             window_title = driver.get_title()
             if not py_a_g_title.startswith(window_title):
                 window_rect = driver.get_window_rect()

seleniumbase/core/sb_cdp.py

@@ -289,6 +289,15 @@ def click_nth_visible_element(self, selector, number):
     def click_link(self, link_text):
         self.find_elements_by_text(link_text, "a")[0].click()
 
+    def go_back(self):
+        self.loop.run_until_complete(self.page.back())
+
+    def go_forward(self):
+        self.loop.run_until_complete(self.page.forward())
+
+    def get_navigation_history(self):
+        return self.loop.run_until_complete(self.page.get_navigation_history())
+
     def __clear_input(self, element):
         return (
             self.loop.run_until_complete(element.clear_input_async())

seleniumbase/fixtures/base_case.py

@@ -1323,6 +1323,9 @@ def get_locale_code(self):
 
     def go_back(self):
         self.__check_scope()
+        if self.__is_cdp_swap_needed():
+            self.cdp.go_back()
+            return
         if hasattr(self, "recorder_mode") and self.recorder_mode:
             self.save_recorded_actions()
         pre_action_url = None
@@ -1348,6 +1351,9 @@ def go_back(self):
 
     def go_forward(self):
         self.__check_scope()
+        if self.__is_cdp_swap_needed():
+            self.cdp.go_forward()
+            return
         if hasattr(self, "recorder_mode") and self.recorder_mode:
             self.save_recorded_actions()
         self.__last_page_load_url = None
@@ -1732,6 +1738,9 @@ def click_partial_link_text(self, partial_link_text, timeout=None):
         if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:
             timeout = self.__get_new_timeout(timeout)
         partial_link_text = self.__get_type_checked_text(partial_link_text)
+        if self.__is_cdp_swap_needed():
+            self.cdp.find_element(partial_link_text, timeout=timeout).click()
+            return
         if not self.is_partial_link_text_present(partial_link_text):
             self.wait_for_partial_link_text_present(
                 partial_link_text, timeout=timeout
@@ -8133,6 +8142,8 @@ def is_connected(self):
     def is_chromium(self):
         """Return True if the browser is Chrome or Edge."""
         self.__check_scope()
+        if self.__is_cdp_swap_needed():
+            return True
         chromium = False
         if (
             "chrome" in self.driver.capabilities

seleniumbase/undetected/cdp_driver/tab.py

@@ -636,6 +636,10 @@ async def forward(self):
         """History forward"""
         await self.send(cdp.runtime.evaluate("window.history.forward()"))
 
+    async def get_navigation_history(self):
+        """Get Navigation History"""
+        return await self.send(cdp.page.get_navigation_history())
+
     async def reload(
         self,
         ignore_cache: Optional[bool] = True,

setup.py

@@ -193,7 +193,7 @@
         'iniconfig==2.0.0',
         'pluggy==1.5.0',
         "py==1.11.0",  # Needed by pytest-html
-        'pytest==8.3.3',
+        'pytest==8.3.4',
         "pytest-html==2.0.1",  # Newer ones had issues
         'pytest-metadata==3.1.1',
         "pytest-ordering==0.6",