diff --git a/README.md b/README.md index 992be728629..f473ef494b7 100755 --- a/README.md +++ b/README.md @@ -8,11 +8,10 @@ -

SeleniumBase
All-in-one Browser Automation Framework:
Web Crawling / Testing / Scraping / Stealth

+

SeleniumBase
A Complete Browser Automation Experience:
Web Crawling / Testing / Scraping / Stealth

-PyPI version CI Status SeleniumBase Docs SeleniumBase PyPI downloads
-YouTube Channel SeleniumBase Facebook Followers SeleniumBase X Followers Discord Status +PyPI version CI Status SeleniumBase Docs
SeleniumBase PyPI downloads YouTube Channel

@@ -50,20 +49,26 @@ 🗂️ CPlans

+

SeleniumBase is the professional toolkit for web automation activities. Built for testing websites, bypassing CAPTCHAs, enhancing productivity, completing tasks, and scaling your business.

+ +-------- + +📚 Learn from [**over 200 examples** in **SeleniumBase/examples/**](https://github.com/seleniumbase/SeleniumBase/tree/master/examples). + +

SeleniumBase Google Search

+ -------- -

⚙️ High-level architecture and special modes:

+

SeleniumBase Chromium architecture and modes:

+ +Stealthy architecture flowchart -Stealthy architecture flowchart +> (*Note that "Regular Mode" also supports Safari/WebKit and Firefox/Gecko.*) - -------- @@ -84,21 +89,21 @@ ```python from seleniumbase import sb_cdp -sb = sb_cdp.Chrome() # Launches a stealth browser +sb = sb_cdp.Chrome() # Launch a stealth browser sb.goto("https://www.planetminecraft.com/account") sb.type('[name="email"]', "Test123") sb.type('[name="password"]', "Pass") -sb.click("input#autologin") # Unchecks a checkbox -sb.solve_captcha() # (as backup if not automatic) +sb.click("input#autologin") # Uncheck a checkbox +sb.solve_captcha() # (fallback if not automatic) sb.assert_element('input[name="login"]') sb.assert_element_not_visible("input[disabled]") -sb.sleep(1) # (only to slow down the live demo) -sb.quit() # (otherwise quits automatically at end) +sb.sleep(1) # (see the live demo more easily) +sb.quit() ```

-Cloudflare Turnstile bypassed successfully -
(Note that the click action only unchecked the "Stay signed in" checkbox.) +Cloudflare Turnstile bypassed successfully +
(Note: In this example, the click only unchecks the "Stay signed in" checkbox.)

-------- @@ -820,6 +825,7 @@ pytest test_coffee_cart.py --trace --rs | --reuse-session # (Reuse browser session for all tests.) --rcs | --reuse-class-session # (Reuse session for tests in class.) --crumbs # (Delete all cookies between tests reusing a session.) +--no-tt # (Skips "--test-type" when using Proxy Mode. Adds stealth.) --disable-beforeunload # (Disable the "beforeunload" event on Chrome.) --window-position=X,Y # (Set the browser's starting window position.) --window-size=WIDTH,HEIGHT # (Set the browser's starting window size.) diff --git a/examples/ReadMe.md b/examples/ReadMe.md index 29697104be0..b82c2cd05e3 100644 --- a/examples/ReadMe.md +++ b/examples/ReadMe.md @@ -2,7 +2,7 @@

Example Tests:

-

(For stealthy examples, see ./cdp_mode/ and ./cdp_mode/playwright/)

+

(For stealthy examples, see ./cdp_mode/ and ./cdp_mode/playwright/)

SeleniumBase Demo Page

diff --git a/examples/boilerplates/samples/google_test.py b/examples/boilerplates/samples/google_test.py index 3435348cd68..d957cbe8ebb 100644 --- a/examples/boilerplates/samples/google_test.py +++ b/examples/boilerplates/samples/google_test.py @@ -4,7 +4,7 @@ from .google_objects import HomePage, ResultsPage except Exception: from google_objects import HomePage, ResultsPage - BaseCase.main(__name__, __file__, "--uc") +BaseCase.main(__name__, __file__, "--uc", "--chromium") class GoogleTests(BaseCase): @@ -14,7 +14,7 @@ def test_google_dot_com(self): print("\n Skipping test in headless mode.") self.skip("Skipping test in headless mode.") if not self.undetectable: - self.get_new_driver(undetectable=True) + self.get_new_driver(undetectable=True, use_chromium=True) self.goto("https://google.com/ncr") self.click_if_visible('button:contains("Accept all")') self.assert_title_contains("Google") diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index e78ecb30c80..719724dcac7 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -235,14 +235,14 @@ with SB(uc=True, test=True, locale="en", ad_block=True) as sb: sb.activate_cdp_mode() sb.goto("https://www.pokemon.com/us/pokedex") sb.sleep(1.5) - sb.click_if_visible("button#onetrust-accept-btn-handler") + sb.click_if_visible("#onetrust-accept-btn-handler", timeout=2) sb.sleep(1.5) sb.click('b:contains("Show Advanced Search")') sb.sleep(2.5) sb.click('span[data-type="type"][data-value="electric"]') - sb.sleep(0.7) + sb.sleep(1) sb.scroll_into_view("a#advSearch") - sb.sleep(0.7) + sb.sleep(1) sb.click("a#advSearch") sb.sleep(1.2) sb.click('img[src*="img/pokedex/detail/025.png"]') @@ -467,6 +467,7 @@ sb.select_option_by_index(dropdown_selector, option) sb.select_option_by_value(dropdown_selector, option) sb.flash(selector, duration=1, color="44CC88", pause=0) sb.highlight(selector, scroll=True, timeout=None) +sb.highlight_if_visible(selector, scroll=True, timeout=0) sb.focus(selector, timeout=None) sb.highlight_overlay(selector) sb.get_parent(element) diff --git a/examples/cdp_mode/playwright/raw_clearcote_sync.py b/examples/cdp_mode/playwright/raw_clearcote_sync.py index 76c7875754d..191d201e739 100644 --- a/examples/cdp_mode/playwright/raw_clearcote_sync.py +++ b/examples/cdp_mode/playwright/raw_clearcote_sync.py @@ -9,9 +9,7 @@ page = browser.contexts[0].pages[0] page.goto("https://www.clearcotelabs.com/audit") sb.click('button[aria-label="Marker 1"]') - sb.click('button[aria-label="Marker 2"]') - sb.click('button[aria-label="Marker 3"]') - sb.press_keys("input", "audit") + sb.press_keys("input", "Audit") sb.set_value('input[type="range"]', "100") sb.click_and_hold("button[data-interaction-hold]") sb.select_option_by_text("select", "Mouse") diff --git a/examples/cdp_mode/raw_async.py b/examples/cdp_mode/raw_async.py index 88618173ddc..aedbf7b925e 100644 --- a/examples/cdp_mode/raw_async.py +++ b/examples/cdp_mode/raw_async.py @@ -53,7 +53,9 @@ async def main(): sb = sb_cdp.Chrome("https://www.priceline.com/") sb.sleep(2.5) sb.internalize_links() # Don't open links in a new tab - sb.click("#link_header_nav_experiences") + sb.click_if_visible('[aria-label="Close Modal"]') + sb.sleep(0.2) + sb.click('a[href*="experiences.priceline"]') sb.sleep(3.5) sb.remove_elements("msm-cookie-banner") sb.sleep(1.5) @@ -70,5 +72,7 @@ async def main(): print("************") cards = sb.select_all('span[data-automation*="product-list-card"]') for card in cards: - print("* %s" % card.text) + card_text = card.text.strip() + if len(card_text) >= 20: + print("* %s" % card_text) sb.quit() diff --git a/examples/cdp_mode/raw_cdp.py b/examples/cdp_mode/raw_cdp.py index 49dc5238dda..4e4e42ff836 100644 --- a/examples/cdp_mode/raw_cdp.py +++ b/examples/cdp_mode/raw_cdp.py @@ -9,7 +9,7 @@ def main(): sb.goto("https://www.priceline.com/") sb.sleep(2) sb.internalize_links() # Don't open links in a new tab - sb.click("#link_header_nav_experiences") + sb.click('a[href*="experiences.priceline"]') sb.sleep(3) sb.remove_elements("msm-cookie-banner") sb.sleep(1) @@ -31,7 +31,9 @@ def main(): sb.sleep(0.2) cards = sb.select_all('span[data-automation*="product-list-card"]') for card in cards: - print("* %s" % card.text) + card_text = card.text.strip() + if len(card_text) >= 20: + print("* %s" % card_text) sb.quit() diff --git a/examples/cdp_mode/raw_cdp_clearcote.py b/examples/cdp_mode/raw_cdp_clearcote.py index 2938c4f0d5b..abe2cd36b11 100644 --- a/examples/cdp_mode/raw_cdp_clearcote.py +++ b/examples/cdp_mode/raw_cdp_clearcote.py @@ -3,9 +3,7 @@ sb = sb_cdp.Chrome() sb.goto("https://www.clearcotelabs.com/audit") sb.click('button[aria-label="Marker 1"]') -sb.click('button[aria-label="Marker 2"]') -sb.click('button[aria-label="Marker 3"]') -sb.press_keys("input", "audit") +sb.press_keys("input", "Audit") sb.set_value('input[type="range"]', "100") sb.click_and_hold("button[data-interaction-hold]") sb.select_option_by_text("select", "Mouse") diff --git a/examples/cdp_mode/raw_cdp_google.py b/examples/cdp_mode/raw_cdp_google.py index fee0e7ed7e6..1bbd331b728 100644 --- a/examples/cdp_mode/raw_cdp_google.py +++ b/examples/cdp_mode/raw_cdp_google.py @@ -1,11 +1,14 @@ from seleniumbase import sb_cdp -sb = sb_cdp.Chrome() +sb = sb_cdp.Chrome(use_chromium=True) sb.goto("https://google.com/ncr") sb.click_if_visible('button:contains("Accept all")') -sb.type('[name="q"]', "SeleniumBase site:github.com") -sb.sleep(0.2) -sb.click('[value="Google Search"]') +sb.press_keys('[name="q"]', "SeleniumBase GitHub Page") +sb.sleep(0.5) +if sb.is_element_visible('[value="Google Search"]'): + sb.click('[value="Google Search"]') +else: + sb.press_keys('[name="q"]', "\n") sb.sleep(1) sb.click_if_visible('h3:contains("seleniumbase/SeleniumBase")') sb.sleep(1) diff --git a/examples/cdp_mode/raw_cdp_mode.py b/examples/cdp_mode/raw_cdp_mode.py index 41de8bf403f..47b18872e0a 100644 --- a/examples/cdp_mode/raw_cdp_mode.py +++ b/examples/cdp_mode/raw_cdp_mode.py @@ -1,15 +1,13 @@ -"""Bypass bot-detection with SeleniumBase CDP Mode. -Assert statements that aren't met raise exceptions. -""" +"""Bypass bot-detection with SeleniumBase CDP Mode.""" from seleniumbase import sb_cdp -sb = sb_cdp.Chrome() # Launches a stealth browser +sb = sb_cdp.Chrome() # Launch a stealth browser sb.goto("https://www.planetminecraft.com/account") sb.type('[name="email"]', "Test123") sb.type('[name="password"]', "Pass") -sb.click("input#autologin") # Unchecks a checkbox -sb.solve_captcha() # (as backup if not automatic) +sb.click("input#autologin") # Uncheck a checkbox +sb.solve_captcha() # (fallback if not automatic) sb.assert_element('input[name="login"]') sb.assert_element_not_visible("input[disabled]") -sb.sleep(1) # (only to slow down the live demo) -sb.quit() # (otherwise quits automatically at end) +sb.sleep(1) # (see the live demo more easily) +sb.quit() diff --git a/examples/cdp_mode/raw_cdp_rastreator.py b/examples/cdp_mode/raw_cdp_rastreator.py new file mode 100644 index 00000000000..4329ad20bd3 --- /dev/null +++ b/examples/cdp_mode/raw_cdp_rastreator.py @@ -0,0 +1,17 @@ +from seleniumbase import sb_cdp + +sb = sb_cdp.Chrome() +sb.goto("https://www.rastreator.com/seguros-de-coche") +sb.hover_element("body") +sb.click_if_visible(".accept-btn", timeout=2) +sb.click('button[type="submit"]') +sb.sleep(1) +sb.click("label:has(#radioThinkingToBuying)") +sb.sleep(1) +sb.click("label:has(#radioNewCar)") +sb.sleep(1) +sb.solve_captcha() +sb.sleep(1) +expected_text = "El coche que quieres asegurar" +sb.highlight(f'h2:contains("{expected_text}")') +sb.sleep(2) diff --git a/examples/cdp_mode/raw_cdp_with_sb.py b/examples/cdp_mode/raw_cdp_with_sb.py index f43e741d792..834d1948617 100644 --- a/examples/cdp_mode/raw_cdp_with_sb.py +++ b/examples/cdp_mode/raw_cdp_with_sb.py @@ -9,7 +9,7 @@ sb.internalize_links() # Don't open links in a new tab sb.click_if_visible('[aria-label="Close Modal"]') sb.sleep(0.2) - sb.click("#link_header_nav_experiences") + sb.click('a[href*="experiences.priceline"]') sb.sleep(3.5) sb.remove_elements("msm-cookie-banner") sb.sleep(1.5) @@ -29,4 +29,6 @@ sb.sleep(0.2) cards = sb.select_all('span[data-automation*="product-list-card"]') for card in cards: - print("* %s" % card.text) + card_text = card.text.strip() + if len(card_text) >= 20: + print("* %s" % card_text) diff --git a/examples/cdp_mode/raw_clearcote.py b/examples/cdp_mode/raw_clearcote.py new file mode 100644 index 00000000000..a0d2bcdf8ee --- /dev/null +++ b/examples/cdp_mode/raw_clearcote.py @@ -0,0 +1,25 @@ +from seleniumbase import SB + +with SB(uc=True, test=True) as sb: + sb.activate_cdp_mode() + sb.goto("https://www.clearcotelabs.com/audit") + sb.click('button[aria-label="Marker 1"]') + sb.press_keys("input", "Audit") + sb.set_value('input[type="range"]', "100") + sb.click_and_hold("button[data-interaction-hold]") + sb.select_option_by_text("select", "Mouse") + sb.click('button:contains("Run the audit")') + sb.sleep(6) + sb.assert_element("div.text-successText", timeout=10) + try: + sb.assert_text("100", "div.text-successText", timeout=5) + sb.highlight("div.text-successText") + print(" ✅ The browser fingerprint is clean! Score: 100") + except Exception: + score = sb.get_text("div.text-successText") + print(f" ❌ Fingerprint tampering detected! Score: {score}") + sb.sleep(1) + folder = "downloaded_files" + file_name = "c_audit_results.pdf" + sb.save_as_pdf(file_name, folder) + print('"./%s/%s" was saved!' % (folder, file_name)) diff --git a/examples/cdp_mode/raw_pokemon.py b/examples/cdp_mode/raw_pokemon.py index a084bc5e928..81c40c39c6e 100644 --- a/examples/cdp_mode/raw_pokemon.py +++ b/examples/cdp_mode/raw_pokemon.py @@ -4,14 +4,14 @@ sb.activate_cdp_mode() sb.goto("https://www.pokemon.com/us/pokedex") sb.sleep(1.5) - sb.click_if_visible("button#onetrust-accept-btn-handler") + sb.click_if_visible("#onetrust-accept-btn-handler", timeout=2) sb.sleep(1.5) sb.click('b:contains("Show Advanced Search")') sb.sleep(2.5) sb.click('span[data-type="type"][data-value="electric"]') - sb.sleep(0.7) + sb.sleep(1) sb.scroll_into_view("a#advSearch") - sb.sleep(0.7) + sb.sleep(1) sb.click("a#advSearch") sb.sleep(1.2) sb.click('img[src*="img/pokedex/detail/025.png"]') diff --git a/examples/cdp_mode/raw_reddit.py b/examples/cdp_mode/raw_reddit.py index 007b6be91a6..51ef1d79403 100644 --- a/examples/cdp_mode/raw_reddit.py +++ b/examples/cdp_mode/raw_reddit.py @@ -1,7 +1,7 @@ """Reddit Search / Bypasses reCAPTCHA.""" from seleniumbase import SB -with SB(uc=True, test=True, use_chromium=True) as sb: +with SB(uc=True, test=True, use_chromium=True, guest=True) as sb: search = "Reddit Scraping" sb.activate_cdp_mode() sb.goto("https://www.reddit.com/r/webscraping/") diff --git a/examples/cdp_mode/raw_trails.py b/examples/cdp_mode/raw_trails.py index 4dc2f180b92..d6b32bf8dbf 100644 --- a/examples/cdp_mode/raw_trails.py +++ b/examples/cdp_mode/raw_trails.py @@ -1,21 +1,31 @@ from seleniumbase import SB -with SB(uc=True, ad_block=True, test=True) as sb: +with SB(uc=True, guest=True, test=True) as sb: sb.activate_cdp_mode() sb.goto("https://www.alltrails.com/") - sb.sleep(3.5) + sb.sleep(4.5) + sb.hover_element("body") + sb.sleep(0.5) sb.click_if_visible("button.osano-cm-close") sb.sleep(0.5) search_box = 'input[aria-describedby="search-instructions"]' search_term = "Thundering Brook Falls" + try: + sb.click('button[class*="ImmersiveSearch"]', timeout=3) + except Exception: + sb.hover_element("body") + sb.sleep(0.5) + sb.click('button[class*="ImmersiveSearch"]') + sb.sleep(0.5) + sb.sleep(0.5) sb.wait_for_element(search_box) sb.sleep(1.5) sb.type(search_box, search_term + " Trail") - sb.sleep(1.5) + sb.sleep(2.5) sb.click('a span:contains("%s")' % search_term) sb.sleep(3.5) sb.click('button:contains("more")') - sb.sleep(0.7) + sb.sleep(0.8) print("Description: (%s)\n" % sb.get_text("h1")) print(sb.get_text('div[class*="Description_description"]')) sb.scroll_to_bottom() @@ -31,8 +41,10 @@ folder = "images_exported" file_name = "thundering_brook_falls.png" sb.scroll_to_top() + sb.sleep(0.6) + sb.click_visible_elements('button[title="Close"]') sb.sleep(0.3) - sb.click_if_visible('[aria-label="Close"]') + sb.scroll_to_top() sb.sleep(1.2) sb.save_screenshot(file_name, folder, selector="body") print('\n"./%s/%s" was saved!' % (folder, file_name)) diff --git a/examples/presenter/ua_presentation_4.py b/examples/presenter/ua_presentation_4.py index 0350b3bc813..15cca3f04f8 100644 --- a/examples/presenter/ua_presentation_4.py +++ b/examples/presenter/ua_presentation_4.py @@ -464,14 +464,14 @@ def test_presentation_4(self): sb.activate_cdp_mode() sb.goto("https://www.pokemon.com/us/pokedex") sb.sleep(1.5) - sb.click_if_visible("button#onetrust-accept-btn-handler") + sb.click_if_visible("#onetrust-accept-btn-handler", timeout=2) sb.sleep(1.5) sb.click('b:contains("Show Advanced Search")') sb.sleep(2.5) sb.click('span[data-type="type"][data-value="electric"]') - sb.sleep(0.7) + sb.sleep(1) sb.scroll_into_view("a#advSearch") - sb.sleep(0.7) + sb.sleep(1) sb.click("a#advSearch") sb.sleep(1.2) sb.click('img[src*="img/pokedex/detail/025.png"]') diff --git a/examples/raw_cdp_mode.py b/examples/raw_cdp_mode.py index dccc48fed2e..e9a62c7c4e2 100644 --- a/examples/raw_cdp_mode.py +++ b/examples/raw_cdp_mode.py @@ -1,7 +1,7 @@ """Bypass bot-detection with SeleniumBase CDP Mode.""" from seleniumbase import sb_cdp -sb = sb_cdp.Chrome() # Launches a stealth browser +sb = sb_cdp.Chrome() # Launch a stealth browser sb.goto("https://browserscan.net/bot-detection") sb.assert_element('strong:contains("Normal")') sb.sleep(1) diff --git a/examples/raw_google.py b/examples/raw_google.py index 28f309499ad..5fc2c853397 100644 --- a/examples/raw_google.py +++ b/examples/raw_google.py @@ -1,11 +1,14 @@ from seleniumbase import SB -with SB(uc=True, test=True) as sb: +with SB(uc=True, test=True, use_chromium=True) as sb: sb.goto("https://google.com/ncr") sb.click_if_visible('button:contains("Accept all")') - sb.type('[name="q"]', "SeleniumBase site:github.com") - sb.sleep(0.2) - sb.click('[value="Google Search"]') + sb.press_keys('[name="q"]', "SeleniumBase GitHub Page") + sb.sleep(0.5) + if sb.is_element_visible('[value="Google Search"]'): + sb.click('[value="Google Search"]') + else: + sb.press_keys('[name="q"]', "\n") sb.sleep(1) sb.click_if_visible('h3:contains("seleniumbase/SeleniumBase")') sb.sleep(1) diff --git a/examples/raw_test_scripts.py b/examples/raw_test_scripts.py index 7cdbccb6b2a..b33e22b2c28 100644 --- a/examples/raw_test_scripts.py +++ b/examples/raw_test_scripts.py @@ -6,8 +6,10 @@ sb.click_if_visible('button:contains("Accept all")') sb.type('[name="q"]', "SeleniumBase on GitHub") sb.click('[value="Google Search"]') - sb.highlight('a[href*="github.com/seleniumbase"]') - sb.sleep(0.5) + sb.sleep(1) + sb.highlight_if_visible( + 'a[href*="github.com/seleniumbase"]', timeout=3 + ) with SB(test=True, rtf=True, demo=True) as sb: sb.goto("seleniumbase.github.io/demo_page") diff --git a/help_docs/cdp_mode_methods.md b/help_docs/cdp_mode_methods.md index dac17cae321..12b27db324a 100644 --- a/help_docs/cdp_mode_methods.md +++ b/help_docs/cdp_mode_methods.md @@ -67,6 +67,7 @@ sb.select_option_by_index(dropdown_selector, option) sb.select_option_by_value(dropdown_selector, option) sb.flash(selector, duration=1, color="44CC88", pause=0) sb.highlight(selector, scroll=True, timeout=None) +sb.highlight_if_visible(selector, scroll=True, timeout=0) sb.focus(selector, timeout=None) sb.highlight_overlay(selector) sb.get_parent(element) diff --git a/help_docs/customizing_test_runs.md b/help_docs/customizing_test_runs.md index a20878c3422..2b1ea057300 100644 --- a/help_docs/customizing_test_runs.md +++ b/help_docs/customizing_test_runs.md @@ -223,6 +223,7 @@ pytest my_first_test.py --settings-file=custom_settings.py --rs | --reuse-session # (Reuse browser session for all tests.) --rcs | --reuse-class-session # (Reuse session for tests in class.) --crumbs # (Delete all cookies between tests reusing a session.) +--no-tt # (Skips "--test-type" when using Proxy Mode. Adds stealth.) --disable-beforeunload # (Disable the "beforeunload" event on Chrome.) --window-position=X,Y # (Set the browser's starting window position.) --window-size=WIDTH,HEIGHT # (Set the browser's starting window size.) diff --git a/help_docs/method_summary.md b/help_docs/method_summary.md index 00956232ad4..3948d6cd129 100644 --- a/help_docs/method_summary.md +++ b/help_docs/method_summary.md @@ -284,7 +284,7 @@ self.highlight_type(selector, text, by="css selector", loops=3, scroll=True, tim # Duplicates: # self.highlight_update_text( # selector, text, by="css selector", loops=3, scroll=True, timeout=None) -self.highlight_if_visible(selector, by="css selector", loops=4, scroll=True) +self.highlight_if_visible(selector, by="css selector", loops=4, scroll=True, timeout=0) self.highlight(selector, by="css selector", loops=4, scroll=True, timeout=None) self.highlight_elements(selector, by="css selector", loops=4, scroll=True, limit=0) self.press_up_arrow(selector="html", times=1, by="css selector") diff --git a/help_docs/syntax_formats.md b/help_docs/syntax_formats.md index 98c2303ea4e..1345b43a714 100644 --- a/help_docs/syntax_formats.md +++ b/help_docs/syntax_formats.md @@ -775,7 +775,10 @@ with SB(uc=True, test=True) as sb: sb.goto("https://google.com/ncr") sb.type('[name="q"]', "SeleniumBase on GitHub\n") sb.highlight('a[href*="github.com/seleniumbase"]') - sb.sleep(0.5) + sb.sleep(1) + sb.highlight_if_visible( + 'a[href*="github.com/seleniumbase"]', timeout=3 + ) with SB(test=True, rtf=True, demo=True) as sb: sb.goto("seleniumbase.github.io/demo_page") @@ -961,14 +964,16 @@ sb.quit() Here's a Pure CDP Mode example that bypasses bot-detection to scrape data from a website: ```python +from seleniumbase import decorators from seleniumbase import sb_cdp +@decorators.print_runtime("CDP Priceline Example") def main(): - url = "https://www.priceline.com/" - sb = sb_cdp.Chrome(url, lang="en") + sb = sb_cdp.Chrome(lang="en") + sb.goto("https://www.priceline.com/") sb.sleep(2) sb.internalize_links() # Don't open links in a new tab - sb.click("#link_header_nav_experiences") + sb.click('a[href*="experiences.priceline"]') sb.sleep(3) sb.remove_elements("msm-cookie-banner") sb.sleep(1) @@ -990,7 +995,9 @@ def main(): sb.sleep(0.2) cards = sb.select_all('span[data-automation*="product-list-card"]') for card in cards: - print("* %s" % card.text) + card_text = card.text.strip() + if len(card_text) >= 20: + print("* %s" % card_text) sb.quit() if __name__ == "__main__": diff --git a/mcp_servers/pyproject.toml b/mcp_servers/pyproject.toml index 9c464d2f1e3..f31b02f2fec 100644 --- a/mcp_servers/pyproject.toml +++ b/mcp_servers/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "seleniumbase-mcp" -version = "1.6.1dev0" +version = "1.7.0dev0" description = "MCP server exposing SeleniumBase CDP Mode as tools for MCP clients." readme = "README.md" requires-python = ">=3.10" diff --git a/mkdocs.yml b/mkdocs.yml index ce0bad5a3e3..4885c9023cd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -2,7 +2,7 @@ site_name: SeleniumBase Docs site_url: https://seleniumbase.io site_author: Michael Mintz -site_description: A complete framework for end-to-end testing with Python, pytest, behave-BDD, and WebDriver. +site_description: A complete framework for end-to-end testing, web scraping, and stealth with Python. # Repository information repo_name: seleniumbase/SeleniumBase repo_url: https://github.com/seleniumbase/SeleniumBase diff --git a/mkdocs_build/requirements.txt b/mkdocs_build/requirements.txt index 731359aeee3..b5f393ad70a 100644 --- a/mkdocs_build/requirements.txt +++ b/mkdocs_build/requirements.txt @@ -1,15 +1,15 @@ # mkdocs dependencies for generating the seleniumbase.io website # For docs generation, you'll need Python 3.11 or newer -regex>=2026.9.29 +regex>=2026.11.9 platformdirs>=4.14.5 -nab>=0.0.26 -backrefs>=8.2 -pymdown-extensions>=12.3 +nab>=0.0.29 +backrefs>=8.3 +pymdown-extensions>=13.1 python-dateutil>=4.0.0 -click>=8.5.0 +click>=8.5.1 tzdata>=2026.5 -Markdown>=3.11.2 +Markdown>=3.11.3 pipdeptree==4.2.14 ghp-import==2.1.0 watchdog==7.1.0 @@ -18,7 +18,7 @@ pathspec==1.1.1 Babel==2.18.0 paginate==0.5.7 mkdocs==1.6.1 -mkdocs-material==9.8.0 +mkdocs-material==9.8.1 mkdocs-exclude-search==0.6.7 mkdocs-simple-hooks==0.1.5 mkdocs-get-deps==0.2.2 diff --git a/requirements.txt b/requirements.txt index ad49ddf32c1..7b9747a2290 100755 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ exceptiongroup>=1.3.1 websockets~=16.1.1;python_version=="3.10" websockets>=17.4;python_version>="3.11" filelock==4.1.0;python_version=="3.10" -filelock>=4.2.9;python_version>="3.11" +filelock>=4.2.10;python_version>="3.11" fasteners>=0.20 mycdp>=1.4.0 pynose>=1.5.5 @@ -25,7 +25,7 @@ pyyaml>=6.0.3 pygments>=2.20.1 pyreadline3>=3.5.4;platform_system=="Windows" tabcompleter>=1.4.1 -pdbp>=1.8.3 +pdbp>=1.9.0 idna>=3.20 charset-normalizer>=3.5.2,<4 urllib3>=2.8.0,<3 @@ -56,12 +56,12 @@ pytest-xdist==3.8.0 parameterized==0.9.0 behave==1.2.6 soupsieve~=2.10.0;python_version<"3.11.5" -soupsieve~=4.0.1;python_version>="3.11.5" +soupsieve~=4.1.1;python_version>="3.11.5" beautifulsoup4~=4.15.0 -pyotp~=2.10.0 +pyotp~=2.11.2 python-xlib==0.33;platform_system=="Linux" PyAutoGUI>=0.9.54;platform_system=="Linux" -markdown-it-py==4.2.0 +markdown-it-py==4.3.0 mdurl==0.1.2 rich>=15.0.0,<16 @@ -69,7 +69,7 @@ rich>=15.0.0,<16 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.) coverage>=7.16.2;python_version=="3.10" -coverage>=7.17.2;python_version>="3.11" +coverage>=7.17.3;python_version>="3.11" pytest-cov>=7.1.0 flake8==7.4.1;python_version=="3.10" flake8==7.5.0;python_version>="3.11" diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 4c366d791c3..0bef3f3f49a 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.57.1" +__version__ = "4.58.0" diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index d9f89e59b30..01d3a167509 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -929,6 +929,7 @@ def uc_open_with_cdp_mode(driver, url=None, **kwargs): cdp.enter_mfa_code = CDPM.enter_mfa_code cdp.flash = CDPM.flash cdp.highlight = CDPM.highlight + cdp.highlight_if_visible = CDPM.highlight_if_visible cdp.focus = CDPM.focus cdp.highlight_overlay = CDPM.highlight_overlay cdp.get_window_position = CDPM.get_window_position @@ -2373,7 +2374,8 @@ def _set_chrome_options( device_pixel_ratio, ): chrome_options = webdriver.ChromeOptions() - if is_using_uc(undetectable, browser_name): + is_uc = is_using_uc(undetectable, browser_name) + if is_uc: from seleniumbase import undetected chrome_options = undetected.ChromeOptions() elif browser_name == constants.Browser.EDGE: @@ -2457,7 +2459,7 @@ def _set_chrome_options( chrome_options.add_argument( "--host-resolver-rules=%s" % host_resolver_rules ) - if mobile_emulator and not is_using_uc(undetectable, browser_name): + if mobile_emulator and not is_uc: emulator_settings = {} device_metrics = {} if ( @@ -2555,7 +2557,7 @@ def _set_chrome_options( pass if dark_mode: chrome_options.add_argument("--enable-features=WebContentsForceDark") - if user_data_dir and not is_using_uc(undetectable, browser_name): + if user_data_dir and not is_uc: abs_path = os.path.abspath(user_data_dir) chrome_options.add_argument("--user-data-dir=%s" % abs_path) if extension_zip: @@ -2646,7 +2648,7 @@ def _set_chrome_options( if proxy_string: if proxy_auth: zip_it = True - if is_using_uc(undetectable, browser_name): + if is_uc: zip_it = False # undetected-chromedriver needs a folder ext chrome_options = _add_chrome_proxy_extension( chrome_options, @@ -2666,7 +2668,7 @@ def _set_chrome_options( elif proxy_pac_url: if proxy_auth: zip_it = True # undetected-chromedriver needs a folder ext - if is_using_uc(undetectable, browser_name): + if is_uc: zip_it = False chrome_options = _add_chrome_proxy_extension( chrome_options, @@ -2679,18 +2681,14 @@ def _set_chrome_options( multi_proxy, ) chrome_options.add_argument("--proxy-pac-url=%s" % proxy_pac_url) - if ( - not is_using_uc(undetectable, browser_name) - or not enable_ws - or proxy_string - ): + if not is_uc or not enable_ws or proxy_string: chrome_options.add_argument("--ignore-certificate-errors") chrome_options.add_argument("--ignore-ssl-errors=yes") if not enable_ws: chrome_options.add_argument("--disable-web-security") if ( IS_LINUX - or (IS_MAC and not is_using_uc(undetectable, browser_name)) + or (IS_MAC and not is_uc) ): chrome_options.add_argument("--no-sandbox") if remote_debug: @@ -2705,13 +2703,10 @@ def _set_chrome_options( if swiftshader: chrome_options.add_argument("--use-gl=angle") chrome_options.add_argument("--use-angle=swiftshader-webgl") - elif ( - not is_using_uc(undetectable, browser_name) - and not enable_3d_apis - ): + elif not is_uc and not enable_3d_apis: chrome_options.add_argument("--disable-gpu") if ( - (not IS_LINUX and is_using_uc(undetectable, browser_name)) + (not IS_LINUX and is_uc) or ( IS_MAC and binary_location @@ -2722,7 +2717,7 @@ def _set_chrome_options( chrome_options.add_argument("--disable-application-cache") if IS_LINUX: chrome_options.add_argument("--disable-dev-shm-usage") - if is_using_uc(undetectable, browser_name): + if is_uc: chrome_options.add_argument("--disable-application-cache") chrome_options.add_argument("--disable-setuid-sandbox") if not binary_location: @@ -2762,9 +2757,7 @@ def _set_chrome_options( ) if "set-binary" in chromium_arg_item and not binary_location: br_app = "google-chrome" - binary_loc = detect_b_ver.get_binary_location( - br_app, is_using_uc(undetectable, browser_name) - ) + binary_loc = detect_b_ver.get_binary_location(br_app, is_uc) if os.path.exists(binary_loc): binary_location = binary_loc elif "disable-features=" in chromium_arg_item: @@ -2789,9 +2782,9 @@ def _set_chrome_options( chrome_options.add_argument("--disable-translate") if binary_location: chrome_options.binary_location = binary_location - if not enable_3d_apis and not is_using_uc(undetectable, browser_name): + if not enable_3d_apis and not is_uc: chrome_options.add_argument("--disable-3d-apis") - if headless or headless2 or is_using_uc(undetectable, browser_name): + if headless or headless2 or is_uc: chrome_options.add_argument("--disable-renderer-backgrounding") chrome_options.add_argument("--disable-background-networking") chrome_options.add_argument("--disable-backgrounding-occluded-windows") @@ -2805,7 +2798,10 @@ def _set_chrome_options( chrome_options.add_argument( '--simulate-outdated-no-au="Tue, 31 Dec 2099 23:59:59 GMT"' ) - chrome_options.add_argument("--disable-ipc-flooding-protection") + if not is_uc or __is_using_fast_mode(): + # Since disabling IPC FP is detectable by sites that look for it, + # we'll now keep it enabled in UC Mode / CDP Mode, unless "fast". + chrome_options.add_argument("--disable-ipc-flooding-protection") chrome_options.add_argument("--disable-password-generation") chrome_options.add_argument("--disable-domain-reliability") chrome_options.add_argument("--disable-breakpad") @@ -2830,7 +2826,7 @@ def _set_chrome_options( included_disabled_features.append("PasskeyAuth") included_disabled_features.append("DialMediaRouteProvider") included_disabled_features.append("WebRtcHideLocalIpsWithMdns") - if is_using_uc(undetectable, browser_name): + if is_uc: included_disabled_features.append("IsolateOrigins") included_disabled_features.append("site-per-process") for item in extra_disabled_features: @@ -2839,20 +2835,30 @@ def _set_chrome_options( d_f_string = ",".join(included_disabled_features) chrome_options.add_argument("--disable-features=%s" % d_f_string) chrome_options.add_argument("--enable-unsafe-extension-debugging") + if is_uc and binary_location and "cft_drivers" in binary_location: + chrome_options.add_argument("--disable-infobars") if ( hasattr(sb_config, "_cdp_browser") and sb_config._cdp_browser == "brave" ): pass # Looks like "--test-type" breaks Brave now else: - chrome_options.add_argument("--test-type") + if not is_uc or ( + proxy_string and not ( + hasattr(sb_config, "_no_tt") and sb_config._no_tt + ) + ): + # Since --test-type is detectable by sites that look for it, + # there's a new no_tt option to skip it in UC Mode / CDP Mode + # when using a proxy. Not used anymore if no proxy otherwise. + chrome_options.add_argument("--test-type") if proxy_auth or sb_config._ext_dirs: - if not is_using_uc(undetectable, browser_name): + if not is_uc: chrome_options.add_argument("--remote-debugging-pipe") chrome_options.enable_webextensions = True chrome_options.enable_bidi = True if ( - is_using_uc(undetectable, browser_name) + is_uc and ( not headless or IS_LINUX # switches to Xvfb (non-headless) @@ -2866,7 +2872,8 @@ def _set_chrome_options( chrome_options.add_argument("--disable-auto-reload") chrome_options.add_argument("--disable-background-timer-throttling") # Prevent new tabs opened by Selenium from being blocked: - chrome_options.add_argument("--disable-popup-blocking") + # (Option might not be needed anymore. Removing for now.) + # chrome_options.add_argument("--disable-popup-blocking") # Skip remaining options that trigger anti-bot services return chrome_options chrome_options.add_argument("--log-level=3") diff --git a/seleniumbase/core/sb_cdp.py b/seleniumbase/core/sb_cdp.py index 47d1eb9931b..066b97ddb44 100644 --- a/seleniumbase/core/sb_cdp.py +++ b/seleniumbase/core/sb_cdp.py @@ -257,12 +257,15 @@ def get_rd_url(self): that SeleniumBase methods can be called from Playwright without encountering event loop error messages such as: Cannot run the event loop while another loop is running. + Also fixes a NodeJS compatibility issue on old versions + of macOS when using Playwright versions 1.65.0 or newer. Also sets an environment variable to hide this warning: Deprecation: "url.parse() behavior is not standardized". (github.com/microsoft/playwright-python/issues/3016)""" from seleniumbase.core import nest_asyncio nest_asyncio.apply() os.environ["NODE_NO_WARNINGS"] = "1" + shared_utils.fix_playwright_node_compatibility() driver = self.driver if hasattr(driver, "cdp_base"): driver = driver.cdp_base @@ -1249,6 +1252,19 @@ def highlight(self, selector, scroll=True, timeout=None): element.flash(0.30, "44CC88", x_offset, y_offset) time.sleep(0.30) + def highlight_if_visible(self, selector, scroll=True, timeout=0): + if self.is_element_visible(selector): + with suppress(Exception): + self.highlight(selector, scroll=scroll, timeout=1) + elif timeout == 0: + return + else: + with suppress(Exception): + self.find_element(selector, timeout=timeout) + self.sleep(0.1) + if self.is_element_visible(selector): + self.highlight(selector, scroll=scroll, timeout=1) + def focus(self, selector, timeout=None): if not timeout: timeout = settings.SMALL_TIMEOUT @@ -2463,8 +2479,8 @@ def _on_a_datadome_slider_page(self, *args, **kwargs): self.loop.run_until_complete(self.page.wait(0.05)) if ( self.is_element_visible( - 'body > iframe[src*="/geo.captcha-delivery.com/captcha/"], ' - 'body > iframe[src*="/geo.captcha-delivery.com/interstitial/"]' + 'iframe[src*="/geo.captcha-delivery.com/captcha/"], ' + 'iframe[src*="/geo.captcha-delivery.com/interstitial/"]' ) ): return True @@ -2551,8 +2567,8 @@ def __gui_click_recaptcha(self, use_cdp=False): def __gui_slide_datadome_captcha(self): iframe = ( - 'body > iframe[src*="/geo.captcha-delivery.com/captcha/"], ' - 'body > iframe[src*="/geo.captcha-delivery.com/interstitial/"]' + 'iframe[src*="/geo.captcha-delivery.com/captcha/"], ' + 'iframe[src*="/geo.captcha-delivery.com/interstitial/"]' ) if not self.is_element_visible(iframe): return False diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index aeff3c134e6..bc56468dc6d 100644 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -4172,6 +4172,7 @@ def get_new_driver( d_width=None, d_height=None, d_p_r=None, + no_tt=None, **kwargs, ): """This method spins up an extra browser for tests that require @@ -4233,6 +4234,7 @@ def get_new_driver( d_width - the device width of the mobile emulator (Chrome-only) d_height - the device height of the mobile emulator (Chrome-only) d_p_r - the device pixel ratio of the mobile emulator (Chrome-only) + no_tt - the option to NOT use "--test-type" (Chromium-only stealth) """ self.__check_scope() if self.browser == "remote" and self.servername == "localhost": @@ -4379,6 +4381,11 @@ def get_new_driver( if external_pdf is None: external_pdf = self.external_pdf test_id = self.__get_test_id() + _existing_no_tt = None + if hasattr(sb_config, "_no_tt"): + _existing_no_tt = sb_config._no_tt + if no_tt is not None: + sb_config._no_tt = no_tt if cap_file is None: cap_file = self.cap_file if cap_string is None: @@ -4522,6 +4529,7 @@ def get_new_driver( self.__dont_record_open = True self.open(new_start_page) self.__dont_record_open = False + sb_config._no_tt = _existing_no_tt if undetectable: if hasattr(new_driver, "cdp"): self.cdp = new_driver.cdp @@ -6649,12 +6657,25 @@ def highlight_type( self.update_text(selector, text, by=by) def highlight_if_visible( - self, selector, by="css selector", loops=4, scroll=True, + self, selector, by="css selector", loops=4, scroll=True, timeout=0, ): """Highlights the element if the element is visible.""" self.__check_scope() + if self.__is_cdp_swap_needed(): + self.cdp.highlight_if_visible( + selector, scroll=scroll, timeout=timeout + ) + return if self.is_element_visible(selector, by=by): self.__highlight(selector, by=by, loops=loops, scroll=scroll) + elif timeout > 0: + with suppress(Exception): + self.wait_for_element_visible( + selector, by=by, timeout=timeout + ) + self.sleep(0.1) + if self.is_element_visible(selector, by=by): + self.__highlight(selector, by=by, loops=loops, scroll=scroll) def __highlight_element(self, element, loops=None, scroll=True): self.__check_scope() diff --git a/seleniumbase/fixtures/shared_utils.py b/seleniumbase/fixtures/shared_utils.py index a5c83ce77fc..7a5b9cb6f51 100644 --- a/seleniumbase/fixtures/shared_utils.py +++ b/seleniumbase/fixtures/shared_utils.py @@ -3,6 +3,7 @@ import os import pathlib import platform +import subprocess import sys import time from contextlib import suppress @@ -13,7 +14,6 @@ def pip_install(package, version=None): import fasteners - import subprocess pip_install_lock = fasteners.InterProcessLock( constants.PipInstall.LOCKFILE @@ -169,6 +169,25 @@ def is_cdp_swap_needed(driver): ) +def fix_playwright_node_compatibility(): + """Use system Node.js on macOS versions below 13.5""" + if platform.system() != "Darwin": + return + mac_version = platform.mac_ver()[0] + if not mac_version: + return + version = tuple(int(x) for x in mac_version.split(".")[:2]) + if version < (13, 5): + with suppress(Exception): + node_path = subprocess.run( + ["which", "node"], + capture_output=True, + text=True, + ).stdout.strip() + if node_path: + os.environ["PLAYWRIGHT_NODEJS_PATH"] = node_path + + def is_chrome_130_or_newer(self, binary_location=None): from seleniumbase.core import detect_b_ver diff --git a/seleniumbase/plugins/driver_manager.py b/seleniumbase/plugins/driver_manager.py index 4be5af844af..20aa5f6f247 100644 --- a/seleniumbase/plugins/driver_manager.py +++ b/seleniumbase/plugins/driver_manager.py @@ -139,6 +139,7 @@ def Driver( cft=None, # Use "Chrome for Testing" chs=None, # Use "Chrome-Headless-Shell" use_chromium=None, # Use base "Chromium" + no_tt=None, # No --test-type to hide infobars in Proxy Mode. ) -> sb_driver.DriverMethods: """ * SeleniumBase Driver as a Python Context Manager or a returnable object. * @@ -490,6 +491,13 @@ def Driver( is_mobile = True else: is_mobile = False + if ( + (no_tt is None and "--no-tt" in sys_argv) + or no_tt + ): + sb_config._no_tt = True + else: + sb_config._no_tt = False test_id = "direct_driver" proxy_string = proxy if proxy_string is None and "--proxy" in arg_join: diff --git a/seleniumbase/plugins/pytest_plugin.py b/seleniumbase/plugins/pytest_plugin.py index e1ef4ff0460..3352fe81ffe 100644 --- a/seleniumbase/plugins/pytest_plugin.py +++ b/seleniumbase/plugins/pytest_plugin.py @@ -123,6 +123,7 @@ def pytest_addoption(parser): --rs | --reuse-session (Reuse browser session for all tests.) --rcs | --reuse-class-session (Reuse session for tests in class.) --crumbs (Delete all cookies between tests reusing a session.) + --no-tt (No --test-type to hide infobars in Proxy Mode. Stealthier.) --disable-beforeunload (Disable the "beforeunload" event on Chrome.) --window-position=X,Y (Set the browser's starting window position.) --window-size=WIDTH,HEIGHT (Set the browser's starting window size.) @@ -1331,6 +1332,20 @@ def pytest_addoption(parser): or "--reuse-class-session"/"--rcs" because tests use a new clean browser if not reusing sessions.""", ) + parser.addoption( + "--no-tt", + "--no_tt", + action="store_true", + dest="_no_tt", + default=False, + help="""The option to not use the "--test-type" option + on Chromium-based browsers, which is generally + used to suppress startup infobars. This option + only takes effect when using proxy in UC Mode. + Setting "--no-tt" improves stealth because + "--test-type" is detectable, however very few + sites are currenly trying to detect it.""", + ) parser.addoption( "--disable-beforeunload", "--disable_beforeunload", @@ -1874,6 +1889,7 @@ def pytest_configure(config): sb_config.reuse_session = True sb_config.shared_driver = None # The default driver for session reuse sb_config.crumbs = config.getoption("crumbs") + sb_config._no_tt = config.getoption("_no_tt") sb_config._disable_beforeunload = config.getoption("_disable_beforeunload") sb_config.window_position = config.getoption("window_position") sb_config.window_size = config.getoption("window_size") diff --git a/seleniumbase/plugins/sb_manager.py b/seleniumbase/plugins/sb_manager.py index 9a11719dc43..e3a544811d6 100644 --- a/seleniumbase/plugins/sb_manager.py +++ b/seleniumbase/plugins/sb_manager.py @@ -124,7 +124,8 @@ def SB( wfa=None, # Shortcut / Duplicate of "wait_for_angularjs". cft=None, # Use "Chrome for Testing" chs=None, # Use "Chrome-Headless-Shell" - use_chromium=None, # Use base "Chromium" + use_chromium=None, # Use unbranded "Chromium" + no_tt=None, # No --test-type to hide infobars in Proxy Mode. save_screenshot=None, # Save a screenshot at the end of each test. no_screenshot=None, # No screenshots saved unless tests directly ask it. page_load_strategy=None, # Set Chrome PLS to "normal", "eager", or "none". @@ -563,6 +564,13 @@ def SB( is_mobile = False if is_mobile: sb_config.mobile_emulator = True + if ( + (no_tt is None and "--no-tt" in sys_argv) + or no_tt + ): + sb_config._no_tt = True + else: + sb_config._no_tt = False proxy_string = proxy if proxy_string is None and "--proxy" in arg_join: if "--proxy=" in arg_join: diff --git a/seleniumbase/plugins/selenium_plugin.py b/seleniumbase/plugins/selenium_plugin.py index 9952b01af01..72d7e03bab3 100644 --- a/seleniumbase/plugins/selenium_plugin.py +++ b/seleniumbase/plugins/selenium_plugin.py @@ -95,6 +95,7 @@ class SeleniumBrowser(Plugin): --guest (Enable Chrome's Guest mode.) --dark (Enable Chrome's Dark mode.) --devtools (Open Chrome's DevTools when the browser opens.) + --no-tt (No --test-type to hide infobars in Proxy Mode. Stealthier.) --disable-beforeunload (Disable the "beforeunload" event on Chrome.) --window-position=X,Y (Set the browser's starting window position.) --window-size=WIDTH,HEIGHT (Set the browser's starting window size.) @@ -991,6 +992,20 @@ def options(self, parser, env): default=False, help="""Using this opens Chrome's DevTools.""", ) + parser.addoption( + "--no-tt", + "--no_tt", + action="store_true", + dest="_no_tt", + default=False, + help="""The option to not use the "--test-type" option + on Chromium-based browsers, which is generally + used to suppress startup infobars. This option + only takes effect when using proxy in UC Mode. + Setting "--no-tt" improves stealth because + "--test-type" is detectable, however very few + sites are currenly trying to detect it.""", + ) parser.addoption( "--disable-beforeunload", "--disable_beforeunload", @@ -1446,6 +1461,7 @@ def beforeTest(self, test): test.test.dashboard = False test.test._multithreaded = False test.test._reuse_session = False + sb_config._no_tt = self.options._no_tt sb_config.recorder_mode = test.test.recorder_mode sb_config.no_screenshot = test.test.no_screenshot_after_test if test.test.servername != "localhost": diff --git a/seleniumbase/undetected/cdp_driver/browser.py b/seleniumbase/undetected/cdp_driver/browser.py index 5e348c59ce6..03d949f0242 100644 --- a/seleniumbase/undetected/cdp_driver/browser.py +++ b/seleniumbase/undetected/cdp_driver/browser.py @@ -298,10 +298,13 @@ def get_rd_port(self): def get_rd_url(self): """Returns the remote-debugging URL, which is used for allowing the Playwright integration to launch stealthy. + Also fixes a NodeJS compatibility issue on old versions + of macOS when using Playwright versions 1.65.0 or newer. Also sets an environment variable to hide this warning: Deprecation: "url.parse() behavior is not standardized". (github.com/microsoft/playwright-python/issues/3016)""" os.environ["NODE_NO_WARNINGS"] = "1" + shared_utils.fix_playwright_node_compatibility() host = self.config.host port = self.config.port return f"http://{host}:{port}" diff --git a/seleniumbase/undetected/cdp_driver/cdp_util.py b/seleniumbase/undetected/cdp_driver/cdp_util.py index 5bb015f4647..5c8b33584a8 100644 --- a/seleniumbase/undetected/cdp_driver/cdp_util.py +++ b/seleniumbase/undetected/cdp_driver/cdp_util.py @@ -308,10 +308,11 @@ async def start( tzone: str | None = None, # Eg "America/New_York", "Asia/Kolkata" geoloc: list | tuple | None = None, # Eg [48.87645, 2.26340] mobile: bool | None = None, # Use Mobile Mode with default args - disable_csp: str | None = None, # Disable content security policy + disable_csp: bool | None = None, # Disable content security policy extension_dir: str | None = None, # Chrome extension directory - use_chromium: str | None = None, # Use the base Chromium browser - cft: str | None = None, # Use the Chrome-for-Testing browser + use_chromium: bool | None = None, # Use unbranded Chromium browser + cft: bool | None = None, # Use the Chrome-for-Testing browser + no_tt: bool | None = None, # No --test-type with proxy to hide infobars. **kwargs: dict | None, ) -> Browser: """ @@ -391,6 +392,13 @@ async def start( sb_config._cdp_mobile_mode = True else: sb_config._cdp_mobile_mode = False + if ( + (no_tt is None and "--no-tt" in sys_argv) + or no_tt + ): + sb_config._no_tt = True + else: + sb_config._no_tt = False if ad_block is None: if "--ad-block" in sys_argv or "--ad_block" in sys_argv: ad_block = True diff --git a/seleniumbase/undetected/cdp_driver/config.py b/seleniumbase/undetected/cdp_driver/config.py index 1be164f4daf..e13c08b1c64 100644 --- a/seleniumbase/undetected/cdp_driver/config.py +++ b/seleniumbase/undetected/cdp_driver/config.py @@ -248,7 +248,6 @@ def __init__( "--disable-password-generation", "--disable-save-password-bubble", "--disable-single-click-autofill", - "--disable-ipc-flooding-protection", "--disable-background-timer-throttling", "--disable-search-engine-choice-screen", "--disable-background-networking", @@ -261,17 +260,32 @@ def __init__( "--disable-renderer-backgrounding", "--disable-dev-shm-usage", ] + if ( + self.browser_executable_path + and ( + "cft_drivers" in self.browser_executable_path + or self.browser_executable_path == "cft" + ) + ): + self._default_browser_args.append("--disable-infobars") if ( hasattr(sb_config, "_cdp_browser") and sb_config._cdp_browser == "brave" ): pass # Looks like "--test-type" breaks Brave now else: - self._default_browser_args.append("--test-type") + if proxy and not ( + hasattr(sb_config, "_no_tt") and sb_config._no_tt + ): + self._default_browser_args.append("--test-type") if IS_LINUX: self._default_browser_args.append("--disable-setuid-sandbox") if mock_keychain: self._default_browser_args.append("--use-mock-keychain") + if hasattr(sb_config, "fast_mode") and sb_config.fast_mode: + self._default_browser_args.append( + "--disable-ipc-flooding-protection" + ) @property def browser_args(self): diff --git a/seleniumbase/undetected/cdp_driver/element.py b/seleniumbase/undetected/cdp_driver/element.py index 984bc600bd0..81839133902 100644 --- a/seleniumbase/undetected/cdp_driver/element.py +++ b/seleniumbase/undetected/cdp_driver/element.py @@ -871,85 +871,115 @@ async def __new_send_keys_async( self, text: str, add_delay: bool = False, fast: bool = False ): """A helper method for send_keys_async()""" - # Map non-alphanumeric symbols to the correct CDP virtual key code. - # Prevents collisions with control keys. Eg. ord('.') = 46 = VK_DELETE. + # Map non-alphanumeric symbols to the correct + # CDP virtual key code and unshifted key. + # Format: char: (key, code, vk, unshifted_char) SYMBOL_MAP = { - ".": ("Period", 190), - ",": ("Comma", 188), - "-": ("Minus", 189), - "=": ("Equal", 187), - "/": ("Slash", 191), - "\\": ("Backslash", 220), - ";": ("Semicolon", 186), - "'": ("Quote", 222), - "`": ("Backquote", 192), - "[": ("BracketLeft", 219), - "]": ("BracketRight", 221), - "!": ("Digit1", 49), - "@": ("Digit2", 50), - "#": ("Digit3", 51), - "$": ("Digit4", 52), - "%": ("Digit5", 53), - "^": ("Digit6", 54), - "&": ("Digit7", 55), - "*": ("Digit8", 56), - "(": ("Digit9", 57), - ")": ("Digit0", 48), - "_": ("Minus", 189), - "+": ("Equal", 187), - "{": ("BracketLeft", 219), - "}": ("BracketRight", 221), - "|": ("Backslash", 220), - ":": ("Semicolon", 186), - '"': ("Quote", 222), - "<": ("Comma", 188), - ">": ("Period", 190), - "?": ("Slash", 191), - "~": ("Backquote", 192), + "!": ("!", "Digit1", 49, "1"), + "@": ("@", "Digit2", 50, "2"), + "#": ("#", "Digit3", 51, "3"), + "$": ("$", "Digit4", 52, "4"), + "%": ("%", "Digit5", 53, "5"), + "^": ("^", "Digit6", 54, "6"), + "&": ("&", "Digit7", 55, "7"), + "*": ("*", "Digit8", 56, "8"), + "(": ("(", "Digit9", 57, "9"), + ")": (")", "Digit0", 48, "0"), + "_": ("_", "Minus", 189, "-"), + "+": ("+", "Equal", 187, "="), + "{": ("{", "BracketLeft", 219, "["), + "}": ("}", "BracketRight", 221, "]"), + "|": ("|", "Backslash", 220, "\\"), + ":": (":", "Semicolon", 186, ";"), + '"': ('"', "Quote", 222, "'"), + "<": ("<", "Comma", 188, ","), + ">": (">", "Period", 190, "."), + "?": ("?", "Slash", 191, "/"), + "~": ("~", "Backquote", 192, "`"), + ".": (".", "Period", 190, "."), + ",": (",", "Comma", 188, ","), + "-": ("-", "Minus", 189, "-"), + "=": ("=", "Equal", 187, "="), + "/": ("/", "Slash", 191, "/"), + "\\": ("\\", "Backslash", 220, "\\"), + ";": (";", "Semicolon", 186, ";"), + "'": ("'", "Quote", 222, "'"), + "`": ("`", "Backquote", 192, "`"), + "[": ("[", "BracketLeft", 219, "["), + "]": ("]", "BracketRight", 221, "]"), } + for char in text: - # 1. Map character to its DOM key attributes + # Determine if Shift is required + needs_shift = ( + char.isupper() + or char in SYMBOL_MAP and char not in ".,-=/\\;`'[]" + ) + modifiers = 8 if needs_shift else 0 if char in ("\r", "\n"): key = "Enter" code = "Enter" vk = 13 text_val = "\r" + unmodified_text = "\r" elif char == "\t": key = "Tab" code = "Tab" vk = 9 text_val = "" + unmodified_text = "" elif char == " ": key = " " code = "Space" vk = 32 text_val = " " + unmodified_text = " " + elif char.isalpha(): + key = char # Uppercase or lowercase letter + code = f"Key{char.upper()}" + vk = ord(char.upper()) + text_val = char + unmodified_text = char.lower() if needs_shift else char + elif char.isdigit(): + key = char + code = f"Digit{char}" + vk = ord(char) + text_val = char + unmodified_text = char + elif char in SYMBOL_MAP: + key, code, vk, unshifted_char = SYMBOL_MAP[char] + text_val = char + unmodified_text = unshifted_char if needs_shift else char else: key = char + code = "" + vk = 0 text_val = char - if char.isalpha(): - code = f"Key{char.upper()}" - vk = ord(char.upper()) - elif char.isdigit(): - code = f"Digit{char}" - vk = ord(char) - elif char in SYMBOL_MAP: - code, vk = SYMBOL_MAP[char] - else: - code = "" - vk = 0 - # 2. Trigger keydown DOM event WITHOUT inserting text + unmodified_text = char + # Dispatch Shift keydown if needed (to ensure trusted event chain) + if needs_shift: + await self._tab.send( + cdp.input_.dispatch_key_event( + type_="rawKeyDown", + key="Shift", + code="ShiftLeft", + windows_virtual_key_code=16, + modifiers=8, + ) + ) + # Trigger rawKeyDown event await self._tab.send( cdp.input_.dispatch_key_event( type_="rawKeyDown", key=key, code=code, windows_virtual_key_code=vk, + modifiers=modifiers, ) ) if not fast: await asyncio.sleep(random.uniform(0.002, 0.003)) - # 3. Trigger keypress DOM event AND insert text + # Trigger char event with proper text and unmodified_text if text_val: await self._tab.send( cdp.input_.dispatch_key_event( @@ -957,11 +987,12 @@ async def __new_send_keys_async( key=key, code=code, text=text_val, - unmodified_text=text_val, + unmodified_text=unmodified_text, windows_virtual_key_code=vk, + modifiers=modifiers, ) ) - # 4. Trigger keyup DOM event + # Trigger keyUp event if not fast: await asyncio.sleep(random.uniform(0.011, 0.014)) await self._tab.send( @@ -970,8 +1001,20 @@ async def __new_send_keys_async( key=key, code=code, windows_virtual_key_code=vk, + modifiers=modifiers, ) ) + # Release Shift if it was pressed + if needs_shift: + await self._tab.send( + cdp.input_.dispatch_key_event( + type_="keyUp", + key="Shift", + code="ShiftLeft", + windows_virtual_key_code=16, + modifiers=0, + ) + ) if add_delay and not fast: await asyncio.sleep(float(0.04 + (random.random() / 110.0))) diff --git a/server.json b/server.json index 5fe6f41d598..51c1ace8e9c 100644 --- a/server.json +++ b/server.json @@ -3,7 +3,7 @@ "name": "io.github.seleniumbase/seleniumbase", "title": "SeleniumBase MCP", "description": "Stealthy browser automation, testing, and web-scraping via CDP Mode.", - "version": "4.57.1", + "version": "4.58.0", "repository": { "url": "https://github.com/seleniumbase/SeleniumBase", "source": "github" @@ -13,7 +13,7 @@ { "registryType": "pypi", "identifier": "seleniumbase", - "version": "4.57.1", + "version": "4.58.0", "transport": { "type": "stdio" }, diff --git a/setup.py b/setup.py index 80ac8007282..b13b2f1775a 100755 --- a/setup.py +++ b/setup.py @@ -174,7 +174,7 @@ 'websockets~=16.1.1;python_version=="3.10"', 'websockets>=17.4;python_version>="3.11"', 'filelock==4.1.0;python_version=="3.10"', - 'filelock>=4.2.9;python_version>="3.11"', + 'filelock>=4.2.10;python_version>="3.11"', 'fasteners>=0.20', 'mycdp>=1.4.0', 'pynose>=1.5.5', @@ -191,7 +191,7 @@ 'pygments>=2.20.1', 'pyreadline3>=3.5.4;platform_system=="Windows"', 'tabcompleter>=1.4.1', - 'pdbp>=1.8.3', + 'pdbp>=1.9.0', 'idna>=3.20', 'charset-normalizer>=3.5.2,<4', 'urllib3>=2.8.0,<3', @@ -222,12 +222,12 @@ 'parameterized==0.9.0', 'behave==1.2.6', # Newer ones had issues 'soupsieve~=2.10.0;python_version<"3.11.5"', - 'soupsieve~=4.0.1;python_version>="3.11.5"', + 'soupsieve~=4.1.1;python_version>="3.11.5"', 'beautifulsoup4~=4.15.0', - 'pyotp~=2.10.0', + 'pyotp~=2.11.2', 'python-xlib==0.33;platform_system=="Linux"', 'PyAutoGUI>=0.9.54;platform_system=="Linux"', - 'markdown-it-py==4.2.0', + 'markdown-it-py==4.3.0', 'mdurl==0.1.2', 'rich>=15.0.0,<16', ], @@ -244,7 +244,7 @@ # Usage: coverage run -m pytest; coverage html; coverage report "coverage": [ 'coverage>=7.16.2;python_version=="3.10"', - 'coverage>=7.17.2;python_version>="3.11"', + 'coverage>=7.17.3;python_version>="3.11"', 'pytest-cov>=7.1.0', ], # pip install -e .[flake8] @@ -299,10 +299,8 @@ # pip install -e .[playwright] # (For the Playwright integration.) "playwright": [ - 'playwright>=1.64.1', + 'playwright>=1.65.2', 'greenlet>=3.5.6', - 'pyee>=13.0.1;python_version<"3.12"', - 'pyee>=14.0.0;python_version>="3.12"', ], # pip install -e .[pyautogui] # (Already a required dependency on Linux now.) @@ -312,7 +310,7 @@ # pip install -e .[uv] # (For the MCP integration and more.) "uv": [ - "uv>=0.13.7" + "uv>=0.13.8" ], }, packages=[