diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index 46f5c283467..43f1f6d4e44 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -563,6 +563,12 @@ sb.save_as_html(name, folder=None) sb.save_screenshot(name, folder=None, selector=None) sb.print_to_pdf(name, folder=None) sb.save_as_pdf(name, folder=None) +sb.save_as_pdf_to_logs(name=None) +sb.save_screenshot_to_logs(name=None, selector=None) +sb.save_page_source_to_logs(name=None) +sb.save_as_html_to_logs(name=None) +sb.save_data_to_logs(data, file_name=None) +sb.append_data_to_logs(data, file_name=None) sb.quit() # (Pure CDP Mode only: `sb_cdp`) ``` @@ -667,7 +673,7 @@ await page.click_if_visible(selector, timeout=0) await page.click_with_offset(selector, x, y, center=False, timeout=5) await page.solve_captcha() await page.click_captcha() # Same as solve_captcha() -await page.reload(ignore_cache=True, script_to_evaluate_on_load=None) +await page.reload(ignore_cache=False, script_to_evaluate_on_load=None) await page.evaluate(expression) await page.js_dumps(obj_name) await page.back() diff --git a/examples/cdp_mode/playwright/raw_etsy_sync.py b/examples/cdp_mode/playwright/raw_etsy_sync.py index 6705ac4b31a..81782f7fe20 100644 --- a/examples/cdp_mode/playwright/raw_etsy_sync.py +++ b/examples/cdp_mode/playwright/raw_etsy_sync.py @@ -5,7 +5,7 @@ sb.goto("https://www.etsy.com/") sb.sleep(1) search = "FIFA Keychains" -required_text = "keychain" +required_text = "Keychain" sb.type('input[data-id="search-query"]', search) sb.sleep(1) sb.click('button[aria-label="Search"]') @@ -18,6 +18,8 @@ page = browser.contexts[0].pages[0] items = page.locator("div.v2-listing-card__info") num = 0 + print('*** Etsy search for "%s":' % search) + print(' (Results must contain "%s")' % required_text) for i in range(items.count()): title = items.nth(i).locator("h3.v2-listing-card__title").inner_text() price = items.nth(i).locator("div.n-listing-card__price").inner_text() diff --git a/examples/cdp_mode/playwright/raw_nordstrom_sync.py b/examples/cdp_mode/playwright/raw_nordstrom_sync.py index 17bd19ee9a5..2ac2d9d8dc2 100644 --- a/examples/cdp_mode/playwright/raw_nordstrom_sync.py +++ b/examples/cdp_mode/playwright/raw_nordstrom_sync.py @@ -15,9 +15,6 @@ search_box = page.locator("input#keyword-search-input") search_box.press_sequentially(search + "\n", delay=80) page.wait_for_timeout(2200) - for i in range(17): - sb.scroll_down(16) - sb.sleep(0.14) print('*** Nordstrom Search for "%s":' % search) unique_item_text = [] items = sb.find_elements("article") @@ -30,3 +27,4 @@ if price: price_text = price.text print("* %s (%s)" % (description.text, price_text)) + item.flash(color="44CC88") diff --git a/examples/cdp_mode/raw_ad_blocking.py b/examples/cdp_mode/raw_ad_blocking.py index 47b6512ff73..ab1016a9d6b 100644 --- a/examples/cdp_mode/raw_ad_blocking.py +++ b/examples/cdp_mode/raw_ad_blocking.py @@ -17,6 +17,8 @@ async def block_urls(tab): "*.doubleclick.net*", "*.fastclick.net*", "*.snigelweb.com*", + "*.viously.com*", + "*.flashb.id*", "*.2mdn.net*", ] )) @@ -27,7 +29,13 @@ async def block_urls(tab): loop.run_until_complete(block_urls(sb.get_active_tab())) sb.goto("https://www.w3schools.com/jquery/default.asp") source = sb.get_page_source() - sb.assert_false("doubleclick.net" in source) - sb.assert_false("google-analytics.com" in source) + try: + sb.assert_false("doubleclick.net" in source) + sb.assert_false("google-analytics.com" in source) + except Exception: + sb.refresh() + source = sb.get_page_source() + sb.assert_false("doubleclick.net" in source) + sb.assert_false("google-analytics.com" in source) sb.post_message("Blocking was successful!") sb.quit() diff --git a/examples/cdp_mode/raw_basic_cdp.py b/examples/cdp_mode/raw_basic_cdp.py index fc1cd118697..2ca49fa6dbb 100644 --- a/examples/cdp_mode/raw_basic_cdp.py +++ b/examples/cdp_mode/raw_basic_cdp.py @@ -12,6 +12,11 @@ links = top_nav.query_selector_all("a") for nav_item in links: print(nav_item.text) +sb.save_as_pdf_to_logs() +sb.save_screenshot_to_logs() +sb.save_page_source_to_logs() +sb.save_data_to_logs("hello!") +sb.append_data_to_logs("more data!") sb.click_link("Sign out") sb.assert_text("signed out", "#top_message") sb.quit() diff --git a/examples/cdp_mode/raw_cdp_etsy.py b/examples/cdp_mode/raw_cdp_etsy.py index 0d35d9bbc3a..5369c00ab31 100644 --- a/examples/cdp_mode/raw_cdp_etsy.py +++ b/examples/cdp_mode/raw_cdp_etsy.py @@ -4,7 +4,7 @@ sb.goto("https://www.etsy.com/") sb.sleep(1) search = "FIFA Keychains" -required_text = "keychain" +required_text = "Keychain" sb.type('input[data-id="search-query"]', search) sb.sleep(1) sb.click('button[aria-label="Search"]') @@ -13,6 +13,8 @@ soup = sb.get_beautiful_soup() items = soup.select("div.v2-listing-card__info") num = 0 +print('*** Etsy search for "%s":' % search) +print(' (Results must contain "%s")' % required_text) for item in items: title_element = item.select_one("h3.v2-listing-card__title") price_element = item.select_one("div.n-listing-card__price") diff --git a/examples/cdp_mode/raw_cdp_fingerprint.py b/examples/cdp_mode/raw_cdp_fingerprint.py index 06e61c9734e..f553daa6c35 100644 --- a/examples/cdp_mode/raw_cdp_fingerprint.py +++ b/examples/cdp_mode/raw_cdp_fingerprint.py @@ -1,6 +1,6 @@ from seleniumbase import sb_cdp -sb = sb_cdp.Chrome() +sb = sb_cdp.Chrome(use_chromium=True) sb.goto("https://demo.fingerprint.com/playground") sb.wait_for_element('a[href*="browser-bot-detection"]') sb.flash('a[href*="browser-bot-detection"]', duration=3, pause=1) diff --git a/examples/cdp_mode/raw_cdp_nordstrom.py b/examples/cdp_mode/raw_cdp_nordstrom.py index 3a91121f197..2a3d5d0ba87 100644 --- a/examples/cdp_mode/raw_cdp_nordstrom.py +++ b/examples/cdp_mode/raw_cdp_nordstrom.py @@ -20,5 +20,5 @@ if price: price_text = price.text print("* %s (%s)" % (description.text, price_text)) - sb.scroll_down(16) + item.flash(color="44CC88") sb.quit() diff --git a/examples/cdp_mode/raw_homedepot.py b/examples/cdp_mode/raw_homedepot.py index 50e4f0a4706..d184b936f8e 100644 --- a/examples/cdp_mode/raw_homedepot.py +++ b/examples/cdp_mode/raw_homedepot.py @@ -17,20 +17,26 @@ sb.click("button#typeahead-search-icon-button") sb.sleep(3.8) sb.click('a[aria-label="%s"]' % category) - sb.sleep(3.2) + sb.sleep(3.8) print('*** Home Depot Search for "%s":' % search) print(' (Results must contain "%s".)' % required_text) unique_item_text = [] - items = sb.find_elements('div[data-testid="product-pod"]') + product_pod = 'div[data-testid="product-pod"]' + sb.wait_for_element(product_pod) + soup = sb.get_beautiful_soup() + items = soup.select(product_pod) for item in items: - if required_text in item.text: - description = item.querySelector( + item_text = item.get_text() + if required_text in item_text: + description = item.select_one( 'span[data-testid="attribute-product-label"]' ) - if description and description.text not in unique_item_text: - unique_item_text.append(description.text) - print("* " + description.text) - price = item.querySelector('[class*="sm:sui-text-4xl"]') - if price: - price_text = "$%s" % price.text - print(" (" + price_text + ")") + if description: + description_text = description.get_text(strip=True) + if description_text not in unique_item_text: + unique_item_text.append(description_text) + print("* " + description_text) + price = item.select_one('[class*="sm:sui-text-4xl"]') + if price: + price_text = "$%s" % price.get_text(strip=True) + print(" (" + price_text + ")") diff --git a/examples/cdp_mode/raw_indeed_jobs.py b/examples/cdp_mode/raw_indeed_jobs.py index f56fcb09189..2529cf38eea 100644 --- a/examples/cdp_mode/raw_indeed_jobs.py +++ b/examples/cdp_mode/raw_indeed_jobs.py @@ -8,7 +8,7 @@ sb.click('button[type="submit"]') sb.sleep(3.6) items = sb.find_visible_elements('[data-testid*="jobcard"]') - print('*** %s:' % search) + print('*** Indeed search for "%s":' % search) for i, item in enumerate(items): print(f"* <====== {i + 1} ======>") print(item.text) diff --git a/examples/cdp_mode/raw_mouser.py b/examples/cdp_mode/raw_mouser.py index 79cc4afa0b9..0f88b404235 100644 --- a/examples/cdp_mode/raw_mouser.py +++ b/examples/cdp_mode/raw_mouser.py @@ -1,15 +1,20 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale="en") as sb: +with SB(uc=True, test=True, guest=True) as sb: sb.activate_cdp_mode() sb.goto("https://www.mouser.com/") - sb.sleep(5) - sb.press_keys('input[name="keyword"]', "FLUKE-TC01B 25HZ") + search_box = 'input[name="keyword"]' + sb.sleep(1.6) + sb.solve_captcha() + sb.sleep(1.8) + sb.wait_for_element(search_box) + sb.sleep(1.2) + sb.press_keys(search_box, "FLUKE-TC01B 25HZ") + sb.sleep(1.2) sb.click('button[type="submit"]') - sb.sleep(2) - sb.highlight("h1") - sb.highlight("span#spnDescription") - sb.highlight("td.ext-price-col") - print(sb.get_text("h1")) - print(sb.get_text("span#spnDescription")) - print(sb.get_text("td.ext-price-col")) + sb.sleep(3.2) + sb.wait_for_element("span#spnDescription") + soup = sb.get_beautiful_soup() + print(soup.select_one("h1").get_text(strip=True)) + print(soup.select_one("span#spnDescription").get_text(strip=True)) + print(soup.select_one("td.ext-price-col").get_text(strip=True)) diff --git a/examples/cdp_mode/raw_muse.py b/examples/cdp_mode/raw_muse.py index aea0c83e1a9..220d4601ae4 100644 --- a/examples/cdp_mode/raw_muse.py +++ b/examples/cdp_mode/raw_muse.py @@ -4,8 +4,8 @@ with SB(uc=True, test=True, guest=True) as sb: sb.activate_cdp_mode() sb.goto("https://muse.jhu.edu/verify") - sb.sleep(1.6) + sb.sleep(0.6) sb.solve_captcha() sb.sleep(4) sb.assert_element('#search_input') - sb.sleep(3) + sb.sleep(2.5) diff --git a/examples/cdp_mode/raw_muse_async.py b/examples/cdp_mode/raw_muse_async.py index c5a97efe5e8..490e81d712b 100644 --- a/examples/cdp_mode/raw_muse_async.py +++ b/examples/cdp_mode/raw_muse_async.py @@ -7,11 +7,11 @@ async def main(): url = "https://muse.jhu.edu/verify" driver = await cdp_driver.start_async(guest=True) page = await driver.get(url) - await page.sleep(2.5) + await page.sleep(1.2) await page.solve_captcha() await page.sleep(4) await page.find('#search_input') - await page.sleep(3) + await page.sleep(2.5) if __name__ == "__main__": asyncio.run(main()) diff --git a/examples/cdp_mode/raw_nordstrom.py b/examples/cdp_mode/raw_nordstrom.py index e5e7bdce6cc..e75e608473f 100644 --- a/examples/cdp_mode/raw_nordstrom.py +++ b/examples/cdp_mode/raw_nordstrom.py @@ -21,4 +21,4 @@ if price: price_text = price.text print("* %s (%s)" % (description.text, price_text)) - sb.scroll_down(16) + item.flash(color="44CC88") diff --git a/examples/cdp_mode/raw_zoro.py b/examples/cdp_mode/raw_zoro.py index fb0baef0b6d..247dd7adcdf 100644 --- a/examples/cdp_mode/raw_zoro.py +++ b/examples/cdp_mode/raw_zoro.py @@ -19,15 +19,20 @@ sb.sleep(3.2) sb.wait_for_element('[data-za="product-cards-list"]', timeout=5) print('*** Zoro Search for "%s":' % search) - print(' (Results must contain "%s".)' % required_text) + print(' (Results must contain "%s")' % required_text) unique_item_text = [] - items = sb.find_elements('[data-za="search-product-card"]') + soup = sb.get_beautiful_soup() + items = soup.select('[data-za="search-product-card"]') for item in items: - if required_text in item.text: - description = item.querySelector("h2") - if description and description.text not in unique_item_text: - unique_item_text.append(description.text) - print("* " + description.text) - price = item.querySelector("div.price-main") - if price: - print(" (" + price.text + ")") + item_text = item.get_text() + if required_text.lower() in item_text.lower(): + description_element = item.select_one("h2") + if description_element: + description_text = description_element.get_text(strip=True) + if description_text.lower() not in unique_item_text: + unique_item_text.append(description_text.lower()) + print("* " + description_text) + price_element = item.select_one("div.price-main") + if price_element: + price_text = " ".join(price_element.get_text().split()) + print(" (" + price_text + ")") diff --git a/examples/test_cdp_ad_blocking.py b/examples/test_cdp_ad_blocking.py index 1576dc014ab..652c9db694e 100644 --- a/examples/test_cdp_ad_blocking.py +++ b/examples/test_cdp_ad_blocking.py @@ -20,10 +20,18 @@ def test_cdp_network_blocking(self): "*.doubleclick.net*", "*.fastclick.net*", "*.snigelweb.com*", + "*.viously.com*", + "*.flashb.id*", "*.2mdn.net*", ]}) self.goto("https://www.w3schools.com/jquery/default.asp") source = self.get_page_source() - self.assert_false("doubleclick.net" in source) - self.assert_false("google-analytics.com" in source) + try: + self.assert_false("doubleclick.net" in source) + self.assert_false("google-analytics.com" in source) + except Exception: + self.refresh() + source = self.get_page_source() + self.assert_false("doubleclick.net" in source) + self.assert_false("google-analytics.com" in source) self.post_message("Blocking was successful!") diff --git a/help_docs/cdp_mode_methods.md b/help_docs/cdp_mode_methods.md index c55b0934f33..e8f5f4987fe 100644 --- a/help_docs/cdp_mode_methods.md +++ b/help_docs/cdp_mode_methods.md @@ -10,7 +10,7 @@ Here's a list of SeleniumBase CDP Mode method definitions, which are defined in sb.get(url, **kwargs) sb.open(url, **kwargs) # Same as sb.get(url, **kwargs) in CDP Mode sb.goto(url, **kwargs) # Same as sb.get(url, **kwargs) in CDP Mode -sb.reload(ignore_cache=True, script_to_evaluate_on_load=None) +sb.reload(ignore_cache=False, script_to_evaluate_on_load=None) sb.refresh(*args, **kwargs) sb.goto_if_not_url(url) sb.get_event_loop() @@ -209,6 +209,12 @@ sb.save_as_html(name, folder=None) sb.save_screenshot(name, folder=None, selector=None) sb.print_to_pdf(name, folder=None) sb.save_as_pdf(name, folder=None) +sb.save_as_pdf_to_logs(name=None) +sb.save_screenshot_to_logs(name=None, selector=None) +sb.save_page_source_to_logs(name=None) +sb.save_as_html_to_logs(name=None) +sb.save_data_to_logs(data, file_name=None) +sb.append_data_to_logs(data, file_name=None) sb.quit() # (Pure CDP Mode only: `sb_cdp`) ``` diff --git a/help_docs/method_summary.md b/help_docs/method_summary.md index 11b2a535fe6..cd9d4dc382e 100644 --- a/help_docs/method_summary.md +++ b/help_docs/method_summary.md @@ -251,6 +251,7 @@ self.save_as_pdf_to_logs(name=None) self.save_data_to_logs(data, file_name=None) self.append_data_to_logs(data, file_name=None) self.save_page_source_to_logs(name=None) +# Duplicates: self.save_as_html_to_logs(name, folder=None) self.save_page_source(name, folder=None) # Duplicates: self.save_as_html(name, folder=None) self.save_cookies(name="cookies.txt") diff --git a/mkdocs_build/requirements.txt b/mkdocs_build/requirements.txt index 4c8325178d8..cc95a93ad99 100644 --- a/mkdocs_build/requirements.txt +++ b/mkdocs_build/requirements.txt @@ -1,7 +1,7 @@ # mkdocs dependencies for generating the seleniumbase.io website # Minimum Python version: 3.10 (for generating docs only) -regex>=2026.6.28 +regex>=2026.7.10 pymdown-extensions>=10.21.3 pipdeptree>=3.1.1 python-dateutil>=2.8.2 @@ -17,5 +17,5 @@ mkdocs==1.6.1 mkdocs-material==9.6.23 mkdocs-exclude-search==0.6.6 mkdocs-simple-hooks==0.1.5 -mkdocs-get-deps==0.2.0 +mkdocs-get-deps==0.2.2 mkdocs-material-extensions==1.3.1 diff --git a/requirements.txt b/requirements.txt index f4ddd7bdac6..c7f789d8786 100755 --- a/requirements.txt +++ b/requirements.txt @@ -8,9 +8,9 @@ attrs>=26.1.0 certifi>=2026.6.17 exceptiongroup>=1.3.1 websockets~=15.0.1;python_version<"3.10" -websockets>=16.0;python_version>="3.10" +websockets>=16.1;python_version>="3.10" filelock~=3.19.1;python_version<"3.10" -filelock>=3.29.6;python_version>="3.10" +filelock>=3.29.7;python_version>="3.10" fasteners>=0.20 mycdp>=1.4.0 pynose>=1.5.5 @@ -46,7 +46,7 @@ wsproto==1.2.0;python_version<"3.10" wsproto~=1.3.2;python_version>="3.10" websocket-client~=1.9.0 selenium==4.32.0;python_version<"3.10" -selenium==4.45.0;python_version>="3.10" +selenium==4.46.0;python_version>="3.10" cssselect==1.3.0;python_version<"3.10" cssselect>=1.4.0,<2;python_version>="3.10" sortedcontainers==2.4.0 @@ -80,7 +80,7 @@ rich>=15.0.0,<16 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.) coverage>=7.10.7;python_version<"3.10" -coverage>=7.15.0;python_version>="3.10" +coverage>=7.15.1;python_version>="3.10" pytest-cov>=7.1.0 flake8==7.3.0 mccabe==0.7.0 diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 9159d1b7954..eff8d496fa8 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.50.6" +__version__ = "4.51.0" diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index 43e7ecdd21f..2b39b66d3ce 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -966,6 +966,12 @@ def uc_open_with_cdp_mode(driver, url=None, **kwargs): cdp.save_screenshot = CDPM.save_screenshot cdp.print_to_pdf = CDPM.print_to_pdf cdp.save_as_pdf = CDPM.save_as_pdf + cdp.save_as_pdf_to_logs = CDPM.save_as_pdf_to_logs + cdp.save_screenshot_to_logs = CDPM.save_screenshot_to_logs + cdp.save_page_source_to_logs = CDPM.save_page_source_to_logs + cdp.save_as_html_to_logs = CDPM.save_as_html_to_logs + cdp.save_data_to_logs = CDPM.save_data_to_logs + cdp.append_data_to_logs = CDPM.append_data_to_logs cdp.page = page # async world cdp.driver = driver.cdp_base # async world cdp.tab = cdp.page # shortcut (original) @@ -3093,6 +3099,7 @@ def get_driver( browser=None, # A duplicate of browser_name to avoid confusion ): sb_config._ext_dirs = [] + sb_config._cdp_launched = False driver_dir = DRIVER_DIR if binary_location == "_chromium_": driver_dir = DRIVER_DIR_CHROMIUM diff --git a/seleniumbase/core/sb_cdp.py b/seleniumbase/core/sb_cdp.py index c6c7cc25daa..e3d7475207b 100644 --- a/seleniumbase/core/sb_cdp.py +++ b/seleniumbase/core/sb_cdp.py @@ -7,6 +7,7 @@ import re import sys import time +import traceback from contextlib import suppress from filelock import FileLock from seleniumbase import config as sb_config @@ -24,6 +25,19 @@ def __init__(self, loop, page, driver): self.loop = loop self.page = page self.driver = driver + self.__initialize_variables() + + def __initialize_variables(self): + self.__page_source_count = 0 + self.__screenshot_count = 0 + self.__saved_pdf_count = 0 + self.__logs_data_count = 0 + self.__last_data_file = None + # Set variables that may be useful to developers + self.log_path = constants.Logs.LATEST + self.log_abspath = os.path.abspath(self.log_path) + self.data_path = os.path.join(self.log_path, self.__get_script_id()) + self.data_abspath = os.path.abspath(self.data_path) def _swap_driver(self, driver): self.driver = driver @@ -169,7 +183,7 @@ def open(self, url, **kwargs): def goto(self, url, **kwargs): self.get(url, **kwargs) - def reload(self, ignore_cache=True, script_to_evaluate_on_load=None): + def reload(self, ignore_cache=False, script_to_evaluate_on_load=None): self.loop.run_until_complete( self.page.reload( ignore_cache=ignore_cache, @@ -3522,8 +3536,9 @@ def save_page_source(self, name, folder=None): html_file.write(rendered_source) html_file.close() - def save_as_html(self, *args, **kwargs): - self.save_page_source(*args, **kwargs) + def save_as_html(self, name, folder=None): + """Same as self.save_page_source()""" + self.save_page_source(name, folder=folder) def save_screenshot(self, name, folder=None, selector=None): filename = name @@ -3545,9 +3560,151 @@ def print_to_pdf(self, name, folder=None): def save_as_pdf(self, *args, **kwargs): self.print_to_pdf(*args, **kwargs) + def save_as_pdf_to_logs(self, name=None): + """Saves the page as a PDF to the "latest_logs/" folder. + Naming is automatic: + If NO NAME provided: "_1_PDF.pdf", "_2_PDF.pdf", etc. + If NAME IS provided, then: "_1_name.pdf", "_2_name.pdf", etc.""" + script_logpath = os.path.join(self.log_path, self.__get_script_id()) + self.__create_log_path_as_needed(script_logpath) + if name: + name = str(name) + self.__saved_pdf_count += 1 + if not name or len(name) == 0: + name = "_%s_PDF.pdf" % self.__saved_pdf_count + else: + pre_name = "_%s_" % self.__saved_pdf_count + if len(name) >= 4 and name[-4:].lower() == ".pdf": + name = name[:-4] + if len(name) == 0: + name = "PDF" + name = "%s%s.pdf" % (pre_name, name) + sb_config._has_logs = True + return self.print_to_pdf(name, folder=script_logpath) + + def save_screenshot_to_logs(self, name=None, selector=None): + """Saves a screenshot of the current page to the "latest_logs/" folder. + Naming is automatic: + If NO NAME provided: "_1_screenshot.png", "_2_screenshot.png", etc. + If NAME IS provided, it becomes: "_1_name.png", "_2_name.png", etc. + The screenshot will include the entire page unless a selector is given. + If a provided selector is not found, then takes a full-page screenshot. + The screenshot will be in PNG format.""" + script_logpath = os.path.join(self.log_path, self.__get_script_id()) + self.__create_log_path_as_needed(script_logpath) + if name: + name = str(name) + self.__screenshot_count += 1 + if not name or len(name) == 0: + name = "_%s_screenshot.png" % self.__screenshot_count + else: + pre_name = "_%s_" % self.__screenshot_count + if len(name) >= 4 and name[-4:].lower() == ".png": + name = name[:-4] + if len(name) == 0: + name = "screenshot" + name = "%s%s.png" % (pre_name, name) + if selector: + if self.is_element_present(selector): + return self.save_screenshot( + name, folder=script_logpath, selector=selector + ) + sb_config._has_logs = True + return self.save_screenshot(name, folder=script_logpath) + + def save_page_source_to_logs(self, name=None): + """Saves the page HTML to the "latest_logs/" folder. + Naming is automatic: + If NO NAME provided: "_1_source.html", "_2_source.html", etc. + If NAME IS provided, then: "_1_name.html", "_2_name.html", etc.""" + script_logpath = os.path.join(self.log_path, self.__get_script_id()) + self.__create_log_path_as_needed(script_logpath) + if name: + name = str(name) + self.__page_source_count += 1 + if not name or len(name) == 0: + name = "_%s_source.html" % self.__page_source_count + else: + pre_name = "_%s_" % self.__page_source_count + if len(name) >= 4 and name[-4:].lower() == ".html": + name = name[:-4] + if len(name) == 0: + name = "source" + name = "%s%s.html" % (pre_name, name) + sb_config._has_logs = True + return self.save_page_source(name, folder=script_logpath) + + def save_as_html_to_logs(self, name=None): + """Same as save_page_source_to_logs().""" + self.save_page_source_to_logs(name=name) + + def save_data_to_logs(self, data, file_name=None): + """Saves data to the "latest_logs/" data folder of the current script. + If no file_name, file_name becomes: "data_1.txt", "data_2.txt", etc. + Useful variables for getting the "latest_logs/" or script data folders: + self.log_path OR self.log_abspath (For the top-level folder) + self.data_path OR self.data_abspath (Individual script folders) + If a file_name is given with no extension, adds ".txt" to the end.""" + script_logpath = os.path.join(self.log_path, self.__get_script_id()) + self.__create_log_path_as_needed(script_logpath) + if file_name: + file_name = str(file_name) + if not file_name or len(file_name) == 0: + self.__logs_data_count += 1 + file_name = "data_%s.txt" % self.__logs_data_count + elif "." not in file_name: + file_name = "%s.txt" % file_name + self.__last_data_file = file_name + sb_config._has_logs = True + destination_folder = script_logpath + page_utils._save_data_as(data, destination_folder, file_name) + + def append_data_to_logs(self, data, file_name=None): + """Saves data to the "latest_logs/" folder of the current script. + If no file_name, file_name becomes the last data file used in + save_data_to_logs() or append_data_to_logs(). + If neither method was called before, creates "data_1.txt".""" + script_logpath = os.path.join(self.log_path, self.__get_script_id()) + self.__create_log_path_as_needed(script_logpath) + if file_name: + file_name = str(file_name) + if (not file_name or len(file_name) == 0) and self.__last_data_file: + file_name = self.__last_data_file + elif not file_name or len(file_name) == 0: + if self.__logs_data_count == 0: + self.__logs_data_count += 1 + file_name = "data_%s.txt" % self.__logs_data_count + elif "." not in file_name: + file_name = "%s.txt" % file_name + self.__last_data_file = file_name + sb_config._has_logs = True + destination_folder = script_logpath + page_utils._append_data_to_file(data, destination_folder, file_name) + + def __get_script_id(self): + if hasattr(sb_config, "_cdp_saved_id") and sb_config._cdp_saved_id: + return sb_config._cdp_saved_id + else: + stack_base = traceback.format_stack()[0].split(", in ")[0] + test_base = stack_base.split(os.sep)[-1] + return test_base.split('"')[0] + + def __create_log_path_as_needed(self, script_logpath): + if not os.path.exists(script_logpath): + try: + os.makedirs(script_logpath) + except Exception: + pass + class Chrome(CDPMethods): def __init__(self, url=None, **kwargs): + stack_base = traceback.format_stack()[0].split(", in ")[0] + test_base = stack_base.split(os.sep)[-1] + filename = test_base.split('"')[0] + linename = ".line_" + test_base.split(", line ")[-1] + context_id = filename.split(".")[0] + linename + sb_config._cdp_saved_id = context_id if not url: url = "about:blank" loop = asyncio.new_event_loop() diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index c87ca8479bf..669f5ef6f6b 100644 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -7543,15 +7543,18 @@ def get_beautiful_soup(self, source=None): """BeautifulSoup is a toolkit for dissecting an HTML document and extracting what you need. It's great for screen-scraping! See: https://www.crummy.com/software/BeautifulSoup/bs4/doc/ """ - from bs4 import BeautifulSoup + if self.__is_cdp_swap_needed(): + return self.cdp.get_beautiful_soup(source=source) + else: + from bs4 import BeautifulSoup - if not source: - with suppress(Exception): - self.wait_for_element_visible( - "body", timeout=settings.MINI_TIMEOUT - ) - source = self.get_page_source() - return BeautifulSoup(source, "html.parser") + if not source: + with suppress(Exception): + self.wait_for_element_visible( + "body", timeout=settings.MINI_TIMEOUT + ) + source = self.get_page_source() + return BeautifulSoup(source, "html.parser") def get_unique_links(self): """Get all unique links in the html of the page source. @@ -9723,6 +9726,10 @@ def save_as_html(self, name, folder=None): """Same as self.save_page_source()""" self.save_page_source(name, folder=folder) + def save_as_html_to_logs(self, name=None): + """Same as save_page_source_to_logs().""" + self.save_page_source_to_logs(name=name) + def input( self, selector, text, by="css selector", timeout=None, retry=False ): diff --git a/seleniumbase/fixtures/js_utils.py b/seleniumbase/fixtures/js_utils.py index 331fe4976bf..7fabb26276a 100644 --- a/seleniumbase/fixtures/js_utils.py +++ b/seleniumbase/fixtures/js_utils.py @@ -855,7 +855,10 @@ def activate_messenger(driver): if not is_jquery_activated(driver): add_js_link(driver, jquery_js) - wait_for_jquery_active(driver, timeout=1.1) + if hasattr(sb_config, "_cdp_launched") and sb_config._cdp_launched: + time.sleep(0.1) + else: + wait_for_jquery_active(driver, timeout=1.1) add_css_link(driver, messenger_css) add_css_link(driver, msgr_theme_flat_css) add_css_link(driver, msgr_theme_future_css) diff --git a/seleniumbase/undetected/cdp_driver/cdp_util.py b/seleniumbase/undetected/cdp_driver/cdp_util.py index 89a0292685a..4d222ebc22f 100644 --- a/seleniumbase/undetected/cdp_driver/cdp_util.py +++ b/seleniumbase/undetected/cdp_driver/cdp_util.py @@ -760,6 +760,7 @@ async def start( sb_config._cdp_platform = None driver.page = driver.main_tab driver.solve_captcha = driver.page.solve_captcha + sb_config._cdp_launched = True return driver diff --git a/seleniumbase/undetected/cdp_driver/tab.py b/seleniumbase/undetected/cdp_driver/tab.py index a1320072d0f..3099f38b02c 100644 --- a/seleniumbase/undetected/cdp_driver/tab.py +++ b/seleniumbase/undetected/cdp_driver/tab.py @@ -691,12 +691,12 @@ async def is_online(self): async def reload( self, - ignore_cache: Optional[bool] = True, + ignore_cache: Optional[bool] = False, script_to_evaluate_on_load: Optional[str] = None, ): """ Reloads the page - :param ignore_cache: When set to True (default), + :param ignore_cache: When set to True, it ignores cache, and re-downloads the items. :param script_to_evaluate_on_load: Script to run on load. """ diff --git a/setup.py b/setup.py index 1456d12c911..7608a8f9368 100755 --- a/setup.py +++ b/setup.py @@ -172,9 +172,9 @@ 'certifi>=2026.6.17', 'exceptiongroup>=1.3.1', 'websockets~=15.0.1;python_version<"3.10"', - 'websockets>=16.0;python_version>="3.10"', + 'websockets>=16.1;python_version>="3.10"', 'filelock~=3.19.1;python_version<"3.10"', - 'filelock>=3.29.6;python_version>="3.10"', + 'filelock>=3.29.7;python_version>="3.10"', 'fasteners>=0.20', 'mycdp>=1.4.0', 'pynose>=1.5.5', @@ -210,7 +210,7 @@ 'wsproto~=1.3.2;python_version>="3.10"', 'websocket-client~=1.9.0', 'selenium==4.32.0;python_version<"3.10"', - 'selenium==4.45.0;python_version>="3.10"', + 'selenium==4.46.0;python_version>="3.10"', 'cssselect==1.3.0;python_version<"3.10"', 'cssselect>=1.4.0,<2;python_version>="3.10"', 'sortedcontainers==2.4.0', @@ -253,7 +253,7 @@ # Usage: coverage run -m pytest; coverage html; coverage report "coverage": [ 'coverage>=7.10.7;python_version<"3.10"', - 'coverage>=7.15.0;python_version>="3.10"', + 'coverage>=7.15.1;python_version>="3.10"', 'pytest-cov>=7.1.0', ], # pip install -e .[flake8]