web-scraping-with-python VS NodeJs-Cucumber-Selenium

import asyncio import pytest from pyppeteer.errors import PageError from urllib.parse import quote import json import os import sys from os import environ from pyppeteer import connect, launch exec_platform = os.getenv('EXEC_PLATFORM') # Get username and access key of the LambdaTest Platform username = environ.get('LT_USERNAME', None) access_key = environ.get('LT_ACCESS_KEY', None) test1_url = 'https://ecommerce-playground.lambdatest.io/' test2_url = 'https://scrapingclub.com/exercise/list_infinite_scroll/' # Usecase - 1 # loc_ecomm_1 = ".order-1.col-lg-6 div:nth-of-type(1) > div:nth-of-type(1) > div:nth-of-type(1) > div:nth-of-type(1) > div:nth-of-type(1) div:nth-of-type(1) > img:nth-of-type(1)" loc_ecomm_1 = "[aria-label='1 / 2'] div:nth-of-type(1) > [alt='Nikon D300']" target_url_1 = "https://ecommerce-playground.lambdatest.io/index.php?route=product/product&product_id=63" # Usecase - 2 (Click on e-commerce sliding banner) loc_ecomm_2 = "[alt='Canon DSLR camera']" target_url_2 = "https://ecommerce-playground.lambdatest.io/index.php?route=product/product&product_id=30" # Usecase - 3 Automating interactions on https://scrapingclub.com/exercise/list_infinite_scroll/ loc_infinite_src_prod1 = ".grid .p-4 [href='/exercise/list_basic_detail/93926-C/']" target_url_3 = "https://scrapingclub.com/exercise/list_basic_detail/93926-C/" # Usecase - 4 Automating interactions on https://scrapingclub.com/exercise/list_infinite_scroll/ # when the images are lazy loaded loc_infinite_src_prod2 = "div:nth-of-type(31) > .p-4 [href='/exercise/list_basic_detail/94967-A/']" target_url_4 = "https://scrapingclub.com/exercise/list_basic_detail/94967-A/" # Set timeout in ms timeOut = 60000 async def scroll_to_element(page, selector): # Scroll until the element is detected await page.evaluateHandle( '''async (selector) => { const element = document.querySelector(selector); if (element) { element.scrollIntoView(); } }''', selector ) return selector async def scroll_carousel(page, scr_count): for scr in range(1, scr_count): elem_next_button = "#mz-carousel-213240 > ul li:nth-child(" + str(scr) + ")" await asyncio.sleep(1) elem_next_button = await page.querySelector(elem_next_button) await elem_next_button.click() # Replica of https://github.com/hjsblogger/web-scraping-with-python/blob/ # main/tests/beautiful-soup/test_infinite_scraping.py#L67C5-L80C18 async def scroll_end_of_page(page): start_height = await page.evaluate('document.documentElement.scrollHeight') while True: # Scroll to the bottom of the page await page.evaluate(f'window.scrollTo(0, {start_height})') # Wait for the content to load await asyncio.sleep(1) # Get the new scroll height scroll_height = await page.evaluate('document.documentElement.scrollHeight') if scroll_height == start_height: # If heights are the same, we reached the end of the page break # Add an additional wait await asyncio.sleep(2) start_height = scroll_height # Additional wait after scrolling await asyncio.sleep(2) @pytest.mark.asyncio @pytest.mark.order(1) async def test_lazy_load_ecomm_1(page): # The time out can be set using the setDefaultNavigationTimeout # It is primarily used for overriding the default page timeout of 30 seconds page.setDefaultNavigationTimeout(timeOut) await page.goto(test1_url, {'waitUntil': 'load', 'timeout': timeOut}) # Set the viewport - Apple MacBook Air 13-inch # Reference - https://codekbyte.com/devices-viewport-sizes/ # await page.setViewport({'width': 1440, 'height': 770}) await asyncio.sleep(2) if exec_platform == 'local': # Scroll until the element is detected elem_button = await scroll_to_element(page, loc_ecomm_1) # await page.click(elem_button) # Wait until the page is loaded # https://miyakogi.github.io/pyppeteer/reference.html#pyppeteer.page.Page.waitForNavigation navigationPromise = asyncio.ensure_future(page.waitForNavigation()) await page.click(elem_button) await navigationPromise elif exec_platform == 'cloud': elem_button = await page.waitForSelector(loc_ecomm_1, {'visible': True}) await asyncio.gather( elem_button.click(), page.waitForNavigation({'waitUntil': 'networkidle2', 'timeout': 30000}), ) # Assert if required, since the test is a simple one; we leave as is :D current_url = page.url print('Current URL is: ' + current_url) try: assert current_url == target_url_1 print("Test Success: Product checkout successful") except PageError as e: print("Test Failure: Could not checkout Product") print("Error Code" + str(e)) @pytest.mark.asyncio @pytest.mark.order(2) async def test_lazy_load_ecomm_2(page): carousel_len = 4 # The time out can be set using the setDefaultNavigationTimeout # It is primarily used for overriding the default page timeout of 30 seconds page.setDefaultNavigationTimeout(timeOut) await page.goto(test1_url, {'waitUntil': 'load', 'timeout': timeOut}) # Set the viewport - Apple MacBook Air 13-inch # Reference - https://codekbyte.com/devices-viewport-sizes/ # await page.setViewport({'width': 1440, 'height': 770}) await asyncio.sleep(2) # Approach 1: Directly click on the third button on the carousel # elem_carousel_banner = await page.querySelector("#mz-carousel-213240 > ul li:nth-child(3)") # await asyncio.sleep(1) # await elem_carousel_banner.click() # Approach 2 (Only for demo): Serially click on every button on carousel await scroll_carousel(page, carousel_len) await asyncio.sleep(1) # elem_prod_1 = await page.querySelector(loc_ecomm_2) elem_prod_1 = await page.waitForSelector(loc_ecomm_2, {'visible': True}) await asyncio.gather( elem_prod_1.click(), page.waitForNavigation({'waitUntil': 'networkidle2', 'timeout': 60000}), ) # Assert if required, since the test is a simple one; we leave as is :D current_url = page.url print('Current URL is: ' + current_url) try: assert current_url == target_url_2 print("Test Success: Product checkout successful") except PageError as e: print("Test Failure: Could not checkout Product") print("Error Code" + str(e)) @pytest.mark.asyncio @pytest.mark.order(3) async def test_lazy_load_infinite_scroll_1(page): # The time out can be set using the setDefaultNavigationTimeout # It is primarily used for overriding the default page timeout of 30 seconds page.setDefaultNavigationTimeout(timeOut) await page.goto(test2_url, {'waitUntil': 'load', 'timeout': timeOut}) # Set the viewport - Apple MacBook Air 13-inch # Reference - https://codekbyte.com/devices-viewport-sizes/ # await page.setViewport({'width': 1440, 'height': 770}) await asyncio.sleep(1) elem_prod1 = await page.querySelector(loc_infinite_src_prod1) await asyncio.gather( elem_prod1.click(), page.waitForNavigation({'waitUntil': 'networkidle2', 'timeout': 60000}), ) # await asyncio.sleep(1) # await elem_carousel_banner.click() # elem_button = scroll_to_element(page, loc_infinite_src_prod1) # print(elem_button) # await asyncio.sleep(2) # await elem_button.click() # Assert if required, since the test is a simple one; we leave as is :D current_url = page.url print('Current URL is: ' + current_url) try: assert current_url == target_url_3 print("Test Success: Product checkout successful") except PageError as e: print("Test Failure: Could not checkout Product") print("Error Code" + str(e)) @pytest.mark.asyncio @pytest.mark.order(4) async def test_lazy_load_infinite_scroll_2(page): # The time out can be set using the setDefaultNavigationTimeout # It is primarily used for overriding the default page timeout of 30 seconds page.setDefaultNavigationTimeout(timeOut) # Tested navigation using LambdaTest YouTube channel # await page.goto("https://www.youtube.com/@LambdaTest/videos", await page.goto(test2_url, {'waitUntil': 'load', 'timeout': timeOut}) # Set the viewport - Apple MacBook Air 13-inch # Reference - https://codekbyte.com/devices-viewport-sizes/ # await page.setViewport({'width': 1440, 'height': 770}) await asyncio.sleep(1) await scroll_end_of_page(page) await page.evaluate('window.scrollTo(0, 0)') await asyncio.sleep(1) # elem_prod = await page.querySelector(loc_infinite_src_prod2) # asyncio.sleep(1) # await asyncio.gather( # elem_prod.click(), # page.waitForNavigation({'waitUntil': 'load', 'timeout': 60000}), # ) elem_button = await scroll_to_element(page, loc_infinite_src_prod2) await asyncio.sleep(1) # await page.click(elem_button) await asyncio.gather( page.click(elem_button), page.waitForNavigation({'waitUntil': 'networkidle2', 'timeout': 60000}), ) # Assert if required, since the test is a simple one; we leave as is :D current_url = page.url print('Current URL is: ' + current_url) try: assert current_url == target_url_4 print("Test Success: Product checkout successful") except PageError as e: print("Test Failure: Could not checkout Product") print("Error Code" + str(e))

The helpers.py file contains the core implementation for scraping the LambdaTest YouTube channel.

/*Selenium Webdriver command to check whether a particular web element is displayed */ driver = new ChromeDriver(); driver.Url = "https://www.lambdatest.com"; /* The Xpath of the Login Button on LambdaTest homepage */ IWebElement web_element = driver.FindElement(By.XPath("//*[@id="navbarSupportedContent"]/ul/li[6]/a")); Boolean element_enabled = web_element.Displayed;

lambdatest.com — Manual, visual, screenshot, and automated browser testing on selenium and cypress, free for Open Source

import asyncio import pytest from pyppeteer.errors import PageError from urllib.parse import quote import json import os import sys from os import environ from pyppeteer import connect, launch # Documentation Link # https://miyakogi.github.io/pyppeteer/reference.html#dialog-class # Interesting reference question # https://stackoverflow.com/questions/75622322/ # why-dialog-popup-alert-doesnt-dismiss-as-they-claim-by-using-pyppeteer-class # Scenario Handling Alerts dialog_test_url = 'https://www.lambdatest.com/selenium-playground/javascript-alert-box-demo' # Locators for different elements # 0 - JS Alert # 1 - Confirm Box # 2 - Prompt Box loc_alert_arr = ['.my-30', '.py-20.ml-10 .btn', 'section:nth-of-type(3) div:nth-of-type(3) .btn'] test_message = 'LambdaTest is a great platform!' # Event event listener for handling JS alert dialog box def handle_js_dialog_box(dialog): asyncio.ensure_future(dialog.accept()) print(f"Dialog message: {dialog.message}") # Event event listener for handling confirm dialog box def handle_confirm_accept_dialog_box(dialog): asyncio.ensure_future(dialog.accept()) print(f"Dialog message: {dialog.message}") # Event event listener for handling confirm dialog box def handle_confirm_dismiss_dialog_box(dialog): asyncio.ensure_future(dialog.dismiss()) print(f"Dialog message: {dialog.message}") # Event event listener for handling prompt dialog box def handle_confirm_prompt_dialog_box(dialog): asyncio.ensure_future(dialog.accept(test_message)) print(f"Dialog message: {dialog.message}") @pytest.mark.asyncio @pytest.mark.order(1) async def test_handling_js_alerts(page): await page.goto(dialog_test_url) # Can be changed with non-blocking sleep await asyncio.sleep(1) page.on('dialog', handle_js_dialog_box) elem_alert = await page.querySelector(loc_alert_arr[0]) # Click on the located element await elem_alert.click() # Wait for the event loop to process events await asyncio.sleep(2) @pytest.mark.asyncio @pytest.mark.order(2) async def test_handling_confirm_accept_alerts(page): await page.goto(dialog_test_url) # Can be changed with non-blocking sleep await asyncio.sleep(1) page.on('dialog', handle_confirm_accept_dialog_box) # Confirm Alert elem_alert = await page.querySelector(loc_alert_arr[1]) # Click on the located element await elem_alert.click() # Wait for the event loop to process events await asyncio.sleep(2) @pytest.mark.asyncio @pytest.mark.order(3) async def test_handling_confirm_dismiss_alerts(page): await page.goto(dialog_test_url) # Can be changed with non-blocking sleep await asyncio.sleep(1) page.on('dialog', handle_confirm_dismiss_dialog_box) await asyncio.sleep(2) # Dismiss Alert elem_alert = await page.querySelector(loc_alert_arr[1]) # Click on the located element await elem_alert.click() # Wait for the event loop to process events await asyncio.sleep(2) @pytest.mark.asyncio @pytest.mark.order(4) async def test_handling_prompt_alerts(page): await page.goto(dialog_test_url) # Can be changed with non-blocking sleep await asyncio.sleep(1) page.on('dialog', handle_confirm_prompt_dialog_box) await asyncio.sleep(1) # Prompt Alert elem_alert = await page.querySelector(loc_alert_arr[2]) # Click on the located element await elem_alert.click() page.on('dialog', handle_confirm_dismiss_dialog_box) await asyncio.sleep(2)

/** * Add the file in your test suite to run tests on LambdaTest. * Import 'test' object from this file in the tests. */ import * as base from "@playwright/test"; import path from "path"; import { chromium } from "@playwright/test" import dotenv from 'dotenv'; dotenv.config(); // LambdaTest capabilities const capabilities = { browserName: "Chrome", // Browsers allowed: 'Chrome', 'MicrosoftEdge', 'pw-chromium', 'pw-firefox' and 'pw-webkit' browserVersion: "latest", "LT:Options": { platform: "Windows 10", build: "Playwright TypeScript iframes", name: "Playwright TypeScript iframes", user: process.env.LT_USERNAME, accessKey: process.env.LT_ACCESS_KEY, network: true, video: true, console: true, tunnel: false, // Add tunnel configuration if testing locally hosted webpage tunnelName: "", // Optional geoLocation: "US", // country code can be fetched from https://www.lambdatest.com/capabilities-generator/ }, }; // Patching the capabilities dynamically according to the project name. const modifyCapabilities = (configName, testName) => { let config = configName.split("@lambdatest")[0]; let [browserName, browserVersion, platform] = config.split(":"); capabilities.browserName = browserName ? browserName : capabilities.browserName; capabilities.browserVersion = browserVersion ? browserVersion : capabilities.browserVersion; capabilities["LT:Options"]["platform"] = platform ? platform : capabilities["LT:Options"]["platform"]; capabilities["LT:Options"]["name"] = testName; }; const getErrorMessage = (obj, keys) => keys.reduce( (obj, key) => (typeof obj == "object" ? obj[key] : undefined), obj ); const test = base.test.extend({ page: async ({ page, playwright }, use, testInfo) => { // Configure LambdaTest platform for cross-browser testing let fileName = testInfo.file.split(path.sep).pop(); if (testInfo.project.name.match(/lambdatest/)) { modifyCapabilities( testInfo.project.name, '${testInfo.title} - ${fileName}' ); const browser = await chromium.connect({ wsEndpoint: 'wss://cdp.lambdatest.com/playwright?capabilities=${encodeURIComponent( JSON.stringify(capabilities) )}', }); const ltPage = await browser.newPage(testInfo.project.use); await use(ltPage); const testStatus = { action: "setTestStatus", arguments: { status: testInfo.status, remark: getErrorMessage(testInfo, ["error", "message"]), }, }; await ltPage.evaluate(() => {}, 'lambdatest_action: ${JSON.stringify(testStatus)}'); await ltPage.close(); // await browser.close(); } else { // Run tests in local in case of local config provided await use(page); } }, }); export default test;

import json import logging import os import subprocess import sys import time import urllib from logging import getLogger from dotenv import load_dotenv from playwright.sync_api import sync_playwright # setup basic logging for our project which will display the time, log level & log message logger = getLogger("webscapper.py") logging.basicConfig( stream=sys.stdout, # uncomment this line to redirect output to console format="%(message)s", level=logging.DEBUG, ) # LambdaTest username & access key are stored in an env file & we fetch it from there using python dotenv module load_dotenv("sample.env") capabilities = { "browserName": "Chrome", # Browsers allowed: `Chrome`, `MicrosoftEdge`, `pw-chromium`, `pw-firefox` and `pw-webkit` "browserVersion": "latest", "LT:Options": { "platform": "Windows 10", "build": "E Commerce Scrape Build", "name": "Scrape Lambda Software Product", "user": os.getenv("LT_USERNAME"), "accessKey": os.getenv("LT_ACCESS_KEY"), "network": False, "video": True, "console": True, "tunnel": False, # Add tunnel configuration if testing locally hosted webpage "tunnelName": "", # Optional "geoLocation": "", # country code can be fetched from https://www.lambdatest.com/capabilities-generator/ }, } def main(): with sync_playwright() as playwright: playwright_version = ( str(subprocess.getoutput("playwright --version")).strip().split(" ")[1] ) capabilities["LT:Options"]["playwrightClientVersion"] = playwright_version lt_cdp_url = ( "wss://cdp.lambdatest.com/playwright?capabilities=" + urllib.parse.quote(json.dumps(capabilities)) ) logger.info(f"Initiating connection to cloud playwright grid") browser = playwright.chromium.connect(lt_cdp_url) # comment above line & uncomment below line to test on local grid # browser = playwright.chromium.launch(headless=False) page = browser.new_page() try: # section to navigate to software category page.goto("https://ecommerce-playground.lambdatest.io/") page.get_by_role("button", name="Shop by Category").click() page.get_by_role("link", name="Software").click() page_to_be_scrapped = page.get_by_role( "combobox", name="Show:" ).select_option( "https://ecommerce-playground.lambdatest.io/index.php?route=product/category&path=17&limit=75" ) page.goto(page_to_be_scrapped[0]) # Since image are lazy-loaded scroll to bottom of page # the range is dynamically decided based on the number of items i.e. we take the range from limit # https://ecommerce-playground.lambdatest.io/index.php?route=product/category&path=17&limit=75 for i in range(int(page_to_be_scrapped[0].split("=")[-1])): page.mouse.wheel(0, 300) i += 1 time.sleep(0.1) # Construct locators to identify name, price & image base_product_row_locator = page.locator("#entry_212408").locator(".row").locator(".product-grid") product_name = base_product_row_locator.get_by_role("heading") product_price = base_product_row_locator.locator(".price-new") product_image = ( base_product_row_locator.locator(".carousel-inner") .locator(".active") .get_by_role("img") ) total_products = base_product_row_locator.count() for product in range(total_products): logger.info( f"\n**** PRODUCT {product+1} ****\n" f"Product Name = {product_name.nth(product).all_inner_texts()[0]}\n" f"Price = {product_price.nth(product).all_inner_texts()[0]}\n" f"Image = {product_image.nth(product).get_attribute('src')}\n" ) status = 'status' remark = 'Scraping Completed' page.evaluate("_ => {}","lambdatest_action: {\"action\": \"setTestStatus\", \"arguments\": {\"status\":\"" + status + "\", \"remark\": \"" + remark + "\"}}") except Exception as ex: logger.error(str(ex)) if __name__ == "__main__": main()

public class SeleniumPlaygroundTests extends BaseTest { @Test public void testMenus() { this.driverManager.getDriver().get("https://www.lambdatest.com/selenium-playground/"); final var mainPage = new MainPage(this.driverManager.getDriver()); assertEquals(mainPage.menuListSize(), 37); }

Tired of manually creating JSON data? Generate random JSON data in seconds with our easy-to-use tool. Try it now and make your development process a breeze: https://www.lambdatest.com/free-online-tools/random-json-generator

To see, how it works step by step refer to our tutorial on using LambdaTest’s WordPress Plugin for cross browser testing.

import java.util.Arrays; import java.util.List; import java.net.MalformedURLException; import java.net.URL; import org.openqa.selenium.remote.DesiredCapabilities; import org.openqa.selenium.remote.RemoteWebDriver; import org.openqa.selenium.chrome.ChromeDriver; import org.openqa.selenium.edge.EdgeDriver; import org.openqa.selenium.firefox.FirefoxDriver; public class DriverManager { private RemoteWebDriver driver; public static void init_driver(String browser) throws MalformedURLException { String username = System.getenv("LT_USERNAME") == null ? "Your LT Username" : System.getenv("LT_USERNAME"); String authkey = System.getenv("LT_ACCESS_KEY") == null ? "Your LT AccessKey" : System.getenv("LT_ACCESS_KEY"); // Create a list of browsers List browsers = Arrays.asList("chrome", "firefox", "edge"); // Loop through the list of browsers and create a driver instance for each one for (String browserName : browsers) { if (browser.equalsIgnoreCase("chrome")) { //To launch the chrome browser on local /*driver = new ChromeDriver();*/ //Setting chrome capabilities DesiredCapabilities ChromeCaps = new DesiredCapabilities(); ChromeCaps.setCapability("platform", "Windows 10"); ChromeCaps.setCapability("browserName", "chrome"); ChromeCaps.setCapability("version", "latest"); ChromeCaps.setCapability("build", "TestNG With Java"); ChromeCaps.setCapability("name", m.getName() + this.getClass().getName()); ChromeCaps.setCapability("plugin", "git-testng"); //To launch the chrome browser on LambdaTest cloud grid String[] Tags = new String[]{"Feature", "Magicleap", "Severe"}; ChromeCaps.setCapability("tags", Tags); driver = new RemoteWebDriver(new URL("https://" + username + ":" + authkey + hub), ChromeCaps); } else if (browser.equalsIgnoreCase("firefox")) { //To launch the firefox browser on local /*driver = new FirefoxDriver();*/ //Setting firefox capabilities DesiredCapabilities FirefoxCaps = new DesiredCapabilities(); FirefoxCaps.setCapability("platform", "Windows 10"); FirefoxCaps.setCapability("browserName", "firefox"); FirefoxCaps.setCapability("version", "114.0"); FirefoxCaps.setCapability("build", "TestNG With Java"); FirefoxCaps.setCapability("name", m.getName() + this.getClass().getName()); FirefoxCaps.setCapability("plugin", "git-testng"); //To launch the firefox browser on LambdaTest cloud grid String[] Tags = new String[]{"Feature", "Magicleap", "Severe"}; FirefoxCaps.setCapability("tags", Tags); driver = new RemoteWebDriver(new URL("https://" + username + ":" + authkey + hub), FirefoxCaps); } else if (browser.equalsIgnoreCase("edge")) { //To launch the edge browser on local /*driver = new EdgeDriver();*/ //Setting edge capabilities DesiredCapabilities EdgeCaps = new DesiredCapabilities(); EdgeCaps.setCapability("platform", "Windows 10"); EdgeCaps.setCapability("browserName", "edge"); EdgeCaps.setCapability("version", "112.0"); EdgeCaps.setCapability("build", "TestNG With Java"); EdgeCaps.setCapability("name", m.getName() + this.getClass().getName()); EdgeCaps.setCapability("plugin", "git-testng"); //To launch the edge browser on LambdaTest cloud grid String[] Tags = new String[]{"Feature", "Magicleap", "Severe"}; EdgeCaps.setCapability("tags", Tags); driver = new RemoteWebDriver(new URL("https://" + username + ":" + authkey + hub), EdgeCaps); } else { throw new IllegalArgumentException("Invalid browser name: " + browser); } // Use the driver instance to navigate to a webpage and perform some actions driver.get("https://www.lambdatest.com"); System.out.println(driver.getTitle()); // Close the driver instance driver.quit(); } } }

from selenium import webdriver from selenium.webdriver.edge.service import Service # Driver Code if __name__ == '__main__': # create service object edgeService = Service( r"D:\\Lambdatest Tools\\edgedriver_win64\\msedgedriver.exe") # create webdriver object edgeDriver = webdriver.Edge(service=edgeService) # open browser and navigate to the website edgeDriver.get('https://www.lambdatest.com')