scrape_products.py


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By


base = "https://www.cardone.com/steering/rack-pinion/?limit=96&_bc_fsnf=1&Part+Type=Rack+and+Pinion+Assembly";

driver = webdriver.Chrome()
p_urls = []
for i in range(11, 15):
    list_url = base + "&page=" + str(i);

    driver.get(list_url);


    for product in driver.find_elements(By.CLASS_NAME, "card-title"):
       p_urls.append(product.find_element(By.CSS_SELECTOR, 'a').get_attribute('href'));


for url in p_urls:
    print(url);