scrape_technical.py


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

import csv 
import sys
import json
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By


driver = webdriver.Chrome()

products = {};
for line in sys.stdin.readlines():

    url = line.strip()
    driver.get(url);

    pname = driver.find_element(By.CSS_SELECTOR, ".productView-info-value--sku").get_attribute("innerHTML");
    print(pname);
    products[pname] = {};
    for technical in driver.find_elements(By.CSS_SELECTOR, ".productView-table.technical .productView-table-row"):
        header = technical.find_element(By.CSS_SELECTOR, ".productView-table-header").get_attribute("innerHTML");
        data = technical.find_element(By.CSS_SELECTOR, ".productView-table-data").get_attribute("innerHTML");
        products[pname][header] = data;

print(json.dumps(products));