I am trying to extract price from Amazon PDb https://www.amazon.co.uk/SheaMoisture-T ... 01HOD3ZVQ/ but it is not getting extracted.
Tried:
- X path
- CSS selector
- Find all method
- Found that javascript is blocking price to be scraped but that is also not working or I have not identified proper js.
Код: Выделить всё
import csv
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
# List of URLs to open
urls = ['https://www.amazon.co.uk/SheaMoisture-Treatment-silicone-sulphate-transitioning/dp/B01HOD3ZVQ/']
# Create a Chrome WebDriver instance with custom user agent
user_agent = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0'
options = webdriver.ChromeOptions()
options.add_argument(f'user-agent={user_agent}')
driver = webdriver.Chrome(options=options)
try:
# Set implicit wait to 20 seconds
driver.implicitly_wait(20)
# Open CSV file for writing
with open('product_info_eu_price.csv', 'w', newline='') as csvfile:
# Define CSV header
fieldnames = ['URL', 'Buy_Box_Price']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
# Write header to CSV file
writer.writeheader()
for i, url in enumerate(urls):
# Open the URL
driver.get(url)
# Wait for 2 minutes after opening the first URL
if i == 0:
print(f"Opened {url}. Waiting for 1 minutes...")
time.sleep(60) # 2-minute delay after the first URL
else:
# Wait for 5 seconds for subsequent URLs
print(f"Opened {url}. Waiting for 5 seconds...")
time.sleep(5)
try:
# Wait for the "Price" element to be present using WebDriverWait
price_element = WebDriverWait(driver, 20).until(
EC.presence_of_element_located((By.XPATH, '//span[contains(@class, "apexPriceToPay")]/span[contains(@class, "a-offscreen")]'))
#EC.presence_of_element_located((By.XPATH, '//*[@id="corePriceDisplay_desktop_feature_div"]/div/div/span[1]/span[1]'))
#EC.presence_of_element_located((By.XPATH, '//*[@id="corePrice_feature_div"]/div/div/span[1]/span[1]'))
)
# Extract text from the "Price" element
price_text = price_element.text.strip()
# Write URL and Ranking to CSV file
writer.writerow({'URL': url, 'Buy_Box_Price': price_text})
except Exception as e:
# If element not found, print an error message, write "Not found" to CSV, and continue to the next URL
print(f"Error: {e}. Element not found on {url}. Writing 'Not found' to CSV.")
writer.writerow({'URL': url, 'Buy_Box_Price': 'Not found'})
except Exception as e:
print(f"Error: {e}")
finally:
# Close the browser
driver.quit()
print("Script completed. Results saved to product_info_eu_price.csv.")
Источник: https://stackoverflow.com/questions/781 ... rom-amazon
