Как бороться со страницей, блокирующей парсинг драматургом? ⇐ Python

[Anonymous]

Я пытаюсь сделать простой парсинг. Это идет страница за страницей и продукт за продуктом. Когда он нажимает на ссылку продукта, он переходит на новую вкладку, удаляет информацию, закрывается и возвращается, чтобы сделать это со следующим продуктом. Проблема в том, что каждый раз, когда он открывает новую вкладку, в браузере появляется следующая ошибка:
Извините, вы заблокированы
Вы не можете получить доступ к vivareal.com. br
он ​​блокирует скребок. Как избежать такого поведения? Вот код:

Код: Выделить всё

import random
import time
import json
import re
import pandas as pd
from playwright.sync_api import sync_playwright

proxy_pool = [
{"server": "156.255.25.149:8080"},
{"server": "45.139.59.216:8080"},
{"server": "143.137.165.150:8080"},
{"server": "154.92.120.99:8080"},
{"server": "185.161.254.156:8080"},
]
# Global variables
quantity_2024 = 0
quantity_2023 = 0
quantity_2022 = 0
quantity_2021 = 0
prices = []
sizes = []
prices_per_m2 = []
spreadsheet_data = []

# Base URL
base_url = "https://www.vivareal.com.br/venda/sp/jacarei/bairros/loteamento-villa-branca/avenida-das-letras/#onde=,S%C3%A3o%20Paulo,Jacare%C3%AD,Bairros,Loteamento%20Villa%20Branca,Avenida%20das%20Letras,,,BR%3ESao%20Paulo%3ENULL%3EJacarei%3EBarrios%3ELoteamento%20Villa%20Branca,,,&quartos=2"

def extract_vivareal_data(page):
global quantity_2024, quantity_2023, quantity_2022, quantity_2021
global prices, sizes, prices_per_m2

page.wait_for_selector("//strong[@class='results-summary__count js-total-records']")
total_properties = int(page.locator("//strong[@class='results-summary__count js-total-records']").text_content())
print(f"Total number of properties: {total_properties}")

while True:
# Wait for the list of properties
page.wait_for_selector("//div[@class='results-list js-results-list']/div")
properties = page.locator("//div[@class='results-list js-results-list']/div").all()

for property_card in properties:
page.wait_for_selector("//span[@class='property-card__detail-value js-property-card-value property-card__detail-area js-property-card-detail-area']").text_content()
size = property_card.locator("//span[@class='property-card__detail-value js-property-card-value property-card__detail-area js-property-card-detail-area']").text_content()
page.wait_for_selector("//div[@class='property-card__price js-property-card-prices js-property-card__price-small']/p")
price = property_card.locator("//div[@class='property-card__price js-property-card-prices js-property-card__price-small']/p").text_content()

# Process the data
size = int(re.sub(r'\D', '', size))
price = int("".join(re.findall(r'\d+', price)))
prices.append(price)
sizes.append(size)
price_per_m2 = price / size
prices_per_m2.append(price_per_m2)
print(f"Size: {size}, Price: {price}, Price/m²: {price_per_m2}")

# Click the property link to extract the year of the ad
links = property_card.locator("//a[@class='property-card__labels-container js-main-info js-listing-labels-link']")
for link in links.all():
with page.expect_popup() as popup_info:
link.click()
ad_page = popup_info.value
ad_page.wait_for_selector("//div[@class='description__info-date big-space']/span[2]")
page.pause()
ad_page.scroll_into_view_if_needed()
ad_year = ad_page.locator("//div[@class='description__info-date big-space']/span[2]").text_content()
print(f"Ad year: {ad_year}")
ad_page.close()

# Count ads by year
if "2024" in ad_year:
quantity_2024 += 1
elif "2023" in ad_year:
quantity_2023 += 1
elif "2022" in ad_year:
quantity_2022 += 1
elif "2021"  in ad_year:
quantity_2021 += 1

# Check if there is a next page
try:
next_button = page.locator("//button[@title='Next page']")
if next_button.is_enabled():
next_button.click()
time.sleep(2)  # Wait for the next page to load
else:
break
except:
print("Last page reached.")
break

# Return the total number of properties
return total_properties

with sync_playwright() as p:
#proxy = random.choice(proxy_pool)
browser = p.chromium.launch(headless=False)
context = browser.new_context(
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
viewport={"width": 1920, "height": 1080}
)
page = browser.new_page()
page.goto(base_url)

start_time = time.time()
total_properties = extract_vivareal_data(page)

# Calculate final metrics
avg_size = sum(sizes) / total_properties
avg_price_total = sum(prices) / total_properties
avg_price_per_m2 = sum(prices_per_m2) / total_properties

spreadsheet_data.append({
'URL': base_url,
'Number of ads': total_properties,
'Average Price': avg_price_total,
'Average Size': avg_size,
'Average Price per m²': avg_price_per_m2,
'Ads from 2024': quantity_2024,
'Ads from 2023': quantity_2023,
'Ads from 2022': quantity_2022,
'Ads from 2021': quantity_2021
})

# Save data to an Excel file
df_spreadsheet = pd.DataFrame(spreadsheet_data)
df_spreadsheet.to_excel("vivareal_catalog.xlsx", index=False)
print("--- %s seconds ---" % (time.time() - start_time))

browser.close()
print("Data saved successfully!")

пытался изменить поведение с помощью пользовательских агентов, но это не сработало.

Подробнее здесь: https://stackoverflow.com/questions/791 ... playwright