Как улучшить селениум скрипт, чтобы не появлялась капча ⇐ Python

[Anonymous]

Моя цель — получить результаты поисковых систем (URL-адреса) с yandex.ru
Я запускаю этот селеновый скрипт. На моем ПК скрипт работает нормально, но на сервере я вижу сложный капчу (см. изображение ниже)
Как улучшить скрипт, чтобы капча не появлялась

Код: Выделить всё

import undetected_chromedriver as uc
import random
from selenium.webdriver.common.by import By
from datetime import datetime
import time
import logging
import traceback
import pathlib

logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[
logging.FileHandler("yandex_parser.log"),
logging.StreamHandler()
]
)

logger = logging.getLogger('ParserLogger')

class YandexParser():
def __init__(self, USE_GUI=True):
self.chrome_options = uc.ChromeOptions()
if not USE_GUI:
self.chrome_options.add_argument('--headless')
self.chrome_options.add_argument('--no-sandbox')
self.chrome_options.add_argument('--disable-dev-shm-usage')
self.chrome_options.add_argument('--start-maximized')
self.chrome_options.add_argument('--disable-blink-features=AutomationControlled')
self.chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36")
# self.chrome_options.add_argument('--proxy-server=http://your-proxy:port')  # Optional
self.driver = uc.Chrome(options=self.chrome_options)
self.driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
'source': '''
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
'''
})
self.driver.get("https://ya.ru/")
time.sleep(random.uniform(1.5, 3.5))

def close(self):
self.driver.quit()

def check_captcha(self):
""" Try to overcome isBot button"""
cur_time = str(datetime.now()).replace(' ', '_')
if "showcaptcha" in self.driver.current_url:
logger.info("Captcha found")
self.driver.save_screenshot(f'screens/img_captcha_{cur_time}.png')

raw_button = self.driver.find_elements(By.XPATH, "//input[@class='CheckboxCaptcha-Button']")
if raw_button:
raw_button[0].click()
logger.info("Button clicked")
time.sleep(random.uniform(1, 2))
self.driver.save_screenshot(f'screens/img_captcha_afterclick_{cur_time}.png')
else:
self.driver.save_screenshot(f'screens/img_{cur_time}.png')

def parse(self, film_name: str):
logger.info(f"Start parse {film_name}")
result_urls = []
try:
self.driver.get(f"https://ya.ru/search/?text={film_name}&lr=213&search_source=yaru_desktop_common&search_domain=yaru")
self.check_captcha()
for i in range(1, 5):
result_urls.extend(self.parse_page(page_id=i))
self.get_next_page()
self.check_captcha()
# Human-like random delay
time.sleep(random.uniform(2, 5))
except Exception:
logger.error(f"Exception in {traceback.format_exc()}")
finally:
logger.info(f"Found {len(result_urls)} for film {film_name}: {result_urls}")

def parse_page(self, page_id):
res = []
urls_raw = self.driver.find_elements(By.XPATH, value='//a[@class="Link Link_theme_normal OrganicTitle-Link organic__url link"]')
for url_raw in urls_raw:
href = url_raw.get_attribute("href")
if href and "yabs.yandex.ru"  not in href:
res.append(href)
logger.info(f"Found {len(res)} urls on page {page_id}")
return res

def get_next_page(self):
next_link_raw = self.driver.find_elements(By.XPATH, '//div[@class="Pager-ListItem Pager-ListItem_type_next"]')
if next_link_raw:
next_link_raw[0].click()
# Human-like random delay
time.sleep(random.uniform(3, 6))

if __name__ == "__main__":
pathlib.Path('screens/').mkdir(exist_ok=True)
parser = YandexParser(USE_GUI=False)  # Default to GUI mode for stealth
films = ["Терминатор смотреть", "Саша Таня смотреть", "Джон Уик смотреть онлайн"]
idx = 0
while True:
try:
film = films[idx]
idx = (idx + 1) % len(films)
parser.parse(film)
time.sleep(random.uniform(8, 15))
except Exception as e:
parser = YandexParser(USE_GUI=False)

Журнал работы на сервере

Код: Выделить всё

2025-05-17 14:53:26,295 [INFO] patching driver executable /home/MY_USER/.local/share/undetected_chromedriver/undetected_chromedriver
2025-05-17 14:53:30,904 [INFO] Start parse Терминатор смотреть
2025-05-17 14:53:33,448 [INFO] Captcha found
2025-05-17 14:53:35,291 [INFO] Button clicked
2025-05-17 14:53:42,194 [INFO] Found 0 urls on page 1
2025-05-17 14:53:42,284 [INFO] Captcha found
2025-05-17 14:53:43,363 [INFO] Button clicked
2025-05-17 14:53:53,826 [INFO] Found 0 urls on page 2
2025-05-17 14:53:53,848 [INFO] Captcha found
2025-05-17 14:53:54,659 [INFO] Button clicked
2025-05-17 14:54:06,023 [INFO] Found 0 urls on page 3

Подробнее здесь: https://stackoverflow.com/questions/796 ... not-appear