scrape.py
Проблема в том, что я запускаю программу, но исходный код совсем не похож на то, что предложил Клод (к которому я прибегнул после попытки отладки самостоятельно). Суть парсинга заключается в буквальном парсинге любого ввода с веб-сайта. Другие программные файлы используются для вызова LangChain и преобразуют данные, которыми манипулирует ИИ, в стандартизированные выходные данные (CRM, данные конкурентов и т. д.). Это решение, созданное мной в одиночку.
import ssl
ssl._create_default_https_context = ssl._create_unverified_context
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
from dotenv import load_dotenv
from selenium.webdriver import Remote, ChromeOptions as Options
from selenium.webdriver.chromium.remote_connection import ChromiumRemoteConnection
from selenium.webdriver.common.by import By
from os import environ
from time import sleep
from bs4 import BeautifulSoup
import os
load_dotenv()
# Custom connection class that disables SSL verification
class NoSSLConnection(ChromiumRemoteConnection):
def _get_connection_manager(self):
return urllib3.PoolManager(cert_reqs='CERT_NONE')
def scrape_website(website):
print("Launching chrome browser...")
AUTH = environ.get('AUTH', '**********')
if not AUTH:
raise Exception("Missing AUTH credentials")
print('Connecting to Browser...')
server_addr = f'https://{AUTH}@brd.superproxy.io:9515'
connection = NoSSLConnection(server_addr, 'goog', 'chrome') #
use custom classdriver = Remote(connection, options=Options())
def cdp(cmd, params={}):
return driver.execute('executeCdpCommand', {
'cmd': cmd,
'params': params,
})['value']
try:
print('Connected! Starting inspect session...')
frames = cdp('Page.getFrameTree')
frame_id = frames['frameTree']['frame']['id']
inspect = cdp('Page.inspect', {'frameId': frame_id})
print(f"Inspect session: {inspect['url']}")
sleep(10)
print(f'Navigating to {website}...')
driver.get(website)
print('Waiting for captcha to solve...')
result = driver.execute('executeCdpCommand', {
'cmd': 'Captcha.waitForSolve',
'params': {'detectTimeout': 10000},
})
print(f"Captcha status: {result['value']['status']}")
sleep(10)
print('Scraping page content...')
paragraphs = driver.find_elements(By.TAG_NAME, 'p')
data = driver.execute_script(
'return arguments[0].map(el => el.innerText)', paragraphs
)
return data
except Exception as e:
print(f"Scraping error: {e}")
return []
finally:
driver.quit()
# ------------------------Reducing the Amount of Characters/Batches That We Need to Submit to Our LLM to Get A Value Response----------------------------------------
# Function extracting the body content of the html
def extract_body_content(html_content):
soup = BeautifulSoup(html_content, "html.parser")
body_content = soup.body
if body_content:
return str(body_content)
return ""
# Function cleaning the extracted body content of the html
def clean_body_content(body_content):
soup = BeautifulSoup(body_content, "html.parser")
for script_or_style in soup("script", "style"): # Gets rid of unnecessary content
script_or_style.extract()
cleaned_content = soup.get_text(separator="\n")
cleaned_content = "\n".join(
line.strip() for line in cleaned_content.splitlines() if line.strip()
)
return cleaned_content
# Spliting this Content Up Into Batches
"""
Using a specific LLM comes with token limits of 8000 Characters
Possible that we have a massive website and the LLM can't take all of it at once
and won't be able to parse all of it.
"""
def split_dom_content(dom_content, max_length = 6000):
return (
dom_content[i : i + max_length ] for i in range(0, len(dom_content), max_length)
)
