Последнее обновление см. внизу
Используя Python 3.12.3 и Selenium, я пытаюсь загрузить больше строк перед очисткой, и я очень новичок в этом процесс. В идеале все или хотя бы как можно больше, но в какой-то момент веб-сайт может автоматически ограничить общее количество отзывов на странице. Щелкнув вручную, я смог щелкнуть по нему еще как минимум 10 раз, и это не помогло. Любая помощь будет оценена по достоинству. Пожалуйста, дайте мне знать, если я могу предоставить какой-либо другой контекст.
Вот снимок экрана раздела, на который я хотел бы нажать: Кнопка «Показать больше»
Я постоянно не могу щелкните этот раздел, и я считаю, что это связано с псевдоэлементом :before, но, кроме того, существует несколько контейнеров div.Button__, из-за которых мои попытки щелкнуть обычно приводят к нажатию кнопок, которых я не хотел.
Вот скрипт, который я использую для получения отзывов, но он не может нажать кнопку «Загрузить больше строк», поэтому я надеюсь восполнить этот пробел с помощью селена:
import requests
import os
from bs4 import BeautifulSoup
import pandas as pd
import time
from selenium import webdriver
from IPython.display import display, Image
driver = webdriver.Chrome()
title = driver.title
for i in range(0,1000,100):
#McCurdy - I'm Glad my Mom Died
response = requests.get('https://www.goodreads.com/book/show/260 ... jE4NDI1%22}')
print(response.status_code)
time.sleep(6)
doc = BeautifulSoup(response.text, 'html.parser')
df = pd.DataFrame(columns=['ReviewDate','ReviewerName','Rating','ReviewText','ReviewerMeta'])
ratings = []
# Loop through all elements found
for tag in book_tags_:
# Get the aria-label attribute from the current element
aria_label = tag.get('aria-label')
# Check if aria-label is not None and contains the expected format
if aria_label and 'Rating ' in aria_label and ' out of 5' in aria_label:
# Split the aria-label to extract the desired text
rating_text = aria_label.split('Rating ')[1].split(' out of 5')[0]
# Append the rating_text to the list of ratings
ratings.append(rating_text)
else:
print(f"Skipping element with aria-label: {aria_label}")
# Create a dataframe from the list of ratings
df = pd.DataFrame({'Rating': ratings})
#This is repeated for additional fields
Вот мой код, который я использую для попытки нажатия кнопки:
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, StaleElementReferenceException, ElementClickInterceptedException
# Initialize WebDriver
driver = webdriver.Chrome() # or another driver you're using
url = "https://www.goodreads.com/book/show/593 ... 5MjY2Mw%22}"
# Open the page
driver.get(url)
time.sleep(20)
def click_show_more():
while True:
try:
# Scroll to the bottom of the page to ensure button is in view
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
# Find the 'Show More' button
show_more_button = WebDriverWait(driver, 10).until(
EC.visibility_of_element_located((By.CSS_SELECTOR, 'div.Divider.Divider--contents.Divider--largeMargin > div.Button__container'))
)
# Scroll the element into view
driver.execute_script("arguments[0].scrollIntoView(true);", show_more_button)
# Ensure the element is clickable
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'div.Divider.Divider--contents.Divider--largeMargin > div.Button__container')))
# Try to click the button using JavaScript if needed
driver.execute_script("arguments[0].click();", show_more_button)
# Optionally, wait for some condition after clicking, e.g., new content to load
WebDriverWait(driver, 10).until(EC.staleness_of(show_more_button))
except TimeoutException:
print("No more 'Show More' button found or timed out.")
break
except StaleElementReferenceException:
print("StaleElementReferenceException: Trying to find the button again.")
continue
except ElementClickInterceptedException:
print("ElementClickInterceptedException: Element is being obstructed.")
continue
except Exception as e:
print(f"Error clicking 'Show More' button: {e}")
break
# Remember to call your function
click_show_more()
После комментария @x1337loser я добавил раздел добавления в список в функцию getdata(), после чего начал получать следующую ошибку: Ошибка ссылки на новый код< /strong>
Я также удалил изображение пользователя из распечатки, добавил словарь данных и записал в csv конец скрипта, но эти шаги, похоже, не вызывают каких-либо проблем. .
См. новый код ниже:
import time
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from bs4 import BeautifulSoup
import pandas as pd
ReviewText = []
ratings = []
Reviewer = []
reviewdt = []
ReviewerId = []
Author_ID = []
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
def get_data(content):
data = content['data']['getReviews']['edges']
for i in data:
id_user = i['node']['creator']['id']
rev_img = i['node']['creator']['imageUrlSquare']
is_author = i['node']['creator']['isAuthor']
follower_count = i['node']['creator']['followersCount']
name = i['node']['creator']['name']
review = BeautifulSoup(i['node']['text'], "lxml").text #removed HTML tag from text
review_create_data = i['node']['createdAt']
result_ms = pandas.to_datetime(review_create_data,unit='ms') #decode timestamp
review_liked = i['node']['likeCount']
rating = i['node']['rating']
#\nreviewer_image: {rev_img} between resultms and \nreviewer_follower
print(f"reviewers_name: {name}\nreviewer_user_id: {id_user}\nIs reviewr is author: {is_author}\nreview_date: {result_ms}\nreviewer_follower: {follower_count}\nreview: {review}\nreview_ratings: {rating}\nreview_liked: {review_liked}\n========================================")
#print(f"Is reviewr is author: {is_author}\nreviewer_follower: {follower_count}\nreview_liked: {review_liked}\n========================================")
ReviewText_content = {review}
rating_text = {rating}
Reviewer_name = {name}
reviewdt_content = {result_ms}
reviewer_id = {id_user}
author = {is_author}
# Append the Reviewer_text to the list of ratings
Reviewer.append(Reviewer_name)
reviewdt.append(reviewdt_content)
ratings.append(rating_text)
ReviewText.append(ReviewText_content)
ReviewerId.append(reviewer_id)
Author_ID.append(author)
def gatherNextPage(resourceId):
url = "https://kxbwmqov6jgg3daaamb744ycu4.apps ... om/graphql"
headers = {
"User-Agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0",
"X-Api-Key": "da2-xpgsdydkbregjhpr6ejzqdhuwy" # The Server added this API KEY automatically and strict checking happened from the client-server request
}
data = {
"operationName": "getReviews",
"variables": {
"filters": {
"resourceType": "WORK",
"resourceId": f"{resourceId}"
},
"pagination": {
"limit": 100
}
},
"query": "query getReviews($filters: BookReviewsFilterInput!, $pagination: PaginationInput) {\n getReviews(filters: $filters, pagination: $pagination) {\n ...BookReviewsFragment\n __typename\n }\n}\n\nfragment BookReviewsFragment on BookReviewsConnection {\n totalCount\n edges {\n node {\n ...ReviewCardFragment\n __typename\n }\n __typename\n }\n pageInfo {\n prevPageToken\n nextPageToken\n __typename\n }\n __typename\n}\n\nfragment ReviewCardFragment on Review {\n __typename\n id\n creator {\n ...ReviewerProfileFragment\n __typename\n }\n recommendFor\n updatedAt\n createdAt\n spoilerStatus\n lastRevisionAt\n text\n rating\n shelving {\n shelf {\n name\n webUrl\n __typename\n }\n taggings {\n tag {\n name\n webUrl\n __typename\n }\n __typename\n }\n webUrl\n __typename\n }\n likeCount\n viewerHasLiked\n commentCount\n}\n\nfragment ReviewerProfileFragment on User {\n id: legacyId\n imageUrlSquare\n isAuthor\n ...SocialUserFragment\n textReviewsCount\n viewerRelationshipStatus {\n isBlockedByViewer\n __typename\n }\n name\n webUrl\n contributor {\n id\n works {\n totalCount\n __typename\n }\n __typename\n }\n __typename\n}\n\nfragment SocialUserFragment on User {\n viewerRelationshipStatus {\n isFollowing\n isFriend\n __typename\n }\n followersCount\n __typename\n}\n"
}
n, data_next = 1, data
while True:
time.sleep(3)
resp = requests.post(url, headers=headers, json=data_next, verify=False)
data = resp.json()
get_data(data)
nextPageToken = data['data']['getReviews']['pageInfo']['nextPageToken']
if not nextPageToken:
break
data_next = {
"operationName": "getReviews",
"variables": {
"filters": {
"resourceType": "WORK",
"resourceId": f"{resourceId}"
},
"pagination": {
"after": f"{nextPageToken}", #nextPageToken from response data, and the total limit is 100 results per request.
"limit": 100
}
},
"query": "query getReviews($filters: BookReviewsFilterInput!, $pagination: PaginationInput) {\n getReviews(filters: $filters, pagination: $pagination) {\n ...BookReviewsFragment\n __typename\n }\n}\n\nfragment BookReviewsFragment on BookReviewsConnection {\n totalCount\n edges {\n node {\n ...ReviewCardFragment\n __typename\n }\n __typename\n }\n pageInfo {\n prevPageToken\n nextPageToken\n __typename\n }\n __typename\n}\n\nfragment ReviewCardFragment on Review {\n __typename\n id\n creator {\n ...ReviewerProfileFragment\n __typename\n }\n recommendFor\n updatedAt\n createdAt\n spoilerStatus\n lastRevisionAt\n text\n rating\n shelving {\n shelf {\n name\n webUrl\n __typename\n }\n taggings {\n tag {\n name\n webUrl\n __typename\n }\n __typename\n }\n webUrl\n __typename\n }\n likeCount\n viewerHasLiked\n commentCount\n}\n\nfragment ReviewerProfileFragment on User {\n id: legacyId\n imageUrlSquare\n isAuthor\n ...SocialUserFragment\n textReviewsCount\n viewerRelationshipStatus {\n isBlockedByViewer\n __typename\n }\n name\n webUrl\n contributor {\n id\n works {\n totalCount\n __typename\n }\n __typename\n }\n __typename\n}\n\nfragment SocialUserFragment on User {\n viewerRelationshipStatus {\n isFollowing\n isFriend\n __typename\n }\n followersCount\n __typename\n}\n"
}
gatherNextPage("kca://work/amzn1.gr.work.v3.JeHZlXvg2e1mD9_k")
data_dict = {
'ReviewDate': reviewdt,
'ReviewerName': Reviewer,
'Rating': ratings,
'ReviewText': ReviewText,
'ReviewerID': ReviewerId,
'IsAuthor': Author_ID
#'ReviewerMeta': reviewer_meta
}
df = pd.DataFrame(data_dict)
df. to_csv('McCurdy_Sample_Sept30.csv', index=False)
Подробнее здесь: https://stackoverflow.com/questions/790 ... re-section
