Код: Выделить всё
import pandas as pd
import requests_html
import time
import sys
# Setting up default HTTP headers to simulate a web browser request
__headers = {
"upgrade-insecure-requests": "1",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"
}
def __get_response_from_session(url, session):
retry_number = 0 # Variable to count the number of retries
while True:
try:
response = session.get(url, headers=__headers) # Attempt to get a response from the URL
return response, False # Return the response and a flag indicating no errors
except RuntimeError:
sys.exit()
except Exception as e:
print("Something went wrong...", flush=True)
if retry_number == 2:
return None, True # Return None and a flag indicating too many retries
time.sleep(2) # Wait for 2 seconds
retry_number += 1
continue
def github(response):
date = None
try:
response_html = response.html
dateList = response_html.find('relative-time') # Attempt to find 'relative-time' elements in the HTML
except Exception as e:
print("Something went wrong:", e, flush=True)
return date
def main():
df_links = pd.read_csv('links2.csv', index_col=0) # Read a CSV file into a DataFrame
df_links.reset_index(inplace=True, drop=True)
session = requests_html.HTMLSession() # Create an HTML session for making requests
try:
for i in range(0, len(df_links.index)):
url = df_links.iloc[i]['hyperlink'] # Get the URL from the DataFrame
print(f"[{i}/{len(df_links.index)}]: {url}", flush=True)
response, over_retry = __get_response_from_session(url, session)
if not over_retry:
try:
status_code = response.status_code
if status_code != 404:
date = github(response) # Try to get the date from the GitHub page
except Exception as e:
print("Something went wrong...", flush=True)
except Exception as e:
print("Something went wrong...", flush=True)
if __name__ == "__main__":
main()
Я попросил друзей смоделировать это, и с ними произошло то же самое.
В целях тестирования репозиторий с CSV доступен здесь: https://github.com/carloseduardobanjar/ ... rawler-bug
Когда я закомментирую строку dateList = response_html.find( 'relative-time'), код работает без сбоев до завершения. Кажется, проблема кроется именно в этой строке.
ps: Я знаю, что код может показаться бессмысленным, но это всего лишь пример, иллюстрирующий проблему.
Подробнее здесь: https://stackoverflow.com/questions/783 ... a-for-loop
