Как я могу ускорить свой код на Python с помощью asyncio? [закрыто] ⇐ Python

[Anonymous]

Я написал скрипт на Python, чтобы получить некоторые данные из API, чтобы иметь возможность записать их в Excel, так как существует много вызовов API, которые я хотел сопоставить с асинхронным способом.
Но теперь это занимает еще больше времени, чем раньше, есть подсказки, почему? В журналах нет ошибок, похоже, он просто останавливается...
Разве я неправильно использовал библиотеку asyncio?
Я поместил код, соответствующий проблеме, ниже
Спасибо, что прочитали мой пост

semaphore = asyncio.Semaphore(15)

client = httpx.AsyncClient(
timeout=httpx.Timeout(30.0),
limits=httpx.Limits(
max_connections=5,
max_keepalive_connections=5
)
)

client2 = httpx.AsyncClient(
timeout=httpx.Timeout(40.0),
limits=httpx.Limits(
max_connections=5,
max_keepalive_connections=5
)
)

async def trace_recup(name, coro):
for attempt in range(10):
try:
print(f" Début {name} attempt {attempt}",flush=True)
async with semaphore:
results= await coro()
print(f" Fin {name}",flush=True)
return results
except asyncio.TimeoutError:
print(f" {name} timeout",flush=True)
except Exception as e:
print(f" Erreur dans {name}: {e}",flush=True)
if attempt == 3:
print(f" Erreur dans {name} le nombre d'essaie total a été atteint: {e}",flush=True)
raise
await asyncio.sleep(1)

async def recup_donnees ( BASEURL,old_id,request_date_start,request_date_end,dates_ranges) :
# recup des données
try:
print(" On va lancer toutes les tâches recup_donnees", flush=True)
results = await asyncio.gather(
trace_recup("Acces", lambda: acces_compte.fill_infosRecup(old_id,BASEURL,client)),
trace_recup("revue client", lambda: acces_compte.fill_Revue_des_acces_et_contacts_clientRecup(old_id,BASEURL,client)),
trace_recup("licence", lambda: agents_installes_compte.fill_Licence_Account_infoRecup(old_id,BASEURL,client)),
trace_recup("agents", lambda: agents_installes_compte.fill_Revue_des_agents_installesRecup(old_id,BASEURL,client)),
trace_recup("appareils", lambda: appareil_non_proteges_compte.fill_Revue_des_appareils_non_protegesRecup(old_id,BASEURL,client)),
trace_recup("reseaux", lambda: appareil_non_proteges_compte.fill_Revue_des_decouvertes_reseauxRecup(old_id,BASEURL,client)),
trace_recup("vul csv", lambda: vul_maj_compte.fill_CSVRecup(BASEURL,old_id,request_date_start,request_date_end,client)),
trace_recup("vul score", lambda: vul_maj_compte.scoreVulRecup(BASEURL,old_id,client)),
trace_recup("vul maj", lambda: vul_maj_compte.fill_Revue_des_vulnerabilites_majeures_1Recup(BASEURL,old_id,client)),
trace_recup("vul maj 2", lambda: vul_maj_compte.fill_Revue_des_vulnerabilites_majeures_2Recup(BASEURL,old_id,client)),
return_exceptions=True
)
results2 = await asyncio.gather(
trace_recup("alertes csv", lambda: alertes__compte.fill_CSVRecup(BASEURL,old_id,request_date_start,request_date_end,client2)),
trace_recup("alertes 1", lambda: alertes__compte.fill_Revue_des_alertes_1Recup(request_date_start,request_date_end,BASEURL,old_id,client2)),
trace_recup("alertes 2", lambda: alertes__compte.fill_Revue_des_alertes_2Recup(dates_ranges,BASEURL,old_id,client2)),
trace_recup("alertes averés", lambda: alertes_averres_compte.fill_Revue_des_alertes_averees_1Recup(request_date_start,request_date_end,BASEURL,old_id,client2)),
trace_recup("alertes fp", lambda: alertes_fp_compte.fill_Revue_des_alertes_fp_1Recup(request_date_start,request_date_end,BASEURL,old_id,client2)),
trace_recup("alertes sus", lambda: alertes_sus_compte.fill_Revue_des_alertes_sus_1Recup(request_date_start,request_date_end,BASEURL,old_id,client2)),
trace_recup("politique override", lambda: politique_secu_compte.fill_Revue_de_la_politique_overrideRecup(BASEURL,old_id,client2)),
trace_recup("politique group", lambda: politique_secu_compte.fill_Revue_de_la_politique_de_securite_groupesRecup(BASEURL,old_id,client2)),
trace_recup("politique site", lambda: politique_secu_compte.fill_Revue_de_la_politique_de_securite_sitesRecup(BASEURL,old_id,client2)),
trace_recup("bilan", lambda: bilen_secu_compte.fill_Revue_bilan_secuRecup(old_id,BASEURL,dates_ranges,client2)),
return_exceptions=True
)

except Exception as e:
print(f" Erreur lors de la récupération des données : {e}")
traceback.print_exc()
return

if results:

(
infos_compte,
revue_acces,
licence_info,
agents_installes,
appareils_non_proteges,
decouvertes_reseaux,
vul_csv,
vul_score,
vul_majeures_1,
vul_majeures_2,
) = results

(
alertes_csv,
alertes_1,
alertes_2,
alertes_averres_1,
alertes_fp_1,
alertes_sus_1,
policy,
policy_groupes,
policy_sites,
bilan
) = results2

return {
"name_client": infos_compte["name_client"],
"users": revue_acces["users"],
"active_agents": licence_info["active_agents"],
"total_agents": licence_info["total_agents"],
"total_complete": licence_info["total_complete"],
"total_control": licence_info["total_control"],
"total_core": licence_info["total_core"],
"agents_version": agents_installes["agents_version"],
"agents_latest_package": agents_installes["agents_latest_package"],
"roguesOs": appareils_non_proteges["roguesOs"],
"content": decouvertes_reseaux["content"],
"dataTooBig": vul_csv["dataTooBig"],
"appCritique": vul_csv["appCritique"],
"totendpoints": vul_score["totendpoints"],
"order_by_endpoint_count": vul_majeures_1["order_by_endpoint_count"],
"order_by_NvdBase_score": vul_majeures_1["order_by_NvdBase_score"],
"applicationsBySite": vul_majeures_2["applicationsBySite"],
"contentAlerte": alertes_1["contentAlerte"],
"analystVerdict": alertes_1["analystVerdict"],
"detectionengine": alertes_1["detectionengine"],
"classification": alertes_1["classification"],
"sites": alertes_1["sites"],
"groups": alertes_1["groups"],
"total_alerts_by_group": alertes_1["total_alerts_by_group"],
"alertByMonth": alertes_2["alertByMonth"],
"alert_tp_by_group" : alertes_averres_1["alert_tp_by_group"],
"alert_fp_by_group" : alertes_fp_1["alert_fp_by_group"],
"alert_sus_by_group" : alertes_sus_1["alert_sus_by_group"],
"policy":policy["policy"],
"policy_groupes":policy_groupes["policy_data_group"],
"policy_sites":policy_sites["policy_data_site"],
"nb_mois":bilan["nb_mois"],
"nb_total_sus_avere":bilan["nb_total_sus_avere"],
"nb_alerte_total":bilan["nb_alerte_total"],
"totendpoints2":bilan["totendpoints"],
}

async def generate_excel_file(old_id, BASEURL, number_of_months,request_date_start,request_date_end):

print(f" generate_excel_file appelé pour {old_id}", flush=True)

name = await clientsInfos_compte.getClientNameById(old_id,BASEURL,client)

dates_ranges = utils_functions.get_dates_ranges(request_date_start, number_of_months)

results = await recup_donnees(BASEURL,old_id,request_date_start,request_date_end,dates_ranges)

await client.aclose()
await client2.aclose()

return name



Подробнее здесь: https://stackoverflow.com/questions/798 ... th-asyncio