Напишите pyspark df в csv и загрузите его в SharePoint. ⇐ Python

[Anonymous]

Код: Выделить всё

# Specify the SharePoint credentials and folder URL
username_shrpt = 'XXXXXXXXXXXXXXXXXXXXX'
from cryptography.fernet import Fernet
import os
fernet = Fernet(bytes(os.getenv('SERVICE_EMAIL_ENCRYPT'),'ascii'))
password_shrpt = fernet.decrypt(bytes(os.getenv('SERVICE_EMAIL_PASS'),'ascii')).decode()
url_shrpt = 'YYYYYYYYYYYYYYY'

new_folder_path = 'ZZZZZZZZZZZZZZZZZ'

# Authenticate into SharePoint
ctx_auth = AuthenticationContext(url_shrpt)
if ctx_auth.acquire_token_for_user(username_shrpt, password_shrpt):
ctx = ClientContext(url_shrpt, ctx_auth)
web = ctx.web
ctx.load(web)
ctx.execute_query()
print('Authenticated into SharePoint as:', web.properties['Title'])
else:
print(ctx_auth.get_last_error())

def write_df_as_csv(ctx, shp_path, df):
"""
Retrieves a single Excel file from SharePoint and converts it to a pandas DataFrame.
:param ctx: Authenticated SharePoint ClientContext.
:param file_path: The server-relative URL of the Excel file in SharePoint.
"""
try:
file_content = io.BytesIO(df.to_csv(index=False).encode())
dir, name = os.path.split(shp_path)
file = ctx.web.get_folder_by_server_relative_url(dir).upload_file(name, file_content).execute_query()
print("Success")
except Exception as e:
print("Error occurred:", e)
####################################################################################
file_name2 = 'Test_AG_Threshold'+str(date.today())+'.csv'
path = new_folder_path+file_name2
df2 = threshold_calculation(master_booking_df)

file_name = 'Test_Threshold_Approved_By'+str(date.today())+'.csv'
shp_path = folder_url_shrpt+file_name
df = process_df()

write_df_as_csv(ctx, path,df2) #write the AG_threshold csv
write_df_as_csv(ctx, shp_path,df) #write the Threshold_approved_by csv

Код в настоящее время преобразует мой фрейм данных в Pandas, но это не будет работать, поскольку мой фрейм данных слишком велик для преобразования в pandas, вместо этого я хочу записать в CSV, используя мой фрейм данных pyspark.< /п>

Подробнее здесь: https://stackoverflow.com/questions/791 ... sharepoint