Код: Выделить всё
import urllib3
import boto3
from azure.storage.blob import BlobServiceClient
from boto3.session import Session
import os
import botocore
from smart_open import open as smart_open
def copy_directory_from_s3_to_blob(s3_bucket, s3_prefix, aws_access_key_id, aws_secret_access_key, azure_connection_string, azure_container):
# Create an S3 client with AWS access key
s3_client = boto3.client('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
print(s3_client)
# List all objects in the S3 directory
response = s3_client.list_objects_v2(Bucket=s3_bucket, Prefix=s3_prefix)
print('response')
# Create a BlobServiceClient using the Azure connection string
blob_service_client = BlobServiceClient.from_connection_string(azure_connection_string)
# Get a reference to the Azure container
container_client = blob_service_client.get_container_client(azure_container)
# Iterate over each object in the S3 directory
for obj in response.get('Contents', []):
# Download the file from S3 using smart_open for direct streaming
s3_key = obj['Key']
temp_file_path = f'/tmp/{os.path.basename(s3_key)}'
with smart_open(f's3://{s3_bucket}/{s3_key}', 'rb', transport_params={
'session': boto3.Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
}) as s3_file:
# Upload the file to Azure Blob Storage
blob_client = container_client.get_blob_client(blob=s3_key)
blob_client.upload_blob(data=s3_file, overwrite=True)
return "Directory copied successfully!"
# Example usage
s3_bucket = 'bucket_name'
s3_prefix = 'bucket_prefix/' # Adjusted prefix to not include leading slash
aws_access_key_id = 'access_keyid'
aws_secret_access_key = 'secret_Access_key'
azure_connection_string = 'connection_string'
azure_container = 'container_name'
result = copy_directory_from_s3_to_blob(s3_bucket, s3_prefix, aws_access_key_id, aws_secret_access_key, azure_connection_string, azure_container)
print(result)
Ошибка
SSLError: проверка SSL не удалась для https://test.s3.amazonaws.com/ ?list-type=2&prefix=test%2F&encoding-type=url Соединение TLS/SSL закрыто (EOF) (_ssl.c:992)
Файл /databricks/python/lib/python3.11/site-packages /urllib3/connectionpool.py:714, в HTTPConnectionPool.urlopen(self, метод, URL, тело, заголовки, повторы, перенаправление, Assert_same_host, timeout,pool_timeout, Release_conn, chunked, body_pos, **response_kw)
713 # Сделайте запрос к объекту подключения httplib.
--> 714 httplib_response = self._make_request(
715 conn,
716 метод,
717 url,
718 таймаут =timeout_obj,
719 body=body,
720 headers=headers,
721 chunked=chunked,
722 )
724 # Если мы собираемся выпустить соединение наконец:, then
725 # ответу не обязательно знать о соединении. В противном случае
726 # он также попытается освободить его, и у нас будет двойная
727 # неразбериха.
Подробнее здесь: https://stackoverflow.com/questions/787 ... databricks
