Ошибка оператора Docker в Airflow MWAA ⇐ Python

[Anonymous]

Мы пытаемся использовать оператор докера в Airflow (MWAA). Мы храним образы докеров в репозитории ECR в той же учетной записи AWS. Группа обеспечения доступности баз данных Airflow с оператором докера завершается с ошибкой ниже
Журнал

Код: Выделить всё

Traceback (most recent call last):
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 714, in urlopen
httplib_response = self._make_request(
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 415, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/connection.py", line 244, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/lib/python3.10/http/client.py", line 1282, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1328, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1277, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1037, in _send_output
self.send(msg)
File "/usr/lib/python3.10/http/client.py", line 975, in send
self.connect()
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/transport/unixconn.py", line 27, in connect
sock.connect(self.unix_socket)
FileNotFoundError: [Errno 2] No such file or directory

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/airflow/.local/lib/python3.10/site-packages/requests/adapters.py", line 486, in send
resp = conn.urlopen(
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 798, in urlopen
retries = retries.increment(
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/util/retry.py", line 550, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/packages/six.py", line 769, in reraise
raise value.with_traceback(tb)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 714, in urlopen
httplib_response = self._make_request(
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 415, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/connection.py", line 244, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/lib/python3.10/http/client.py", line 1282, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1328, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1277, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1037, in _send_output
self.send(msg)
File "/usr/lib/python3.10/http/client.py", line 975, in send
self.connect()
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/transport/unixconn.py", line 27, in connect
sock.connect(self.unix_socket)
urllib3.exceptions.ProtocolError:  ('Connection aborted.', FileNotFoundError(2, 'No such file or directory'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/api/client.py", line 214, in _retrieve_server_version
return self.version(api_version=False)["ApiVersion"]
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/api/daemon.py", line 181, in version
return self._result(self._get(url), json=True)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/utils/decorators.py", line 46, in inner
return f(self, *args, **kwargs)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/api/client.py", line 237, in _get
return self.get(url, **self._set_request_timeout(kwargs))
File "/usr/local/airflow/.local/lib/python3.10/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/requests/adapters.py", line 501, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/airflow/.local/lib/python3.10/site-packages/airflow/providers/docker/operators/docker.py", line 459, in execute
if self.force_pull or not self.cli.images(name=self.image):
File "/usr/local/airflow/.local/lib/python3.10/site-packages/airflow/providers/docker/operators/docker.py", line 329, in cli
return self.hook.api_client
File "/usr/lib/python3.10/functools.py", line 981, in __get__
val = self.func(instance)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/airflow/providers/docker/hooks/docker.py", line 120, in api_client
client = APIClient(
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/api/client.py", line 197, in __init__
self._version = self._retrieve_server_version()
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/api/client.py", line 221, in _retrieve_server_version
raise DockerException(
docker.errors.DockerException: Error while fetching server API version: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory'))

Версия Airflow: v2.6.3
Код

Код: Выделить всё

# dbt docker image name from Airflow variables
dbt_image = (
"{{ var.value.get('DBT_IMAGE') }}"
)

refresh_function = PythonOperator(
task_id="refresh_ecr_creds", python_callable=refresh_ecr_credentials
)

run_dbt_task = DockerOperator(
task_id="dbt_run_fct_sample",
image=dbt_image,
api_version="auto",
auto_remove=True,
command=["run", "-s", "fct_sample"],  # Replace with dbt commands
dag=dag,
docker_conn_id=ECR_CONNECTION,
environment=SNOWFLAKE_ENV_VAR,
)

Я попробовал запустить локальную настройку воздушного потока, и все работало нормально, однако MWAA по-прежнему вызывает у меня проблемы. Я проверил разрешения на листинг и получение изображений из ECR, они установлены правильно.

Подробнее здесь: https://stackoverflow.com/questions/771 ... rflow-mwaa