[Python] Docker Operator fail in Airflow MWAA

We are trying to use docker operator in Airflow(MWAA). We are storing docker images in ECR repository in same AWS account. The Airflow DAG with docker operator fails with below error

Log

Traceback (most recent call last):
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 714, in urlopen
httplib_response = self._make_request(
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 415, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/connection.py", line 244, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/lib/python3.10/http/client.py", line 1282, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1328, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1277, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1037, in _send_output
self.send(msg)
File "/usr/lib/python3.10/http/client.py", line 975, in send
self.connect()
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/transport/unixconn.py", line 27, in connect
sock.connect(self.unix_socket)
FileNotFoundError: [Errno 2] No such file or directory

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/airflow/.local/lib/python3.10/site-packages/requests/adapters.py", line 486, in send
resp = conn.urlopen(
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 798, in urlopen
retries = retries.increment(
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/util/retry.py", line 550, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/packages/six.py", line 769, in reraise
raise value.with_traceback(tb)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 714, in urlopen
httplib_response = self._make_request(
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 415, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/urllib3/connection.py", line 244, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/lib/python3.10/http/client.py", line 1282, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1328, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1277, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1037, in _send_output
self.send(msg)
File "/usr/lib/python3.10/http/client.py", line 975, in send
self.connect()
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/transport/unixconn.py", line 27, in connect
sock.connect(self.unix_socket)
urllib3.exceptions.ProtocolError: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/api/client.py", line 214, in _retrieve_server_version
return self.version(api_version=False)["ApiVersion"]
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/api/daemon.py", line 181, in version
return self._result(self._get(url), json=True)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/utils/decorators.py", line 46, in inner
return f(self, *args, **kwargs)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/api/client.py", line 237, in _get
return self.get(url, **self._set_request_timeout(kwargs))
File "/usr/local/airflow/.local/lib/python3.10/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/requests/adapters.py", line 501, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/local/airflow/.local/lib/python3.10/site-packages/airflow/providers/docker/operators/docker.py", line 459, in execute
if self.force_pull or not self.cli.images(name=self.image):
File "/usr/local/airflow/.local/lib/python3.10/site-packages/airflow/providers/docker/operators/docker.py", line 329, in cli
return self.hook.api_client
File "/usr/lib/python3.10/functools.py", line 981, in __get__
val = self.func(instance)
File "/usr/local/airflow/.local/lib/python3.10/site-packages/airflow/providers/docker/hooks/docker.py", line 120, in api_client
client = APIClient(
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/api/client.py", line 197, in __init__
self._version = self._retrieve_server_version()
File "/usr/local/airflow/.local/lib/python3.10/site-packages/docker/api/client.py", line 221, in _retrieve_server_version
raise DockerException(
docker.errors.DockerException: Error while fetching server API version: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory'))


Airflow Version: v2.6.3

Code

# dbt docker image name from Airflow variables
dbt_image = (
"{{ var.value.get('DBT_IMAGE') }}"
)

refresh_function = PythonOperator(
task_id="refresh_ecr_creds", python_callable=refresh_ecr_credentials
)

run_dbt_task = DockerOperator(
task_id="dbt_run_fct_sample",
image=dbt_image,
api_version="auto",
auto_remove=True,
command=["run", "-s", "fct_sample"], # Replace with dbt commands
dag=dag,
docker_conn_id=ECR_CONNECTION,
environment=SNOWFLAKE_ENV_VAR,
)


I tried running local airflow setup and it worked fine, However MWAA still giving me issues. I have verified permissions for listing and pulling images from ECR, they have been set appropriately.

Continue reading...