[Python] Download large file from s3 using aioboto3 and aiofiles is really slow

I have high-load system, where many users could upload large files (+1gb). After uploading, sometimes, I need to download them from S3 to calculate some meta information. Currently I'm using this code to do this (look at fetch_file):

def _get_file_extension(key) -> str
...

class S3Storage:
def __init__(self, *args, **kwargs):
...
self._config = botocore.config.Config(
read_timeout=read_timeout,
connect_timeout=connect_timeout,
retries={
"total_max_attempts": ...,
"max_attempts": ...,
},
signature_version="v4",
)
self._session = self._create_session()

def _create_session():
return aioboto3.Session()

def _create_client(self):
return self._session.client(
service_name="s3",
endpoint_url=self._endpoint_url,
region_name=self._region,
aws_access_key_id=self._access_key_id,
aws_secret_access_key=self._secret_access_key,
config=self._config,
)

async def fetch_file(self, key: str, bucket: str | None = None) -> str:
try:
async with self._create_client() as client:
response = await client.get_object(Bucket=bucket or self._bucket, Key=key)
async with aiofiles.tempfile.NamedTemporaryFile(
"wb",
suffix=_get_file_extension(key),
delete=False,
) as file:
async for chunk in response["Body"]:
await file.write(chunk)
return str(file.name)
except botocore.exceptions.ClientError as e:
...
except aiohttp.ServerTimeoutError as e:
...
except (
botocore.exceptions.BotoCoreError,
aiohttp.ClientError,
) as e:
...


For some reason, this realisation is really slow. After profiling I've found out that it takes 300 seconds to download 1gb file. All 300 seconds were spent exactly at fetch_file method.

Could someone help me to figure out, why it takes so long to download file from s3 and what I could do with this.

Continue reading...