[Python] Fetch data concurrently within for loop using asnycio

I would like to optimize my current function called process_coordinates.

import asyncio
from aiohttp import ClientSession
from tqdm.asyncio import tqdm
import pandas as pd
from streetlevel import streetview


data_dict = {
1: {'country': 'AZE', 'longitude': 48.84328388521507, 'latitude': 39.75349231850633},
2: {'country': 'AZE', 'longitude': 46.42568983461019, 'latitude': 41.74686028542068},
3: {'country': 'AZE', 'longitude': 46.77391084838791, 'latitude': 41.05880534123746},
4: {'country': 'AZE', 'longitude': 46.57032078287734, 'latitude': 39.312871420751485},
5: {'country': 'AZE', 'longitude': 47.26319069021316, 'latitude': 41.28950907274436},
6: {'country': 'AZE', 'longitude': 46.49956247696345, 'latitude': 40.96062005058899},
7: {'country': 'AZE', 'longitude': 48.291171317357815, 'latitude': 38.939108897065445},
8: {'country': 'AZE', 'longitude': 47.12081533506723, 'latitude': 39.681052295694656},
}


async def process_coordinates(data_dict):
pano_dict = {}

async with ClientSession() as session:
for k, v in tqdm(data_dict.items(), desc="Parsing coordinates", unit="coordinate"):
pano = await streetview.find_panorama_async(v['latitude'], v['longitude'], session)

if pano is not None:
pano_dict[k] = {
'longitude': pano.lon,
'latitude': pano.lat,
'date': str(pano.date)
}
else:
pano_dict[k] = {
'longitude': None,
'latitude': None,
'date': None
}

return pano_dict


async def main(data_dict):
pano_results = await process_coordinates(data_dict)

df = pd.DataFrame.from_dict(pano_results, orient='index')

df.reset_index(inplace=True)
df.rename(columns={'index': 'uuid'}, inplace=True)

df.to_parquet('results.parquet', index=False)



asyncio.run(main(data_dict))


Right now, it is capable of iterating over one coordinate per iteration and request information for the fiven coordinate via find_panorama_async. I would like to speed things up, and request information for four coordinates in parallell. The response is stored as a dictionary.

Thanks!

Continue reading...