[Python] Django - Test asynchronous method causes psycopg2.InterfaceError: connection already...

I want to test some async method that I use on an async view, but I want to be very clear that right now I am not testing my view, I already saw the documentation on django but it is only for views....

So I have this class containing multiple method and even contacting database asynchronously.

class SessionService:
""" Session service.

Attributes:
uuid (str): system uuid
gcloud (str): console gcloud_id
start (str): start time of the period
end (str): end time of the period
"""

def __init__(self, uuid: str, gcloud: str, start: str, end: str):
self.system_uuid: str = uuid
self.gcloud_id: str = gcloud
self.start: str = start
self.end: str = end

@database_sync_to_async
def __vision_data(self) -> Tuple[List[str], List[str], List[int]]:
""" Async method to get vision module's data from a uuid

Returns
----------
QuerySet
List where 1st element are gcloud id, 2nd UUID and last system IDs
"""
return System.objects.get_vision_conf_data(self.system_uuid)

@async_property
async def __vision_system_ids(self) -> List[int]:
""" get visions Ids

Returns:
List[int]
"""
data = await self.__vision_data()
return data[2]

async def get_session_list(self) -> List[Dict[str, Union[str, bool, int]]]:
""" Get all session for a period for a system
Returns:
List[Dict[str, Union[str, bool, int]]]
"""
spraying_session: SprayingSessionSumUpGetter = SprayingSessionSumUpGetter(self.gcloud_id, self.start,
self.end)
vision_systems_ids: List[int] = await self.__vision_system_ids
sessions: List[Dict[str, Union[str, pd.Timestamp]]] = await spraying_session.get_sum_up(
vision_systems_ids)
return sessions


I have two different situations, with the same problem at the end.

Situation ONE:

I am now trying to test get_session_list. To do so, I use TestCase from django.test .

My tests inherite from another class because I need to set a System instance and it depends on A LOT of other table so I manage it on another test Class SystemSetUpTest So, I have:

class SystemSetUpTest(TestCase):
@staticmethod
def setUp(): # pylint: disable=too-many-locals
create_everything_needed

@staticmethod
def tearDown():
delete_all_stuff

class SystemServicesTest(SystemSetUpTest):
def setUp(self):
super().setUp()


this is the test causing issue:

async def test_getting_sessions_list(self):
uuid: str = self.uuid

console_gcloud_id = 'console'
start_period = '2021-05-06'
end_period = '2021-05-07'
service = SessionService(uuid, console_gcloud_id, start_period, end_period)
print(await service.get_session_list())


and this is the complete Trace:

Traceback (most recent call last):
File "/app/systems/tests/test_set_up.py", line 60, in tearDown
User.objects.filter(email="jul@gmail.com").delete()
File "/usr/local/lib/python3.8/dist-packages/django/db/models/query.py", line 745, in delete
collector.collect(del_query)
File "/usr/local/lib/python3.8/dist-packages/django/db/models/deletion.py", line 243, in collect
new_objs = self.add(objs, source, nullable,
File "/usr/local/lib/python3.8/dist-packages/django/db/models/deletion.py", line 107, in add
if not objs:
File "/usr/local/lib/python3.8/dist-packages/django/db/models/query.py", line 284, in __bool__
self._fetch_all()
File "/usr/local/lib/python3.8/dist-packages/django/db/models/query.py", line 1324, in _fetch_all
self._result_cache = list(self._iterable_class(self))
File "/usr/local/lib/python3.8/dist-packages/django/db/models/query.py", line 51, in __iter__
results = compiler.execute_sql(chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size)
File "/usr/local/lib/python3.8/dist-packages/django/db/models/sql/compiler.py", line 1167, in execute_sql
cursor = self.connection.cursor()
File "/usr/local/lib/python3.8/dist-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/django/db/backends/base/base.py", line 259, in cursor
return self._cursor()
File "/usr/local/lib/python3.8/dist-packages/django/db/backends/base/base.py", line 237, in _cursor
return self._prepare_cursor(self.create_cursor(name))
File "/usr/local/lib/python3.8/dist-packages/django/db/utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/usr/local/lib/python3.8/dist-packages/django/db/backends/base/base.py", line 237, in _cursor
return self._prepare_cursor(self.create_cursor(name))
File "/usr/local/lib/python3.8/dist-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/django/db/backends/postgresql/base.py", line 236, in create_cursor
cursor = self.connection.cursor()
django.db.utils.InterfaceError: connection already closed


Basically it can tearDown the database, we can see it from:

File "/app/systems/tests/test_set_up.py", line 60, in tearDown User.objects.filter(email="jul@gmail.com").delete()
​

situation TWO class SystemServicesTest inherit direclty from TestCase and I set up my system directly in it's setUp method,

then the stack trace is:

Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/asgiref/sync.py", line 223, in __call__
return call_result.result()
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 437, in result
return self.__get_result()
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
raise self._exception
File "/usr/local/lib/python3.8/dist-packages/asgiref/sync.py", line 292, in main_wrap
result = await self.awaitable(*args, **kwargs)
File "/app/systems/tests/test_services.py", line 41, in test_getting_sessions_list
print(await service.get_session_list())
File "/app/systems/services/session.py", line 85, in get_session_list
vision_systems_ids: List[int] = await self.__vision_system_ids
File "/usr/local/lib/python3.8/dist-packages/async_property/base.py", line 37, in get_value
return await self._fget(instance)
File "/app/systems/services/session.py", line 47, in __vision_system_ids
data = await self.__vision_data()
File "/usr/local/lib/python3.8/dist-packages/asgiref/sync.py", line 444, in __call__
ret = await asyncio.wait_for(future, timeout=None)
File "/usr/lib/python3.8/asyncio/tasks.py", line 455, in wait_for
return await fut
File "/usr/local/lib/python3.8/dist-packages/asgiref/current_thread_executor.py", line 22, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.8/dist-packages/channels/db.py", line 13, in thread_handler
return super().thread_handler(loop, *args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/asgiref/sync.py", line 486, in thread_handler
return func(*args, **kwargs)
File "/app/systems/services/session.py", line 38, in __vision_data
return System.objects.get_vision_conf_data(self.system_uuid)
File "/app/systems/models/manager/manager.py", line 130, in get_vision_conf_data
gcloud: List[str] = [data[0] for data in visions]
File "/usr/local/lib/python3.8/dist-packages/django/db/models/query.py", line 280, in __iter__
self._fetch_all()
File "/usr/local/lib/python3.8/dist-packages/django/db/models/query.py", line 1324, in _fetch_all
self._result_cache = list(self._iterable_class(self))
File "/usr/local/lib/python3.8/dist-packages/django/db/models/query.py", line 140, in __iter__
return compiler.results_iter(tuple_expected=True, chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size)
File "/usr/local/lib/python3.8/dist-packages/django/db/models/sql/compiler.py", line 1124, in results_iter
results = self.execute_sql(MULTI, chunked_fetch=chunked_fetch, chunk_size=chunk_size)
File "/usr/local/lib/python3.8/dist-packages/django/db/models/sql/compiler.py", line 1167, in execute_sql
cursor = self.connection.cursor()
File "/usr/local/lib/python3.8/dist-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/django/db/backends/base/base.py", line 259, in cursor
return self._cursor()
File "/usr/local/lib/python3.8/dist-packages/django/db/backends/base/base.py", line 237, in _cursor
return self._prepare_cursor(self.create_cursor(name))
File "/usr/local/lib/python3.8/dist-packages/django/db/utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/usr/local/lib/python3.8/dist-packages/django/db/backends/base/base.py", line 237, in _cursor
return self._prepare_cursor(self.create_cursor(name))
File "/usr/local/lib/python3.8/dist-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/django/db/backends/postgresql/base.py", line 236, in create_cursor
cursor = self.connection.cursor()
django.db.utils.InterfaceError: connection already closed


The problem seems to come from:

File "/app/systems/models/manager/manager.py", line 130, in get_vision_conf_data gcloud: List[str] = [data[0] for data in visions]
​

Here is the full method:

def get_vision_conf_data(self, system_uuid: str) -> Tuple[List[str], List[str], List[int]]:
""" Get vision conf data for a system
Parameters
----------
system_uuid : str
The system uuid
Returns
-------
Tuple[List[str], List[str], List[int]]
List[str]: List of all visions gcloud id
List[str]: List of all visions uuid
List[str]: List of all visions system_id
"""
visions: QuerySet = self.get_queryset().get_vision_conf_details(system_uuid)

gcloud: List[str] = [data[0] for data in visions]
uuid: List[str] = [data[1] for data in visions]
system_id: List[int] = [data[2] for data in visions]

return gcloud, uuid, system_id


where get_vision_conf_details is a call to my db.

How should I test my class ? Any help would be great ! thanks.

Continue reading...