[Python] Django ORM sporadically dies when ran in fastAPI endpoints using gunicorn [duplicate]

Other related Questions mention using either django.db's close_old_connections method or looping through the django connections and calling close_if_unusable_or_obsolete on each.

I've tried doing this as a a fastapi middlewear and the issue persists just the same:

class OptOutOfRandomConnectionCrashFastapiMiddleware(BaseHTTPMiddleware):
def __init__(self, app: ASGIApp, *args, **kwargs) -> None:
super().__init__(app)

async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
# Pre-processing
response = await call_next(request)
app = request.app
if not hasattr(app.state, "last_pruned"):
app.state.last_pruned = 0

if app.state.last_pruned + 5 < time.time():
start = time.time()
app.state.last_pruned = start
for conn in connections.all():
conn.close_if_unusable_or_obsolete()
print(f"Pruned all connections. Took {round((time.time() - start) * 1000, 1)} miliseconds")


# Post-processing
return response


I'm using the django ORM outside of a django app, in async fastapi endpoints that I'm running with gunicorn.

All works fine except that once in a blue moon I get these odd errors where a worker seemingly "goes defunct" and is unable to process any requests due to database connections dropping.

I've attached a stack trace bellow but they really make no sense whatsoerver to me.

I'm using postgres without any short timeouts (database side) and the way I setup the connectiong enables healthchecks at every request, which I'd assume would get rid of any "stale connection" style issues:

DATABASES = {
"default": dj_database_url.config(
default="postgres://postgresass@localhost:5432/db_name",
conn_max_age=300,
conn_health_checks=True,
)
}


I'm curious if anyone has any idea as to how I'd go about debugging this? It's really making the django ORM unusable due to apps going down spontanously.

Stacktrace bellow is an example of the kind of error I get here:

Traceback (most recent call last):
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/uvicorn/protocols/http/h11_impl.py", line 403, in
run_asgi
result = await app( # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in
__call__
return await self.app(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/fastapi/applications.py", line 1054, in __call__
await super().__call__(scope, receive, send)
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/starlette/applications.py", line 113, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/starlette/middleware/errors.py", line 187, in __c
all__
raise exc
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/starlette/middleware/errors.py", line 165, in __c
all__
await self.app(scope, receive, _send)
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/starlette/middleware/cors.py", line 85, in __call
__
await self.app(scope, receive, send)
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/honeybadger/contrib/asgi.py", line 109, in _run_a
sgi3 File "/home/ubuntu/py_venv/lib/python3.12/site-packages/honeybadger/contrib/asgi.py", line 118, in _run_a
pp
raise exc from None
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/honeybadger/contrib/asgi.py", line 115, in _run_a
pp
return await callback()
^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/starlette/middleware/exceptions.py", line 62, in
__call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wra
pped_app
raise exc
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wra
pped_app
await app(scope, receive, sender)
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/starlette/routing.py", line 715, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/starlette/routing.py", line 735, in app
await route.handle(scope, receive, send)
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/starlette/routing.py", line 288, in handle
await self.app(scope, receive, send)
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/starlette/routing.py", line 76, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wra
pped_app
raise exc
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wra
pped_app
await app(scope, receive, sender)
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/starlette/routing.py", line 73, in app
response = await f(request)
^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/fastapi/routing.py", line 301, in app
raw_response = await run_endpoint_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/fastapi/routing.py", line 212, in run_endpoint_fu
nction
return await dependant.call(**values)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/stateshift/main.py", line 181, in email_exists
user = await User.objects.filter(email=email).afirst() ^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/fastapi/routing.py", line 301, in app
raw_response = await run_endpoint_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/fastapi/routing.py", line 212, in run_endpoint_function
return await dependant.call(**values)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/stateshift/main.py", line 181, in email_exists
user = await User.objects.filter(email=email).afirst()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/django/db/models/query.py", line 1101, in afirst
return await sync_to_async(self.first)()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/asgiref/sync.py", line 468, in __call__
ret = await asyncio.shield(exec_coro)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/asgiref/sync.py", line 522, in thread_handler
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/django/db/models/query.py", line 1097, in first
for obj in queryset[:1]:
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/django/db/models/query.py", line 400, in __iter__
self._fetch_all()
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/django/db/models/query.py", line 1928, in _fetch_all
self._result_cache = list(self._iterable_class(self))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/django/db/models/query.py", line 91, in __iter__
results = compiler.execute_sql(
^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/django/db/models/sql/compiler.py", line 1572, in execute_sql
cursor = self.connection.cursor()
^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/django/db/backends/base/base.py", line 320, in cursor
return self._cursor()
^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/django/db/backends/base/base.py", line 297, in _cursor
with self.wrap_database_errors:
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/django/db/utils.py", line 91, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/django/db/backends/base/base.py", line 298, in _cursor
return self._prepare_cursor(self.create_cursor(name))
^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/py_venv/lib/python3.12/site-packages/django/db/backends/postgresql/base.py", line 429, in create_cursor
cursor = self.connection.cursor()

Continue reading...