From de475a9a1f608045c25f17d37a4e5aae288a8927 Mon Sep 17 00:00:00 2001 From: Buddhaprakash Patil Date: Fri, 7 Aug 2026 22:59:31 +0530 Subject: [PATCH] fix(litellm): close Token Spy client when worker is cancelled ODSTokenSpyCallback._run() created an httpx.AsyncClient inside an async-with each time the worker task was (re)created. Under cancellation (LiteLLM model swap, gunicorn recycle, asyncio shutdown race) the __aexit__ could be skipped, leaking sockets and file descriptors that compounded with every restart. Create the client explicitly and close it in a finally block so aclose runs on both cancellation and normal exit. --- .../services/litellm/ods_token_spy_callback.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ods/extensions/services/litellm/ods_token_spy_callback.py b/ods/extensions/services/litellm/ods_token_spy_callback.py index 59b113615..70d035c66 100644 --- a/ods/extensions/services/litellm/ods_token_spy_callback.py +++ b/ods/extensions/services/litellm/ods_token_spy_callback.py @@ -195,9 +195,8 @@ async def _run(self) -> None: timeout = max( 0.1, float(os.environ.get("ODS_LITELLM_TELEMETRY_TIMEOUT", "3")) ) - async with httpx.AsyncClient( - follow_redirects=False, timeout=timeout - ) as client: + client = httpx.AsyncClient(follow_redirects=False, timeout=timeout) + try: while True: event = await self.queue.get() try: @@ -215,6 +214,13 @@ async def _run(self) -> None: self._warn(f"Token Spy telemetry unavailable: {exc}") finally: self.queue.task_done() + finally: + # Release connections and file descriptors even when the worker is + # cancelled (LiteLLM model swap / gunicorn recycle / asyncio + # shutdown). Relying only on async-with __aexit__ can leave the + # client unclosed when a task is cancelled mid-cycle, compounding + # a leak across restarts. + await client.aclose() def _warn(self, message: str) -> None: now = time.monotonic()