diff --git a/openrag/services/inference/vllm_client.py b/openrag/services/inference/vllm_client.py index ef5befaf3..8138c61f3 100644 --- a/openrag/services/inference/vllm_client.py +++ b/openrag/services/inference/vllm_client.py @@ -115,6 +115,22 @@ def _log_safe_error_detail(exc: BaseException) -> dict: # --------------------------------------------------------------------------- +def _strip_falsy_logprobs(payload: dict) -> dict: + """Drop a falsy ``logprobs`` (and its dependent ``top_logprobs``) from *payload*. + + ``logprobs: false`` is already the OpenAI default, so sending it adds + nothing — but strict providers whose schema lacks the field reject it by + name whatever the value, e.g. Gemini. Without + this, the config default (``LLMParamsConfig.logprobs = False``) lands in + ``self._defaults`` and is sent on every request. A truthy value is a + deliberate opt-in and is forwarded as-is. + """ + if not payload.get("logprobs"): + payload.pop("logprobs", None) + payload.pop("top_logprobs", None) + return payload + + @llm_registry.register("vllm") class VLLMClient(LLM): """OpenAI-compatible LLM client backed by vLLM. @@ -192,6 +208,7 @@ def _chat_payload_kwargs(self, kwargs: dict) -> dict: chat_template_kwargs = dict(payload_kwargs.get("chat_template_kwargs") or {}) chat_template_kwargs.setdefault("enable_thinking", enable_thinking) payload_kwargs["chat_template_kwargs"] = chat_template_kwargs + payload_kwargs = _strip_falsy_logprobs(payload_kwargs) return payload_kwargs @with_circuit_breaker("llm") @@ -200,6 +217,7 @@ async def generate(self, prompt: str, **kwargs) -> dict: base_url, model, headers = self._resolve_overrides(kwargs) kwargs.pop("metadata", None) payload = {**self._defaults, **kwargs, "model": model, "prompt": prompt} + payload = _strip_falsy_logprobs(payload) try: resp = await self._client.post(f"{base_url}/completions", json=payload, headers=headers) resp.raise_for_status() diff --git a/tests/unit/services/inference/test_vllm_client.py b/tests/unit/services/inference/test_vllm_client.py index a20ba34e2..14bd500d3 100644 --- a/tests/unit/services/inference/test_vllm_client.py +++ b/tests/unit/services/inference/test_vllm_client.py @@ -233,6 +233,63 @@ def capture(req: httpx.Request) -> httpx.Response: assert "max_retries" not in captured + @pytest.mark.asyncio + async def test_falsy_config_logprobs_not_forwarded_on_chat(self): + """The DI container forwards LLMParamsConfig.logprobs (default False) + into self._defaults. `logprobs: false` is the OpenAI default, so it adds + nothing — but strict providers whose schema lacks the field reject it by + name whatever the value, e.g. Gemini""" + captured: dict = {} + + def capture(req: httpx.Request) -> httpx.Response: + captured.update(json.loads(req.content)) + return _chat_response() + + await self._make_client(capture, logprobs=False).chat([{"role": "user", "content": "hi"}]) + + assert "logprobs" not in captured + + @pytest.mark.asyncio + async def test_falsy_config_logprobs_not_forwarded_on_generate(self): + captured: dict = {} + + def capture(req: httpx.Request) -> httpx.Response: + captured.update(json.loads(req.content)) + return _completions_response() + + await self._make_client(capture, logprobs=False).generate("hi") + + assert "logprobs" not in captured + + @pytest.mark.asyncio + async def test_falsy_request_logprobs_dropped_with_top_logprobs(self): + """A client sending `logprobs: false` alongside `top_logprobs` must not + leak either: top_logprobs is only meaningful when logprobs is on.""" + captured: dict = {} + + def capture(req: httpx.Request) -> httpx.Response: + captured.update(json.loads(req.content)) + return _chat_response() + + await self._make_client(capture).chat([{"role": "user", "content": "hi"}], logprobs=False, top_logprobs=3) + + assert "logprobs" not in captured + assert "top_logprobs" not in captured + + @pytest.mark.asyncio + async def test_truthy_logprobs_forwarded(self): + """An explicit opt-in must keep flowing through unchanged.""" + captured: dict = {} + + def capture(req: httpx.Request) -> httpx.Response: + captured.update(json.loads(req.content)) + return _chat_response() + + await self._make_client(capture).chat([{"role": "user", "content": "hi"}], logprobs=True, top_logprobs=5) + + assert captured["logprobs"] is True + assert captured["top_logprobs"] == 5 + @pytest.mark.asyncio async def test_trailing_slash_stripped(self): c = VLLMClient(endpoint="http://vllm:8000/v1/", model_name="m")